diff options
| author | Alex Gorrod <alexg@wiredtiger.com> | 2016-02-29 16:12:10 +1100 |
|---|---|---|
| committer | Alex Gorrod <alexg@wiredtiger.com> | 2016-02-29 16:13:10 +1100 |
| commit | ff0846809805c5a9a961314d3b6ec9ed7bbe0947 (patch) | |
| tree | 9e48c798ee5718e97e13d064c64f15524b92bdd2 /src/third_party | |
| parent | 81efb319f05f4ec1b5bd4523b13694241e967fc2 (diff) | |
| download | mongo-ff0846809805c5a9a961314d3b6ec9ed7bbe0947.tar.gz | |
Import wiredtiger-wiredtiger-2.7.0-829-g4531b92.tar.gz from wiredtiger branch mongodb-3.4
ref: 4f38287..4531b92
SERVER-22784 Coverity analysis defect 77722: Unused value
SERVER-22831 Low query rate with heavy cache pressure and an idle collection
WT-2173 test/format cache stuck full
WT-2264 Checkpoints cannot keep up with inserts
WT-2349 Add ability to open databases read-only
WT-2376 Modules should compile without including wt_internal.h header file
WT-2382 Problem with custom collator for 'u' format with join cursor
WT-2394 Long Unit Test for test_compact02 failed.
WT-2395 Recovery failure with an LSM tree
WT-2399 Add test case that verifies cursor traversal
WT-2405 test utility error handling.
WT-2407 recovery lint
WT-2409 Minor Perf Regression in LSM
WT-2410 Casting function pointers to different types
WT-2411 LSM drop hang
WT-2412 Truncate error tag is incorrect
WT-2417 Windows Jenkins task is failing
WT-2419 Tests fail to compile on windows due to new util functions
WT-2420 LSM failed to read bytes
WT-2423 Session reference count could be wrong if adding the handle fails
WT-2425 evict-btree read through performance drop
WT-2428 Make statistics logging compatible with MongoDB
WT-2429 Add a statistic that tracks aggressive mode in eviction
Diffstat (limited to 'src/third_party')
131 files changed, 3952 insertions, 1905 deletions
diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 3cb20ff2b26..2544bbd371c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -134,9 +134,11 @@ config_free(CONFIG *cfg) } cleanup_truncate_config(cfg); + free(cfg->base_uri); free(cfg->ckptthreads); + free(cfg->partial_config); free(cfg->popthreads); - free(cfg->base_uri); + free(cfg->reopen_config); free(cfg->workers); free(cfg->workload); } @@ -157,13 +159,19 @@ config_compress(CONFIG *cfg) cfg->compress_ext = NULL; cfg->compress_table = NULL; } else if (strcmp(s, "lz4") == 0) { +#ifndef HAVE_BUILTIN_EXTENSION_LZ4 cfg->compress_ext = LZ4_EXT; +#endif cfg->compress_table = LZ4_BLK; } else if (strcmp(s, "snappy") == 0) { +#ifndef HAVE_BUILTIN_EXTENSION_SNAPPY cfg->compress_ext = SNAPPY_EXT; +#endif cfg->compress_table = SNAPPY_BLK; } else if (strcmp(s, "zlib") == 0) { +#ifndef HAVE_BUILTIN_EXTENSION_ZLIB cfg->compress_ext = ZLIB_EXT; +#endif cfg->compress_table = ZLIB_BLK; } else { fprintf(stderr, @@ -634,6 +642,9 @@ config_opt_str(CONFIG *cfg, const char *name, const char *value) int config_sanity(CONFIG *cfg) { + WORKLOAD *workp; + u_int i; + /* Various intervals should be less than the run-time. */ if (cfg->run_time > 0 && ((cfg->checkpoint_threads != 0 && @@ -660,6 +671,17 @@ config_sanity(CONFIG *cfg) "Invalid pareto distribution - should be a percentage\n"); return (EINVAL); } + + if (cfg->readonly && cfg->workload != NULL) + for (i = 0, workp = cfg->workload; + i < cfg->workload_cnt; ++i, ++workp) + if (workp->insert != 0 || workp->update != 0 || + workp->truncate != 0) { + fprintf(stderr, + "Invalid workload: insert, update or " + "truncate specified with readonly\n"); + return (EINVAL); + } return (0); } diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf new file mode 100644 index 00000000000..d79af2b762b --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf @@ -0,0 +1,12 @@ +# wtperf options file: evict btree configuration +conn_config="cache_size=50M" +table_config="type=file" +icount=10000000 +report_interval=5 +run_time=120 +populate_threads=1 +readonly=true +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf new file mode 100644 index 00000000000..fe45c0e93b6 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf @@ -0,0 +1,13 @@ +# wtperf options file: evict lsm configuration +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" +compact=true +icount=10000000 +report_interval=5 +run_time=120 +populate_threads=1 +readonly=true +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index b2e68198e9a..b6e10762d8c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -33,6 +33,7 @@ static const CONFIG default_cfg = { "WT_TEST", /* home */ "WT_TEST", /* monitor dir */ NULL, /* partial logging */ + NULL, /* reopen config */ NULL, /* base_uri */ NULL, /* uris */ NULL, /* helium_mount */ @@ -1517,7 +1518,7 @@ close_reopen(CONFIG *cfg) { int ret; - if (!cfg->reopen_connection) + if (!cfg->readonly && !cfg->reopen_connection) return (0); /* * Reopen the connection. We do this so that the workload phase always @@ -1533,7 +1534,7 @@ close_reopen(CONFIG *cfg) return (ret); } if ((ret = wiredtiger_open( - cfg->home, NULL, cfg->conn_config, &cfg->conn)) != 0) { + cfg->home, NULL, cfg->reopen_config, &cfg->conn)) != 0) { lprintf(cfg, ret, 0, "Re-opening the connection failed"); return (ret); } @@ -2297,9 +2298,25 @@ main(int argc, char *argv[]) req_len = strlen(cfg->table_config) + strlen(LOG_PARTIAL_CONFIG) + 1; cfg->partial_config = dcalloc(req_len, 1); - snprintf((char *)cfg->partial_config, req_len, "%s%s", - (char *)cfg->table_config, LOG_PARTIAL_CONFIG); + snprintf(cfg->partial_config, req_len, "%s%s", + cfg->table_config, LOG_PARTIAL_CONFIG); } + /* + * Set the config for reopen. If readonly add in that string. + * If not readonly then just copy the original conn_config. + */ + if (cfg->readonly) + req_len = strlen(cfg->conn_config) + + strlen(READONLY_CONFIG) + 1; + else + req_len = strlen(cfg->conn_config) + 1; + cfg->reopen_config = dcalloc(req_len, 1); + if (cfg->readonly) + snprintf(cfg->reopen_config, req_len, "%s%s", + cfg->conn_config, READONLY_CONFIG); + else + snprintf(cfg->reopen_config, req_len, "%s", + cfg->conn_config); /* Sanity-check the configuration. */ if ((ret = config_sanity(cfg)) != 0) diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index 929880b0aef..65ee4963276 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -138,6 +138,7 @@ typedef struct { } THROTTLE_CONFIG; #define LOG_PARTIAL_CONFIG ",log=(enabled=false)" +#define READONLY_CONFIG ",readonly=true" /* * NOTE: If you add any fields to this structure here, you must also add * an initialization in wtperf.c in the default_cfg. @@ -145,7 +146,8 @@ typedef struct { struct __config { /* Configuration structure */ const char *home; /* WiredTiger home */ const char *monitor_dir; /* Monitor output dir */ - const char *partial_config; /* Config string for partial logging */ + char *partial_config; /* Config string for partial logging */ + char *reopen_config; /* Config string for conn reopen */ char *base_uri; /* Object URI */ char **uris; /* URIs if multiple tables */ const char *helium_mount; /* Optional Helium mount point */ diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index 60bbaff56e5..ecc1f216299 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -145,6 +145,10 @@ DEF_OPT_AS_UINT32(random_range, 0, "insert operations") DEF_OPT_AS_BOOL(random_value, 0, "generate random content for the value") DEF_OPT_AS_UINT32(read_range, 0, "scan a range of keys after each search") +DEF_OPT_AS_BOOL(readonly, 0, + "reopen the connection between populate and workload phases in readonly " + "mode. Requires reopen_connection turned on (default). Requires that " + "read be the only workload specified") DEF_OPT_AS_BOOL(reopen_connection, 1, "close and reopen the connection between populate and workload phases") DEF_OPT_AS_UINT32(report_interval, 2, diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index bc4283a4876..14258666d84 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -31,6 +31,7 @@ test/fops test/format test/huge test/packing +test/readonly test/recovery test/salvage test/thread diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index c386c0b345d..5575bd9f790 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -522,6 +522,9 @@ connection_runtime_config = [ the statistics log server uses a session from the configured session_max''', type='category', subconfig=[ + Config('json', 'false', r''' + encode statistics in JSON format''', + type='boolean'), Config('on_close', 'false', r'''log statistics on database close''', type='boolean'), Config('path', '"WiredTigerStat.%d.%H"', r''' @@ -538,7 +541,8 @@ connection_runtime_config = [ type='list'), Config('timestamp', '"%b %d %H:%M:%S"', r''' a timestamp prepended to each log record, may contain strftime - conversion specifications'''), + conversion specifications, when \c json is configured, defaults + to \c "%FT%Y.000Z"'''), Config('wait', '0', r''' seconds to wait between each write of the log records; setting this value above 0 configures statistics logging''', @@ -655,6 +659,11 @@ wiredtiger_open_common = connection_runtime_config + [ RPC server for primary processes and use RPC for secondary processes). <b>Not yet supported in WiredTiger</b>''', type='boolean'), + Config('readonly', 'false', r''' + open connection in read-only mode. The database must exist. All + methods that may modify a database are disabled. See @ref readonly + for more information''', + type='boolean'), Config('session_max', '100', r''' maximum expected number of sessions (including server threads)''', diff --git a/src/third_party/wiredtiger/dist/api_err.py b/src/third_party/wiredtiger/dist/api_err.py index 09332d508a2..a17c68ee196 100644 --- a/src/third_party/wiredtiger/dist/api_err.py +++ b/src/third_party/wiredtiger/dist/api_err.py @@ -56,6 +56,8 @@ errors = [ This error is generated when wiredtiger_open is configured to run in-memory, and an insert or update operation requires more than the configured cache size to complete.''', undoc=True), + Error('WT_PERM_DENIED', -31808, + 'permission denied (internal)', undoc=True), ] # Update the #defines in the wiredtiger.in file. diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index d5784630ab8..f500e3b1ae1 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -99,6 +99,7 @@ flags = { 'CONN_LOG_SERVER_RUN', 'CONN_LSM_MERGE', 'CONN_PANIC', + 'CONN_READONLY', 'CONN_SERVER_ASYNC', 'CONN_SERVER_CHECKPOINT', 'CONN_SERVER_LSM', diff --git a/src/third_party/wiredtiger/dist/s_export b/src/third_party/wiredtiger/dist/s_export index 1212b5b2c1f..8a2c701d27f 100755 --- a/src/third_party/wiredtiger/dist/s_export +++ b/src/third_party/wiredtiger/dist/s_export @@ -12,10 +12,7 @@ Darwin) *) # We require GNU nm, which may not be installed. type nm > /dev/null 2>&1 && - (nm --version | grep 'GNU nm') > /dev/null 2>&1 || { - echo 'skipped: GNU nm not found' - exit 0 - } + (nm --version | grep 'GNU nm') > /dev/null 2>&1 || exit 0 NM='nm --extern-only --defined-only --print-file-name $f' ;; esac @@ -28,7 +25,9 @@ check() sed 's/.* //' | egrep -v '^__wt') | sort | - uniq -u > $t + uniq -u | + egrep -v \ + 'zlib_extension_init|lz4_extension_init|snappy_extension_init' > $t test -s $t && { echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 7a8f3a9b0bd..c8769abdfbe 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -107,11 +107,13 @@ FALLTHROUGH FH FLD FLSv +FLv FNV FORALL FOREACH FULLFSYNC FindFirstFile +Fixup Fk FlushFileBuffers Fprintf @@ -211,6 +213,7 @@ MySecret NEEDKEY NEEDVALUE NOLL +NOLOCK NONINFRINGEMENT NOTFOUND NOTREACHED @@ -244,6 +247,7 @@ Preload Prepend Qsort RCS +RDNOLOCK RECNO REF's REFs @@ -256,6 +260,7 @@ RNG RPC RUNDIR Radu +Readonly Rebalance RedHat Redistributions @@ -329,6 +334,7 @@ VxWorks WAL WIREDTIGER WRLSN +WRNOLOCK WakeAllConditionVariable Wconditional WeakHashLen @@ -431,6 +437,8 @@ cfg cfkos change's changelog +chdir +checkfmt checkpointed checkpointer checkpointing @@ -438,6 +446,7 @@ checksum checksums children's chk +chmod chongo cip cjoin @@ -502,6 +511,7 @@ datasets datasource datastore dbc +dbs dcalloc decile deciles @@ -671,6 +681,7 @@ inline inmem insertK insertV +inserters instantiation intl intnum @@ -717,6 +728,7 @@ libwiredtiger llll llu loadtext +localTime localtime logf logmgr @@ -795,6 +807,7 @@ nhex nlpo nocase nocrypto +nolock nonliteral noop nop @@ -872,6 +885,7 @@ rS rb rbrace rbracket +rdonly rduppo readlock readonly @@ -901,6 +915,7 @@ rng rocksdb rotN rotn +rp rpc run's runtime @@ -1057,9 +1072,12 @@ vsize vsnprintf vtype vunpack +vw +waitpid walk's warmup wb +wiredTiger wiredtiger workFactor wrapup diff --git a/src/third_party/wiredtiger/dist/stat.py b/src/third_party/wiredtiger/dist/stat.py index 6dcfccfeab5..7961bf7053f 100644 --- a/src/third_party/wiredtiger/dist/stat.py +++ b/src/third_party/wiredtiger/dist/stat.py @@ -98,11 +98,11 @@ for line in open('../src/include/wiredtiger.in', 'r'): f.close() compare_srcfile(tmp_file, '../src/include/wiredtiger.in') -def print_func(name, handle, list): +def print_func(name, handle, statlist): '''Print the structures/functions for the stat.c file.''' f.write('\n') f.write('static const char * const __stats_' + name + '_desc[] = {\n') - for l in list: + for l in statlist: f.write('\t"' + l.desc + '",\n') f.write('};\n') @@ -143,7 +143,7 @@ void __wt_stat_''' + name + '_clear_single(WT_' + name.upper() + '''_STATS *stats) { ''') - for l in sorted(list): + for l in statlist: # no_clear: don't clear the value. if 'no_clear' in l.flags: f.write('\t\t/* not clearing ' + l.name + ' */\n') @@ -170,7 +170,7 @@ __wt_stat_''' + name + '''_aggregate_single( WT_''' + name.upper() + '_STATS *from, WT_' + name.upper() + '''_STATS *to) { ''') - for l in sorted(list): + for l in statlist: if 'max_aggregate' in l.flags: o = '\tif (from->' + l.name + ' > to->' + l.name + ')\n' +\ '\t\tto->' + l.name + ' = from->' + l.name + ';\n' @@ -190,11 +190,11 @@ __wt_stat_''' + name + '''_aggregate( # Connection level aggregation does not currently have any computation # of a maximum value; I'm leaving in support for it, but don't declare # a temporary variable until it's needed. - for l in sorted(list): + for l in statlist: if 'max_aggregate' in l.flags: f.write('\tint64_t v;\n\n') break; - for l in sorted(list): + for l in statlist: if 'max_aggregate' in l.flags: o = '\tif ((v = WT_STAT_READ(from, ' + l.name + ')) > ' +\ 'to->' + l.name + ')\n' diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 41a93961079..be4bacaece7 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -157,6 +157,7 @@ connection_stats = [ CacheStat('cache_bytes_overflow', 'tracked bytes belonging to overflow pages in the cache', 'no_clear,no_scale'), CacheStat('cache_bytes_read', 'bytes read into cache'), CacheStat('cache_bytes_write', 'bytes written from cache'), + CacheStat('cache_eviction_aggressive_set', 'eviction currently operating in aggressive mode', 'no_clear,no_scale'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'), CacheStat('cache_eviction_clean', 'unmodified pages evicted'), @@ -315,7 +316,7 @@ connection_stats = [ YieldStat('page_sleep', 'page acquire time sleeping (usecs)'), ] -connection_stats = sorted(connection_stats, key=attrgetter('name')) +connection_stats = sorted(connection_stats, key=attrgetter('desc')) ########################################## # Data source statistics @@ -457,7 +458,7 @@ dsrc_stats = [ TxnStat('txn_update_conflict', 'update conflicts'), ] -dsrc_stats = sorted(dsrc_stats, key=attrgetter('name')) +dsrc_stats = sorted(dsrc_stats, key=attrgetter('desc')) ########################################## # Cursor Join statistics @@ -468,4 +469,4 @@ join_stats = [ JoinStat('bloom_false_positive', 'bloom filter false positives'), ] -join_stats = sorted(join_stats, key=attrgetter('name')) +join_stats = sorted(join_stats, key=attrgetter('desc')) diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index 587204efff1..72fd98aff7b 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -12,6 +12,7 @@ noinst_PROGRAMS = \ ex_cursor \ ex_data_source \ ex_encrypt \ + ex_event_handler \ ex_extending \ ex_extractor \ ex_hello \ diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 418c99ad6a3..1c036b75461 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -346,8 +346,7 @@ cursor_ops(WT_SESSION *session) cursor->set_key(cursor, key); if ((ret = cursor->remove(cursor)) != 0) { fprintf(stderr, - "cursor.remove: %s\n", - cursor->session->strerror(cursor->session, ret)); + "cursor.remove: %s\n", wiredtiger_strerror(ret)); return (ret); } /*! [Display an error] */ @@ -359,7 +358,8 @@ cursor_ops(WT_SESSION *session) cursor->set_key(cursor, key); if ((ret = cursor->remove(cursor)) != 0) { fprintf(stderr, - "cursor.remove: %s\n", session->strerror(session, ret)); + "cursor.remove: %s\n", + cursor->session->strerror(cursor->session, ret)); return (ret); } /*! [Display an error thread safe] */ diff --git a/src/third_party/wiredtiger/examples/c/ex_event_handler.c b/src/third_party/wiredtiger/examples/c/ex_event_handler.c new file mode 100644 index 00000000000..ba6807cd56d --- /dev/null +++ b/src/third_party/wiredtiger/examples/c/ex_event_handler.c @@ -0,0 +1,136 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_event_handler.c + * Demonstrate how to use the WiredTiger event handler mechanism. + * + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> + +static const char *home; + +int handle_wiredtiger_error( + WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +int handle_wiredtiger_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); + +/*! [Function event_handler] */ +/* + * Create our own event handler structure to allow us to pass context through + * to event handler callbacks. For this to work the WiredTiger event handler + * must appear first in our custom event handler structure. + */ +typedef struct { + WT_EVENT_HANDLER h; + const char *app_id; +} CUSTOM_EVENT_HANDLER; + +/* + * handle_wiredtiger_error -- + * Function to handle error callbacks from WiredTiger. + */ +int +handle_wiredtiger_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) +{ + CUSTOM_EVENT_HANDLER *custom_handler; + + /* Cast the handler back to our custom handler. */ + custom_handler = (CUSTOM_EVENT_HANDLER *)handler; + + /* Report the error on the console. */ + fprintf(stderr, + "app_id %s, thread context %p, error %d, message %s\n", + custom_handler->app_id, session, error, message); + + return (0); +} + +/* + * handle_wiredtiger_message -- + * Function to handle message callbacks from WiredTiger. + */ +int +handle_wiredtiger_message( + WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) +{ + /* Cast the handler back to our custom handler. */ + printf("app id %s, thread context %p, message %s\n", + ((CUSTOM_EVENT_HANDLER *)handler)->app_id, session, message); + + return (0); +} +/*! [Function event_handler] */ + +static int +config_event_handler() +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + /*! [Configure event_handler] */ + CUSTOM_EVENT_HANDLER event_handler; + + event_handler.h.handle_error = handle_wiredtiger_error; + event_handler.h.handle_message = handle_wiredtiger_message; + /* Set handlers to NULL to use the default handler. */ + event_handler.h.handle_progress = NULL; + event_handler.h.handle_close = NULL; + event_handler.app_id = "example_event_handler"; + + ret = wiredtiger_open(home, + (WT_EVENT_HANDLER *)&event_handler, "create", &conn); + /*! [Configure event_handler] */ + + /* Make an invalid API call, to ensure the event handler works. */ + (void)conn->open_session(conn, NULL, "isolation=invalid", &session); + + if (ret == 0) + ret = conn->close(conn, NULL); + + return (ret); +} + +int +main(void) +{ + /* + * Create a clean test directory for this run of the test program if the + * environment variable isn't already set (as is done by make check). + */ + if (getenv("WIREDTIGER_HOME") == NULL) { + home = "WT_HOME"; + (void)system("rm -rf WT_HOME && mkdir WT_HOME"); + } else + home = NULL; + + return (config_event_handler()); +} diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java index 09db8e0fd56..5fe767d49bf 100644 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java +++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_all.java @@ -326,6 +326,22 @@ public static int cursor_ops(Session session) /*! [Display an error] */ } + { + /*! [Display an error thread safe] */ + try { + String key = "non-existent key"; + cursor.putKeyString(key); + if ((ret = cursor.remove()) != 0) { + System.err.println( + "cursor.remove: " + wiredtiger.wiredtiger_strerror(ret)); + return (ret); + } + } catch (WiredTigerException wte) { /* Catch severe errors. */ + System.err.println("cursor.remove exception: " + wte); + } + /*! [Display an error thread safe] */ + } + /*! [Close the cursor] */ ret = cursor.close(); /*! [Close the cursor] */ diff --git a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c index 062307b721a..35159d0fa76 100644 --- a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c @@ -26,13 +26,15 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <wt_internal.h> - #include <lz4.h> #include <errno.h> #include <stdlib.h> #include <string.h> +#include <wiredtiger_config.h> +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + /* Local compressor structure. */ typedef struct { WT_COMPRESSOR compressor; /* Must come first */ @@ -62,6 +64,22 @@ typedef struct { uint32_t unused; /* Guaranteed to be 0 */ } LZ4_PREFIX; +#ifdef WORDS_BIGENDIAN +/* + * lz4_bswap32 -- + * 32-bit unsigned little-endian to/from big-endian value. + */ +static inline uint32_t +lz4_bswap32(uint32_t v) +{ + return ( + ((v << 24) & 0xff000000) | + ((v << 8) & 0x00ff0000) | + ((v >> 8) & 0x0000ff00) | + ((v >> 24) & 0x000000ff) + ); +} + /* * lz4_prefix_swap -- * The additional information is written in little-endian format, handle @@ -70,15 +88,12 @@ typedef struct { static inline void lz4_prefix_swap(LZ4_PREFIX *prefix) { -#ifdef WORDS_BIGENDIAN - prefix->compressed_len = __wt_bswap32(prefix->compressed_len); - prefix->uncompressed_len = __wt_bswap32(prefix->uncompressed_len); - prefix->useful_len = __wt_bswap32(prefix->useful_len); - prefix->unused = __wt_bswap32(prefix->unused); -#else - WT_UNUSED(prefix); -#endif + prefix->compressed_len = lz4_bswap32(prefix->compressed_len); + prefix->uncompressed_len = lz4_bswap32(prefix->uncompressed_len); + prefix->useful_len = lz4_bswap32(prefix->useful_len); + prefix->unused = lz4_bswap32(prefix->unused); } +#endif /* * lz4_error -- @@ -127,7 +142,9 @@ lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, prefix.uncompressed_len = (uint32_t)src_len; prefix.useful_len = (uint32_t)src_len; prefix.unused = 0; +#ifdef WORDS_BIGENDIAN lz4_prefix_swap(&prefix); +#endif memcpy(dst, &prefix, sizeof(LZ4_PREFIX)); *result_lenp = (size_t)lz4_len + sizeof(LZ4_PREFIX); @@ -163,7 +180,9 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, * decompressed bytes to return from the start of the source buffer. */ memcpy(&prefix, src, sizeof(LZ4_PREFIX)); +#ifdef WORDS_BIGENDIAN lz4_prefix_swap(&prefix); +#endif /* * Decompress, starting after the prefix bytes. Use safe decompression: @@ -278,7 +297,9 @@ lz4_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, prefix.uncompressed_len = (uint32_t)sourceSize; prefix.useful_len = offsets[slot]; prefix.unused = 0; +#ifdef WORDS_BIGENDIAN lz4_prefix_swap(&prefix); +#endif memcpy(dst, &prefix, sizeof(LZ4_PREFIX)); *result_slotsp = slot; diff --git a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c index fcefb8bb575..981e334a2de 100644 --- a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c @@ -26,13 +26,15 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <wt_internal.h> - #include <snappy-c.h> #include <errno.h> #include <stdlib.h> #include <string.h> +#include <wiredtiger_config.h> +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + /* Local compressor structure. */ typedef struct { WT_COMPRESSOR compressor; /* Must come first */ @@ -40,6 +42,27 @@ typedef struct { WT_EXTENSION_API *wt_api; /* Extension API */ } SNAPPY_COMPRESSOR; +#ifdef WORDS_BIGENDIAN +/* + * snappy_bswap64 -- + * 64-bit unsigned little-endian to/from big-endian value. + */ +static inline uint64_t +snappy_bswap64(uint64_t v) +{ + return ( + ((v << 56) & 0xff00000000000000UL) | + ((v << 40) & 0x00ff000000000000UL) | + ((v << 24) & 0x0000ff0000000000UL) | + ((v << 8) & 0x000000ff00000000UL) | + ((v >> 8) & 0x00000000ff000000UL) | + ((v >> 24) & 0x0000000000ff0000UL) | + ((v >> 40) & 0x000000000000ff00UL) | + ((v >> 56) & 0x00000000000000ffUL) + ); +} +#endif + /* * wt_snappy_error -- * Output an error message, and return a standard error code. @@ -109,7 +132,7 @@ wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, * Store the value in little-endian format. */ #ifdef WORDS_BIGENDIAN - snaplen = __wt_bswap64(snaplen); + snaplen = snappy_bswap64(snaplen); #endif *(size_t *)dst = snaplen; } else @@ -142,7 +165,7 @@ wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, */ snaplen = *(size_t *)src; #ifdef WORDS_BIGENDIAN - snaplen = __wt_bswap64(snaplen); + snaplen = snappy_bswap64(snaplen); #endif if (snaplen + sizeof(size_t) > src_len) { (void)wt_api->err_printf(wt_api, diff --git a/src/third_party/wiredtiger/src/block/block_mgr.c b/src/third_party/wiredtiger/src/block/block_mgr.c index dceaae8bb99..0bb75d129e1 100644 --- a/src/third_party/wiredtiger/src/block/block_mgr.c +++ b/src/third_party/wiredtiger/src/block/block_mgr.c @@ -69,6 +69,21 @@ __bm_checkpoint(WT_BM *bm, } /* + * __bm_checkpoint_readonly -- + * Write a buffer into a block, creating a checkpoint; readonly version. + */ +static int +__bm_checkpoint_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_cksum) +{ + WT_UNUSED(buf); + WT_UNUSED(ckptbase); + WT_UNUSED(data_cksum); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_checkpoint_load -- * Load a checkpoint. */ @@ -113,6 +128,16 @@ __bm_checkpoint_resolve(WT_BM *bm, WT_SESSION_IMPL *session) } /* + * __bm_checkpoint_resolve_readonly -- + * Resolve the checkpoint; readonly version. + */ +static int +__bm_checkpoint_resolve_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + +/* * __bm_checkpoint_unload -- * Unload a checkpoint point. */ @@ -161,6 +186,16 @@ __bm_compact_end(WT_BM *bm, WT_SESSION_IMPL *session) } /* + * __bm_compact_end_readonly -- + * End a block manager compaction; readonly version. + */ +static int +__bm_compact_end_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + +/* * __bm_compact_page_skip -- * Return if a page is useful for compaction. */ @@ -173,6 +208,21 @@ __bm_compact_page_skip(WT_BM *bm, WT_SESSION_IMPL *session, } /* + * __bm_compact_page_skip_readonly -- + * Return if a page is useful for compaction; readonly version. + */ +static int +__bm_compact_page_skip_readonly(WT_BM *bm, WT_SESSION_IMPL *session, + const uint8_t *addr, size_t addr_size, bool *skipp) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_size); + WT_UNUSED(skipp); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_compact_skip -- * Return if a file can be compacted. */ @@ -183,6 +233,18 @@ __bm_compact_skip(WT_BM *bm, WT_SESSION_IMPL *session, bool *skipp) } /* + * __bm_compact_skip_readonly -- + * Return if a file can be compacted; readonly version. + */ +static int +__bm_compact_skip_readonly(WT_BM *bm, WT_SESSION_IMPL *session, bool *skipp) +{ + WT_UNUSED(skipp); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_compact_start -- * Start a block manager compaction. */ @@ -193,6 +255,16 @@ __bm_compact_start(WT_BM *bm, WT_SESSION_IMPL *session) } /* + * __bm_compact_start_readonly -- + * Start a block manager compaction; readonly version. + */ +static int +__bm_compact_start_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + +/* * __bm_free -- * Free a block of space to the underlying file. */ @@ -204,6 +276,20 @@ __bm_free(WT_BM *bm, } /* + * __bm_free_readonly -- + * Free a block of space to the underlying file; readonly version. + */ +static int +__bm_free_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_size); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_is_mapped -- * Return if the file is mapped into memory. */ @@ -226,6 +312,31 @@ __bm_salvage_end(WT_BM *bm, WT_SESSION_IMPL *session) } /* + * __bm_salvage_end_readonly -- + * End a block manager salvage; readonly version. + */ +static int +__bm_salvage_end_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + +/* + * __bm_salvage_next_readonly -- + * Return the next block from the file; readonly version. + */ +static int +__bm_salvage_next_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *eofp) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_sizep); + WT_UNUSED(eofp); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_salvage_next -- * Return the next block from the file. */ @@ -248,6 +359,16 @@ __bm_salvage_start(WT_BM *bm, WT_SESSION_IMPL *session) } /* + * __bm_salvage_start_readonly -- + * Start a block manager salvage; readonly version. + */ +static int +__bm_salvage_start_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + +/* * __bm_salvage_valid -- * Inform salvage a block is valid. */ @@ -260,6 +381,21 @@ __bm_salvage_valid(WT_BM *bm, } /* + * __bm_salvage_valid_readonly -- + * Inform salvage a block is valid; readonly version. + */ +static int +__bm_salvage_valid_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, uint8_t *addr, size_t addr_size, bool valid) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_size); + WT_UNUSED(valid); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_stat -- * Block-manager statistics. */ @@ -283,6 +419,18 @@ __bm_sync(WT_BM *bm, WT_SESSION_IMPL *session, bool async) } /* + * __bm_sync_readonly -- + * Flush a file to disk; readonly version. + */ +static int +__bm_sync_readonly(WT_BM *bm, WT_SESSION_IMPL *session, bool async) +{ + WT_UNUSED(async); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_verify_addr -- * Verify an address. */ @@ -327,6 +475,23 @@ __bm_write(WT_BM *bm, WT_SESSION_IMPL *session, } /* + * __bm_write_readonly -- + * Write a buffer into a block, returning the block's address cookie; + * readonly version. + */ +static int +__bm_write_readonly(WT_BM *bm, WT_SESSION_IMPL *session, + WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_cksum) +{ + WT_UNUSED(buf); + WT_UNUSED(addr); + WT_UNUSED(addr_sizep); + WT_UNUSED(data_cksum); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_write_size -- * Return the buffer size required to write a block. */ @@ -337,84 +502,68 @@ __bm_write_size(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep) } /* + * __bm_write_size_readonly -- + * Return the buffer size required to write a block; readonly version. + */ +static int +__bm_write_size_readonly(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep) +{ + WT_UNUSED(sizep); + + return (__bm_readonly(bm, session)); +} + +/* * __bm_method_set -- * Set up the legal methods. */ static void __bm_method_set(WT_BM *bm, bool readonly) { + bm->addr_invalid = __bm_addr_invalid; + bm->addr_string = __bm_addr_string; + bm->block_header = __bm_block_header; + bm->checkpoint = __bm_checkpoint; + bm->checkpoint_load = __bm_checkpoint_load; + bm->checkpoint_resolve = __bm_checkpoint_resolve; + bm->checkpoint_unload = __bm_checkpoint_unload; + bm->close = __bm_close; + bm->compact_end = __bm_compact_end; + bm->compact_page_skip = __bm_compact_page_skip; + bm->compact_skip = __bm_compact_skip; + bm->compact_start = __bm_compact_start; + bm->free = __bm_free; + bm->is_mapped = __bm_is_mapped; + bm->preload = __wt_bm_preload; + bm->read = __wt_bm_read; + bm->salvage_end = __bm_salvage_end; + bm->salvage_next = __bm_salvage_next; + bm->salvage_start = __bm_salvage_start; + bm->salvage_valid = __bm_salvage_valid; + bm->size = __wt_block_manager_size; + bm->stat = __bm_stat; + bm->sync = __bm_sync; + bm->verify_addr = __bm_verify_addr; + bm->verify_end = __bm_verify_end; + bm->verify_start = __bm_verify_start; + bm->write = __bm_write; + bm->write_size = __bm_write_size; + if (readonly) { - bm->addr_invalid = __bm_addr_invalid; - bm->addr_string = __bm_addr_string; - bm->block_header = __bm_block_header; - bm->checkpoint = (int (*)(WT_BM *, WT_SESSION_IMPL *, - WT_ITEM *, WT_CKPT *, bool))__bm_readonly; - bm->checkpoint_load = __bm_checkpoint_load; - bm->checkpoint_resolve = - (int (*)(WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->checkpoint_unload = __bm_checkpoint_unload; - bm->close = __bm_close; - bm->compact_end = - (int (*)(WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->compact_page_skip = (int (*)(WT_BM *, WT_SESSION_IMPL *, - const uint8_t *, size_t, bool *))__bm_readonly; - bm->compact_skip = (int (*) - (WT_BM *, WT_SESSION_IMPL *, bool *))__bm_readonly; - bm->compact_start = - (int (*)(WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->free = (int (*)(WT_BM *, - WT_SESSION_IMPL *, const uint8_t *, size_t))__bm_readonly; - bm->is_mapped = __bm_is_mapped; - bm->preload = __wt_bm_preload; - bm->read = __wt_bm_read; - bm->salvage_end = (int (*) - (WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->salvage_next = (int (*)(WT_BM *, WT_SESSION_IMPL *, - uint8_t *, size_t *, bool *))__bm_readonly; - bm->salvage_start = (int (*) - (WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->salvage_valid = (int (*)(WT_BM *, - WT_SESSION_IMPL *, uint8_t *, size_t, bool))__bm_readonly; - bm->size = __wt_block_manager_size; - bm->stat = __bm_stat; - bm->sync = - (int (*)(WT_BM *, WT_SESSION_IMPL *, bool))__bm_readonly; - bm->verify_addr = __bm_verify_addr; - bm->verify_end = __bm_verify_end; - bm->verify_start = __bm_verify_start; - bm->write = (int (*)(WT_BM *, WT_SESSION_IMPL *, - WT_ITEM *, uint8_t *, size_t *, bool))__bm_readonly; - bm->write_size = (int (*) - (WT_BM *, WT_SESSION_IMPL *, size_t *))__bm_readonly; - } else { - bm->addr_invalid = __bm_addr_invalid; - bm->addr_string = __bm_addr_string; - bm->block_header = __bm_block_header; - bm->checkpoint = __bm_checkpoint; - bm->checkpoint_load = __bm_checkpoint_load; - bm->checkpoint_resolve = __bm_checkpoint_resolve; - bm->checkpoint_unload = __bm_checkpoint_unload; - bm->close = __bm_close; - bm->compact_end = __bm_compact_end; - bm->compact_page_skip = __bm_compact_page_skip; - bm->compact_skip = __bm_compact_skip; - bm->compact_start = __bm_compact_start; - bm->free = __bm_free; - bm->is_mapped = __bm_is_mapped; - bm->preload = __wt_bm_preload; - bm->read = __wt_bm_read; - bm->salvage_end = __bm_salvage_end; - bm->salvage_next = __bm_salvage_next; - bm->salvage_start = __bm_salvage_start; - bm->salvage_valid = __bm_salvage_valid; - bm->size = __wt_block_manager_size; - bm->stat = __bm_stat; - bm->sync = __bm_sync; - bm->verify_addr = __bm_verify_addr; - bm->verify_end = __bm_verify_end; - bm->verify_start = __bm_verify_start; - bm->write = __bm_write; - bm->write_size = __bm_write_size; + bm->checkpoint = __bm_checkpoint_readonly; + bm->checkpoint_resolve = __bm_checkpoint_resolve_readonly; + bm->compact_end = __bm_compact_end_readonly; + bm->compact_page_skip = __bm_compact_page_skip_readonly; + bm->compact_skip = __bm_compact_skip_readonly; + bm->compact_start = __bm_compact_start_readonly; + bm->free = __bm_free_readonly; + bm->salvage_end = __bm_salvage_end_readonly; + bm->salvage_next = __bm_salvage_next_readonly; + bm->salvage_start = __bm_salvage_start_readonly; + bm->salvage_valid = __bm_salvage_valid_readonly; + bm->sync = __bm_sync_readonly; + bm->write = __bm_write_readonly; + bm->write_size = __bm_write_size_readonly; } } diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index 12df19a7e04..9cc56c56452 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -96,14 +96,13 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) WT_BTREE *btree; WT_DECL_RET; WT_REF *ref; - bool block_manager_begin, skip; + bool skip; WT_UNUSED(cfg); btree = S2BT(session); bm = btree->bm; ref = NULL; - block_manager_begin = false; WT_STAT_FAST_DATA_INCR(session, session_compact); @@ -123,24 +122,12 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) * We need to ensure we don't race with page reconciliation as it's * writing the page modify information. * - * There are three ways we call reconciliation: checkpoints, threads - * writing leaf pages (usually in preparation for a checkpoint or if - * closing a file), and eviction. - * - * We're holding the schema lock which serializes with checkpoints. - */ - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)); - - /* - * Get the tree handle's flush lock which blocks threads writing leaf - * pages. + * There are two ways we call reconciliation: checkpoints and eviction. + * Get the tree's flush lock which blocks threads writing pages for + * checkpoints. */ __wt_spin_lock(session, &btree->flush_lock); - /* Start compaction. */ - WT_ERR(bm->compact_start(bm, session)); - block_manager_begin = true; - /* Walk the tree reviewing pages to see if they should be re-written. */ for (;;) { /* @@ -170,9 +157,6 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) err: if (ref != NULL) WT_TRET(__wt_page_release(session, ref, 0)); - if (block_manager_begin) - WT_TRET(bm->compact_end(bm, session)); - /* Unblock threads writing leaf pages. */ __wt_spin_unlock(session, &btree->flush_lock); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 2db3ca7d984..52152a2fcac 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -36,7 +36,8 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) btree = S2BT(session); /* Checkpoint files are readonly. */ - readonly = dhandle->checkpoint != NULL; + readonly = (dhandle->checkpoint != NULL || + F_ISSET(S2C(session), WT_CONN_READONLY)); /* Get the checkpoint information for this name/checkpoint pair. */ WT_CLEAR(ckpt); diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index bbfb06c636f..5273f0ee2c3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -23,12 +23,11 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_REF *walk; WT_TXN *txn; uint64_t internal_bytes, internal_pages, leaf_bytes, leaf_pages; - uint64_t saved_snap_min; + uint64_t oldest_id, saved_snap_min; uint32_t flags; bool evict_reset; btree = S2BT(session); - walk = NULL; txn = &session->txn; saved_snap_min = WT_SESSION_TXN_STATE(session)->snap_min; @@ -56,6 +55,15 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) return (0); } + /* + * Save the oldest transaction ID we need to keep around. + * Otherwise, in a busy system, we could be updating pages so + * fast that write leaves never catches up. We deliberately + * have no transaction running at this point that would keep + * the oldest ID from moving forwards as we walk the tree. + */ + oldest_id = __wt_txn_oldest_id(session); + flags |= WT_READ_NO_WAIT | WT_READ_SKIP_INTL; for (walk = NULL;;) { WT_ERR(__wt_tree_walk(session, &walk, flags)); @@ -64,13 +72,13 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) /* * Write dirty pages if nobody beat us to it. Don't - * try to write the hottest pages: checkpoint will have - * to visit them anyway. + * try to write hot pages (defined as pages that have + * been updated since the write phase leaves started): + * checkpoint will have to visit them anyway. */ page = walk->page; if (__wt_page_is_modified(page) && - __wt_txn_visible_all( - session, page->modify->update_txn)) { + WT_TXNID_LT(page->modify->update_txn, oldest_id)) { if (txn->isolation == WT_ISO_READ_COMMITTED) __wt_txn_get_snapshot(session); leaf_bytes += page->memory_footprint; diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index 55b11d7b2d1..bb8a750d848 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -583,14 +583,14 @@ restart: /* break; } WT_ERR(ret); + couple = ref; /* * A new page: configure for traversal of any internal * page's children, else return the leaf page. */ if (WT_PAGE_IS_INTERNAL(ref->page)) { -descend: couple = ref; - empty_internal = true; +descend: empty_internal = true; /* * There's a split race when a cursor is setting @@ -649,7 +649,6 @@ descend: couple = ref; */ if (skipleafcntp != NULL || LF_ISSET(WT_READ_SKIP_LEAF)) { - couple = ref; if (LF_ISSET(WT_READ_SKIP_LEAF)) break; if (*skipleafcntp > 0) { diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index 1ef8dd32bb4..3549e41e80d 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -58,6 +58,8 @@ __wt_las_create(WT_SESSION_IMPL *session) conn = S2C(session); + if (F_ISSET(conn, WT_CONN_READONLY)) + return (0); /* * Done at startup: we cannot do it on demand because we require the * schema lock to create and drop the table, and it may not always be diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 879de670695..6e88f9b4d14 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -99,6 +99,7 @@ static const WT_CONFIG_CHECK static const WT_CONFIG_CHECK confchk_wiredtiger_open_statistics_log_subconfigs[] = { + { "json", "boolean", NULL, NULL, NULL, 0 }, { "on_close", "boolean", NULL, NULL, NULL, 0 }, { "path", "string", NULL, NULL, NULL, 0 }, { "sources", "list", NULL, NULL, NULL, 0 }, @@ -146,7 +147,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evictserver\",\"fileops\",\"log\",\"lsm\"," @@ -544,6 +545,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -554,7 +556,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -624,6 +626,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -634,7 +637,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -701,6 +704,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -711,7 +715,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -776,6 +780,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -786,7 +791,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -853,7 +858,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_max=100MB,path=,prealloc=,recover=on,zero_fill=0)," "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(on_close=0," + "statistics=none,statistics_log=(json=0,on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", confchk_WT_CONNECTION_reconfigure, 18 @@ -1017,14 +1022,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=0,log=(archive=,compressor=,enabled=0,file_max=100MB," "path=,prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),use_environment=," "use_environment_priv=0,verbose=,write_through=", - confchk_wiredtiger_open, 37 + confchk_wiredtiger_open, 38 }, { "wiredtiger_open_all", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1038,15 +1043,15 @@ static const WT_CONFIG_ENTRY config_entries[] = { "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=0,log=(archive=,compressor=,enabled=0,file_max=100MB," "path=,prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),use_environment=," "use_environment_priv=0,verbose=,version=(major=0,minor=0)," "write_through=", - confchk_wiredtiger_open_all, 38 + confchk_wiredtiger_open_all, 39 }, { "wiredtiger_open_basecfg", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1059,14 +1064,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "log=(archive=,compressor=,enabled=0,file_max=100MB,path=," "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 32 + confchk_wiredtiger_open_basecfg, 33 }, { "wiredtiger_open_usercfg", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1079,14 +1084,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "log=(archive=,compressor=,enabled=0,file_max=100MB,path=," "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 31 + confchk_wiredtiger_open_usercfg, 32 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/conn/api_strerror.c b/src/third_party/wiredtiger/src/conn/api_strerror.c index edb11957556..87864f7f4b0 100644 --- a/src/third_party/wiredtiger/src/conn/api_strerror.c +++ b/src/third_party/wiredtiger/src/conn/api_strerror.c @@ -40,6 +40,8 @@ __wt_wiredtiger_error(int error) return ("WT_RUN_RECOVERY: recovery must be run to continue"); case WT_CACHE_FULL: return ("WT_CACHE_FULL: operation would overflow cache"); + case WT_PERM_DENIED: + return ("WT_PERM_DENIED: permission denied (internal)"); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 27977de63b2..bb67185f5c9 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1109,6 +1109,29 @@ __conn_config_append(const char *cfg[], const char *config) } /* + * __conn_config_readonly -- + * Append an entry to a config stack that overrides some settings + * when read-only is configured. + */ +static void +__conn_config_readonly(const char *cfg[]) +{ + const char *readonly; + + /* + * Override certain settings. In general we override the options + * whose default conflicts. Other settings at odds will return + * an error and will be checked when those settings are processed. + */ + readonly="checkpoint=(wait=0)," + "config_base=false," + "create=false," + "log=(archive=false,prealloc=false)," + "lsm_manager=(merge=false),"; + __conn_config_append(cfg, readonly); +} + +/* * __conn_config_check_version -- * Check if a configuration version isn't compatible. */ @@ -1382,7 +1405,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_FH *fh; size_t len; wt_off_t size; - bool exist, is_create; + bool bytelock, exist, is_create; char buf[256]; conn = S2C(session); @@ -1391,6 +1414,10 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_config_gets(session, cfg, "create", &cval)); is_create = cval.val != 0; + if (F_ISSET(conn, WT_CONN_READONLY)) + is_create = false; + + bytelock = true; __wt_spin_lock(session, &__wt_process.spinlock); /* @@ -1448,47 +1475,89 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) exist = false; if (!is_create) WT_ERR(__wt_exist(session, WT_WIREDTIGER, &exist)); - WT_ERR(__wt_open(session, - WT_SINGLETHREAD, is_create || exist, false, 0, &conn->lock_fh)); + ret = __wt_open(session, + WT_SINGLETHREAD, is_create || exist, false, 0, &conn->lock_fh); /* - * Lock a byte of the file: if we don't get the lock, some other process - * is holding it, we're done. The file may be zero-length, and that's - * OK, the underlying call supports locking past the end-of-file. + * If this is a read-only connection and we cannot grab the lock + * file, check if it is because there is not write permission or + * if the file does not exist. If so, then ignore the error. + * XXX Ignoring the error does allow multiple read-only + * connections to exist at the same time on a read-only directory. */ - if (__wt_bytelock(conn->lock_fh, (wt_off_t)0, true) != 0) - WT_ERR_MSG(session, EBUSY, - "WiredTiger database is already being managed by another " - "process"); + if (F_ISSET(conn, WT_CONN_READONLY)) { + /* + * If we got an expected permission or non-existence error + * then skip the byte lock. + */ + ret = __wt_map_error_rdonly(ret); + if (ret == WT_NOTFOUND || ret == WT_PERM_DENIED) { + bytelock = false; + ret = 0; + } + } + WT_ERR(ret); + if (bytelock) { + /* + * Lock a byte of the file: if we don't get the lock, some other + * process is holding it, we're done. The file may be + * zero-length, and that's OK, the underlying call supports + * locking past the end-of-file. + */ + if (__wt_bytelock(conn->lock_fh, (wt_off_t)0, true) != 0) + WT_ERR_MSG(session, EBUSY, + "WiredTiger database is already being managed by " + "another process"); - /* - * If the size of the lock file is non-zero, we created it (or won a - * locking race with the thread that created it, it doesn't matter). - * - * Write something into the file, zero-length files make me nervous. - * - * The test against the expected length is sheer paranoia (the length - * should be 0 or correct), but it shouldn't hurt. - */ + /* + * If the size of the lock file is non-zero, we created it (or + * won a locking race with the thread that created it, it + * doesn't matter). + * + * Write something into the file, zero-length files make me + * nervous. + * + * The test against the expected length is sheer paranoia (the + * length should be 0 or correct), but it shouldn't hurt. + */ #define WT_SINGLETHREAD_STRING "WiredTiger lock file\n" - WT_ERR(__wt_filesize(session, conn->lock_fh, &size)); - if (size != strlen(WT_SINGLETHREAD_STRING)) - WT_ERR(__wt_write(session, conn->lock_fh, (wt_off_t)0, - strlen(WT_SINGLETHREAD_STRING), WT_SINGLETHREAD_STRING)); + WT_ERR(__wt_filesize(session, conn->lock_fh, &size)); + if (size != strlen(WT_SINGLETHREAD_STRING)) + WT_ERR(__wt_write(session, conn->lock_fh, (wt_off_t)0, + strlen(WT_SINGLETHREAD_STRING), + WT_SINGLETHREAD_STRING)); + + } /* We own the lock file, optionally create the WiredTiger file. */ - WT_ERR(__wt_open(session, WT_WIREDTIGER, is_create, false, 0, &fh)); + ret = __wt_open(session, WT_WIREDTIGER, is_create, false, 0, &fh); /* - * Lock the WiredTiger file (for backward compatibility reasons as - * described above). Immediately release the lock, it's just a test. + * If we're read-only, check for success as well as handled errors. + * Even if we're able to open the WiredTiger file successfully, we + * do not try to lock it. The lock file test above is the only + * one we do for read-only. */ - if (__wt_bytelock(fh, (wt_off_t)0, true) != 0) { - WT_ERR_MSG(session, EBUSY, - "WiredTiger database is already being managed by another " - "process"); + if (F_ISSET(conn, WT_CONN_READONLY)) { + ret = __wt_map_error_rdonly(ret); + if (ret == 0 || ret == WT_NOTFOUND || ret == WT_PERM_DENIED) + ret = 0; + WT_ERR(ret); + } else { + WT_ERR(ret); + + /* + * Lock the WiredTiger file (for backward compatibility reasons + * as described above). Immediately release the lock, it's + * just a test. + */ + if (__wt_bytelock(fh, (wt_off_t)0, true) != 0) { + WT_ERR_MSG(session, EBUSY, + "WiredTiger database is already being managed by " + "another process"); + } + WT_ERR(__wt_bytelock(fh, (wt_off_t)0, false)); } - WT_ERR(__wt_bytelock(fh, (wt_off_t)0, false)); /* * We own the database home, figure out if we're creating it. There are @@ -1502,11 +1571,21 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) conn->is_new = exist ? 0 : 1; if (conn->is_new) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_ERR_MSG(session, EINVAL, "Creating a new database is" + " incompatible with read-only configuration."); len = (size_t)snprintf(buf, sizeof(buf), "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING); WT_ERR(__wt_write(session, fh, (wt_off_t)0, len, buf)); WT_ERR(__wt_fsync(session, fh)); } else { + /* + * Although exclusive and the read-only configuration settings + * are at odds, we do not have to check against read-only here + * because it falls out from earlier code in this function + * preventing creation and confirming the database + * already exists. + */ WT_ERR(__wt_config_gets(session, cfg, "exclusive", &cval)); if (cval.val != 0) WT_ERR_MSG(session, EEXIST, @@ -1736,6 +1815,7 @@ __conn_write_base_config(WT_SESSION_IMPL *session, const char *cfg[]) "exclusive=," "in_memory=," "log=(recover=)," + "readonly=," "use_environment_priv=," "verbose=,", &base_config)); WT_ERR(__wt_config_init(session, &parser, base_config)); @@ -1808,7 +1888,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const WT_NAME_FLAG *ft; WT_SESSION_IMPL *session; bool config_base_set; - const char *enc_cfg[] = { NULL, NULL }; + const char *enc_cfg[] = { NULL, NULL }, *merge_cfg; char version[64]; /* Leave lots of space for optional additional configuration. */ @@ -1819,6 +1899,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, conn = NULL; session = NULL; + merge_cfg = NULL; WT_RET(__wt_library_init()); @@ -1860,6 +1941,16 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, session, cval.str, cval.len, &conn->error_prefix)); /* + * We need to look for read-only early so that we can use it + * in __conn_single and whether to use the base config file. + * XXX that means we can only make the choice in __conn_single if the + * user passes it in via the config string to wiredtiger_open. + */ + WT_ERR(__wt_config_gets(session, cfg, "readonly", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_READONLY); + + /* * XXX ideally, we would check "in_memory" here, so we could completely * avoid having a database directory. However, it can be convenient to * pass "in_memory" via the WIREDTIGER_CONFIG environment variable, and @@ -1883,6 +1974,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * 4. the config passed in by the application * 5. user configuration file (optional) * 6. environment variable settings (optional) + * 7. overrides for a read-only connection * * Clear the entries we added to the stack, we're going to build it in * order. @@ -1898,8 +1990,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, (int)sizeof(version), ENOMEM); __conn_config_append(cfg, version); - /* Ignore the base_config file if we config_base set to false. */ - if (config_base_set) + /* Ignore the base_config file if config_base_set is false. */ + if (config_base_set || F_ISSET(conn, WT_CONN_READONLY)) WT_ERR( __conn_config_file(session, WT_BASECONFIG, false, cfg, i1)); __conn_config_append(cfg, config); @@ -1909,7 +2001,35 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, /* * Merge the full configuration stack and save it for reconfiguration. */ - WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); + WT_ERR(__wt_config_merge(session, cfg, NULL, &merge_cfg)); + /* + * The read-only setting may have been set in a configuration file. + * Get it again so that we can override other configuration settings + * before they are processed by the subsystems. + */ + WT_ERR(__wt_config_gets(session, cfg, "readonly", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_READONLY); + if (F_ISSET(conn, WT_CONN_READONLY)) { + /* + * Create a new stack with the merged configuration as the + * base. The read-only string will use entry 1 and then + * we'll merge it again. + */ + cfg[0] = merge_cfg; + cfg[1] = NULL; + cfg[2] = NULL; + /* + * We override some configuration settings for read-only. + * Other settings that conflict with and are an error with + * read-only are tested in their individual locations later. + */ + __conn_config_readonly(cfg); + WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); + } else { + conn->cfg = merge_cfg; + merge_cfg = NULL; + } /* * Configuration ... @@ -2082,6 +2202,7 @@ err: /* Discard the scratch buffers. */ __wt_scr_free(session, &i2); __wt_scr_free(session, &i3); + __wt_free(session, merge_cfg); /* * We may have allocated scratch memory when using the dummy session or * the subsequently created real session, and we don't want to tie down diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 3bea24be508..f1b35571533 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -355,13 +355,25 @@ err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); /* * __conn_btree_apply_internal -- - * Apply a function to the open btree handles. + * Apply a function to an open data handle. */ static int __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_DECL_RET; + bool skip; + + /* Always apply the name function, if supplied. */ + skip = false; + if (name_func != NULL) + WT_RET(name_func(session, dhandle->name, &skip)); + + /* If there is no file function, don't bother locking the handle */ + if (file_func == NULL || skip) + return (0); /* * We need to pull the handle into the session handle cache and make @@ -372,7 +384,7 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, dhandle->name, dhandle->checkpoint, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); - WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); if (WT_META_TRACKING(session)) WT_TRET(__wt_meta_track_handle_lock(session, false)); else @@ -385,9 +397,10 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, * Apply a function to all open btree handles with the given URI. */ int -__wt_conn_btree_apply(WT_SESSION_IMPL *session, - bool apply_checkpoints, const char *uri, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) +__wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; @@ -404,23 +417,26 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, if (uri != NULL) { bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; - TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD) && - strcmp(uri, dhandle->name) == 0 && - (apply_checkpoints || dhandle->checkpoint == NULL)) - WT_RET(__conn_btree_apply_internal( - session, dhandle, func, cfg)); + TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || + F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->checkpoint != NULL || + strcmp(uri, dhandle->name) != 0) + continue; + WT_RET(__conn_btree_apply_internal( + session, dhandle, file_func, name_func, cfg)); + } } else { - TAILQ_FOREACH(dhandle, &conn->dhqh, q) - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD) && - (apply_checkpoints || - dhandle->checkpoint == NULL) && - WT_PREFIX_MATCH(dhandle->name, "file:") && - !WT_IS_METADATA(session, dhandle)) - WT_RET(__conn_btree_apply_internal( - session, dhandle, func, cfg)); + TAILQ_FOREACH(dhandle, &conn->dhqh, q) { + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || + F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->checkpoint != NULL || + !WT_PREFIX_MATCH(dhandle->name, "file:") || + WT_IS_METADATA(session, dhandle)) + continue; + WT_RET(__conn_btree_apply_internal( + session, dhandle, file_func, name_func, cfg)); + } } return (0); diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index b33ec18dfca..16717597f4d 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -124,7 +124,8 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) * underlying file-close code uses the mutex to guard lists of * open files. */ - WT_TRET(__wt_close(session, &conn->lock_fh)); + if (conn->lock_fh) + WT_TRET(__wt_close(session, &conn->lock_fh)); /* Remove from the list of connections. */ __wt_spin_lock(session, &__wt_process.spinlock); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index fa3928a8539..5999cf20b3b 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -133,8 +133,13 @@ __logmgr_config( FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR); WT_RET(__wt_config_gets(session, cfg, "log.zero_fill", &cval)); - if (cval.val != 0) + if (cval.val != 0) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_RET_MSG(session, EINVAL, + "Read-only configuration incompatible with " + "zero-filling log files"); FLD_SET(conn->log_flags, WT_CONN_LOG_ZERO_FILL); + } WT_RET(__logmgr_sync_cfg(session, cfg)); return (0); diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index 08ad105c725..9ccfa8ad8db 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -86,6 +86,11 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) conn->stat_usecs = (uint64_t)cval.val * WT_MILLION; WT_RET(__wt_config_gets( + session, cfg, "statistics_log.json", &cval)); + if (cval.val != 0) + FLD_SET(conn->stat_flags, WT_CONN_STAT_JSON); + + WT_RET(__wt_config_gets( session, cfg, "statistics_log.on_close", &cval)); if (cval.val != 0) FLD_SET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE); @@ -97,6 +102,14 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) if (!*runp && !FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE)) return (0); + /* + * If any statistics logging is done, this must not be a read-only + * connection. + */ + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_RET_MSG(session, EINVAL, + "Read-only configuration incompatible with statistics " + "logging"); WT_RET(__wt_config_gets(session, cfg, "statistics_log.sources", &cval)); WT_RET(__wt_config_subinit(session, &objectconf, &cval)); for (cnt = 0; (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt) @@ -132,9 +145,24 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) WT_ERR(__wt_config_gets(session, cfg, "statistics_log.path", &cval)); WT_ERR(__wt_nfilename(session, cval.str, cval.len, &conn->stat_path)); - WT_ERR(__wt_config_gets( - session, cfg, "statistics_log.timestamp", &cval)); - WT_ERR(__wt_strndup(session, cval.str, cval.len, &conn->stat_format)); + /* + * When using JSON format, use the same timestamp format as MongoDB by + * default. + */ + if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) { + ret = __wt_config_gets( + session, &cfg[1], "statistics_log.timestamp", &cval); + if (ret == WT_NOTFOUND) + WT_ERR(__wt_strdup( + session, "%FT%T.000Z", &conn->stat_format)); + WT_ERR_NOTFOUND_OK(ret); + } + if (conn->stat_format == NULL) { + WT_ERR(__wt_config_gets( + session, cfg, "statistics_log.timestamp", &cval)); + WT_ERR(__wt_strndup( + session, cval.str, cval.len, &conn->stat_format)); + } err: __stat_sources_free(session, &sources); return (ret); @@ -149,22 +177,25 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) { WT_CONNECTION_IMPL *conn; WT_CURSOR *cursor; - WT_CURSOR_STAT *cst; WT_DECL_ITEM(tmp); WT_DECL_RET; - int64_t *stats; - int i; - const char *desc, *uri; + int64_t val; + size_t prefixlen; + const char *desc, *endprefix, *valstr, *uri; const char *cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL }; + bool first, groupfirst; conn = S2C(session); + cursor = NULL; + + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + first = groupfirst = true; /* Build URI and configuration string. */ if (conn_stats) uri = "statistics:"; else { - WT_RET(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_buf_fmt(session, tmp, "statistics:%s", name)); uri = tmp->data; } @@ -175,31 +206,54 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) * If we don't find an underlying object, silently ignore it, the object * may exist only intermittently. */ - switch (ret = __wt_curstat_open(session, uri, NULL, cfg, &cursor)) { - case 0: - cst = (WT_CURSOR_STAT *)cursor; - for (stats = cst->stats, i = 0; i < cst->stats_count; ++i) { - if (conn_stats) - WT_ERR(__wt_stat_connection_desc(cst, i, - &desc)); - else - WT_ERR(__wt_stat_dsrc_desc(cst, i, &desc)); + if ((ret = __wt_curstat_open(session, uri, NULL, cfg, &cursor)) != 0) { + if (ret == EBUSY || ret == ENOENT || ret == WT_NOTFOUND) + ret = 0; + goto err; + } + + if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) { + WT_ERR(__wt_fprintf(conn->stat_fp, + "{\"version\":\"%s\",\"localTime\":\"%s\"", + WIREDTIGER_VERSION_STRING, conn->stat_stamp)); + WT_ERR(__wt_fprintf(conn->stat_fp, ",\"wiredTiger\":{")); + while ((ret = cursor->next(cursor)) == 0) { + WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + /* Check if we are starting a new section. */ + endprefix = strchr(desc, ':'); + prefixlen = WT_PTRDIFF(endprefix, desc); + WT_ASSERT(session, endprefix != NULL); + if (first || + tmp->size != prefixlen || + strncmp(desc, tmp->data, tmp->size) != 0) { + WT_ERR(__wt_buf_set( + session, tmp, desc, prefixlen)); + WT_ERR(__wt_fprintf(conn->stat_fp, + "%s\"%.*s\":{", first ? "" : "},", + (int)prefixlen, desc)); + first = false; + groupfirst = true; + } + WT_ERR(__wt_fprintf(conn->stat_fp, + "%s\"%s\":%" PRId64, + groupfirst ? "" : ",", endprefix + 2, val)); + groupfirst = false; + } + WT_ERR_NOTFOUND_OK(ret); + WT_ERR(__wt_fprintf(conn->stat_fp, "}}}\n")); + } else { + while ((ret = cursor->next(cursor)) == 0) { + WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); WT_ERR(__wt_fprintf(conn->stat_fp, "%s %" PRId64 " %s %s\n", - conn->stat_stamp, stats[i], name, desc)); + conn->stat_stamp, val, name, desc)); } - WT_ERR(cursor->close(cursor)); - break; - case EBUSY: - case ENOENT: - case WT_NOTFOUND: - ret = 0; - break; - default: - break; + WT_ERR_NOTFOUND_OK(ret); } err: __wt_scr_free(session, &tmp); + if (cursor != NULL) + WT_TRET(cursor->close(cursor)); return (ret); } @@ -342,7 +396,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) if (conn->stat_sources != NULL) { WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_conn_btree_apply( - session, false, NULL, __statlog_apply, NULL)); + session, NULL, __statlog_apply, NULL, NULL)); WT_RET(ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index d7d74da48d4..b097a8c08aa 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -8,12 +8,12 @@ #include "wt_internal.h" -static int __backup_all(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); +static int __backup_all(WT_SESSION_IMPL *); static int __backup_cleanup_handles(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_file_create(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool); -static int __backup_list_all_append(WT_SESSION_IMPL *, const char *[]); static int __backup_list_append( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); +static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *); static int __backup_start( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *[]); static int __backup_stop(WT_SESSION_IMPL *); @@ -103,22 +103,22 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_notsup, /* get-value */ - __wt_cursor_notsup, /* set-key */ - __wt_cursor_notsup, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curbackup_next, /* next */ - __wt_cursor_notsup, /* prev */ - __curbackup_reset, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curbackup_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value_notsup, /* get-value */ + __wt_cursor_set_key_notsup, /* set-key */ + __wt_cursor_set_value_notsup, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curbackup_next, /* next */ + __wt_cursor_notsup, /* prev */ + __curbackup_reset, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curbackup_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_BACKUP *cb; WT_DECL_RET; @@ -241,7 +241,7 @@ __backup_start( if (!target_list) { WT_ERR(__backup_log_append(session, cb, true)); - WT_ERR(__backup_all(session, cb)); + WT_ERR(__backup_all(session)); } /* Add the hot backup and standard WiredTiger files to the list. */ @@ -332,55 +332,14 @@ __backup_stop(WT_SESSION_IMPL *session) * Backup all objects in the database. */ static int -__backup_all(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) +__backup_all(WT_SESSION_IMPL *session) { - WT_CONFIG_ITEM cval; - WT_CURSOR *cursor; WT_DECL_RET; - const char *key, *value; - - cursor = NULL; - - /* Copy all of the metadata entries to the hot backup file. */ - WT_RET(__wt_metadata_cursor(session, &cursor)); - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_key(cursor, &key)); - WT_ERR(cursor->get_value(cursor, &value)); - WT_ERR(__wt_fprintf(cb->bfp, "%s\n%s\n", key, value)); - - /* - * While reading the metadata file, check there are no "sources" - * or "types" which can't support hot backup. This checks for - * a data source that's non-standard, which can't be backed up, - * but is also sanity checking: if there's an entry backed by - * anything other than a file or lsm entry, we're confused. - */ - if ((ret = __wt_config_getones( - session, value, "type", &cval)) == 0 && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file") && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm")) - WT_ERR_MSG(session, ENOTSUP, - "hot backup is not supported for objects of " - "type %.*s", (int)cval.len, cval.str); - WT_ERR_NOTFOUND_OK(ret); - if ((ret =__wt_config_getones( - session, value, "source", &cval)) == 0 && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file:") && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm:")) - WT_ERR_MSG(session, ENOTSUP, - "hot backup is not supported for objects of " - "source %.*s", (int)cval.len, cval.str); - WT_ERR_NOTFOUND_OK(ret); - } - WT_ERR_NOTFOUND_OK(ret); - - WT_ERR(__wt_metadata_cursor_release(session, &cursor)); /* Build a list of the file objects that need to be copied. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = - __wt_meta_btree_apply(session, __backup_list_all_append, NULL)); + __wt_meta_apply_all(session, NULL, __backup_list_uri_append, NULL)); -err: WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); } @@ -430,11 +389,11 @@ __backup_uri(WT_SESSION_IMPL *session, */ if (WT_PREFIX_MATCH(uri, "log:")) { *log_only = !target_list; - WT_ERR(__wt_backup_list_uri_append(session, uri, NULL)); + WT_ERR(__backup_list_uri_append(session, uri, NULL)); } else { *log_only = false; WT_ERR(__wt_schema_worker(session, - uri, NULL, __wt_backup_list_uri_append, cfg, 0)); + uri, NULL, __backup_list_uri_append, cfg, 0)); } } WT_ERR_NOTFOUND_OK(ret); @@ -471,12 +430,12 @@ __wt_backup_file_remove(WT_SESSION_IMPL *session) } /* - * __wt_backup_list_uri_append -- + * __backup_list_uri_append -- * Append a new file name to the list, allocate space as necessary. * Called via the schema_worker function. */ -int -__wt_backup_list_uri_append( +static int +__backup_list_uri_append( WT_SESSION_IMPL *session, const char *name, bool *skip) { WT_CURSOR_BACKUP *cb; @@ -485,11 +444,31 @@ __wt_backup_list_uri_append( cb = session->bkp_cursor; WT_UNUSED(skip); + /* + * While reading the metadata file, check there are no data sources + * that can't support hot backup. This checks for a data source that's + * non-standard, which can't be backed up, but is also sanity checking: + * if there's an entry backed by anything other than a file or lsm + * entry, we're confused. + */ if (WT_PREFIX_MATCH(name, "log:")) { WT_RET(__backup_log_append(session, cb, false)); return (0); } + if (!WT_PREFIX_MATCH(name, "file:") && + !WT_PREFIX_MATCH(name, "colgroup:") && + !WT_PREFIX_MATCH(name, "index:") && + !WT_PREFIX_MATCH(name, "lsm:") && + !WT_PREFIX_MATCH(name, "table:")) + WT_RET_MSG(session, ENOTSUP, + "hot backup is not supported for objects of type %s", + name); + + /* Ignore the lookaside table. */ + if (strcmp(name, WT_LAS_URI) == 0) + return (0); + /* Add the metadata entry to the backup file. */ WT_RET(__wt_metadata_search(session, name, &value)); WT_RET(__wt_fprintf(cb->bfp, "%s\n%s\n", name, value)); @@ -503,34 +482,6 @@ __wt_backup_list_uri_append( } /* - * __backup_list_all_append -- - * Append a new file name to the list, allocate space as necessary. - * Called via the __wt_meta_btree_apply function. - */ -static int -__backup_list_all_append(WT_SESSION_IMPL *session, const char *cfg[]) -{ - WT_CURSOR_BACKUP *cb; - const char *name; - - WT_UNUSED(cfg); - - cb = session->bkp_cursor; - name = session->dhandle->name; - - /* Ignore files in the process of being bulk-loaded. */ - if (F_ISSET(S2BT(session), WT_BTREE_BULK)) - return (0); - - /* Ignore the lookaside table. */ - if (strcmp(name, WT_LAS_URI) == 0) - return (0); - - /* Add the file to the list of files to be copied. */ - return (__backup_list_append(session, cb, name)); -} - -/* * __backup_list_append -- * Append a new file name to the list, allocate space as necessary. */ @@ -541,7 +492,6 @@ __backup_list_append( WT_CURSOR_BACKUP_ENTRY *p; WT_DATA_HANDLE *old_dhandle; WT_DECL_RET; - bool need_handle; const char *name; /* Leave a NULL at the end to mark the end of the list. */ @@ -551,11 +501,26 @@ __backup_list_append( p[0].name = p[1].name = NULL; p[0].handle = p[1].handle = NULL; - need_handle = false; name = uri; + + /* + * If it's a file in the database, get a handle for the underlying + * object (this handle blocks schema level operations, for example + * WT_SESSION.drop or an LSM file discard after level merging). + * + * If the handle is busy (e.g., it is being bulk-loaded), silently skip + * it. We have a special fake checkpoint in the metadata, and recovery + * will recreate an empty file. + */ if (WT_PREFIX_MATCH(uri, "file:")) { - need_handle = true; name += strlen("file:"); + + old_dhandle = session->dhandle; + ret = __wt_session_get_btree(session, uri, NULL, NULL, 0); + p->handle = session->dhandle; + session->dhandle = old_dhandle; + if (ret != 0) + return (ret == EBUSY ? 0 : ret); } /* @@ -569,20 +534,6 @@ __backup_list_append( */ WT_RET(__wt_strdup(session, name, &p->name)); - /* - * If it's a file in the database, get a handle for the underlying - * object (this handle blocks schema level operations, for example - * WT_SESSION.drop or an LSM file discard after level merging). - */ - if (need_handle) { - old_dhandle = session->dhandle; - if ((ret = - __wt_session_get_btree(session, uri, NULL, NULL, 0)) == 0) - p->handle = session->dhandle; - session->dhandle = old_dhandle; - WT_RET(ret); - } - ++cb->list_next; return (0); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c index 1b2fec0eb89..e0d270e4245 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_config.c +++ b/src/third_party/wiredtiger/src/cursor/cur_config.c @@ -27,21 +27,21 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __wt_cursor_notsup, /* next */ - __wt_cursor_notsup, /* prev */ - __wt_cursor_noop, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_noop, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ __curconfig_close); WT_CURSOR_CONFIG *cconfig; WT_CURSOR *cursor; diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index 2a598c99523..804c24a3d2e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -449,22 +449,22 @@ __wt_curds_open( const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curds_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curds_next, /* next */ - __curds_prev, /* prev */ - __curds_reset, /* reset */ - __curds_search, /* search */ - __curds_search_near, /* search-near */ - __curds_insert, /* insert */ - __curds_update, /* update */ - __curds_remove, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curds_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curds_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curds_next, /* next */ + __curds_prev, /* prev */ + __curds_reset, /* reset */ + __curds_search, /* search */ + __curds_search_near, /* search-near */ + __curds_insert, /* insert */ + __curds_update, /* update */ + __curds_remove, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curds_close); /* close */ WT_CONFIG_ITEM cval, metadata; WT_CURSOR *cursor, *source; WT_CURSOR_DATA_SOURCE *data_source; diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 3324efd96cc..a7b1c98871a 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -348,22 +348,22 @@ int __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __curdump_get_key, /* get-key */ - __curdump_get_value, /* get-value */ - __curdump_set_key, /* set-key */ - __curdump_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curdump_next, /* next */ - __curdump_prev, /* prev */ - __curdump_reset, /* reset */ - __curdump_search, /* search */ - __curdump_search_near, /* search-near */ - __curdump_insert, /* insert */ - __curdump_update, /* update */ - __curdump_remove, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curdump_close); /* close */ + __curdump_get_key, /* get-key */ + __curdump_get_value, /* get-value */ + __curdump_set_key, /* set-key */ + __curdump_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curdump_next, /* next */ + __curdump_prev, /* prev */ + __curdump_reset, /* reset */ + __curdump_search, /* search */ + __curdump_search_near, /* search-near */ + __curdump_insert, /* insert */ + __curdump_update, /* update */ + __curdump_remove, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curdump_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_DUMP *cdump; WT_CURSOR_JSON *json; diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 8bbe1cc8eda..fac903b4770 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -397,22 +397,22 @@ __wt_curfile_create(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curfile_compare, /* compare */ - __curfile_equals, /* equals */ - __curfile_next, /* next */ - __curfile_prev, /* prev */ - __curfile_reset, /* reset */ - __curfile_search, /* search */ - __curfile_search_near, /* search-near */ - __curfile_insert, /* insert */ - __curfile_update, /* update */ - __curfile_remove, /* remove */ - __wt_cursor_reconfigure, /* reconfigure */ - __curfile_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curfile_compare, /* compare */ + __curfile_equals, /* equals */ + __curfile_next, /* next */ + __curfile_prev, /* prev */ + __curfile_reset, /* reset */ + __curfile_search, /* search */ + __curfile_search_near, /* search-near */ + __curfile_insert, /* insert */ + __curfile_update, /* update */ + __curfile_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ + __curfile_close); /* close */ WT_BTREE *btree; WT_CONFIG_ITEM cval; WT_CURSOR *cursor; diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index 6822055131a..dbe8046ca21 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -386,22 +386,22 @@ __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __curindex_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __curindex_set_value, /* set-value */ - __curindex_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curindex_next, /* next */ - __curindex_prev, /* prev */ - __curindex_reset, /* reset */ - __curindex_search, /* search */ - __curindex_search_near, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curindex_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __curindex_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __curindex_set_value, /* set-value */ + __curindex_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curindex_next, /* next */ + __curindex_prev, /* prev */ + __curindex_reset, /* reset */ + __curindex_search, /* search */ + __curindex_search_near, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curindex_close); /* close */ WT_CURSOR_INDEX *cindex; WT_CURSOR *cursor; WT_DECL_ITEM(tmp); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 797e6e5879a..fa6dd5c32f7 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -583,22 +583,22 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_CURJOIN_EXTRACTOR extract_cursor; WT_CURSOR *c; WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __wt_cursor_notsup, /* next */ - __wt_cursor_notsup, /* prev */ - __wt_cursor_notsup, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __curjoin_extract_insert, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* reconfigure */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_notsup, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __curjoin_extract_insert, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup); /* close */ WT_DECL_RET; WT_INDEX *idx; WT_ITEM *key, v; @@ -797,22 +797,22 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __curjoin_get_key, /* get-key */ - __curjoin_get_value, /* get-value */ - __wt_cursor_notsup, /* set-key */ - __wt_cursor_notsup, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curjoin_next, /* next */ - __wt_cursor_notsup, /* prev */ - __curjoin_reset, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curjoin_close); /* close */ + __curjoin_get_key, /* get-key */ + __curjoin_get_value, /* get-value */ + __wt_cursor_set_key_notsup, /* set-key */ + __wt_cursor_set_value_notsup, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curjoin_next, /* next */ + __wt_cursor_notsup, /* prev */ + __curjoin_reset, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curjoin_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_JOIN *cjoin; WT_DECL_ITEM(tmp); diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 3fcd8a86066..47436ac7237 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -347,22 +347,22 @@ __wt_curlog_open(WT_SESSION_IMPL *session, { WT_CONNECTION_IMPL *conn; WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curlog_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curlog_next, /* next */ - __wt_cursor_notsup, /* prev */ - __curlog_reset, /* reset */ - __curlog_search, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curlog_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curlog_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curlog_next, /* next */ + __wt_cursor_notsup, /* prev */ + __curlog_reset, /* reset */ + __curlog_search, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curlog_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_LOG *cl; WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index df66ef34ddd..df2cc3f546e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -448,22 +448,22 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curmetadata_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curmetadata_next, /* next */ - __curmetadata_prev, /* prev */ - __curmetadata_reset, /* reset */ - __curmetadata_search, /* search */ - __curmetadata_search_near, /* search-near */ - __curmetadata_insert, /* insert */ - __curmetadata_update, /* update */ - __curmetadata_remove, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curmetadata_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curmetadata_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curmetadata_next, /* next */ + __curmetadata_prev, /* prev */ + __curmetadata_reset, /* reset */ + __curmetadata_search, /* search */ + __curmetadata_search_near, /* search-near */ + __curmetadata_insert, /* insert */ + __curmetadata_update, /* update */ + __curmetadata_remove, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curmetadata_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_METADATA *mdc; WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index bb492c66ace..34e64b34ccb 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -200,8 +200,6 @@ __curstat_next(WT_CURSOR *cursor) if (cst->notinitialized) { WT_ERR(__wt_curstat_init( session, cursor->internal_uri, NULL, cst->cfg, cst)); - if (cst->next_set != NULL) - WT_ERR((*cst->next_set)(session, cst, true, true)); cst->notinitialized = false; } @@ -244,8 +242,6 @@ __curstat_prev(WT_CURSOR *cursor) if (cst->notinitialized) { WT_ERR(__wt_curstat_init( session, cursor->internal_uri, NULL, cst->cfg, cst)); - if (cst->next_set != NULL) - WT_ERR((*cst->next_set)(session, cst, false, true)); cst->notinitialized = false; } @@ -449,7 +445,6 @@ __curstat_join_next_set(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst, WT_JOIN_STATS_GROUP *join_group; ssize_t pos; - WT_ASSERT(session, WT_STREQ(cst->iface.uri, "statistics:join")); join_group = &cst->u.join_stats_group; cjoin = join_group->join_cursor; if (init) @@ -542,25 +537,31 @@ __wt_curstat_init(WT_SESSION_IMPL *session, dsrc_uri = uri + strlen("statistics:"); if (WT_STREQ(dsrc_uri, "join")) - return (__curstat_join_init(session, curjoin, cfg, cst)); + WT_RET(__curstat_join_init(session, curjoin, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) - return ( + else if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) + WT_RET( __wt_curstat_colgroup_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "file:")) - return (__curstat_file_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "file:")) + WT_RET(__curstat_file_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "index:")) - return (__wt_curstat_index_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "index:")) + WT_RET(__wt_curstat_index_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "lsm:")) - return (__wt_curstat_lsm_init(session, dsrc_uri, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "lsm:")) + WT_RET(__wt_curstat_lsm_init(session, dsrc_uri, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "table:")) - return (__wt_curstat_table_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "table:")) + WT_RET(__wt_curstat_table_init(session, dsrc_uri, cfg, cst)); - return (__wt_bad_object_type(session, uri)); + else + return (__wt_bad_object_type(session, uri)); + + if (cst->next_set != NULL) + WT_RET((*cst->next_set)(session, cst, false, true)); + + return (0); } /* @@ -573,22 +574,22 @@ __wt_curstat_open(WT_SESSION_IMPL *session, { WT_CONNECTION_IMPL *conn; WT_CURSOR_STATIC_INIT(iface, - __curstat_get_key, /* get-key */ - __curstat_get_value, /* get-value */ - __curstat_set_key, /* set-key */ - __curstat_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curstat_next, /* next */ - __curstat_prev, /* prev */ - __curstat_reset, /* reset */ - __curstat_search, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curstat_close); /* close */ + __curstat_get_key, /* get-key */ + __curstat_get_value, /* get-value */ + __curstat_set_key, /* set-key */ + __curstat_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curstat_next, /* next */ + __curstat_prev, /* prev */ + __curstat_reset, /* reset */ + __curstat_search, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curstat_close); /* close */ WT_CONFIG_ITEM cval, sval; WT_CURSOR *cursor; WT_CURSOR_STAT *cst; diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 051f36c8854..7839971f975 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -9,27 +9,108 @@ #include "wt_internal.h" /* + * __wt_cursor_noop -- + * Cursor noop. + */ +int +__wt_cursor_noop(WT_CURSOR *cursor) +{ + WT_UNUSED(cursor); + + return (0); +} + +/* * __wt_cursor_notsup -- * Unsupported cursor actions. */ int __wt_cursor_notsup(WT_CURSOR *cursor) { - WT_UNUSED(cursor); + WT_SESSION_IMPL *session; - return (ENOTSUP); + session = (WT_SESSION_IMPL *)cursor->session; + WT_RET_MSG(session, ENOTSUP, "Unsupported cursor operation"); } /* - * __wt_cursor_noop -- - * Cursor noop. + * __wt_cursor_get_value_notsup -- + * WT_CURSOR.get_value not-supported. */ int -__wt_cursor_noop(WT_CURSOR *cursor) +__wt_cursor_get_value_notsup(WT_CURSOR *cursor, ...) { - WT_UNUSED(cursor); + return (__wt_cursor_notsup(cursor)); +} - return (0); +/* + * __wt_cursor_set_key_notsup -- + * WT_CURSOR.set_key not-supported. + */ +void +__wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...) +{ + (void)__wt_cursor_notsup(cursor); +} + +/* + * __wt_cursor_set_value_notsup -- + * WT_CURSOR.set_value not-supported. + */ +void +__wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...) +{ + (void)__wt_cursor_notsup(cursor); +} + +/* + * __wt_cursor_compare_notsup -- + * Unsupported cursor comparison. + */ +int +__wt_cursor_compare_notsup(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) +{ + WT_UNUSED(b); + WT_UNUSED(cmpp); + + return (__wt_cursor_notsup(a)); +} + +/* + * __wt_cursor_equals_notsup -- + * Unsupported cursor equality. + */ +int +__wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) +{ + WT_UNUSED(other); + WT_UNUSED(equalp); + + return (__wt_cursor_notsup(cursor)); +} + +/* + * __wt_cursor_search_near_notsup -- + * Unsupported cursor search-near. + */ +int +__wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) +{ + WT_UNUSED(exact); + + return (__wt_cursor_notsup(cursor)); +} + +/* + * __wt_cursor_reconfigure_notsup -- + * Unsupported cursor reconfiguration. + */ +int +__wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config) +{ + WT_UNUSED(config); + + return (__wt_cursor_notsup(cursor)); } /* @@ -46,13 +127,12 @@ __wt_cursor_set_notsup(WT_CURSOR *cursor) * cursors in a session. Reconfigure is left open in case it's possible * in the future to change these configurations. */ - cursor->compare = - (int (*)(WT_CURSOR *, WT_CURSOR *, int *))__wt_cursor_notsup; + cursor->compare = __wt_cursor_compare_notsup; cursor->next = __wt_cursor_notsup; cursor->prev = __wt_cursor_notsup; cursor->reset = __wt_cursor_noop; cursor->search = __wt_cursor_notsup; - cursor->search_near = (int (*)(WT_CURSOR *, int *))__wt_cursor_notsup; + cursor->search_near = __wt_cursor_search_near_notsup; cursor->insert = __wt_cursor_notsup; cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; @@ -628,7 +708,7 @@ __wt_cursor_init(WT_CURSOR *cursor, } else { WT_RET( __wt_config_gets_def(session, cfg, "readonly", 0, &cval)); - if (cval.val != 0) { + if (cval.val != 0 || F_ISSET(S2C(session), WT_CONN_READONLY)) { cursor->insert = __wt_cursor_notsup; cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index d986577f640..9eb88ec6fcd 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -79,22 +79,22 @@ __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, WT_CURSOR *cur, WT_CURSOR_TABLE *ctable, int (*f)(WT_CURSOR *)) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __wt_cursor_notsup, /* next */ - __wt_cursor_notsup, /* prev */ - __wt_cursor_notsup, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __curextract_insert, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* reconfigure */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_notsup, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __curextract_insert, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup); /* close */ WT_CURSOR_EXTRACTOR extract_cursor; WT_DECL_RET; WT_ITEM key, value; @@ -842,22 +842,22 @@ __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_curtable_get_key, /* get-key */ - __wt_curtable_get_value, /* get-value */ - __wt_curtable_set_key, /* set-key */ - __wt_curtable_set_value, /* set-value */ - __curtable_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curtable_next, /* next */ - __curtable_prev, /* prev */ - __curtable_reset, /* reset */ - __curtable_search, /* search */ - __curtable_search_near, /* search-near */ - __curtable_insert, /* insert */ - __curtable_update, /* update */ - __curtable_remove, /* remove */ - __wt_cursor_reconfigure, /* reconfigure */ - __curtable_close); /* close */ + __wt_curtable_get_key, /* get-key */ + __wt_curtable_get_value, /* get-value */ + __wt_curtable_set_key, /* set-key */ + __wt_curtable_set_value, /* set-value */ + __curtable_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curtable_next, /* next */ + __curtable_prev, /* prev */ + __curtable_reset, /* reset */ + __curtable_search, /* search */ + __curtable_search_near, /* search-near */ + __curtable_insert, /* insert */ + __curtable_update, /* update */ + __curtable_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ + __curtable_close); /* close */ WT_CONFIG_ITEM cval; WT_CURSOR *cursor; WT_CURSOR_TABLE *ctable; diff --git a/src/third_party/wiredtiger/src/docs/error-handling.dox b/src/third_party/wiredtiger/src/docs/error-handling.dox index d1291e38ff0..d91a126ee21 100644 --- a/src/third_party/wiredtiger/src/docs/error-handling.dox +++ b/src/third_party/wiredtiger/src/docs/error-handling.dox @@ -55,14 +55,32 @@ This error is generated when wiredtiger_open is configured to return an error if @if IGNORE_BUILT_BY_API_ERR_END @endif -The ::wiredtiger_strerror function returns the standard message -associated with any WiredTiger, ISO C99, or POSIX 1003.1-2001 function: +@section error_translation Translating errors + +The WT_SESSION::strerror and ::wiredtiger_strerror functions return the +standard text message associated with any WiredTiger, ISO C, or POSIX +standard API. + +@snippet ex_all.c Display an error thread safe @snippet ex_all.c Display an error +Note that ::wiredtiger_strerror is not thread-safe. + @m_if{c} +@section error_handling_event Error handling using the WT_EVENT_HANDLER + More complex error handling can be configured by passing an implementation of WT_EVENT_HANDLER to ::wiredtiger_open or WT_CONNECTION::open_session. + +For example, both informational and error messages might be passed to an +application-specific logging function that added a timestamp and logged +the message to a file, and error messages might additionally be output to +the \c stderr file stream. + +@snippet ex_event_handler.c Function event_handler +@snippet ex_event_handler.c Configure event_handler + @m_endif */ diff --git a/src/third_party/wiredtiger/src/docs/programming.dox b/src/third_party/wiredtiger/src/docs/programming.dox index 5d79edd660b..f717f4ed1fe 100644 --- a/src/third_party/wiredtiger/src/docs/programming.dox +++ b/src/third_party/wiredtiger/src/docs/programming.dox @@ -30,6 +30,7 @@ each of which is ordered by one or more columns. <h2>Programming notes</h2> - @subpage threads - @subpage namespace +- @subpage readonly @m_if{c} - @subpage signals @m_endif diff --git a/src/third_party/wiredtiger/src/docs/readonly.dox b/src/third_party/wiredtiger/src/docs/readonly.dox new file mode 100644 index 00000000000..9935f5d1b17 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/readonly.dox @@ -0,0 +1,55 @@ +/*! @m_page{{c,java},readonly,Database read-only mode} + +WiredTiger supports read-only mode databases. When a database is opened +in read-only mode, all modifications are disabled on the WT_CONNECTION +handle, any sessions opened in that connection and any cursors opened +in any of those sessions. For example, all cursor or session handle +methods that modify the database will instead return errors. + +When a database is opened in read-only mode, the database directory and +content must already exist and have been shutdown cleanly. + +@section readonly_config Database read-only configuration considerations + +The \c readonly configuration affects other configuration settings. +Where a default setting contradicts read-only operation, WiredTiger +defaults are overridden to perform in a read-only mode. For example, LSM +tree merges are turned off when LSM trees are configured, and log file +archiving is disabled when logging is configured. + +Where a user configured setting contradicts read-only operation, WiredTiger +will return an error. For example, statistics logging or zero-filling +log files is not allowed in read-only mode, and attempting to configure +them will return an error. + +@section readonly_recovery Readonly configuration and recovery + +Because recovery modifies the database, recovery cannot be done in +read-only mode. A ::wiredtiger_open call to open a database in read-only +mode will fail if the database was not cleanly shutdown and recovery is +required. + +@section readonly_logging Readonly configuration and logging + +If logging is enabled on the database when opened in read-only mode, log +file archiving and log file pre-allocation are disabled and the log files +will not be modified any way. + +@section readonly_lsm Readonly configuration and LSM trees + +If LSM trees are in use, read-only mode turns off all modification. +Internal LSM operations such as merging, creating new chunks, creating +bloom filters and dropping old chunks are disabled. + +@section readonly_handles Readonly configuration and multiple database handles + +One unusual affect of read-only operations is the potential for multiple +read-only database handles open on the same database at the same time. +WiredTiger prevents multiple connection handles by writing a lock file, +and this locking is done even in read-only mode. However, if the lock +file cannot be written, opening in read-only mode is still allowed to +proceed. For that reason, multiple read-only connection handles could +be open at the same time. Normal locking occurs if the lock file can be +written in read-only mode, preventing multiple database connections. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index ac71214f8b1..efc306568cd 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -13,7 +13,6 @@ Christoph Collet's Coverity Coverity's -crc DB's DBTs Datastore @@ -81,7 +80,6 @@ Seward's SiH TXT URIs -vpmsum WiredTiger WiredTiger's WiredTigerCheckpoint @@ -155,6 +153,7 @@ control's copydoc cpp crashless +crc cursortype customerABC cv @@ -381,6 +380,7 @@ rVv rdbms rdlock readlock +readonly realclean realloc realloc'd @@ -423,6 +423,7 @@ src ssd startsync statlog +stderr str strerror strftime @@ -479,6 +480,7 @@ valuefmt vec versa vm +vpmsum warmup whitespace wiredtiger diff --git a/src/third_party/wiredtiger/src/docs/wtperf.dox b/src/third_party/wiredtiger/src/docs/wtperf.dox index 1f0d1533ac4..6f3d2f87ee0 100644 --- a/src/third_party/wiredtiger/src/docs/wtperf.dox +++ b/src/third_party/wiredtiger/src/docs/wtperf.dox @@ -212,6 +212,10 @@ insert operations generate random content for the value @par read_range (unsigned int, default=0) scan a range of keys after each search +@par readonly (boolean, default=false) +reopen the connection between populate and workload phases in readonly +mode. Requires reopen_connection turned on (default). Requires that +read be the only workload specified @par reopen_connection (boolean, default=true) close and reopen the connection between populate and workload phases @par report_interval (unsigned int, default=2) diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 8ef7164dbc6..35b12e2b685 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -477,6 +477,7 @@ __evict_update_work(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; + WT_STAT_FAST_CONN_SET(session, cache_eviction_aggressive_set, 0); /* Clear previous state. */ cache->state = 0; @@ -534,8 +535,11 @@ __evict_update_work(WT_SESSION_IMPL *session) return (false); -done: if (F_ISSET(cache, WT_CACHE_STUCK)) +done: if (F_ISSET(cache, WT_CACHE_STUCK)) { + WT_STAT_FAST_CONN_SET(session, + cache_eviction_aggressive_set, 1); FLD_SET(cache->state, WT_EVICT_PASS_AGGRESSIVE); + } return (true); } @@ -594,8 +598,11 @@ __evict_pass(WT_SESSION_IMPL *session) if (!__evict_update_work(session)) break; - if (loop > 10) + if (loop > 10) { + WT_STAT_FAST_CONN_SET(session, + cache_eviction_aggressive_set, 1); FLD_SET(cache->state, WT_EVICT_PASS_AGGRESSIVE); + } /* * Start a worker if we have capacity and we haven't reached @@ -1267,7 +1274,8 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) /* Pages we no longer need (clean or dirty), are found money. */ if (__wt_page_is_empty(page) || - F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) + F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || + page->read_gen == WT_READGEN_OLDEST) goto fast; /* Skip clean pages if appropriate. */ @@ -1280,14 +1288,13 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) * eviction, skip anything that isn't marked. */ if (FLD_ISSET(cache->state, WT_EVICT_PASS_WOULD_BLOCK) && - page->memory_footprint < btree->splitmempage && - page->read_gen != WT_READGEN_OLDEST) + page->memory_footprint < btree->splitmempage) continue; /* Limit internal pages to 50% unless we get aggressive. */ if (WT_PAGE_IS_INTERNAL(page) && - ++internal_pages > WT_EVICT_WALK_PER_FILE / 2 && - !FLD_ISSET(cache->state, WT_EVICT_PASS_AGGRESSIVE)) + !FLD_ISSET(cache->state, WT_EVICT_PASS_AGGRESSIVE) && + internal_pages >= (int)(evict - start) / 2) continue; /* @@ -1332,6 +1339,9 @@ fast: /* If the page can't be evicted, give up. */ __evict_init_candidate(session, evict, ref); ++evict; + if (WT_PAGE_IS_INTERNAL(page)) + ++internal_pages; + WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" PRIu64, page, page->memory_footprint)); } @@ -1392,8 +1402,9 @@ __evict_get_ref( } /* - * The eviction server only tries to evict half of the pages before - * looking for more. + * Only evict half of the pages before looking for more. The remainder + * are left to eviction workers (if configured), or application threads + * if necessary. */ candidates = cache->evict_candidates; if (is_server && candidates > 1) diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 72c07eaa05d..f0d4752cc83 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -471,8 +471,7 @@ __evict_review( LF_SET(WT_EVICT_IN_MEMORY | WT_EVICT_UPDATE_RESTORE); else if (page->read_gen == WT_READGEN_OLDEST) LF_SET(WT_EVICT_UPDATE_RESTORE); - else if (F_ISSET(session, WT_SESSION_INTERNAL) && - F_ISSET(S2C(session)->cache, WT_CACHE_STUCK)) + else if (F_ISSET(S2C(session)->cache, WT_CACHE_STUCK)) LF_SET(WT_EVICT_LOOKASIDE); } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index b0edcef718b..2255056fcf6 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -299,9 +299,10 @@ struct __wt_connection_impl { #define WT_CONN_STAT_ALL 0x01 /* "all" statistics configured */ #define WT_CONN_STAT_CLEAR 0x02 /* clear after gathering */ #define WT_CONN_STAT_FAST 0x04 /* "fast" statistics configured */ -#define WT_CONN_STAT_NONE 0x08 /* don't gather statistics */ -#define WT_CONN_STAT_ON_CLOSE 0x10 /* output statistics on close */ -#define WT_CONN_STAT_SIZE 0x20 /* "size" statistics configured */ +#define WT_CONN_STAT_JSON 0x08 /* output JSON format */ +#define WT_CONN_STAT_NONE 0x10 /* don't gather statistics */ +#define WT_CONN_STAT_ON_CLOSE 0x20 /* output statistics on close */ +#define WT_CONN_STAT_SIZE 0x40 /* "size" statistics configured */ uint32_t stat_flags; /* Connection statistics */ diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index f9bd20c8ba1..48db8b9ec23 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -31,22 +31,22 @@ NULL, /* uri */ \ NULL, /* key_format */ \ NULL, /* value_format */ \ - (int (*)(WT_CURSOR *, ...))(get_key), \ - (int (*)(WT_CURSOR *, ...))(get_value), \ - (void (*)(WT_CURSOR *, ...))(set_key), \ - (void (*)(WT_CURSOR *, ...))(set_value), \ - (int (*)(WT_CURSOR *, WT_CURSOR *, int *))(compare), \ - (int (*)(WT_CURSOR *, WT_CURSOR *, int *))(equals), \ + get_key, \ + get_value, \ + set_key, \ + set_value, \ + compare, \ + equals, \ next, \ prev, \ reset, \ search, \ - (int (*)(WT_CURSOR *, int *))(search_near), \ + search_near, \ insert, \ update, \ remove, \ close, \ - (int (*)(WT_CURSOR *, const char *))(reconfigure), \ + reconfigure, \ { NULL, NULL }, /* TAILQ_ENTRY q */ \ 0, /* recno key */ \ { 0 }, /* recno raw buffer */ \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 07b4adfe698..55b0b8cd7ff 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -252,7 +252,7 @@ extern int __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint); extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags); -extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, bool apply_checkpoints, const char *uri, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); +extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]); extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *uri, bool force); extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session); @@ -276,7 +276,6 @@ extern int __wt_sweep_create(WT_SESSION_IMPL *session); extern int __wt_sweep_destroy(WT_SESSION_IMPL *session); extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); extern int __wt_backup_file_remove(WT_SESSION_IMPL *session); -extern int __wt_backup_list_uri_append( WT_SESSION_IMPL *session, const char *name, bool *skip); extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); extern int __wt_curds_open( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp); @@ -302,8 +301,15 @@ extern int __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_C extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); extern int __wt_curstat_init(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *curjoin, const char *cfg[], WT_CURSOR_STAT *cst); extern int __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp); -extern int __wt_cursor_notsup(WT_CURSOR *cursor); extern int __wt_cursor_noop(WT_CURSOR *cursor); +extern int __wt_cursor_notsup(WT_CURSOR *cursor); +extern int __wt_cursor_get_value_notsup(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...); +extern int __wt_cursor_compare_notsup(WT_CURSOR *a, WT_CURSOR *b, int *cmpp); +extern int __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp); +extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact); +extern int __wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config); extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key); extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...); @@ -439,7 +445,7 @@ extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk); extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args); -extern int __wt_meta_btree_apply(WT_SESSION_IMPL *session, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); +extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt); extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep); extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname); @@ -488,6 +494,7 @@ extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret); extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh); extern int __wt_errno(void); +extern int __wt_map_error_rdonly(int error); extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); extern int __wt_exist(WT_SESSION_IMPL *session, const char *filename, bool *existp); extern void __wt_fallocate_config(WT_SESSION_IMPL *session, WT_FH *fh); @@ -550,7 +557,6 @@ extern int __wt_struct_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v); extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char *fmt, ...); extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...); -extern int __wt_struct_unpack_size(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, size_t *resultp); extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf); extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell); extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); @@ -610,6 +616,7 @@ extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const ch extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str); extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len); extern int __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags); +extern int __wt_session_notsup(WT_SESSION *wt_session); extern int __wt_session_reset_cursors(WT_SESSION_IMPL *session, bool free_buffers); extern int __wt_session_copy_values(WT_SESSION_IMPL *session); extern int __wt_session_release_resources(WT_SESSION_IMPL *session); @@ -619,8 +626,8 @@ extern int __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const ch extern int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop); extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, bool open_metadata, WT_SESSION_IMPL **sessionp); extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp); -extern int __wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp); extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config); +extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config); extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp); extern int __wt_session_release_btree(WT_SESSION_IMPL *session); extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], uint32_t flags); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index e610c07f432..a6f42a9938f 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -12,13 +12,14 @@ #define WT_CONN_LOG_SERVER_RUN 0x00000080 #define WT_CONN_LSM_MERGE 0x00000100 #define WT_CONN_PANIC 0x00000200 -#define WT_CONN_SERVER_ASYNC 0x00000400 -#define WT_CONN_SERVER_CHECKPOINT 0x00000800 -#define WT_CONN_SERVER_LSM 0x00001000 -#define WT_CONN_SERVER_RUN 0x00002000 -#define WT_CONN_SERVER_STATISTICS 0x00004000 -#define WT_CONN_SERVER_SWEEP 0x00008000 -#define WT_CONN_WAS_BACKUP 0x00010000 +#define WT_CONN_READONLY 0x00000400 +#define WT_CONN_SERVER_ASYNC 0x00000800 +#define WT_CONN_SERVER_CHECKPOINT 0x00001000 +#define WT_CONN_SERVER_LSM 0x00002000 +#define WT_CONN_SERVER_RUN 0x00004000 +#define WT_CONN_SERVER_STATISTICS 0x00008000 +#define WT_CONN_SERVER_SWEEP 0x00010000 +#define WT_CONN_WAS_BACKUP 0x00020000 #define WT_EVICTING 0x00000001 #define WT_EVICT_IN_MEMORY 0x00000002 #define WT_EVICT_LOOKASIDE 0x00000004 diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 5dadb1b1484..4d3ca758dc7 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -201,10 +201,6 @@ (((const char *)str)[0] == ((const char *)pfx)[0] && \ strncmp((str), (pfx), strlen(pfx)) == 0) -/* Check if a non-nul-terminated string matches a prefix. */ -#define WT_PREFIX_MATCH_LEN(str, len, pfx) \ - ((len) >= strlen(pfx) && WT_PREFIX_MATCH(str, pfx)) - /* Check if a string matches a prefix, and move past it. */ #define WT_PREFIX_SKIP(str, pfx) \ (WT_PREFIX_MATCH(str, pfx) ? ((str) += strlen(pfx), 1) : 0) diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index b3c475805a4..7fdb7fc2548 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -126,11 +126,24 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { void *block_manager; /* Block-manager support */ int (*block_manager_cleanup)(WT_SESSION_IMPL *); - /* Checkpoint support */ + /* Checkpoint handles */ WT_DATA_HANDLE **ckpt_handle; /* Handle list */ u_int ckpt_handle_next; /* Next empty slot */ size_t ckpt_handle_allocated; /* Bytes allocated */ + /* + * Operations acting on handles. + * + * The preferred pattern is to gather all of the required handles at + * the beginning of an operation, then drop any other locks, perform + * the operation, then release the handles. This cannot be easily + * merged with the list of checkpoint handles because some operations + * (such as compact) do checkpoints internally. + */ + WT_DATA_HANDLE **op_handle; /* Handle list */ + u_int op_handle_next; /* Next empty slot */ + size_t op_handle_allocated; /* Bytes allocated */ + void *reconcile; /* Reconciliation support */ int (*reconcile_cleanup)(WT_SESSION_IMPL *); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 51d2fa332e7..8bc6c37b53e 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -227,12 +227,22 @@ __wt_stats_clear(void *stats_arg, int slot) */ #define WT_CONNECTION_STATS_BASE 1000 struct __wt_connection_stats { - int64_t async_alloc_race; - int64_t async_alloc_view; + int64_t lsm_work_queue_app; + int64_t lsm_work_queue_manager; + int64_t lsm_rows_merged; + int64_t lsm_checkpoint_throttle; + int64_t lsm_merge_throttle; + int64_t lsm_work_queue_switch; + int64_t lsm_work_units_discarded; + int64_t lsm_work_units_done; + int64_t lsm_work_units_created; + int64_t lsm_work_queue_max; int64_t async_cur_queue; + int64_t async_max_queue; + int64_t async_alloc_race; int64_t async_flush; + int64_t async_alloc_view; int64_t async_full; - int64_t async_max_queue; int64_t async_nowork; int64_t async_op_alloc; int64_t async_op_compact; @@ -240,55 +250,64 @@ struct __wt_connection_stats { int64_t async_op_remove; int64_t async_op_search; int64_t async_op_update; - int64_t block_byte_map_read; - int64_t block_byte_read; - int64_t block_byte_write; - int64_t block_map_read; int64_t block_preload; int64_t block_read; int64_t block_write; - int64_t cache_bytes_dirty; - int64_t cache_bytes_internal; + int64_t block_byte_read; + int64_t block_byte_write; + int64_t block_map_read; + int64_t block_byte_map_read; int64_t cache_bytes_inuse; - int64_t cache_bytes_leaf; - int64_t cache_bytes_max; - int64_t cache_bytes_overflow; int64_t cache_bytes_read; int64_t cache_bytes_write; - int64_t cache_eviction_app; int64_t cache_eviction_checkpoint; - int64_t cache_eviction_clean; - int64_t cache_eviction_deepen; - int64_t cache_eviction_dirty; - int64_t cache_eviction_fail; - int64_t cache_eviction_force; - int64_t cache_eviction_force_delete; - int64_t cache_eviction_force_fail; - int64_t cache_eviction_hazard; - int64_t cache_eviction_internal; - int64_t cache_eviction_maximum_page_size; + int64_t cache_eviction_aggressive_set; int64_t cache_eviction_queue_empty; int64_t cache_eviction_queue_not_empty; int64_t cache_eviction_server_evicting; int64_t cache_eviction_server_not_evicting; int64_t cache_eviction_slow; - int64_t cache_eviction_split_internal; - int64_t cache_eviction_split_leaf; - int64_t cache_eviction_walk; int64_t cache_eviction_worker_evicting; - int64_t cache_inmem_split; + int64_t cache_eviction_force_fail; + int64_t cache_eviction_hazard; int64_t cache_inmem_splittable; + int64_t cache_inmem_split; + int64_t cache_eviction_internal; + int64_t cache_eviction_split_internal; + int64_t cache_eviction_split_leaf; int64_t cache_lookaside_insert; int64_t cache_lookaside_remove; - int64_t cache_overhead; - int64_t cache_pages_dirty; + int64_t cache_bytes_max; + int64_t cache_eviction_maximum_page_size; + int64_t cache_eviction_dirty; + int64_t cache_eviction_deepen; + int64_t cache_write_lookaside; int64_t cache_pages_inuse; + int64_t cache_eviction_force; + int64_t cache_eviction_force_delete; + int64_t cache_eviction_app; int64_t cache_read; int64_t cache_read_lookaside; + int64_t cache_eviction_fail; + int64_t cache_eviction_walk; int64_t cache_write; - int64_t cache_write_lookaside; int64_t cache_write_restore; + int64_t cache_overhead; + int64_t cache_bytes_internal; + int64_t cache_bytes_leaf; + int64_t cache_bytes_overflow; + int64_t cache_bytes_dirty; + int64_t cache_pages_dirty; + int64_t cache_eviction_clean; + int64_t file_open; + int64_t memory_allocation; + int64_t memory_free; + int64_t memory_grow; int64_t cond_wait; + int64_t rwlock_read; + int64_t rwlock_write; + int64_t read_io; + int64_t write_io; int64_t cursor_create; int64_t cursor_insert; int64_t cursor_next; @@ -298,96 +317,78 @@ struct __wt_connection_stats { int64_t cursor_restart; int64_t cursor_search; int64_t cursor_search_near; - int64_t cursor_truncate; int64_t cursor_update; + int64_t cursor_truncate; int64_t dh_conn_handle_count; - int64_t dh_session_handles; - int64_t dh_session_sweeps; - int64_t dh_sweep_close; int64_t dh_sweep_ref; + int64_t dh_sweep_close; int64_t dh_sweep_remove; int64_t dh_sweep_tod; int64_t dh_sweeps; - int64_t file_open; - int64_t log_buffer_size; + int64_t dh_session_handles; + int64_t dh_session_sweeps; + int64_t log_slot_switch_busy; + int64_t log_slot_closes; + int64_t log_slot_races; + int64_t log_slot_transitions; + int64_t log_slot_joins; + int64_t log_slot_unbuffered; int64_t log_bytes_payload; int64_t log_bytes_written; - int64_t log_close_yields; - int64_t log_compress_len; - int64_t log_compress_mem; - int64_t log_compress_small; - int64_t log_compress_write_fails; - int64_t log_compress_writes; + int64_t log_zero_fills; int64_t log_flush; + int64_t log_compress_writes; + int64_t log_compress_write_fails; + int64_t log_compress_small; + int64_t log_release_write_lsn; + int64_t log_scans; + int64_t log_scan_rereads; + int64_t log_write_lsn; + int64_t log_sync; + int64_t log_sync_dir; + int64_t log_writes; + int64_t log_slot_consolidated; int64_t log_max_filesize; - int64_t log_prealloc_files; int64_t log_prealloc_max; int64_t log_prealloc_missed; + int64_t log_prealloc_files; int64_t log_prealloc_used; - int64_t log_release_write_lsn; int64_t log_scan_records; - int64_t log_scan_rereads; - int64_t log_scans; - int64_t log_slot_closes; + int64_t log_compress_mem; + int64_t log_buffer_size; + int64_t log_compress_len; int64_t log_slot_coalesced; - int64_t log_slot_consolidated; - int64_t log_slot_joins; - int64_t log_slot_races; - int64_t log_slot_switch_busy; - int64_t log_slot_transitions; - int64_t log_slot_unbuffered; - int64_t log_sync; - int64_t log_sync_dir; - int64_t log_write_lsn; - int64_t log_writes; - int64_t log_zero_fills; - int64_t lsm_checkpoint_throttle; - int64_t lsm_merge_throttle; - int64_t lsm_rows_merged; - int64_t lsm_work_queue_app; - int64_t lsm_work_queue_manager; - int64_t lsm_work_queue_max; - int64_t lsm_work_queue_switch; - int64_t lsm_work_units_created; - int64_t lsm_work_units_discarded; - int64_t lsm_work_units_done; - int64_t memory_allocation; - int64_t memory_free; - int64_t memory_grow; - int64_t page_busy_blocked; - int64_t page_forcible_evict_blocked; - int64_t page_locked_blocked; - int64_t page_read_blocked; - int64_t page_sleep; - int64_t read_io; - int64_t rec_page_delete; + int64_t log_close_yields; int64_t rec_page_delete_fast; int64_t rec_pages; int64_t rec_pages_eviction; + int64_t rec_page_delete; int64_t rec_split_stashed_bytes; int64_t rec_split_stashed_objects; - int64_t rwlock_read; - int64_t rwlock_write; int64_t session_cursor_open; int64_t session_open; + int64_t page_busy_blocked; + int64_t page_forcible_evict_blocked; + int64_t page_locked_blocked; + int64_t page_read_blocked; + int64_t page_sleep; + int64_t txn_snapshots_created; + int64_t txn_snapshots_dropped; int64_t txn_begin; - int64_t txn_checkpoint; - int64_t txn_checkpoint_generation; int64_t txn_checkpoint_running; + int64_t txn_checkpoint_generation; int64_t txn_checkpoint_time_max; int64_t txn_checkpoint_time_min; int64_t txn_checkpoint_time_recent; int64_t txn_checkpoint_time_total; - int64_t txn_commit; + int64_t txn_checkpoint; int64_t txn_fail_cache; - int64_t txn_pinned_checkpoint_range; int64_t txn_pinned_range; + int64_t txn_pinned_checkpoint_range; int64_t txn_pinned_snapshot_range; - int64_t txn_rollback; - int64_t txn_snapshots_created; - int64_t txn_snapshots_dropped; int64_t txn_sync; - int64_t write_io; + int64_t txn_commit; + int64_t txn_rollback; }; /* @@ -395,102 +396,102 @@ struct __wt_connection_stats { */ #define WT_DSRC_STATS_BASE 2000 struct __wt_dsrc_stats { - int64_t allocation_size; - int64_t block_alloc; - int64_t block_checkpoint_size; - int64_t block_extension; - int64_t block_free; - int64_t block_magic; - int64_t block_major; - int64_t block_minor; - int64_t block_reuse_bytes; - int64_t block_size; - int64_t bloom_count; int64_t bloom_false_positive; int64_t bloom_hit; int64_t bloom_miss; int64_t bloom_page_evict; int64_t bloom_page_read; + int64_t bloom_count; + int64_t lsm_chunk_count; + int64_t lsm_generation_max; + int64_t lsm_lookup_no_bloom; + int64_t lsm_checkpoint_throttle; + int64_t lsm_merge_throttle; int64_t bloom_size; + int64_t block_extension; + int64_t block_alloc; + int64_t block_free; + int64_t block_checkpoint_size; + int64_t allocation_size; + int64_t block_reuse_bytes; + int64_t block_magic; + int64_t block_major; + int64_t block_size; + int64_t block_minor; int64_t btree_checkpoint_generation; - int64_t btree_column_deleted; int64_t btree_column_fix; int64_t btree_column_internal; int64_t btree_column_rle; + int64_t btree_column_deleted; int64_t btree_column_variable; - int64_t btree_compact_rewrite; - int64_t btree_entries; int64_t btree_fixed_len; - int64_t btree_maximum_depth; int64_t btree_maxintlkey; int64_t btree_maxintlpage; int64_t btree_maxleafkey; int64_t btree_maxleafpage; int64_t btree_maxleafvalue; + int64_t btree_maximum_depth; + int64_t btree_entries; int64_t btree_overflow; + int64_t btree_compact_rewrite; int64_t btree_row_internal; int64_t btree_row_leaf; int64_t cache_bytes_read; int64_t cache_bytes_write; int64_t cache_eviction_checkpoint; - int64_t cache_eviction_clean; - int64_t cache_eviction_deepen; - int64_t cache_eviction_dirty; int64_t cache_eviction_fail; int64_t cache_eviction_hazard; + int64_t cache_inmem_splittable; + int64_t cache_inmem_split; int64_t cache_eviction_internal; int64_t cache_eviction_split_internal; int64_t cache_eviction_split_leaf; - int64_t cache_inmem_split; - int64_t cache_inmem_splittable; + int64_t cache_eviction_dirty; + int64_t cache_read_overflow; int64_t cache_overflow_value; + int64_t cache_eviction_deepen; + int64_t cache_write_lookaside; int64_t cache_read; int64_t cache_read_lookaside; - int64_t cache_read_overflow; int64_t cache_write; - int64_t cache_write_lookaside; int64_t cache_write_restore; - int64_t compress_raw_fail; - int64_t compress_raw_fail_temporary; - int64_t compress_raw_ok; + int64_t cache_eviction_clean; int64_t compress_read; int64_t compress_write; int64_t compress_write_fail; int64_t compress_write_too_small; - int64_t cursor_create; - int64_t cursor_insert; + int64_t compress_raw_fail_temporary; + int64_t compress_raw_fail; + int64_t compress_raw_ok; int64_t cursor_insert_bulk; + int64_t cursor_create; int64_t cursor_insert_bytes; + int64_t cursor_remove_bytes; + int64_t cursor_update_bytes; + int64_t cursor_insert; int64_t cursor_next; int64_t cursor_prev; int64_t cursor_remove; - int64_t cursor_remove_bytes; int64_t cursor_reset; int64_t cursor_restart; int64_t cursor_search; int64_t cursor_search_near; int64_t cursor_truncate; int64_t cursor_update; - int64_t cursor_update_bytes; - int64_t lsm_checkpoint_throttle; - int64_t lsm_chunk_count; - int64_t lsm_generation_max; - int64_t lsm_lookup_no_bloom; - int64_t lsm_merge_throttle; int64_t rec_dictionary; + int64_t rec_page_delete_fast; + int64_t rec_suffix_compression; int64_t rec_multiblock_internal; - int64_t rec_multiblock_leaf; - int64_t rec_multiblock_max; int64_t rec_overflow_key_internal; + int64_t rec_prefix_compression; + int64_t rec_multiblock_leaf; int64_t rec_overflow_key_leaf; + int64_t rec_multiblock_max; int64_t rec_overflow_value; - int64_t rec_page_delete; - int64_t rec_page_delete_fast; int64_t rec_page_match; int64_t rec_pages; int64_t rec_pages_eviction; - int64_t rec_prefix_compression; - int64_t rec_suffix_compression; + int64_t rec_page_delete; int64_t session_compact; int64_t session_cursor_open; int64_t txn_update_conflict; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 46f2ff3e5f1..40e2a6175d6 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -266,6 +266,8 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) } F_SET(txn, WT_TXN_RUNNING); + if (F_ISSET(S2C(session), WT_CONN_READONLY)) + F_SET(txn, WT_TXN_READONLY); return (false); } diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 767c176b53f..0c314e0705f 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -828,7 +828,8 @@ struct __wt_session { * @snippet ex_all.c Display an error thread safe * * @param session the session handle - * @param error a return value from a WiredTiger function + * @param error a return value from a WiredTiger, ISO C, or POSIX + * standard API * @returns a string representation of the error */ const char *__F(strerror)(WT_SESSION *session, int error); @@ -1409,7 +1410,7 @@ struct __wt_session { * if <code>NULL</code>, the truncate continues to the end of the * object * @configempty{WT_SESSION.truncate, see dist/api_data.py} - * @ebusy_errors + * @errors */ int __F(truncate)(WT_SESSION *session, const char *name, @@ -1893,8 +1894,10 @@ struct __wt_connection { * information. Enabling the statistics log server uses a session from * the configured session_max., a set of related configuration options * defined below.} - * @config{ on_close, log - * statistics on database close., a boolean flag; default \c false.} + * @config{ json, encode + * statistics in JSON format., a boolean flag; default \c false.} + * @config{ on_close, log statistics on database + * close., a boolean flag; default \c false.} * @config{ path, the pathname to a file into * which the log records are written\, may contain ISO C standard * strftime conversion specifications. If the value is not an absolute @@ -1908,7 +1911,8 @@ struct __wt_connection { * empty.} * @config{ timestamp, a timestamp * prepended to each log record\, may contain strftime conversion - * specifications., a string; default \c "%b %d %H:%M:%S".} + * specifications\, when \c json is configured\, defaults to \c + * "%FT%Y.000Z"., a string; default \c "%b %d %H:%M:%S".} * @config{ wait, seconds to wait between each * write of the log records; setting this value above 0 configures * statistics logging., an integer between 0 and 100000; default \c 0.} @@ -1982,7 +1986,8 @@ struct __wt_connection { * * @param connection the connection handle * @param errhandler An error handler. If <code>NULL</code>, the - * connection's error handler is used + * connection's error handler is used. See @ref error_handling_event + * for more information. * @configstart{WT_CONNECTION.open_session, see dist/api_data.py} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c @@ -2143,7 +2148,8 @@ struct __wt_connection { * @param home The path to the database home directory. See @ref home * for more information. * @param errhandler An error handler. If <code>NULL</code>, a builtin error - * handler is installed that writes error messages to stderr + * handler is installed that writes error messages to stderr. See + * @ref error_handling_event for more information. * @configstart{wiredtiger_open, see dist/api_data.py} * @config{async = (, asynchronous operations configuration options., a set of * related configuration options defined below.} @@ -2326,6 +2332,9 @@ struct __wt_connection { * start an RPC server for primary processes and use RPC for secondary * processes). <b>Not yet supported in WiredTiger</b>., a boolean flag; default * \c false.} + * @config{readonly, open connection in read-only mode. The database must + * exist. All methods that may modify a database are disabled. See @ref + * readonly for more information., a boolean flag; default \c false.} * @config{session_max, maximum expected number of sessions (including server * threads)., an integer greater than or equal to 1; default \c 100.} * @config{shared_cache = (, shared cache configuration options. A database @@ -2363,23 +2372,26 @@ struct __wt_connection { * maintain\, to a file. See @ref statistics for more information. Enabling * the statistics log server uses a session from the configured session_max., a * set of related configuration options defined below.} - * @config{ on_close, log statistics on database close., - * a boolean flag; default \c false.} - * @config{ path, the - * pathname to a file into which the log records are written\, may contain ISO C - * standard strftime conversion specifications. If the value is not an absolute - * path name\, the file is created relative to the database home., a string; - * default \c "WiredTigerStat.%d.%H".} - * @config{ sources, - * if non-empty\, include statistics for the list of data source URIs\, if they - * are open at the time of the statistics logging. The list may include URIs + * @config{ json, encode statistics in JSON format., a + * boolean flag; default \c false.} + * @config{ on_close, + * log statistics on database close., a boolean flag; default \c false.} + * @config{ path, the pathname to a file into which the + * log records are written\, may contain ISO C standard strftime conversion + * specifications. If the value is not an absolute path name\, the file is + * created relative to the database home., a string; default \c + * "WiredTigerStat.%d.%H".} + * @config{ sources, if + * non-empty\, include statistics for the list of data source URIs\, if they are + * open at the time of the statistics logging. The list may include URIs * matching a single data source ("table:mytable")\, or a URI matching all data * sources of a particular type ("table:")., a list of strings; default empty.} * @config{ timestamp, a timestamp prepended to each log - * record\, may contain strftime conversion specifications., a string; default - * \c "%b %d %H:%M:%S".} - * @config{ wait, seconds to wait - * between each write of the log records; setting this value above 0 configures + * record\, may contain strftime conversion specifications\, when \c json is + * configured\, defaults to \c "%FT%Y.000Z"., a string; default \c "%b %d + * %H:%M:%S".} + * @config{ wait, seconds to wait between + * each write of the log records; setting this value above 0 configures * statistics logging., an integer between 0 and 100000; default \c 0.} * @config{ * ),,} @@ -2431,11 +2443,12 @@ int wiredtiger_open(const char *home, WT_CONNECTION **connectionp); /*! - * Return information about a WiredTiger error as a string, not thread-safe. + * Return information about a WiredTiger error as a string (see + * WT_SESSION::strerror for a thread-safe API). * * @snippet ex_all.c Display an error * - * @param error a return value from a WiredTiger call + * @param error a return value from a WiredTiger, ISO C, or POSIX standard API * @returns a string representation of the error */ const char *wiredtiger_strerror(int error); @@ -2474,7 +2487,7 @@ struct __wt_async_callback { struct __wt_event_handler { /*! * Callback to handle error messages; by default, error messages are - * written to the stderr stream. + * written to the stderr stream. See @ref error_handling. * * Errors that require the application to exit and restart will have * their \c error value set to \c WT_PANIC. The application can exit @@ -2488,8 +2501,9 @@ struct __wt_event_handler { * @param session the WiredTiger session handle in use when the error * was generated. The handle may have been created by the application * or automatically by WiredTiger. - * @param error a WiredTiger, C99 or POSIX error code, which can - * be converted to a string using ::wiredtiger_strerror + * @param error a return value from a WiredTiger, ISO C, or + * POSIX standard API, which can be converted to a string using + * WT_SESSION::strerror * @param message an error string */ int (*handle_error)(WT_EVENT_HANDLER *handler, @@ -2497,7 +2511,7 @@ struct __wt_event_handler { /*! * Callback to handle informational messages; by default, informational - * messages are written to the stdout stream. + * messages are written to the stdout stream. See @ref error_handling. * * Message handler returns are not ignored: if the handler returns * non-zero, the error may cause the WiredTiger function posting the @@ -2513,7 +2527,7 @@ struct __wt_event_handler { /*! * Callback to handle progress messages; by default, no progress - * messages are written. + * messages are written. See @ref error_handling. * * Progress handler returns are not ignored: if the handler returns * non-zero, the error may cause the WiredTiger function posting the @@ -2998,6 +3012,10 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); */ #define WT_CACHE_FULL -31807 /*! @endcond */ +/*! @cond internal */ +/*! Permission denied (internal). */ +#define WT_PERM_DENIED -31808 +/*! @endcond */ /* * Error return section: END * DO NOT EDIT: automatically built by dist/api_err.py. @@ -3688,329 +3706,331 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * keys. See @ref data_statistics for more information. * @{ */ -/*! async: number of allocation state races */ -#define WT_STAT_CONN_ASYNC_ALLOC_RACE 1000 -/*! async: number of operation slots viewed for allocation */ -#define WT_STAT_CONN_ASYNC_ALLOC_VIEW 1001 +/*! LSM: application work units currently queued */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1000 +/*! LSM: merge work units currently queued */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1001 +/*! LSM: rows merged in an LSM tree */ +#define WT_STAT_CONN_LSM_ROWS_MERGED 1002 +/*! LSM: sleep for LSM checkpoint throttle */ +#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1003 +/*! LSM: sleep for LSM merge throttle */ +#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1004 +/*! LSM: switch work units currently queued */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1005 +/*! LSM: tree maintenance operations discarded */ +#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1006 +/*! LSM: tree maintenance operations executed */ +#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1007 +/*! LSM: tree maintenance operations scheduled */ +#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1008 +/*! LSM: tree queue hit maximum */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1009 /*! async: current work queue length */ -#define WT_STAT_CONN_ASYNC_CUR_QUEUE 1002 +#define WT_STAT_CONN_ASYNC_CUR_QUEUE 1010 +/*! async: maximum work queue length */ +#define WT_STAT_CONN_ASYNC_MAX_QUEUE 1011 +/*! async: number of allocation state races */ +#define WT_STAT_CONN_ASYNC_ALLOC_RACE 1012 /*! async: number of flush calls */ -#define WT_STAT_CONN_ASYNC_FLUSH 1003 +#define WT_STAT_CONN_ASYNC_FLUSH 1013 +/*! async: number of operation slots viewed for allocation */ +#define WT_STAT_CONN_ASYNC_ALLOC_VIEW 1014 /*! async: number of times operation allocation failed */ -#define WT_STAT_CONN_ASYNC_FULL 1004 -/*! async: maximum work queue length */ -#define WT_STAT_CONN_ASYNC_MAX_QUEUE 1005 +#define WT_STAT_CONN_ASYNC_FULL 1015 /*! async: number of times worker found no work */ -#define WT_STAT_CONN_ASYNC_NOWORK 1006 +#define WT_STAT_CONN_ASYNC_NOWORK 1016 /*! async: total allocations */ -#define WT_STAT_CONN_ASYNC_OP_ALLOC 1007 +#define WT_STAT_CONN_ASYNC_OP_ALLOC 1017 /*! async: total compact calls */ -#define WT_STAT_CONN_ASYNC_OP_COMPACT 1008 +#define WT_STAT_CONN_ASYNC_OP_COMPACT 1018 /*! async: total insert calls */ -#define WT_STAT_CONN_ASYNC_OP_INSERT 1009 +#define WT_STAT_CONN_ASYNC_OP_INSERT 1019 /*! async: total remove calls */ -#define WT_STAT_CONN_ASYNC_OP_REMOVE 1010 +#define WT_STAT_CONN_ASYNC_OP_REMOVE 1020 /*! async: total search calls */ -#define WT_STAT_CONN_ASYNC_OP_SEARCH 1011 +#define WT_STAT_CONN_ASYNC_OP_SEARCH 1021 /*! async: total update calls */ -#define WT_STAT_CONN_ASYNC_OP_UPDATE 1012 -/*! block-manager: mapped bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1013 -/*! block-manager: bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_READ 1014 -/*! block-manager: bytes written */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1015 -/*! block-manager: mapped blocks read */ -#define WT_STAT_CONN_BLOCK_MAP_READ 1016 +#define WT_STAT_CONN_ASYNC_OP_UPDATE 1022 /*! block-manager: blocks pre-loaded */ -#define WT_STAT_CONN_BLOCK_PRELOAD 1017 +#define WT_STAT_CONN_BLOCK_PRELOAD 1023 /*! block-manager: blocks read */ -#define WT_STAT_CONN_BLOCK_READ 1018 +#define WT_STAT_CONN_BLOCK_READ 1024 /*! block-manager: blocks written */ -#define WT_STAT_CONN_BLOCK_WRITE 1019 -/*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1020 -/*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1021 +#define WT_STAT_CONN_BLOCK_WRITE 1025 +/*! block-manager: bytes read */ +#define WT_STAT_CONN_BLOCK_BYTE_READ 1026 +/*! block-manager: bytes written */ +#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1027 +/*! block-manager: mapped blocks read */ +#define WT_STAT_CONN_BLOCK_MAP_READ 1028 +/*! block-manager: mapped bytes read */ +#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1029 /*! cache: bytes currently in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INUSE 1022 -/*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1023 -/*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1024 -/*! cache: tracked bytes belonging to overflow pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_OVERFLOW 1025 +#define WT_STAT_CONN_CACHE_BYTES_INUSE 1030 /*! cache: bytes read into cache */ -#define WT_STAT_CONN_CACHE_BYTES_READ 1026 +#define WT_STAT_CONN_CACHE_BYTES_READ 1031 /*! cache: bytes written from cache */ -#define WT_STAT_CONN_CACHE_BYTES_WRITE 1027 -/*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1028 +#define WT_STAT_CONN_CACHE_BYTES_WRITE 1032 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1029 -/*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1030 -/*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1031 -/*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1032 -/*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1033 -/*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1034 -/*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1035 -/*! cache: failed eviction of pages that exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1036 -/*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1037 -/*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1038 -/*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1039 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1033 +/*! cache: eviction currently operating in aggressive mode */ +#define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1034 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1040 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1035 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1041 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1036 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1042 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1037 /*! cache: eviction server populating queue, but not evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1043 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1038 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1044 -/*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1045 -/*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1046 -/*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1047 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1039 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1048 -/*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1049 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1040 +/*! cache: failed eviction of pages that exceeded the in-memory maximum */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1041 +/*! cache: hazard pointer blocked page eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1042 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1050 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1043 +/*! cache: in-memory page splits */ +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1044 +/*! cache: internal pages evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1045 +/*! cache: internal pages split during eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1046 +/*! cache: leaf pages split during eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1047 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1051 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1048 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1052 -/*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1053 -/*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1054 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1049 +/*! cache: maximum bytes configured */ +#define WT_STAT_CONN_CACHE_BYTES_MAX 1050 +/*! cache: maximum page size at eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1051 +/*! cache: modified pages evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1052 +/*! cache: page split during eviction deepened the tree */ +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1053 +/*! cache: page written requiring lookaside records */ +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1054 /*! cache: pages currently held in the cache */ #define WT_STAT_CONN_CACHE_PAGES_INUSE 1055 +/*! cache: pages evicted because they exceeded the in-memory maximum */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1056 +/*! cache: pages evicted because they had chains of deleted items */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1057 +/*! cache: pages evicted by application threads */ +#define WT_STAT_CONN_CACHE_EVICTION_APP 1058 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1056 +#define WT_STAT_CONN_CACHE_READ 1059 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1057 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1060 +/*! cache: pages selected for eviction unable to be evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1061 +/*! cache: pages walked for eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1062 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1058 -/*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1059 +#define WT_STAT_CONN_CACHE_WRITE 1063 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1060 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1064 +/*! cache: percentage overhead */ +#define WT_STAT_CONN_CACHE_OVERHEAD 1065 +/*! cache: tracked bytes belonging to internal pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1066 +/*! cache: tracked bytes belonging to leaf pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1067 +/*! cache: tracked bytes belonging to overflow pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_OVERFLOW 1068 +/*! cache: tracked dirty bytes in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1069 +/*! cache: tracked dirty pages in the cache */ +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1070 +/*! cache: unmodified pages evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1071 +/*! connection: files currently open */ +#define WT_STAT_CONN_FILE_OPEN 1072 +/*! connection: memory allocations */ +#define WT_STAT_CONN_MEMORY_ALLOCATION 1073 +/*! connection: memory frees */ +#define WT_STAT_CONN_MEMORY_FREE 1074 +/*! connection: memory re-allocations */ +#define WT_STAT_CONN_MEMORY_GROW 1075 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1061 +#define WT_STAT_CONN_COND_WAIT 1076 +/*! connection: pthread mutex shared lock read-lock calls */ +#define WT_STAT_CONN_RWLOCK_READ 1077 +/*! connection: pthread mutex shared lock write-lock calls */ +#define WT_STAT_CONN_RWLOCK_WRITE 1078 +/*! connection: total read I/Os */ +#define WT_STAT_CONN_READ_IO 1079 +/*! connection: total write I/Os */ +#define WT_STAT_CONN_WRITE_IO 1080 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1062 +#define WT_STAT_CONN_CURSOR_CREATE 1081 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1063 +#define WT_STAT_CONN_CURSOR_INSERT 1082 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1064 +#define WT_STAT_CONN_CURSOR_NEXT 1083 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1065 +#define WT_STAT_CONN_CURSOR_PREV 1084 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1066 +#define WT_STAT_CONN_CURSOR_REMOVE 1085 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1067 +#define WT_STAT_CONN_CURSOR_RESET 1086 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1068 +#define WT_STAT_CONN_CURSOR_RESTART 1087 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1069 +#define WT_STAT_CONN_CURSOR_SEARCH 1088 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1070 -/*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1071 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1089 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1072 +#define WT_STAT_CONN_CURSOR_UPDATE 1090 +/*! cursor: truncate calls */ +#define WT_STAT_CONN_CURSOR_TRUNCATE 1091 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1073 -/*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1074 -/*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1075 -/*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1076 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1092 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1077 +#define WT_STAT_CONN_DH_SWEEP_REF 1093 +/*! data-handle: connection sweep dhandles closed */ +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1094 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1078 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1095 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1079 +#define WT_STAT_CONN_DH_SWEEP_TOD 1096 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1080 -/*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1081 -/*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1082 +#define WT_STAT_CONN_DH_SWEEPS 1097 +/*! data-handle: session dhandles swept */ +#define WT_STAT_CONN_DH_SESSION_HANDLES 1098 +/*! data-handle: session sweep attempts */ +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1099 +/*! log: busy returns attempting to switch slots */ +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1100 +/*! log: consolidated slot closures */ +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1101 +/*! log: consolidated slot join races */ +#define WT_STAT_CONN_LOG_SLOT_RACES 1102 +/*! log: consolidated slot join transitions */ +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1103 +/*! log: consolidated slot joins */ +#define WT_STAT_CONN_LOG_SLOT_JOINS 1104 +/*! log: consolidated slot unbuffered writes */ +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1105 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1083 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1106 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1084 -/*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1085 -/*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1086 -/*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1087 -/*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1088 -/*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1089 -/*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1090 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1107 +/*! log: log files manually zero-filled */ +#define WT_STAT_CONN_LOG_ZERO_FILLS 1108 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1091 +#define WT_STAT_CONN_LOG_FLUSH 1109 +/*! log: log records compressed */ +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1110 +/*! log: log records not compressed */ +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1111 +/*! log: log records too small to compress */ +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1112 +/*! log: log release advances write LSN */ +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1113 +/*! log: log scan operations */ +#define WT_STAT_CONN_LOG_SCANS 1114 +/*! log: log scan records requiring two reads */ +#define WT_STAT_CONN_LOG_SCAN_REREADS 1115 +/*! log: log server thread advances write LSN */ +#define WT_STAT_CONN_LOG_WRITE_LSN 1116 +/*! log: log sync operations */ +#define WT_STAT_CONN_LOG_SYNC 1117 +/*! log: log sync_dir operations */ +#define WT_STAT_CONN_LOG_SYNC_DIR 1118 +/*! log: log write operations */ +#define WT_STAT_CONN_LOG_WRITES 1119 +/*! log: logging bytes consolidated */ +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1120 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1092 -/*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1093 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1121 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1094 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1122 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1095 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1123 +/*! log: pre-allocated log files prepared */ +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1124 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1096 -/*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1097 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1125 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1098 -/*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1099 -/*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1100 -/*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1101 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1126 +/*! log: total in-memory size of compressed records */ +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1127 +/*! log: total log buffer size */ +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1128 +/*! log: total size of compressed records */ +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1129 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1102 -/*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1103 -/*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1104 -/*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1105 -/*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1106 -/*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1107 -/*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1108 -/*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1109 -/*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1110 -/*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1111 -/*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1112 -/*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1113 -/*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1114 -/*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1115 -/*! LSM: rows merged in an LSM tree */ -#define WT_STAT_CONN_LSM_ROWS_MERGED 1116 -/*! LSM: application work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1117 -/*! LSM: merge work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1118 -/*! LSM: tree queue hit maximum */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1119 -/*! LSM: switch work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1120 -/*! LSM: tree maintenance operations scheduled */ -#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1121 -/*! LSM: tree maintenance operations discarded */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1122 -/*! LSM: tree maintenance operations executed */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1123 -/*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1124 -/*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1125 -/*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1126 -/*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1127 -/*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1128 -/*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1129 -/*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1130 -/*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1131 -/*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1132 -/*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1133 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1130 +/*! log: yields waiting for previous log file close */ +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1131 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1134 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1132 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1135 +#define WT_STAT_CONN_REC_PAGES 1133 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1136 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1134 +/*! reconciliation: pages deleted */ +#define WT_STAT_CONN_REC_PAGE_DELETE 1135 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1137 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1136 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1138 -/*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1139 -/*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1140 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1137 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1141 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1138 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1142 +#define WT_STAT_CONN_SESSION_OPEN 1139 +/*! thread-yield: page acquire busy blocked */ +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1140 +/*! thread-yield: page acquire eviction blocked */ +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1141 +/*! thread-yield: page acquire locked blocked */ +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1142 +/*! thread-yield: page acquire read blocked */ +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1143 +/*! thread-yield: page acquire time sleeping (usecs) */ +#define WT_STAT_CONN_PAGE_SLEEP 1144 +/*! transaction: number of named snapshots created */ +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1145 +/*! transaction: number of named snapshots dropped */ +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1146 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1143 -/*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1144 -/*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1145 +#define WT_STAT_CONN_TXN_BEGIN 1147 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1146 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1148 +/*! transaction: transaction checkpoint generation */ +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1149 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1147 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1150 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1148 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1151 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1149 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1152 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1150 -/*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1151 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1153 +/*! transaction: transaction checkpoints */ +#define WT_STAT_CONN_TXN_CHECKPOINT 1154 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1152 -/*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1153 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1155 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1154 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1156 +/*! transaction: transaction range of IDs currently pinned by a checkpoint */ +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1157 /*! transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1155 -/*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1156 -/*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1157 -/*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1158 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1158 /*! transaction: transaction sync calls */ #define WT_STAT_CONN_TXN_SYNC 1159 -/*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1160 +/*! transaction: transactions committed */ +#define WT_STAT_CONN_TXN_COMMIT 1160 +/*! transaction: transactions rolled back */ +#define WT_STAT_CONN_TXN_ROLLBACK 1161 /*! * @} @@ -4018,200 +4038,200 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * @anchor statistics_dsrc * @{ */ -/*! block-manager: file allocation unit size */ -#define WT_STAT_DSRC_ALLOCATION_SIZE 2000 -/*! block-manager: blocks allocated */ -#define WT_STAT_DSRC_BLOCK_ALLOC 2001 -/*! block-manager: checkpoint size */ -#define WT_STAT_DSRC_BLOCK_CHECKPOINT_SIZE 2002 -/*! block-manager: allocations requiring file extension */ -#define WT_STAT_DSRC_BLOCK_EXTENSION 2003 -/*! block-manager: blocks freed */ -#define WT_STAT_DSRC_BLOCK_FREE 2004 -/*! block-manager: file magic number */ -#define WT_STAT_DSRC_BLOCK_MAGIC 2005 -/*! block-manager: file major version number */ -#define WT_STAT_DSRC_BLOCK_MAJOR 2006 -/*! block-manager: minor version number */ -#define WT_STAT_DSRC_BLOCK_MINOR 2007 -/*! block-manager: file bytes available for reuse */ -#define WT_STAT_DSRC_BLOCK_REUSE_BYTES 2008 -/*! block-manager: file size in bytes */ -#define WT_STAT_DSRC_BLOCK_SIZE 2009 -/*! LSM: bloom filters in the LSM tree */ -#define WT_STAT_DSRC_BLOOM_COUNT 2010 /*! LSM: bloom filter false positives */ -#define WT_STAT_DSRC_BLOOM_FALSE_POSITIVE 2011 +#define WT_STAT_DSRC_BLOOM_FALSE_POSITIVE 2000 /*! LSM: bloom filter hits */ -#define WT_STAT_DSRC_BLOOM_HIT 2012 +#define WT_STAT_DSRC_BLOOM_HIT 2001 /*! LSM: bloom filter misses */ -#define WT_STAT_DSRC_BLOOM_MISS 2013 +#define WT_STAT_DSRC_BLOOM_MISS 2002 /*! LSM: bloom filter pages evicted from cache */ -#define WT_STAT_DSRC_BLOOM_PAGE_EVICT 2014 +#define WT_STAT_DSRC_BLOOM_PAGE_EVICT 2003 /*! LSM: bloom filter pages read into cache */ -#define WT_STAT_DSRC_BLOOM_PAGE_READ 2015 +#define WT_STAT_DSRC_BLOOM_PAGE_READ 2004 +/*! LSM: bloom filters in the LSM tree */ +#define WT_STAT_DSRC_BLOOM_COUNT 2005 +/*! LSM: chunks in the LSM tree */ +#define WT_STAT_DSRC_LSM_CHUNK_COUNT 2006 +/*! LSM: highest merge generation in the LSM tree */ +#define WT_STAT_DSRC_LSM_GENERATION_MAX 2007 +/*! LSM: queries that could have benefited from a Bloom filter that did + * not exist */ +#define WT_STAT_DSRC_LSM_LOOKUP_NO_BLOOM 2008 +/*! LSM: sleep for LSM checkpoint throttle */ +#define WT_STAT_DSRC_LSM_CHECKPOINT_THROTTLE 2009 +/*! LSM: sleep for LSM merge throttle */ +#define WT_STAT_DSRC_LSM_MERGE_THROTTLE 2010 /*! LSM: total size of bloom filters */ -#define WT_STAT_DSRC_BLOOM_SIZE 2016 +#define WT_STAT_DSRC_BLOOM_SIZE 2011 +/*! block-manager: allocations requiring file extension */ +#define WT_STAT_DSRC_BLOCK_EXTENSION 2012 +/*! block-manager: blocks allocated */ +#define WT_STAT_DSRC_BLOCK_ALLOC 2013 +/*! block-manager: blocks freed */ +#define WT_STAT_DSRC_BLOCK_FREE 2014 +/*! block-manager: checkpoint size */ +#define WT_STAT_DSRC_BLOCK_CHECKPOINT_SIZE 2015 +/*! block-manager: file allocation unit size */ +#define WT_STAT_DSRC_ALLOCATION_SIZE 2016 +/*! block-manager: file bytes available for reuse */ +#define WT_STAT_DSRC_BLOCK_REUSE_BYTES 2017 +/*! block-manager: file magic number */ +#define WT_STAT_DSRC_BLOCK_MAGIC 2018 +/*! block-manager: file major version number */ +#define WT_STAT_DSRC_BLOCK_MAJOR 2019 +/*! block-manager: file size in bytes */ +#define WT_STAT_DSRC_BLOCK_SIZE 2020 +/*! block-manager: minor version number */ +#define WT_STAT_DSRC_BLOCK_MINOR 2021 /*! btree: btree checkpoint generation */ -#define WT_STAT_DSRC_BTREE_CHECKPOINT_GENERATION 2017 -/*! btree: column-store variable-size deleted values */ -#define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2018 +#define WT_STAT_DSRC_BTREE_CHECKPOINT_GENERATION 2022 /*! btree: column-store fixed-size leaf pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_FIX 2019 +#define WT_STAT_DSRC_BTREE_COLUMN_FIX 2023 /*! btree: column-store internal pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2020 +#define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2024 /*! btree: column-store variable-size RLE encoded values */ -#define WT_STAT_DSRC_BTREE_COLUMN_RLE 2021 +#define WT_STAT_DSRC_BTREE_COLUMN_RLE 2025 +/*! btree: column-store variable-size deleted values */ +#define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2026 /*! btree: column-store variable-size leaf pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2022 -/*! btree: pages rewritten by compaction */ -#define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2023 -/*! btree: number of key/value pairs */ -#define WT_STAT_DSRC_BTREE_ENTRIES 2024 +#define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2027 /*! btree: fixed-record size */ -#define WT_STAT_DSRC_BTREE_FIXED_LEN 2025 -/*! btree: maximum tree depth */ -#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2026 +#define WT_STAT_DSRC_BTREE_FIXED_LEN 2028 /*! btree: maximum internal page key size */ -#define WT_STAT_DSRC_BTREE_MAXINTLKEY 2027 +#define WT_STAT_DSRC_BTREE_MAXINTLKEY 2029 /*! btree: maximum internal page size */ -#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2028 +#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2030 /*! btree: maximum leaf page key size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2029 +#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2031 /*! btree: maximum leaf page size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2030 +#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2032 /*! btree: maximum leaf page value size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2031 +#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2033 +/*! btree: maximum tree depth */ +#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2034 +/*! btree: number of key/value pairs */ +#define WT_STAT_DSRC_BTREE_ENTRIES 2035 /*! btree: overflow pages */ -#define WT_STAT_DSRC_BTREE_OVERFLOW 2032 +#define WT_STAT_DSRC_BTREE_OVERFLOW 2036 +/*! btree: pages rewritten by compaction */ +#define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2037 /*! btree: row-store internal pages */ -#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2033 +#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2038 /*! btree: row-store leaf pages */ -#define WT_STAT_DSRC_BTREE_ROW_LEAF 2034 +#define WT_STAT_DSRC_BTREE_ROW_LEAF 2039 /*! cache: bytes read into cache */ -#define WT_STAT_DSRC_CACHE_BYTES_READ 2035 +#define WT_STAT_DSRC_CACHE_BYTES_READ 2040 /*! cache: bytes written from cache */ -#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2036 +#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2041 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2037 -/*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2038 -/*! cache: page split during eviction deepened the tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2039 -/*! cache: modified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2040 +#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2042 /*! cache: data source pages selected for eviction unable to be evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2041 +#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2043 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2042 +#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2044 +/*! cache: in-memory page passed criteria to be split */ +#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2045 +/*! cache: in-memory page splits */ +#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2046 /*! cache: internal pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2043 +#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2047 /*! cache: internal pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2044 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2048 /*! cache: leaf pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2045 -/*! cache: in-memory page splits */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2046 -/*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2047 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2049 +/*! cache: modified pages evicted */ +#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2050 +/*! cache: overflow pages read into cache */ +#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2051 /*! cache: overflow values cached in memory */ -#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2048 +#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2052 +/*! cache: page split during eviction deepened the tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2053 +/*! cache: page written requiring lookaside records */ +#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2054 /*! cache: pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ 2049 +#define WT_STAT_DSRC_CACHE_READ 2055 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2050 -/*! cache: overflow pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2051 +#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2056 /*! cache: pages written from cache */ -#define WT_STAT_DSRC_CACHE_WRITE 2052 -/*! cache: page written requiring lookaside records */ -#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2053 +#define WT_STAT_DSRC_CACHE_WRITE 2057 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2054 -/*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2055 -/*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2056 -/*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2057 +#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2058 +/*! cache: unmodified pages evicted */ +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2059 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2058 +#define WT_STAT_DSRC_COMPRESS_READ 2060 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2059 +#define WT_STAT_DSRC_COMPRESS_WRITE 2061 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2060 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2062 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2061 -/*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2062 -/*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2063 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2063 +/*! compression: raw compression call failed, additional data available */ +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2064 +/*! compression: raw compression call failed, no additional data available */ +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2065 +/*! compression: raw compression call succeeded */ +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2066 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2064 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2067 +/*! cursor: create calls */ +#define WT_STAT_DSRC_CURSOR_CREATE 2068 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2065 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2069 +/*! cursor: cursor-remove key bytes removed */ +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2070 +/*! cursor: cursor-update value bytes updated */ +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2071 +/*! cursor: insert calls */ +#define WT_STAT_DSRC_CURSOR_INSERT 2072 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2066 +#define WT_STAT_DSRC_CURSOR_NEXT 2073 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2067 +#define WT_STAT_DSRC_CURSOR_PREV 2074 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2068 -/*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2069 +#define WT_STAT_DSRC_CURSOR_REMOVE 2075 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2070 +#define WT_STAT_DSRC_CURSOR_RESET 2076 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2071 +#define WT_STAT_DSRC_CURSOR_RESTART 2077 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2072 +#define WT_STAT_DSRC_CURSOR_SEARCH 2078 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2073 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2079 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2074 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2080 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2075 -/*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2076 -/*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_DSRC_LSM_CHECKPOINT_THROTTLE 2077 -/*! LSM: chunks in the LSM tree */ -#define WT_STAT_DSRC_LSM_CHUNK_COUNT 2078 -/*! LSM: highest merge generation in the LSM tree */ -#define WT_STAT_DSRC_LSM_GENERATION_MAX 2079 -/*! LSM: queries that could have benefited from a Bloom filter that did - * not exist */ -#define WT_STAT_DSRC_LSM_LOOKUP_NO_BLOOM 2080 -/*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_DSRC_LSM_MERGE_THROTTLE 2081 +#define WT_STAT_DSRC_CURSOR_UPDATE 2081 /*! reconciliation: dictionary matches */ #define WT_STAT_DSRC_REC_DICTIONARY 2082 +/*! reconciliation: fast-path pages deleted */ +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2083 +/*! reconciliation: internal page key bytes discarded using suffix + * compression */ +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2084 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2083 -/*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2084 -/*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2085 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2085 /*! reconciliation: internal-page overflow keys */ #define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2086 +/*! reconciliation: leaf page key bytes discarded using prefix compression */ +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2087 +/*! reconciliation: leaf page multi-block writes */ +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2088 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2087 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2089 +/*! reconciliation: maximum blocks required for a page */ +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2090 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2088 -/*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2089 -/*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2090 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2091 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2091 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2092 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2092 +#define WT_STAT_DSRC_REC_PAGES 2093 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2093 -/*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2094 -/*! reconciliation: internal page key bytes discarded using suffix - * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2095 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2094 +/*! reconciliation: pages deleted */ +#define WT_STAT_DSRC_REC_PAGE_DELETE 2095 /*! session: object compaction */ #define WT_STAT_DSRC_SESSION_COMPACT 2096 /*! session: open cursor count */ diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index ce2d7191491..03145d8408c 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -1088,28 +1088,36 @@ __wt_log_open(WT_SESSION_IMPL *session) WT_RET(__wt_open(session, conn->log_path, false, false, WT_FILE_TYPE_DIRECTORY, &log->log_dir_fh)); } - /* - * Clean up any old interim pre-allocated files. - * We clean up these files because settings have changed upon reboot - * and we want those settings to take effect right away. - */ - WT_ERR(__log_get_files(session, - WT_LOG_TMPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove(session, WT_LOG_TMPNAME, lognum)); - } - __wt_log_files_free(session, logfiles, logcount); - logfiles = NULL; - logcount = 0; - WT_ERR(__log_get_files(session, - WT_LOG_PREPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove(session, WT_LOG_PREPNAME, lognum)); + + if (!F_ISSET(conn, WT_CONN_READONLY)) { + /* + * Clean up any old interim pre-allocated files. We clean + * up these files because settings have changed upon reboot + * and we want those settings to take effect right away. + */ + WT_ERR(__log_get_files(session, + WT_LOG_TMPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove( + session, WT_LOG_TMPNAME, lognum)); + } + __wt_log_files_free(session, logfiles, logcount); + logfiles = NULL; + logcount = 0; + WT_ERR(__log_get_files(session, + WT_LOG_PREPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove( + session, WT_LOG_PREPNAME, lognum)); + } + __wt_log_files_free(session, logfiles, logcount); + logfiles = NULL; } - __wt_log_files_free(session, logfiles, logcount); - logfiles = NULL; + /* * Now look at the log files and set our LSNs. */ @@ -1132,9 +1140,11 @@ __wt_log_open(WT_SESSION_IMPL *session) * Start logging at the beginning of the next log file, no matter * where the previous log file ends. */ - WT_WITH_SLOT_LOCK(session, log, ret, - ret = __log_newfile(session, true, NULL)); - WT_ERR(ret); + if (!F_ISSET(conn, WT_CONN_READONLY)) { + WT_WITH_SLOT_LOCK(session, log, ret, + ret = __log_newfile(session, true, NULL)); + WT_ERR(ret); + } /* If we found log files, save the new state. */ if (logcount > 0) { @@ -1163,20 +1173,24 @@ __wt_log_close(WT_SESSION_IMPL *session) if (log->log_close_fh != NULL && log->log_close_fh != log->log_fh) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing old log %s", log->log_close_fh->name)); - WT_RET(__wt_fsync(session, log->log_close_fh)); + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_RET(__wt_fsync(session, log->log_close_fh)); WT_RET(__wt_close(session, &log->log_close_fh)); } if (log->log_fh != NULL) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing log %s", log->log_fh->name)); - WT_RET(__wt_fsync(session, log->log_fh)); + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_RET(__wt_fsync(session, log->log_fh)); WT_RET(__wt_close(session, &log->log_fh)); log->log_fh = NULL; } if (log->log_dir_fh != NULL) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing log directory %s", log->log_dir_fh->name)); - WT_RET(__wt_directory_sync_fh(session, log->log_dir_fh)); + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_RET( + __wt_directory_sync_fh(session, log->log_dir_fh)); WT_RET(__wt_close(session, &log->log_dir_fh)); log->log_dir_fh = NULL; } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index f76b2bfd9ac..0197b6481f4 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1501,22 +1501,22 @@ __wt_clsm_open(WT_SESSION_IMPL *session, { WT_CONFIG_ITEM cval; WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __clsm_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __clsm_next, /* next */ - __clsm_prev, /* prev */ - __clsm_reset, /* reset */ - __clsm_search, /* search */ - __clsm_search_near, /* search-near */ - __clsm_insert, /* insert */ - __clsm_update, /* update */ - __clsm_remove, /* remove */ - __wt_cursor_reconfigure, /* reconfigure */ - __wt_clsm_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __clsm_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __clsm_next, /* next */ + __clsm_prev, /* prev */ + __clsm_reset, /* reset */ + __clsm_search, /* search */ + __clsm_search_near, /* search-near */ + __clsm_insert, /* insert */ + __clsm_update, /* update */ + __clsm_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ + __wt_clsm_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_LSM *clsm; WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index dac8d987328..cf581475d2c 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -212,6 +212,10 @@ __wt_lsm_manager_start(WT_SESSION_IMPL *session) conn = S2C(session); manager = &conn->lsm_manager; + if (F_ISSET(conn, WT_CONN_READONLY)) { + manager->lsm_workers = 0; + return (0); + } /* * We need at least a manager, a switch thread and a generic * worker. @@ -284,6 +288,8 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) manager = &conn->lsm_manager; removed = 0; + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || + manager->lsm_workers == 0); if (manager->lsm_workers > 0) { /* * Stop the main LSM manager thread first. @@ -616,6 +622,7 @@ __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, manager = &S2C(session)->lsm_manager; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); /* * Don't add merges or bloom filter creates if merges * or bloom filters are disabled in the tree. diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index ab18e41a2f5..7c188bf3dc7 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -420,26 +420,32 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, WT_ERR_MSG(session, EINVAL, "LSM merge_min must be less than or equal to merge_max"); - /* - * Set up the config for each chunk. - * - * Make the memory_page_max double the chunk size, so application - * threads don't immediately try to force evict the chunk when the - * worker thread clears the NO_EVICTION flag. - */ - WT_ERR(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__wt_buf_fmt(session, buf, - "%s,key_format=u,value_format=u,memory_page_max=%" PRIu64, - config, 2 * lsm_tree->chunk_max)); - WT_ERR(__wt_strndup( - session, buf->data, buf->size, &lsm_tree->file_config)); - - /* Create the first chunk and flush the metadata. */ - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); - - /* Discard our partially populated handle. */ - ret = __lsm_tree_discard(session, lsm_tree, false); - lsm_tree = NULL; + if (!F_ISSET(S2C(session), WT_CONN_READONLY)) { + /* + * Set up the config for each chunk. + * + * Make the memory_page_max double the chunk size, so + * application threads don't immediately try to force evict + * the chunk when the worker thread clears the NO_EVICTION flag. + */ + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, + "%s,key_format=u,value_format=u,memory_page_max=%" PRIu64, + config, 2 * lsm_tree->chunk_max)); + WT_ERR(__wt_strndup( + session, buf->data, buf->size, &lsm_tree->file_config)); + + /* Create the first chunk and flush the metadata. */ + WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + + /* Discard our partially populated handle. */ + ret = __lsm_tree_discard(session, lsm_tree, false); + lsm_tree = NULL; + } else { + F_CLR(lsm_tree, WT_LSM_TREE_MERGES); + FLD_SET(lsm_tree->bloom, WT_LSM_BLOOM_OFF); + FLD_CLR(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST); + } /* * Open our new tree and add it to the handle cache. Don't discard on @@ -1455,8 +1461,7 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session, continue; WT_ERR(__wt_schema_worker(session, chunk->uri, file_func, name_func, cfg, open_flags)); - if (name_func == __wt_backup_list_uri_append && - F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) + if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) WT_ERR(__wt_schema_worker(session, chunk->bloom_uri, file_func, name_func, cfg, open_flags)); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 099bde176f7..7723818f607 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -341,20 +341,13 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * time, and our checkpoint operation should be very quick. */ WT_ERR(__wt_meta_track_on(session)); - F_SET(session, WT_SESSION_LOCK_NO_WAIT); WT_WITH_CHECKPOINT_LOCK(session, ret, WT_WITH_SCHEMA_LOCK(session, ret, ret = __wt_schema_worker( session, chunk->uri, __wt_checkpoint, NULL, NULL, 0))); WT_TRET(__wt_meta_track_off(session, false, ret != 0)); - F_CLR(session, WT_SESSION_LOCK_NO_WAIT); - if (ret != 0) { - if (ret == EBUSY) { - ret = 0; - goto err; - } + if (ret != 0) WT_ERR_MSG(session, ret, "LSM checkpoint"); - } /* Now the file is written, get the chunk size. */ WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); diff --git a/src/third_party/wiredtiger/src/meta/meta_apply.c b/src/third_party/wiredtiger/src/meta/meta_apply.c index 7722cd55fbd..fb483c21dd9 100644 --- a/src/third_party/wiredtiger/src/meta/meta_apply.c +++ b/src/third_party/wiredtiger/src/meta/meta_apply.c @@ -15,22 +15,26 @@ */ static inline int __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_DECL_RET; const char *uri; - int cmp; + bool skip; - cursor->set_key(cursor, "file:"); - if ((ret = cursor->search_near(cursor, &cmp)) == 0 && cmp < 0) - ret = cursor->next(cursor); - for (; ret == 0; ret = cursor->next(cursor)) { + while ((ret = cursor->next(cursor)) == 0) { WT_RET(cursor->get_key(cursor, &uri)); - if (!WT_PREFIX_MATCH(uri, "file:")) - break; if (strcmp(uri, WT_METAFILE_URI) == 0) continue; + skip = false; + if (name_func != NULL) + WT_RET(name_func(session, uri, &skip)); + + if (file_func == NULL || skip || !WT_PREFIX_MATCH(uri, "file:")) + continue; + /* * We need to pull the handle into the session handle cache * and make sure it's referenced to stop other internal code @@ -40,7 +44,7 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, if ((ret = __wt_session_get_btree( session, uri, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); - WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); if (WT_META_TRACKING(session)) WT_TRET(__wt_meta_track_handle_lock( session, false)); @@ -54,20 +58,22 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, } /* - * __wt_meta_btree_apply -- + * __wt_meta_apply_all -- * Apply a function to all files listed in the metadata, apart from the * metadata file. */ int -__wt_meta_btree_apply(WT_SESSION_IMPL *session, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) +__wt_meta_apply_all(WT_SESSION_IMPL *session, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_CURSOR *cursor; WT_DECL_RET; WT_RET(__wt_metadata_cursor(session, &cursor)); - WT_SAVE_DHANDLE(session, - ret = __meta_btree_apply(session, cursor, func, cfg)); + WT_SAVE_DHANDLE(session, ret = + __meta_btree_apply(session, cursor, file_func, name_func, cfg)); WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 3d27f0b5845..471bb65cac0 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -113,8 +113,9 @@ __metadata_load_bulk(WT_SESSION_IMPL *session) WT_DECL_RET; uint32_t allocsize; bool exist; - const char *filecfg[] = { WT_CONFIG_BASE(session, file_meta), NULL }; - const char *key; + const char *filecfg[] = { + WT_CONFIG_BASE(session, file_meta), NULL, NULL }; + const char *key, *value; /* * If a file was being bulk-loaded during the hot backup, it will appear @@ -135,6 +136,8 @@ __metadata_load_bulk(WT_SESSION_IMPL *session) * If the file doesn't exist, assume it's a bulk-loaded file; * retrieve the allocation size and re-create the file. */ + WT_ERR(cursor->get_value(cursor, &value)); + filecfg[1] = value; WT_ERR(__wt_direct_io_size_check( session, filecfg, "allocation_size", &allocsize)); WT_ERR(__wt_block_manager_create(session, key, allocsize)); diff --git a/src/third_party/wiredtiger/src/os_posix/os_errno.c b/src/third_party/wiredtiger/src/os_posix/os_errno.c index a58ae88447e..a0f1202c6ef 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_errno.c +++ b/src/third_party/wiredtiger/src/os_posix/os_errno.c @@ -23,6 +23,22 @@ __wt_errno(void) } /* + * __wt_map_error_rdonly -- + * Map an error into a WiredTiger error code specific for + * read-only operation which intercepts based on certain types + * of failures. + */ +int +__wt_map_error_rdonly(int error) +{ + if (error == ENOENT) + return (WT_NOTFOUND); + else if (error == EACCES) + return (WT_PERM_DENIED); + return (error); +} + +/* * __wt_strerror -- * POSIX implementation of WT_SESSION.strerror and wiredtiger_strerror. */ diff --git a/src/third_party/wiredtiger/src/os_posix/os_fallocate.c b/src/third_party/wiredtiger/src/os_posix/os_fallocate.c index 9d160afd179..bf20a99bdef 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fallocate.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fallocate.c @@ -115,6 +115,7 @@ __wt_fallocate( { WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); switch (fh->fallocate_available) { /* * Check for already configured handles and make the configured call. diff --git a/src/third_party/wiredtiger/src/os_posix/os_fsync.c b/src/third_party/wiredtiger/src/os_posix/os_fsync.c index f5afddc557b..0bd0359338b 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fsync.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fsync.c @@ -60,6 +60,7 @@ __wt_directory_sync_fh(WT_SESSION_IMPL *session, WT_FH *fh) #ifdef __linux__ WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); if ((ret = __wt_handle_sync(fh->fd)) == 0) return (0); WT_RET_MSG(session, ret, "%s: fsync", fh->name); @@ -108,6 +109,7 @@ __wt_directory_sync(WT_SESSION_IMPL *session, const char *path) if (ret != 0) WT_RET_MSG(session, ret, "%s: open", path); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); if ((ret = __wt_handle_sync(fd)) != 0) WT_ERR_MSG(session, ret, "%s: fsync", path); @@ -134,6 +136,9 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: fsync", fh->name)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); if ((ret = __wt_handle_sync(fh->fd)) == 0) return (0); WT_RET_MSG(session, ret, "%s fsync error", fh->name); @@ -149,6 +154,7 @@ __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh) #ifdef HAVE_SYNC_FILE_RANGE WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_RET(__wt_verbose( session, WT_VERB_FILEOPS, "%s: sync_file_range", fh->name)); diff --git a/src/third_party/wiredtiger/src/os_posix/os_ftruncate.c b/src/third_party/wiredtiger/src/os_posix/os_ftruncate.c index 2af90512f26..94d6cba3bf5 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_ftruncate.c +++ b/src/third_party/wiredtiger/src/os_posix/os_ftruncate.c @@ -17,6 +17,7 @@ __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t len) { WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_SYSCALL_RETRY(ftruncate(fh->fd, len), ret); if (ret == 0) return (0); diff --git a/src/third_party/wiredtiger/src/os_posix/os_open.c b/src/third_party/wiredtiger/src/os_posix/os_open.c index b085676c53b..219b26c2fa1 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_open.c +++ b/src/third_party/wiredtiger/src/os_posix/os_open.c @@ -73,7 +73,16 @@ __wt_open(WT_SESSION_IMPL *session, goto setupfh; } - f = O_RDWR; + /* + * If this is a read-only connection, open all files read-only + * except the lock file. + */ + if (F_ISSET(conn, WT_CONN_READONLY) && + !WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))) + f = O_RDONLY; + else + f = O_RDWR; #ifdef O_BINARY /* Windows clones: we always want to treat the file as a binary. */ f |= O_BINARY; @@ -94,6 +103,9 @@ __wt_open(WT_SESSION_IMPL *session, #endif if (ok_create) { + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || + WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); f |= O_CREAT; if (exclusive) f |= O_EXCL; diff --git a/src/third_party/wiredtiger/src/os_posix/os_remove.c b/src/third_party/wiredtiger/src/os_posix/os_remove.c index bc244c12e46..eb2e37fdc38 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_remove.c +++ b/src/third_party/wiredtiger/src/os_posix/os_remove.c @@ -21,6 +21,7 @@ __remove_file_check(WT_SESSION_IMPL *session, const char *name) uint64_t bucket; conn = S2C(session); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY)); fh = NULL; bucket = __wt_hash_city64(name, strlen(name)) % WT_HASH_ARRAY_SIZE; diff --git a/src/third_party/wiredtiger/src/os_posix/os_rename.c b/src/third_party/wiredtiger/src/os_posix/os_rename.c index 301190305c4..8ec4ee3aa23 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_rename.c +++ b/src/third_party/wiredtiger/src/os_posix/os_rename.c @@ -21,6 +21,7 @@ __wt_rename(WT_SESSION_IMPL *session, const char *from, const char *to) WT_RET(__wt_verbose( session, WT_VERB_FILEOPS, "rename %s to %s", from, to)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); from_path = to_path = NULL; WT_RET(__wt_filename(session, from, &from_path)); diff --git a/src/third_party/wiredtiger/src/os_posix/os_rw.c b/src/third_party/wiredtiger/src/os_posix/os_rw.c index 8733bfe0f53..3d49fa7e712 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_rw.c +++ b/src/third_party/wiredtiger/src/os_posix/os_rw.c @@ -65,6 +65,9 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || diff --git a/src/third_party/wiredtiger/src/os_win/os_errno.c b/src/third_party/wiredtiger/src/os_win/os_errno.c index 6a9daf8443f..590fcdc9d44 100644 --- a/src/third_party/wiredtiger/src/os_win/os_errno.c +++ b/src/third_party/wiredtiger/src/os_win/os_errno.c @@ -17,11 +17,13 @@ static const int windows_error_offset = -29000; * Windows errors are from 0 - 15999 according to the documentation */ static DWORD -__wt_map_error_to_windows_error(int error) { - /* Ensure we do not exceed the error range - Also validate he do not get any COM errors - (which are negative integers) - */ +__wt_map_error_to_windows_error(int error) +{ + /* + * Ensure we do not exceed the error range + * Also validate we do not get any COM errors + * (which are negative integers) + */ WT_ASSERT(NULL, error < 0); return (error + -(windows_error_offset)); @@ -32,11 +34,28 @@ __wt_map_error_to_windows_error(int error) { * Return a positive integer, a decoded Windows error */ static int -__wt_map_windows_error_to_error(DWORD winerr) { +__wt_map_windows_error_to_error(DWORD winerr) +{ return (winerr + windows_error_offset); } /* + * __wt_map_error_rdonly -- + * Map an error into a WiredTiger error code specific for + * read-only operation which intercepts based on certain types + * of failures. + */ +int +__wt_map_error_rdonly(int winerr) +{ + if (winerr == ERROR_FILE_NOT_FOUND) + return (WT_NOTFOUND); + else if (winerr == ERROR_ACCESS_DENIED) + return (WT_PERM_DENIED); + return (winerr); +} + +/* * __wt_errno -- * Return errno, or WT_ERROR if errno not set. */ diff --git a/src/third_party/wiredtiger/src/os_win/os_fallocate.c b/src/third_party/wiredtiger/src/os_win/os_fallocate.c index cdc7a1c46ee..a324687ca73 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fallocate.c +++ b/src/third_party/wiredtiger/src/os_win/os_fallocate.c @@ -35,6 +35,7 @@ int __wt_fallocate( WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, wt_off_t len) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(fh); WT_UNUSED(offset); diff --git a/src/third_party/wiredtiger/src/os_win/os_fsync.c b/src/third_party/wiredtiger/src/os_win/os_fsync.c index 913b7ca5a4e..c196fc6c06a 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fsync.c +++ b/src/third_party/wiredtiger/src/os_win/os_fsync.c @@ -15,6 +15,7 @@ int __wt_directory_sync_fh(WT_SESSION_IMPL *session, WT_FH *fh) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(fh); return (0); @@ -27,6 +28,7 @@ __wt_directory_sync_fh(WT_SESSION_IMPL *session, WT_FH *fh) int __wt_directory_sync(WT_SESSION_IMPL *session, const char *path) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(path); return (0); @@ -44,6 +46,9 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: FlushFileBuffers", fh->name)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); if ((ret = FlushFileBuffers(fh->filehandle)) == FALSE) WT_RET_MSG(session, __wt_errno(), "%s FlushFileBuffers error", fh->name); @@ -58,6 +63,7 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) int __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(fh); diff --git a/src/third_party/wiredtiger/src/os_win/os_ftruncate.c b/src/third_party/wiredtiger/src/os_win/os_ftruncate.c index 0c11b5509b7..88fcf9542c1 100644 --- a/src/third_party/wiredtiger/src/os_win/os_ftruncate.c +++ b/src/third_party/wiredtiger/src/os_win/os_ftruncate.c @@ -18,6 +18,7 @@ __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t len) WT_DECL_RET; LARGE_INTEGER largeint; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); largeint.QuadPart = len; if ((ret = SetFilePointerEx( diff --git a/src/third_party/wiredtiger/src/os_win/os_open.c b/src/third_party/wiredtiger/src/os_win/os_open.c index 3ec53daf001..f10582c5bd1 100644 --- a/src/third_party/wiredtiger/src/os_win/os_open.c +++ b/src/third_party/wiredtiger/src/os_win/os_open.c @@ -58,7 +58,17 @@ __wt_open(WT_SESSION_IMPL *session, WT_RET(__wt_filename(session, name, &path)); - share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE; + /* + * If this is a read-only connection, open all files read-only + * except the lock file. + */ + if (F_ISSET(conn, WT_CONN_READONLY) && + !WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))) + share_mode = FILE_SHARE_READ; + else + share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE; + /* * Security: * The application may spawn a new process, and we don't want another @@ -72,6 +82,9 @@ __wt_open(WT_SESSION_IMPL *session, dwCreationDisposition = 0; if (ok_create) { + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || + WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); dwCreationDisposition = CREATE_NEW; if (exclusive) dwCreationDisposition = CREATE_ALWAYS; diff --git a/src/third_party/wiredtiger/src/os_win/os_remove.c b/src/third_party/wiredtiger/src/os_win/os_remove.c index 5682a25d7f2..84f1dd86674 100644 --- a/src/third_party/wiredtiger/src/os_win/os_remove.c +++ b/src/third_party/wiredtiger/src/os_win/os_remove.c @@ -21,6 +21,7 @@ __remove_file_check(WT_SESSION_IMPL *session, const char *name) uint64_t bucket; conn = S2C(session); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY)); fh = NULL; bucket = __wt_hash_city64(name, strlen(name)) % WT_HASH_ARRAY_SIZE; diff --git a/src/third_party/wiredtiger/src/os_win/os_rename.c b/src/third_party/wiredtiger/src/os_win/os_rename.c index 829ab1d16e9..b4be2dba24c 100644 --- a/src/third_party/wiredtiger/src/os_win/os_rename.c +++ b/src/third_party/wiredtiger/src/os_win/os_rename.c @@ -22,6 +22,7 @@ __wt_rename(WT_SESSION_IMPL *session, const char *from, const char *to) WT_RET(__wt_verbose( session, WT_VERB_FILEOPS, "rename %s to %s", from, to)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); from_path = to_path = NULL; WT_RET(__wt_filename(session, from, &from_path)); diff --git a/src/third_party/wiredtiger/src/os_win/os_rw.c b/src/third_party/wiredtiger/src/os_win/os_rw.c index 49f011001a4..a9537a648f9 100644 --- a/src/third_party/wiredtiger/src/os_win/os_rw.c +++ b/src/third_party/wiredtiger/src/os_win/os_rw.c @@ -74,6 +74,9 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || diff --git a/src/third_party/wiredtiger/src/packing/pack_impl.c b/src/third_party/wiredtiger/src/packing/pack_impl.c index 2bd850bfc9a..5dbb0f33842 100644 --- a/src/third_party/wiredtiger/src/packing/pack_impl.c +++ b/src/third_party/wiredtiger/src/packing/pack_impl.c @@ -107,36 +107,6 @@ __wt_struct_unpack(WT_SESSION_IMPL *session, } /* - * __wt_struct_unpack_size -- - * Determine the packed size of a buffer matching the format. - */ -int -__wt_struct_unpack_size(WT_SESSION_IMPL *session, - const void *buffer, size_t size, const char *fmt, size_t *resultp) -{ - WT_DECL_PACK_VALUE(pv); - WT_DECL_RET; - WT_PACK pack; - const uint8_t *p, *end; - - p = buffer; - end = p + size; - - WT_RET(__pack_init(session, &pack, fmt)); - while ((ret = __pack_next(&pack, &pv)) == 0) - WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); - - /* Be paranoid - __pack_write should never overflow. */ - WT_ASSERT(session, p <= end); - - if (ret != WT_NOTFOUND) - return (ret); - - *resultp = WT_PTRDIFF(p, buffer); - return (0); -} - -/* * __wt_struct_repack -- * Return the subset of the packed buffer that represents part of * the format. If the result is not contiguous in the existing diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 5511674dc5e..2414229681b 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -13,6 +13,20 @@ static int __session_snapshot(WT_SESSION *, const char *); static int __session_rollback_transaction(WT_SESSION *, const char *); /* + * __wt_session_notsup -- + * Unsupported session method. + */ +int +__wt_session_notsup(WT_SESSION *wt_session) +{ + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + WT_RET_MSG(session, ENOTSUP, "Unsupported session method"); +} + +/* * __wt_session_reset_cursors -- * Reset all open cursors. */ @@ -495,6 +509,20 @@ err: API_END_RET_NOTFOUND_MAP(session, ret); } /* + * __session_create_readonly -- + * WT_SESSION->create method; readonly version. + */ +static int +__session_create_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_log_flush -- * WT_SESSION->log_flush method. */ @@ -532,6 +560,18 @@ err: API_END_RET(session, ret); } /* + * __session_log_flush_readonly -- + * WT_SESSION->log_flush method; readonly version. + */ +static int +__session_log_flush_readonly(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_log_printf -- * WT_SESSION->log_printf method. */ @@ -554,6 +594,19 @@ err: API_END_RET(session, ret); } /* + * __session_log_printf_readonly -- + * WT_SESSION->log_printf method; readonly version. + */ +static int +__session_log_printf_readonly(WT_SESSION *wt_session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) +{ + WT_UNUSED(fmt); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_rebalance -- * WT_SESSION->rebalance method. */ @@ -567,9 +620,6 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR(ENOTSUP); - /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, ret, WT_WITH_SCHEMA_LOCK(session, ret, @@ -580,6 +630,20 @@ err: API_END_RET_NOTFOUND_MAP(session, ret); } /* + * __session_rebalance_readonly -- + * WT_SESSION->rebalance method; readonly version. + */ +static int +__session_rebalance_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_rename -- * WT_SESSION->rename method. */ @@ -606,6 +670,21 @@ err: API_END_RET_NOTFOUND_MAP(session, ret); } /* + * __session_rename_readonly -- + * WT_SESSION->rename method; readonly version. + */ +static int +__session_rename_readonly(WT_SESSION *wt_session, + const char *uri, const char *newuri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(newuri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_reset -- * WT_SESSION->reset method. */ @@ -681,6 +760,20 @@ err: /* Note: drop operations cannot be unrolled (yet?). */ } /* + * __session_drop_readonly -- + * WT_SESSION->drop method; readonly version. + */ +static int +__session_drop_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_join -- * WT_SESSION->join method. */ @@ -825,6 +918,20 @@ err: API_END_RET_NOTFOUND_MAP(session, ret); } /* + * __session_salvage_readonly -- + * WT_SESSION->salvage method; readonly version. + */ +static int +__session_salvage_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __wt_session_range_truncate -- * Session handling of a range truncate. */ @@ -1006,6 +1113,22 @@ err: TXN_API_END_RETRY(session, ret, 0); } /* + * __session_truncate_readonly -- + * WT_SESSION->truncate method; readonly version. + */ +static int +__session_truncate_readonly(WT_SESSION *wt_session, + const char *uri, WT_CURSOR *start, WT_CURSOR *stop, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(start); + WT_UNUSED(stop); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_upgrade -- * WT_SESSION->upgrade method. */ @@ -1028,6 +1151,20 @@ err: API_END_RET_NOTFOUND_MAP(session, ret); } /* + * __session_upgrade_readonly -- + * WT_SESSION->upgrade method; readonly version. + */ +static int +__session_upgrade_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_verify -- * WT_SESSION->verify method. */ @@ -1249,6 +1386,18 @@ err: API_END_RET(session, ret); } /* + * __session_transaction_sync_readonly -- + * WT_SESSION->transaction_sync method; readonly version. + */ +static int +__session_transaction_sync_readonly(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_checkpoint -- * WT_SESSION->checkpoint method. */ @@ -1297,6 +1446,18 @@ err: API_END_RET_NOTFOUND_MAP(session, ret); } /* + * __session_checkpoint_readonly -- + * WT_SESSION->checkpoint method; readonly version. + */ +static int +__session_checkpoint_readonly(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + +/* * __session_snapshot -- * WT_SESSION->snapshot method. */ @@ -1382,6 +1543,33 @@ __open_session(WT_CONNECTION_IMPL *conn, __session_snapshot, __session_transaction_pinned_range, __session_transaction_sync + }, stds_readonly = { + NULL, + NULL, + __session_close, + __session_reconfigure, + __session_strerror, + __session_open_cursor, + __session_create_readonly, + __wt_session_compact_readonly, + __session_drop_readonly, + __session_join, + __session_log_flush_readonly, + __session_log_printf_readonly, + __session_rebalance_readonly, + __session_rename_readonly, + __session_reset, + __session_salvage_readonly, + __session_truncate_readonly, + __session_upgrade_readonly, + __session_verify, + __session_begin_transaction, + __session_commit_transaction, + __session_rollback_transaction, + __session_checkpoint_readonly, + __session_snapshot, + __session_transaction_pinned_range, + __session_transaction_sync_readonly }; WT_DECL_RET; WT_SESSION_IMPL *session, *session_ret; @@ -1421,7 +1609,8 @@ __open_session(WT_CONNECTION_IMPL *conn, conn->session_cnt = i + 1; session_ret->id = i; - session_ret->iface = stds; + session_ret->iface = + F_ISSET(conn, WT_CONN_READONLY) ? stds_readonly : stds; session_ret->iface.connection = &conn->iface; WT_ERR(__wt_cond_alloc(session, "session", false, &session_ret->cond)); diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 5abccbd1366..2a53ad58f52 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -97,13 +97,13 @@ */ /* - * __wt_compact_uri_analyze -- + * __compact_uri_analyze -- * Extract information relevant to deciding what work compact needs to * do from a URI that is part of a table schema. * Called via the schema_worker function. */ -int -__wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) +static int +__compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) { /* * Add references to schema URI objects to the list of objects to be @@ -120,6 +120,61 @@ __wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) } /* + * __compact_start -- + * Start object compaction. + */ +static int +__compact_start(WT_SESSION_IMPL *session) +{ + WT_BM *bm; + + bm = S2BT(session)->bm; + return (bm->compact_start(bm, session)); +} + +/* + * __compact_end -- + * End object compaction. + */ +static int +__compact_end(WT_SESSION_IMPL *session) +{ + WT_BM *bm; + + bm = S2BT(session)->bm; + return (bm->compact_end(bm, session)); +} + +/* + * __compact_handle_append -- + * Gather a file handle to be compacted. + * Called via the schema_worker function. + */ +static int +__compact_handle_append(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_DECL_RET; + + WT_UNUSED(cfg); + + /* Make sure there is space for the next entry. */ + WT_RET(__wt_realloc_def(session, &session->op_handle_allocated, + session->op_handle_next + 1, &session->op_handle)); + + WT_RET(__wt_session_get_btree( + session, session->dhandle->name, NULL, NULL, 0)); + + /* Set compact active on the handle. */ + if ((ret = __compact_start(session)) != 0) { + WT_TRET(__wt_session_release_btree(session)); + return (ret); + } + + session->op_handle[session->op_handle_next++] = session->dhandle; + return (0); +} + +/* * __session_compact_check_timeout -- * Check if the timeout has been exceeded. */ @@ -143,21 +198,25 @@ __session_compact_check_timeout( * Function to alternate between checkpoints and compaction calls. */ static int -__compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +__compact_file(WT_SESSION_IMPL *session, const char *cfg[]) { struct timespec start_time; + WT_DATA_HANDLE *dhandle; WT_DECL_ITEM(t); WT_DECL_RET; int i; const char *checkpoint_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_checkpoint), NULL, NULL }; + dhandle = session->dhandle; + /* * Force the checkpoint: we don't want to skip it because the work we * need to have done is done in the underlying block manager. */ WT_ERR(__wt_scr_alloc(session, 128, &t)); - WT_ERR(__wt_buf_fmt(session, t, "target=(\"%s\"),force=1", uri)); + WT_ERR(__wt_buf_fmt( + session, t, "target=(\"%s\"),force=1", dhandle->name)); checkpoint_cfg[1] = t->data; WT_ERR(__wt_epoch(session, &start_time)); @@ -173,9 +232,8 @@ __compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); session->compact_state = WT_COMPACT_RUNNING; - WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, uri, __wt_compact, NULL, cfg, 0)); + WT_WITH_DHANDLE(session, dhandle, + ret = __wt_compact(session, cfg)); WT_ERR(ret); if (session->compact_state != WT_COMPACT_SUCCESS) break; @@ -193,6 +251,7 @@ err: session->compact_state = WT_COMPACT_NONE; /* * __wt_session_compact -- + * WT_SESSION.compact method. */ int __wt_session_compact( @@ -203,6 +262,7 @@ __wt_session_compact( WT_DECL_RET; WT_SESSION_IMPL *session; WT_TXN *txn; + u_int i; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, compact, config, cfg); @@ -227,10 +287,10 @@ __wt_session_compact( WT_ERR(__wt_config_gets(session, cfg, "timeout", &cval)); session->compact->max_time = (uint64_t)cval.val; - /* Find the types of data sources are being compacted. */ + /* Find the types of data sources being compacted. */ WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, uri, NULL, __wt_compact_uri_analyze, cfg, 0)); + ret = __wt_schema_worker(session, uri, + __compact_handle_append, __compact_uri_analyze, cfg, 0)); WT_ERR(ret); if (session->compact->lsm_count != 0) @@ -247,11 +307,25 @@ __wt_session_compact( WT_ERR_MSG(session, EINVAL, " File compaction not permitted in a transaction"); - WT_ERR(__compact_file(session, uri, cfg)); + for (i = 0; i < session->op_handle_next; ++i) { + WT_WITH_DHANDLE(session, session->op_handle[i], + ret = __compact_file(session, cfg)); + WT_ERR(ret); + } } err: session->compact = NULL; + for (i = 0; i < session->op_handle_next; ++i) { + WT_WITH_DHANDLE(session, session->op_handle[i], + WT_TRET(__compact_end(session))); + WT_WITH_DHANDLE(session, session->op_handle[i], + WT_TRET(__wt_session_release_btree(session))); + } + + __wt_free(session, session->op_handle); + session->op_handle_allocated = session->op_handle_next = 0; + /* * Release common session resources (for example, checkpoint may acquire * significant reconciliation structures/memory). @@ -260,3 +334,17 @@ err: session->compact = NULL; API_END_RET_NOTFOUND_MAP(session, ret); } + +/* + * __wt_session_compact_readonly -- + * WT_SESSION.compact method; readonly version. + */ +int +__wt_session_compact_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 1ee3342442c..242d9ac5cc4 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -15,24 +15,21 @@ static int __session_dhandle_sweep(WT_SESSION_IMPL *); * Add a handle to the session's cache. */ static int -__session_add_dhandle( - WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE **dhandle_cachep) +__session_add_dhandle(WT_SESSION_IMPL *session) { WT_DATA_HANDLE_CACHE *dhandle_cache; uint64_t bucket; + /* Allocate a handle cache entry. */ WT_RET(__wt_calloc_one(session, &dhandle_cache)); + dhandle_cache->dhandle = session->dhandle; bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; TAILQ_INSERT_HEAD(&session->dhandles, dhandle_cache, q); TAILQ_INSERT_HEAD(&session->dhhash[bucket], dhandle_cache, hashq); - if (dhandle_cachep != NULL) - *dhandle_cachep = dhandle_cache; - - /* Sweep the handle list to remove any dead handles. */ - return (__session_dhandle_sweep(session)); + return (0); } /* @@ -450,14 +447,23 @@ __session_get_dhandle( return (0); } + /* Sweep the handle list to remove any dead handles. */ + WT_RET(__session_dhandle_sweep(session)); + /* * We didn't find a match in the session cache, search the shared * handle list and cache the handle we find. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = __session_find_shared_dhandle(session, uri, checkpoint)); - if (ret == 0) - ret = __session_add_dhandle(session, NULL); + WT_RET(ret); + + /* + * Fixup the reference count on failure (we incremented the reference + * count while holding the handle-list lock). + */ + if ((ret = __session_add_dhandle(session)) != 0) + (void)__wt_atomic_sub32(&session->dhandle->session_ref, 1); return (ret); } diff --git a/src/third_party/wiredtiger/src/support/cksum.c b/src/third_party/wiredtiger/src/support/cksum.c index 805602a3aca..0b086753406 100644 --- a/src/third_party/wiredtiger/src/support/cksum.c +++ b/src/third_party/wiredtiger/src/support/cksum.c @@ -1319,10 +1319,8 @@ __wt_cksum_init(void) __wt_cksum_func = __wt_cksum_hw; else __wt_cksum_func = __wt_cksum_sw; - -// SERVER-22773 - disabled as hardware CRC32 is corrupting memory -//#elif defined(__powerpc64__) -// __wt_cksum_func = __wt_cksum_hw; +#elif defined(__powerpc64__) + __wt_cksum_func = __wt_cksum_hw; #else __wt_cksum_func = __wt_cksum_sw; #endif diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 7a615131628..0df38bfe6b0 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -3,102 +3,102 @@ #include "wt_internal.h" static const char * const __stats_dsrc_desc[] = { - "block-manager: file allocation unit size", - "block-manager: blocks allocated", - "block-manager: checkpoint size", - "block-manager: allocations requiring file extension", - "block-manager: blocks freed", - "block-manager: file magic number", - "block-manager: file major version number", - "block-manager: minor version number", - "block-manager: file bytes available for reuse", - "block-manager: file size in bytes", - "LSM: bloom filters in the LSM tree", "LSM: bloom filter false positives", "LSM: bloom filter hits", "LSM: bloom filter misses", "LSM: bloom filter pages evicted from cache", "LSM: bloom filter pages read into cache", + "LSM: bloom filters in the LSM tree", + "LSM: chunks in the LSM tree", + "LSM: highest merge generation in the LSM tree", + "LSM: queries that could have benefited from a Bloom filter that did not exist", + "LSM: sleep for LSM checkpoint throttle", + "LSM: sleep for LSM merge throttle", "LSM: total size of bloom filters", + "block-manager: allocations requiring file extension", + "block-manager: blocks allocated", + "block-manager: blocks freed", + "block-manager: checkpoint size", + "block-manager: file allocation unit size", + "block-manager: file bytes available for reuse", + "block-manager: file magic number", + "block-manager: file major version number", + "block-manager: file size in bytes", + "block-manager: minor version number", "btree: btree checkpoint generation", - "btree: column-store variable-size deleted values", "btree: column-store fixed-size leaf pages", "btree: column-store internal pages", "btree: column-store variable-size RLE encoded values", + "btree: column-store variable-size deleted values", "btree: column-store variable-size leaf pages", - "btree: pages rewritten by compaction", - "btree: number of key/value pairs", "btree: fixed-record size", - "btree: maximum tree depth", "btree: maximum internal page key size", "btree: maximum internal page size", "btree: maximum leaf page key size", "btree: maximum leaf page size", "btree: maximum leaf page value size", + "btree: maximum tree depth", + "btree: number of key/value pairs", "btree: overflow pages", + "btree: pages rewritten by compaction", "btree: row-store internal pages", "btree: row-store leaf pages", "cache: bytes read into cache", "cache: bytes written from cache", "cache: checkpoint blocked page eviction", - "cache: unmodified pages evicted", - "cache: page split during eviction deepened the tree", - "cache: modified pages evicted", "cache: data source pages selected for eviction unable to be evicted", "cache: hazard pointer blocked page eviction", + "cache: in-memory page passed criteria to be split", + "cache: in-memory page splits", "cache: internal pages evicted", "cache: internal pages split during eviction", "cache: leaf pages split during eviction", - "cache: in-memory page splits", - "cache: in-memory page passed criteria to be split", + "cache: modified pages evicted", + "cache: overflow pages read into cache", "cache: overflow values cached in memory", + "cache: page split during eviction deepened the tree", + "cache: page written requiring lookaside records", "cache: pages read into cache", "cache: pages read into cache requiring lookaside entries", - "cache: overflow pages read into cache", "cache: pages written from cache", - "cache: page written requiring lookaside records", "cache: pages written requiring in-memory restoration", - "compression: raw compression call failed, no additional data available", - "compression: raw compression call failed, additional data available", - "compression: raw compression call succeeded", + "cache: unmodified pages evicted", "compression: compressed pages read", "compression: compressed pages written", "compression: page written failed to compress", "compression: page written was too small to compress", - "cursor: create calls", - "cursor: insert calls", + "compression: raw compression call failed, additional data available", + "compression: raw compression call failed, no additional data available", + "compression: raw compression call succeeded", "cursor: bulk-loaded cursor-insert calls", + "cursor: create calls", "cursor: cursor-insert key and value bytes inserted", + "cursor: cursor-remove key bytes removed", + "cursor: cursor-update value bytes updated", + "cursor: insert calls", "cursor: next calls", "cursor: prev calls", "cursor: remove calls", - "cursor: cursor-remove key bytes removed", "cursor: reset calls", "cursor: restarted searches", "cursor: search calls", "cursor: search near calls", "cursor: truncate calls", "cursor: update calls", - "cursor: cursor-update value bytes updated", - "LSM: sleep for LSM checkpoint throttle", - "LSM: chunks in the LSM tree", - "LSM: highest merge generation in the LSM tree", - "LSM: queries that could have benefited from a Bloom filter that did not exist", - "LSM: sleep for LSM merge throttle", "reconciliation: dictionary matches", + "reconciliation: fast-path pages deleted", + "reconciliation: internal page key bytes discarded using suffix compression", "reconciliation: internal page multi-block writes", - "reconciliation: leaf page multi-block writes", - "reconciliation: maximum blocks required for a page", "reconciliation: internal-page overflow keys", + "reconciliation: leaf page key bytes discarded using prefix compression", + "reconciliation: leaf page multi-block writes", "reconciliation: leaf-page overflow keys", + "reconciliation: maximum blocks required for a page", "reconciliation: overflow values written", - "reconciliation: pages deleted", - "reconciliation: fast-path pages deleted", "reconciliation: page checksum matches", "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", - "reconciliation: leaf page key bytes discarded using prefix compression", - "reconciliation: internal page key bytes discarded using suffix compression", + "reconciliation: pages deleted", "session: object compaction", "session: open cursor count", "transaction: update conflicts", @@ -132,6 +132,18 @@ __wt_stat_dsrc_init(WT_DATA_HANDLE *handle) void __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) { + stats->bloom_false_positive = 0; + stats->bloom_hit = 0; + stats->bloom_miss = 0; + stats->bloom_page_evict = 0; + stats->bloom_page_read = 0; + stats->bloom_count = 0; + stats->lsm_chunk_count = 0; + stats->lsm_generation_max = 0; + stats->lsm_lookup_no_bloom = 0; + stats->lsm_checkpoint_throttle = 0; + stats->lsm_merge_throttle = 0; + stats->bloom_size = 0; stats->block_extension = 0; stats->block_alloc = 0; stats->block_free = 0; @@ -145,9 +157,9 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) /* not clearing btree_checkpoint_generation */ stats->btree_column_fix = 0; stats->btree_column_internal = 0; + stats->btree_column_rle = 0; stats->btree_column_deleted = 0; stats->btree_column_variable = 0; - stats->btree_column_rle = 0; stats->btree_fixed_len = 0; stats->btree_maxintlkey = 0; stats->btree_maxintlpage = 0; @@ -202,18 +214,6 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cursor_search_near = 0; stats->cursor_truncate = 0; stats->cursor_update = 0; - stats->bloom_false_positive = 0; - stats->bloom_hit = 0; - stats->bloom_miss = 0; - stats->bloom_page_evict = 0; - stats->bloom_page_read = 0; - stats->bloom_count = 0; - stats->lsm_chunk_count = 0; - stats->lsm_generation_max = 0; - stats->lsm_lookup_no_bloom = 0; - stats->lsm_checkpoint_throttle = 0; - stats->lsm_merge_throttle = 0; - stats->bloom_size = 0; stats->rec_dictionary = 0; stats->rec_page_delete_fast = 0; stats->rec_suffix_compression = 0; @@ -246,6 +246,19 @@ void __wt_stat_dsrc_aggregate_single( WT_DSRC_STATS *from, WT_DSRC_STATS *to) { + to->bloom_false_positive += from->bloom_false_positive; + to->bloom_hit += from->bloom_hit; + to->bloom_miss += from->bloom_miss; + to->bloom_page_evict += from->bloom_page_evict; + to->bloom_page_read += from->bloom_page_read; + to->bloom_count += from->bloom_count; + to->lsm_chunk_count += from->lsm_chunk_count; + if (from->lsm_generation_max > to->lsm_generation_max) + to->lsm_generation_max = from->lsm_generation_max; + to->lsm_lookup_no_bloom += from->lsm_lookup_no_bloom; + to->lsm_checkpoint_throttle += from->lsm_checkpoint_throttle; + to->lsm_merge_throttle += from->lsm_merge_throttle; + to->bloom_size += from->bloom_size; to->block_extension += from->block_extension; to->block_alloc += from->block_alloc; to->block_free += from->block_free; @@ -263,9 +276,9 @@ __wt_stat_dsrc_aggregate_single( to->btree_checkpoint_generation += from->btree_checkpoint_generation; to->btree_column_fix += from->btree_column_fix; to->btree_column_internal += from->btree_column_internal; + to->btree_column_rle += from->btree_column_rle; to->btree_column_deleted += from->btree_column_deleted; to->btree_column_variable += from->btree_column_variable; - to->btree_column_rle += from->btree_column_rle; if (from->btree_fixed_len > to->btree_fixed_len) to->btree_fixed_len = from->btree_fixed_len; if (from->btree_maxintlkey > to->btree_maxintlkey) @@ -328,19 +341,6 @@ __wt_stat_dsrc_aggregate_single( to->cursor_search_near += from->cursor_search_near; to->cursor_truncate += from->cursor_truncate; to->cursor_update += from->cursor_update; - to->bloom_false_positive += from->bloom_false_positive; - to->bloom_hit += from->bloom_hit; - to->bloom_miss += from->bloom_miss; - to->bloom_page_evict += from->bloom_page_evict; - to->bloom_page_read += from->bloom_page_read; - to->bloom_count += from->bloom_count; - to->lsm_chunk_count += from->lsm_chunk_count; - if (from->lsm_generation_max > to->lsm_generation_max) - to->lsm_generation_max = from->lsm_generation_max; - to->lsm_lookup_no_bloom += from->lsm_lookup_no_bloom; - to->lsm_checkpoint_throttle += from->lsm_checkpoint_throttle; - to->lsm_merge_throttle += from->lsm_merge_throttle; - to->bloom_size += from->bloom_size; to->rec_dictionary += from->rec_dictionary; to->rec_page_delete_fast += from->rec_page_delete_fast; to->rec_suffix_compression += from->rec_suffix_compression; @@ -367,6 +367,21 @@ __wt_stat_dsrc_aggregate( { int64_t v; + to->bloom_false_positive += WT_STAT_READ(from, bloom_false_positive); + to->bloom_hit += WT_STAT_READ(from, bloom_hit); + to->bloom_miss += WT_STAT_READ(from, bloom_miss); + to->bloom_page_evict += WT_STAT_READ(from, bloom_page_evict); + to->bloom_page_read += WT_STAT_READ(from, bloom_page_read); + to->bloom_count += WT_STAT_READ(from, bloom_count); + to->lsm_chunk_count += WT_STAT_READ(from, lsm_chunk_count); + if ((v = WT_STAT_READ(from, lsm_generation_max)) > + to->lsm_generation_max) + to->lsm_generation_max = v; + to->lsm_lookup_no_bloom += WT_STAT_READ(from, lsm_lookup_no_bloom); + to->lsm_checkpoint_throttle += + WT_STAT_READ(from, lsm_checkpoint_throttle); + to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); + to->bloom_size += WT_STAT_READ(from, bloom_size); to->block_extension += WT_STAT_READ(from, block_extension); to->block_alloc += WT_STAT_READ(from, block_alloc); to->block_free += WT_STAT_READ(from, block_free); @@ -387,10 +402,10 @@ __wt_stat_dsrc_aggregate( to->btree_column_fix += WT_STAT_READ(from, btree_column_fix); to->btree_column_internal += WT_STAT_READ(from, btree_column_internal); + to->btree_column_rle += WT_STAT_READ(from, btree_column_rle); to->btree_column_deleted += WT_STAT_READ(from, btree_column_deleted); to->btree_column_variable += WT_STAT_READ(from, btree_column_variable); - to->btree_column_rle += WT_STAT_READ(from, btree_column_rle); if ((v = WT_STAT_READ(from, btree_fixed_len)) > to->btree_fixed_len) to->btree_fixed_len = v; if ((v = WT_STAT_READ(from, btree_maxintlkey)) > to->btree_maxintlkey) @@ -467,21 +482,6 @@ __wt_stat_dsrc_aggregate( to->cursor_search_near += WT_STAT_READ(from, cursor_search_near); to->cursor_truncate += WT_STAT_READ(from, cursor_truncate); to->cursor_update += WT_STAT_READ(from, cursor_update); - to->bloom_false_positive += WT_STAT_READ(from, bloom_false_positive); - to->bloom_hit += WT_STAT_READ(from, bloom_hit); - to->bloom_miss += WT_STAT_READ(from, bloom_miss); - to->bloom_page_evict += WT_STAT_READ(from, bloom_page_evict); - to->bloom_page_read += WT_STAT_READ(from, bloom_page_read); - to->bloom_count += WT_STAT_READ(from, bloom_count); - to->lsm_chunk_count += WT_STAT_READ(from, lsm_chunk_count); - if ((v = WT_STAT_READ(from, lsm_generation_max)) > - to->lsm_generation_max) - to->lsm_generation_max = v; - to->lsm_lookup_no_bloom += WT_STAT_READ(from, lsm_lookup_no_bloom); - to->lsm_checkpoint_throttle += - WT_STAT_READ(from, lsm_checkpoint_throttle); - to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); - to->bloom_size += WT_STAT_READ(from, bloom_size); to->rec_dictionary += WT_STAT_READ(from, rec_dictionary); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_suffix_compression += @@ -509,12 +509,22 @@ __wt_stat_dsrc_aggregate( } static const char * const __stats_connection_desc[] = { - "async: number of allocation state races", - "async: number of operation slots viewed for allocation", + "LSM: application work units currently queued", + "LSM: merge work units currently queued", + "LSM: rows merged in an LSM tree", + "LSM: sleep for LSM checkpoint throttle", + "LSM: sleep for LSM merge throttle", + "LSM: switch work units currently queued", + "LSM: tree maintenance operations discarded", + "LSM: tree maintenance operations executed", + "LSM: tree maintenance operations scheduled", + "LSM: tree queue hit maximum", "async: current work queue length", + "async: maximum work queue length", + "async: number of allocation state races", "async: number of flush calls", + "async: number of operation slots viewed for allocation", "async: number of times operation allocation failed", - "async: maximum work queue length", "async: number of times worker found no work", "async: total allocations", "async: total compact calls", @@ -522,55 +532,64 @@ static const char * const __stats_connection_desc[] = { "async: total remove calls", "async: total search calls", "async: total update calls", - "block-manager: mapped bytes read", - "block-manager: bytes read", - "block-manager: bytes written", - "block-manager: mapped blocks read", "block-manager: blocks pre-loaded", "block-manager: blocks read", "block-manager: blocks written", - "cache: tracked dirty bytes in the cache", - "cache: tracked bytes belonging to internal pages in the cache", + "block-manager: bytes read", + "block-manager: bytes written", + "block-manager: mapped blocks read", + "block-manager: mapped bytes read", "cache: bytes currently in the cache", - "cache: tracked bytes belonging to leaf pages in the cache", - "cache: maximum bytes configured", - "cache: tracked bytes belonging to overflow pages in the cache", "cache: bytes read into cache", "cache: bytes written from cache", - "cache: pages evicted by application threads", "cache: checkpoint blocked page eviction", - "cache: unmodified pages evicted", - "cache: page split during eviction deepened the tree", - "cache: modified pages evicted", - "cache: pages selected for eviction unable to be evicted", - "cache: pages evicted because they exceeded the in-memory maximum", - "cache: pages evicted because they had chains of deleted items", - "cache: failed eviction of pages that exceeded the in-memory maximum", - "cache: hazard pointer blocked page eviction", - "cache: internal pages evicted", - "cache: maximum page size at eviction", + "cache: eviction currently operating in aggressive mode", "cache: eviction server candidate queue empty when topping up", "cache: eviction server candidate queue not empty when topping up", "cache: eviction server evicting pages", "cache: eviction server populating queue, but not evicting pages", "cache: eviction server unable to reach eviction goal", - "cache: internal pages split during eviction", - "cache: leaf pages split during eviction", - "cache: pages walked for eviction", "cache: eviction worker thread evicting pages", - "cache: in-memory page splits", + "cache: failed eviction of pages that exceeded the in-memory maximum", + "cache: hazard pointer blocked page eviction", "cache: in-memory page passed criteria to be split", + "cache: in-memory page splits", + "cache: internal pages evicted", + "cache: internal pages split during eviction", + "cache: leaf pages split during eviction", "cache: lookaside table insert calls", "cache: lookaside table remove calls", - "cache: percentage overhead", - "cache: tracked dirty pages in the cache", + "cache: maximum bytes configured", + "cache: maximum page size at eviction", + "cache: modified pages evicted", + "cache: page split during eviction deepened the tree", + "cache: page written requiring lookaside records", "cache: pages currently held in the cache", + "cache: pages evicted because they exceeded the in-memory maximum", + "cache: pages evicted because they had chains of deleted items", + "cache: pages evicted by application threads", "cache: pages read into cache", "cache: pages read into cache requiring lookaside entries", + "cache: pages selected for eviction unable to be evicted", + "cache: pages walked for eviction", "cache: pages written from cache", - "cache: page written requiring lookaside records", "cache: pages written requiring in-memory restoration", + "cache: percentage overhead", + "cache: tracked bytes belonging to internal pages in the cache", + "cache: tracked bytes belonging to leaf pages in the cache", + "cache: tracked bytes belonging to overflow pages in the cache", + "cache: tracked dirty bytes in the cache", + "cache: tracked dirty pages in the cache", + "cache: unmodified pages evicted", + "connection: files currently open", + "connection: memory allocations", + "connection: memory frees", + "connection: memory re-allocations", "connection: pthread mutex condition wait calls", + "connection: pthread mutex shared lock read-lock calls", + "connection: pthread mutex shared lock write-lock calls", + "connection: total read I/Os", + "connection: total write I/Os", "cursor: cursor create calls", "cursor: cursor insert calls", "cursor: cursor next calls", @@ -580,96 +599,78 @@ static const char * const __stats_connection_desc[] = { "cursor: cursor restarted searches", "cursor: cursor search calls", "cursor: cursor search near calls", - "cursor: truncate calls", "cursor: cursor update calls", + "cursor: truncate calls", "data-handle: connection data handles currently active", - "data-handle: session dhandles swept", - "data-handle: session sweep attempts", - "data-handle: connection sweep dhandles closed", "data-handle: connection sweep candidate became referenced", + "data-handle: connection sweep dhandles closed", "data-handle: connection sweep dhandles removed from hash list", "data-handle: connection sweep time-of-death sets", "data-handle: connection sweeps", - "connection: files currently open", - "log: total log buffer size", + "data-handle: session dhandles swept", + "data-handle: session sweep attempts", + "log: busy returns attempting to switch slots", + "log: consolidated slot closures", + "log: consolidated slot join races", + "log: consolidated slot join transitions", + "log: consolidated slot joins", + "log: consolidated slot unbuffered writes", "log: log bytes of payload data", "log: log bytes written", - "log: yields waiting for previous log file close", - "log: total size of compressed records", - "log: total in-memory size of compressed records", - "log: log records too small to compress", - "log: log records not compressed", - "log: log records compressed", + "log: log files manually zero-filled", "log: log flush operations", + "log: log records compressed", + "log: log records not compressed", + "log: log records too small to compress", + "log: log release advances write LSN", + "log: log scan operations", + "log: log scan records requiring two reads", + "log: log server thread advances write LSN", + "log: log sync operations", + "log: log sync_dir operations", + "log: log write operations", + "log: logging bytes consolidated", "log: maximum log file size", - "log: pre-allocated log files prepared", "log: number of pre-allocated log files to create", "log: pre-allocated log files not ready and missed", + "log: pre-allocated log files prepared", "log: pre-allocated log files used", - "log: log release advances write LSN", "log: records processed by log scan", - "log: log scan records requiring two reads", - "log: log scan operations", - "log: consolidated slot closures", + "log: total in-memory size of compressed records", + "log: total log buffer size", + "log: total size of compressed records", "log: written slots coalesced", - "log: logging bytes consolidated", - "log: consolidated slot joins", - "log: consolidated slot join races", - "log: busy returns attempting to switch slots", - "log: consolidated slot join transitions", - "log: consolidated slot unbuffered writes", - "log: log sync operations", - "log: log sync_dir operations", - "log: log server thread advances write LSN", - "log: log write operations", - "log: log files manually zero-filled", - "LSM: sleep for LSM checkpoint throttle", - "LSM: sleep for LSM merge throttle", - "LSM: rows merged in an LSM tree", - "LSM: application work units currently queued", - "LSM: merge work units currently queued", - "LSM: tree queue hit maximum", - "LSM: switch work units currently queued", - "LSM: tree maintenance operations scheduled", - "LSM: tree maintenance operations discarded", - "LSM: tree maintenance operations executed", - "connection: memory allocations", - "connection: memory frees", - "connection: memory re-allocations", - "thread-yield: page acquire busy blocked", - "thread-yield: page acquire eviction blocked", - "thread-yield: page acquire locked blocked", - "thread-yield: page acquire read blocked", - "thread-yield: page acquire time sleeping (usecs)", - "connection: total read I/Os", - "reconciliation: pages deleted", + "log: yields waiting for previous log file close", "reconciliation: fast-path pages deleted", "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", + "reconciliation: pages deleted", "reconciliation: split bytes currently awaiting free", "reconciliation: split objects currently awaiting free", - "connection: pthread mutex shared lock read-lock calls", - "connection: pthread mutex shared lock write-lock calls", "session: open cursor count", "session: open session count", + "thread-yield: page acquire busy blocked", + "thread-yield: page acquire eviction blocked", + "thread-yield: page acquire locked blocked", + "thread-yield: page acquire read blocked", + "thread-yield: page acquire time sleeping (usecs)", + "transaction: number of named snapshots created", + "transaction: number of named snapshots dropped", "transaction: transaction begins", - "transaction: transaction checkpoints", - "transaction: transaction checkpoint generation", "transaction: transaction checkpoint currently running", + "transaction: transaction checkpoint generation", "transaction: transaction checkpoint max time (msecs)", "transaction: transaction checkpoint min time (msecs)", "transaction: transaction checkpoint most recent time (msecs)", "transaction: transaction checkpoint total time (msecs)", - "transaction: transactions committed", + "transaction: transaction checkpoints", "transaction: transaction failures due to cache overflow", - "transaction: transaction range of IDs currently pinned by a checkpoint", "transaction: transaction range of IDs currently pinned", + "transaction: transaction range of IDs currently pinned by a checkpoint", "transaction: transaction range of IDs currently pinned by named snapshots", - "transaction: transactions rolled back", - "transaction: number of named snapshots created", - "transaction: number of named snapshots dropped", "transaction: transaction sync calls", - "connection: total write I/Os", + "transaction: transactions committed", + "transaction: transactions rolled back", }; int @@ -700,6 +701,16 @@ __wt_stat_connection_init(WT_CONNECTION_IMPL *handle) void __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) { + /* not clearing lsm_work_queue_app */ + /* not clearing lsm_work_queue_manager */ + stats->lsm_rows_merged = 0; + stats->lsm_checkpoint_throttle = 0; + stats->lsm_merge_throttle = 0; + /* not clearing lsm_work_queue_switch */ + stats->lsm_work_units_discarded = 0; + stats->lsm_work_units_done = 0; + stats->lsm_work_units_created = 0; + stats->lsm_work_queue_max = 0; stats->async_cur_queue = 0; /* not clearing async_max_queue */ stats->async_alloc_race = 0; @@ -724,6 +735,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_bytes_read = 0; stats->cache_bytes_write = 0; stats->cache_eviction_checkpoint = 0; + /* not clearing cache_eviction_aggressive_set */ stats->cache_eviction_queue_empty = 0; stats->cache_eviction_queue_not_empty = 0; stats->cache_eviction_server_evicting = 0; @@ -821,16 +833,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->log_compress_len = 0; stats->log_slot_coalesced = 0; stats->log_close_yields = 0; - /* not clearing lsm_work_queue_app */ - /* not clearing lsm_work_queue_manager */ - stats->lsm_rows_merged = 0; - stats->lsm_checkpoint_throttle = 0; - stats->lsm_merge_throttle = 0; - /* not clearing lsm_work_queue_switch */ - stats->lsm_work_units_discarded = 0; - stats->lsm_work_units_done = 0; - stats->lsm_work_units_created = 0; - stats->lsm_work_queue_max = 0; stats->rec_page_delete_fast = 0; stats->rec_pages = 0; stats->rec_pages_eviction = 0; @@ -876,6 +878,21 @@ void __wt_stat_connection_aggregate( WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *to) { + to->lsm_work_queue_app += WT_STAT_READ(from, lsm_work_queue_app); + to->lsm_work_queue_manager += + WT_STAT_READ(from, lsm_work_queue_manager); + to->lsm_rows_merged += WT_STAT_READ(from, lsm_rows_merged); + to->lsm_checkpoint_throttle += + WT_STAT_READ(from, lsm_checkpoint_throttle); + to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); + to->lsm_work_queue_switch += + WT_STAT_READ(from, lsm_work_queue_switch); + to->lsm_work_units_discarded += + WT_STAT_READ(from, lsm_work_units_discarded); + to->lsm_work_units_done += WT_STAT_READ(from, lsm_work_units_done); + to->lsm_work_units_created += + WT_STAT_READ(from, lsm_work_units_created); + to->lsm_work_queue_max += WT_STAT_READ(from, lsm_work_queue_max); to->async_cur_queue += WT_STAT_READ(from, async_cur_queue); to->async_max_queue += WT_STAT_READ(from, async_max_queue); to->async_alloc_race += WT_STAT_READ(from, async_alloc_race); @@ -901,6 +918,8 @@ __wt_stat_connection_aggregate( to->cache_bytes_write += WT_STAT_READ(from, cache_bytes_write); to->cache_eviction_checkpoint += WT_STAT_READ(from, cache_eviction_checkpoint); + to->cache_eviction_aggressive_set += + WT_STAT_READ(from, cache_eviction_aggressive_set); to->cache_eviction_queue_empty += WT_STAT_READ(from, cache_eviction_queue_empty); to->cache_eviction_queue_not_empty += @@ -1018,21 +1037,6 @@ __wt_stat_connection_aggregate( to->log_compress_len += WT_STAT_READ(from, log_compress_len); to->log_slot_coalesced += WT_STAT_READ(from, log_slot_coalesced); to->log_close_yields += WT_STAT_READ(from, log_close_yields); - to->lsm_work_queue_app += WT_STAT_READ(from, lsm_work_queue_app); - to->lsm_work_queue_manager += - WT_STAT_READ(from, lsm_work_queue_manager); - to->lsm_rows_merged += WT_STAT_READ(from, lsm_rows_merged); - to->lsm_checkpoint_throttle += - WT_STAT_READ(from, lsm_checkpoint_throttle); - to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); - to->lsm_work_queue_switch += - WT_STAT_READ(from, lsm_work_queue_switch); - to->lsm_work_units_discarded += - WT_STAT_READ(from, lsm_work_units_discarded); - to->lsm_work_units_done += WT_STAT_READ(from, lsm_work_units_done); - to->lsm_work_units_created += - WT_STAT_READ(from, lsm_work_units_created); - to->lsm_work_queue_max += WT_STAT_READ(from, lsm_work_queue_max); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_pages += WT_STAT_READ(from, rec_pages); to->rec_pages_eviction += WT_STAT_READ(from, rec_pages_eviction); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 85102ae8cfe..4bb8ccdc6f0 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -155,8 +155,8 @@ __checkpoint_apply_all(WT_SESSION_IMPL *session, const char *cfg[], ckpt_closed = cval.len != 0; } WT_ERR(ckpt_closed ? - __wt_meta_btree_apply(session, op, cfg) : - __wt_conn_btree_apply(session, false, NULL, op, cfg)); + __wt_meta_apply_all(session, op, NULL, cfg) : + __wt_conn_btree_apply(session, NULL, op, NULL, cfg)); } if (fullp != NULL) diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index e6bd8a8d755..f41691bbc3b 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -449,6 +449,18 @@ __wt_txn_recover(WT_SESSION_IMPL *session) */ if (!was_backup) { r.metadata_only = true; + /* + * If this is a read-only connection, check if the checkpoint + * LSN in the metadata file is up to date, indicating a clean + * shutdown. + */ + if (F_ISSET(conn, WT_CONN_READONLY)) { + WT_ERR(__wt_log_needs_recovery( + session, &metafile->ckpt_lsn, &needs_rec)); + if (needs_rec) + WT_ERR_MSG(session, WT_RUN_RECOVERY, + "Read-only database needs recovery"); + } if (WT_IS_INIT_LSN(&metafile->ckpt_lsn)) WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r)); @@ -492,8 +504,17 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * return an error if the user does not want automatic * recovery. */ - if (needs_rec && FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR)) + if (needs_rec && + (FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR) || + F_ISSET(conn, WT_CONN_READONLY))) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_ERR_MSG(session, WT_RUN_RECOVERY, + "Read-only database needs recovery"); WT_ERR(WT_RUN_RECOVERY); + } + + if (F_ISSET(conn, WT_CONN_READONLY)) + goto done; /* * Recovery can touch more data than fits in cache, so it relies on @@ -504,7 +525,8 @@ __wt_txn_recover(WT_SESSION_IMPL *session) eviction_started = true; /* - * Always run recovery even if it was a clean shutdown. + * Always run recovery even if it was a clean shutdown only if + * this is not a read-only connection. * We can consider skipping it in the future. */ if (WT_IS_INIT_LSN(&r.ckpt_lsn)) diff --git a/src/third_party/wiredtiger/test/bloom/test_bloom.c b/src/third_party/wiredtiger/test/bloom/test_bloom.c index 04fc8d1c371..183dc3d2d42 100644 --- a/src/third_party/wiredtiger/test/bloom/test_bloom.c +++ b/src/third_party/wiredtiger/test/bloom/test_bloom.c @@ -55,6 +55,8 @@ void usage(void); extern char *__wt_optarg; extern int __wt_optind; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { @@ -129,11 +131,9 @@ setup(void) "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", g.progname, g.c_cache, g.config_open == NULL ? "" : g.config_open); - if ((ret = wiredtiger_open(NULL, NULL, config, &conn)) != 0) - testutil_die(ret, "wiredtiger_open"); + testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - testutil_die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); g.wt_conn = conn; g.wt_session = session; @@ -153,9 +153,8 @@ run(void) /* Use the internal session handle to access private APIs. */ sess = (WT_SESSION_IMPL *)g.wt_session; - if ((ret = __wt_bloom_create( - sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)) != 0) - testutil_die(ret, "__wt_bloom_create"); + testutil_check(__wt_bloom_create( + sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)); item.size = g.c_key_max; for (i = 0; i < g.c_ops; i++) { @@ -164,8 +163,7 @@ run(void) testutil_die(ret, "__wt_bloom_insert: %d", i); } - if ((ret = __wt_bloom_finalize(bloomp)) != 0) - testutil_die(ret, "__wt_bloom_finalize"); + testutil_check(__wt_bloom_finalize(bloomp)); for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; @@ -174,18 +172,15 @@ run(void) testutil_die(ret, "__wt_bloom_get"); } } - if ((ret = __wt_bloom_close(bloomp)) != 0) - testutil_die(ret, "__wt_bloom_close"); - - if ((ret = g.wt_session->checkpoint(g.wt_session, NULL)) != 0) - testutil_die(ret, "WT_SESSION.checkpoint"); - if ((ret = __wt_bloom_open( - sess, uri, g.c_factor, g.c_k, NULL, &bloomp)) != 0) - testutil_die(ret, "__wt_bloom_open"); + testutil_check(__wt_bloom_close(bloomp)); + + testutil_check(g.wt_session->checkpoint(g.wt_session, NULL)); + testutil_check(__wt_bloom_open( + sess, uri, g.c_factor, g.c_k, NULL, &bloomp)); + for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; - if ((ret = __wt_bloom_get(bloomp, &item)) != 0) - testutil_die(ret, "__wt_bloom_get"); + testutil_check(__wt_bloom_get(bloomp, &item)); } /* @@ -194,33 +189,33 @@ run(void) */ item.size = g.c_key_max + 10; item.data = calloc(item.size, 1); + if (item.data == NULL) + testutil_die(ENOMEM, "value buffer malloc"); memset((void *)item.data, 'a', item.size); for (i = 0, fp = 0; i < g.c_ops; i++) { ((uint8_t *)item.data)[i % item.size] = 'a' + ((uint8_t)rand() % 26); if ((ret = __wt_bloom_get(bloomp, &item)) == 0) ++fp; + if (ret != 0 && ret != WT_NOTFOUND) + testutil_die(ret, "__wt_bloom_get"); } free((void *)item.data); printf("Out of %d ops, got %d false positives, %.4f%%\n", g.c_ops, fp, 100.0 * fp/g.c_ops); - if ((ret = __wt_bloom_drop(bloomp, NULL)) != 0) - testutil_die(ret, "__wt_bloom_drop"); + testutil_check(__wt_bloom_drop(bloomp, NULL)); } void cleanup(void) { uint32_t i; - int ret; for (i = 0; i < g.c_ops; i++) free(g.entries[i]); free(g.entries); - if ((ret = g.wt_session->close(g.wt_session, NULL)) != 0) - testutil_die(ret, "WT_SESSION.close"); - if ((g.wt_conn->close(g.wt_conn, NULL)) != 0) - testutil_die(ret, "WT_CONNECTION.close"); + testutil_check(g.wt_session->close(g.wt_session, NULL)); + testutil_check(g.wt_conn->close(g.wt_conn, NULL)); } /* diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 1914ad0188a..0f28a86b675 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -41,6 +41,8 @@ static int wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c index 14709a2e88e..68d2f092c60 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -43,6 +43,8 @@ static void wt_shutdown(SHARED_CONFIG *); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c index e5dd76fa1a1..5dc7194b5fb 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c @@ -86,7 +86,7 @@ load(SHARED_CONFIG *cfg, const char *name) for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) { if (cfg->ftype == ROW) { snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); - cursor->set_key(cursor, &keybuf); + cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); value->data = valuebuf; @@ -103,8 +103,10 @@ load(SHARED_CONFIG *cfg, const char *name) /* Setup the starting key range for the workload phase. */ cfg->key_range = cfg->nkeys; - cursor->close(cursor); - session->checkpoint(session, NULL); + if ((ret = cursor->close(cursor)) != 0) + testutil_die(ret, "cursor.close"); + if ((ret = session->checkpoint(session, NULL)) != 0) + testutil_die(ret, "session.checkpoint"); if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c index 9077f500594..d44505ab2f3 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c @@ -171,11 +171,9 @@ static inline void reverse_scan_op( SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { - uint64_t i; + uint64_t i, initial_key_range, prev_key, this_key; int ret; char *strkey; - uint64_t initial_key_range; - uint64_t prev_key, this_key; WT_UNUSED(session); WT_UNUSED(s); @@ -184,7 +182,8 @@ reverse_scan_op( prev_key = this_key = 0; /* Reset the cursor */ - cursor->reset(cursor); + if ((ret = cursor->reset(cursor)) != 0) + testutil_die(ret, "cursor.reset"); /* Save the key range. */ initial_key_range = cfg->key_range - cfg->append_inserters; @@ -197,10 +196,13 @@ reverse_scan_op( } if (cfg->ftype == ROW) { - cursor->get_key(cursor, &strkey); + if ((ret = cursor->get_key(cursor, &strkey)) != 0) + testutil_die(ret, "cursor.get_key"); this_key = (uint64_t)atol(strkey); } else - cursor->get_key(cursor, (uint64_t*)&this_key); + if ((ret = cursor->get_key( + cursor, (uint64_t *)&this_key)) != 0) + testutil_die(ret, "cursor.get_key"); if (i == 0 && this_key < initial_key_range) testutil_die(ret, @@ -227,17 +229,19 @@ reverse_scan(void *arg) SHARED_CONFIG *cfg; WT_CURSOR *cursor; WT_SESSION *session; - int id, ret; - char tid[128]; + uintmax_t id; uint64_t i; + int ret; + char tid[128]; - id = (int)(uintptr_t)arg; + id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; __wt_thread_id(tid, sizeof(tid)); __wt_random_init(&s->rnd); - printf(" reverse scan thread %2d starting: tid: %s, file: %s\n", + printf(" reverse scan thread %2" PRIuMAX + " starting: tid: %s, file: %s\n", id, tid, s->name); __wt_yield(); /* Get all the threads created. */ @@ -254,7 +258,8 @@ reverse_scan(void *arg) if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); - printf(" reverse scan thread %2d stopping: tid: %s, file: %s\n", + printf(" reverse scan thread %2" PRIuMAX + " stopping: tid: %s, file: %s\n", id, tid, s->name); /* Notify all other threads to finish once the first thread is done */ @@ -283,7 +288,7 @@ append_insert_op( keyno = __wt_atomic_add64(&cfg->key_range, 1); if (cfg->ftype == ROW) { snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); - cursor->set_key(cursor, &keybuf); + cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -311,17 +316,18 @@ append_insert(void *arg) SHARED_CONFIG *cfg; WT_CURSOR *cursor; WT_SESSION *session; + uintmax_t id; uint64_t i; - int id, ret; + int ret; char tid[128]; - id = (int)(uintptr_t)arg; + id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; __wt_thread_id(tid, sizeof(tid)); __wt_random_init(&s->rnd); - printf("write thread %2d starting: tid: %s, file: %s\n", + printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", id, tid, s->name); __wt_yield(); /* Get all the threads created. */ @@ -337,7 +343,7 @@ append_insert(void *arg) if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); - printf("write thread %2d stopping: tid: %s, file: %s\n", + printf("write thread %2" PRIuMAX " stopping: tid: %s, file: %s\n", id, tid, s->name); /* Notify all other threads to finish once the first thread is done */ diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index 0881c23d7d4..24994404c7c 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -50,6 +50,8 @@ static void wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 748494bf841..56657940514 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -37,20 +37,18 @@ check_copy(void) { WT_CONNECTION *conn; WT_SESSION *session; - int ret; wts_open(g.home_backup, 0, &conn); - if ((ret = conn->open_session( - conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session: %s", g.home_backup); + testutil_checkfmt( + conn->open_session(conn, NULL, NULL, &session), + "%s", g.home_backup); - ret = session->verify(session, g.uri, NULL); - if (ret != 0) - die(ret, "session.verify: %s: %s", g.home_backup, g.uri); + testutil_checkfmt( + session->verify(session, g.uri, NULL), + "%s: %s", g.home_backup, g.uri); - if ((ret = conn->close(conn, NULL)) != 0) - die(ret, "connection.close: %s", g.home_backup); + testutil_checkfmt(conn->close(conn, NULL), "%s", g.home_backup); } /* @@ -62,14 +60,12 @@ copy_file(const char *name) { size_t len; char *cmd; - int ret; len = strlen(g.home) + strlen(g.home_backup) + strlen(name) * 2 + 20; cmd = dmalloc(len); (void)snprintf(cmd, len, "cp %s/%s %s/%s", g.home, name, g.home_backup, name); - if ((ret = system(cmd)) != 0) - die(ret, "backup copy: %s", cmd); + testutil_checkfmt(system(cmd), "backup copy: %s", cmd); free(cmd); } @@ -96,8 +92,7 @@ backup(void *arg) return (NULL); /* Open a session. */ - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform a backup at somewhere under 10 seconds (so we get at @@ -113,12 +108,12 @@ backup(void *arg) break; /* Lock out named checkpoints */ - if ((ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_wrlock: backup lock"); + testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); /* Re-create the backup directory. */ - if ((ret = system(g.home_backup_init)) != 0) - die(ret, "backup directory creation failed"); + testutil_checkfmt( + system(g.home_backup_init), + "%s", "backup directory creation failed"); /* * open_cursor can return EBUSY if a metadata operation is @@ -128,26 +123,21 @@ backup(void *arg) "backup:", NULL, NULL, &backup_cursor)) == EBUSY) sleep(1); if (ret != 0) - die(ret, "session.open_cursor: backup"); + testutil_die(ret, "session.open_cursor: backup"); while ((ret = backup_cursor->next(backup_cursor)) == 0) { - if ((ret = - backup_cursor->get_key(backup_cursor, &key)) != 0) - die(ret, "cursor.get_key"); + testutil_check( + backup_cursor->get_key(backup_cursor, &key)); copy_file(key); } - if ((ret = backup_cursor->close(backup_cursor)) != 0) - die(ret, "cursor.close"); - - if ((ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_unlock: backup lock"); + testutil_check(backup_cursor->close(backup_cursor)); + testutil_check(pthread_rwlock_unlock(&g.backup_lock)); check_copy(); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); return (NULL); } diff --git a/src/third_party/wiredtiger/test/format/bdb.c b/src/third_party/wiredtiger/test/format/bdb.c index d7b4bca62f2..823fc8ff888 100644 --- a/src/third_party/wiredtiger/test/format/bdb.c +++ b/src/third_party/wiredtiger/test/format/bdb.c @@ -128,7 +128,7 @@ bdb_np(int next, if ((ret = dbc->get(dbc, &key, &value, next ? DB_NEXT : DB_PREV)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.get: %s: {%.*s}", + testutil_die(ret, "dbc.get: %s: {%.*s}", next ? "DB_NEXT" : "DB_PREV", (int)key.size, (char *)key.data); *notfoundp = 1; @@ -154,7 +154,7 @@ bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp) *notfoundp = 0; if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.get: DB_SET: {%.*s}", + testutil_die(ret, "dbc.get: DB_SET: {%.*s}", (int)key.size, (char *)key.data); *notfoundp = 1; } else { @@ -178,7 +178,7 @@ bdb_update(const void *arg_key, size_t arg_key_size, *notfoundp = 0; if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) { if (ret != DB_NOTFOUND) { - die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", + testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", (int)key.size, (char *)key.data, (int)value.size, (char *)value.data); } @@ -204,7 +204,7 @@ bdb_remove(uint64_t keyno, int *notfoundp) if ((ret = dbc->del(dbc, 0)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.del: {%.*s}", + testutil_die(ret, "dbc.del: {%.*s}", (int)key.size, (char *)key.data); *notfoundp = 1; } diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index 28189e25b65..64b005d294f 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -37,13 +37,11 @@ wts_load(void) WT_SESSION *session; uint8_t *keybuf, *valbuf; bool is_bulk; - int ret; conn = g.wts_conn; keybuf = valbuf = NULL; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, @@ -61,9 +59,8 @@ wts_load(void) if (g.c_reverse) is_bulk = false; - if ((ret = session->open_cursor(session, g.uri, NULL, - is_bulk ? "bulk,append" : NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor(session, g.uri, NULL, + is_bulk ? "bulk,append" : NULL, &cursor)); /* Set up the key/value buffers. */ key_gen_setup(&keybuf); @@ -120,8 +117,7 @@ wts_load(void) break; } - if ((ret = cursor->insert(cursor)) != 0) - die(ret, "cursor.insert"); + testutil_check(cursor->insert(cursor)); #ifdef HAVE_BERKELEY_DB if (SINGLETHREADED) @@ -129,15 +125,13 @@ wts_load(void) #endif } - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + testutil_check(cursor->close(cursor)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== bulk load stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); free(keybuf); free(valbuf); diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index fdfa597e07e..a75ee4f2adf 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -48,8 +48,7 @@ compact(void *arg) /* Open a session. */ conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform compaction at somewhere under 15 seconds (so we get at @@ -66,11 +65,10 @@ compact(void *arg) if ((ret = session->compact( session, g.uri, NULL)) != 0 && ret != WT_ROLLBACK) - die(ret, "session.compact"); + testutil_die(ret, "session.compact"); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); return (NULL); } diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index d431546f254..042316d8344 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -138,9 +138,10 @@ config_setup(void) /* Required shared libraries. */ if (DATASOURCE("helium") && access(HELIUM_PATH, R_OK) != 0) - die(errno, "Levyx/helium shared library: %s", HELIUM_PATH); + testutil_die(errno, + "Levyx/helium shared library: %s", HELIUM_PATH); if (DATASOURCE("kvsbdb") && access(KVS_BDB_PATH, R_OK) != 0) - die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH); + testutil_die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH); /* Some data-sources don't support user-specified collations. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) @@ -199,14 +200,15 @@ config_setup(void) if (!config_is_perm("key_max") && g.c_key_max < g.c_key_min) g.c_key_max = g.c_key_min; if (g.c_key_min > g.c_key_max) - die(EINVAL, "key_min may not be larger than key_max"); + testutil_die(EINVAL, "key_min may not be larger than key_max"); if (!config_is_perm("value_min") && g.c_value_min > g.c_value_max) g.c_value_min = g.c_value_max; if (!config_is_perm("value_max") && g.c_value_max < g.c_value_min) g.c_value_max = g.c_value_min; if (g.c_value_min > g.c_value_max) - die(EINVAL, "value_min may not be larger than value_max"); + testutil_die(EINVAL, + "value_min may not be larger than value_max"); /* Reset the key count. */ g.key_cnt = 0; @@ -412,7 +414,7 @@ config_lrt(void) */ if (g.type == FIX) { if (g.c_long_running_txn && config_is_perm("long_running_txn")) - die(EINVAL, + testutil_die(EINVAL, "long_running_txn not supported with fixed-length " "column store"); g.c_long_running_txn = 0; @@ -453,7 +455,7 @@ config_print(int error_display) fp = stdout; else if ((fp = fopen(g.home_config, "w")) == NULL) - die(errno, "fopen: %s", g.home_config); + testutil_die(errno, "fopen: %s", g.home_config); fprintf(fp, "############################################\n"); fprintf(fp, "# RUN PARAMETERS\n"); @@ -487,7 +489,7 @@ config_file(const char *name) char *p, buf[256]; if ((fp = fopen(name, "r")) == NULL) - die(errno, "fopen: %s", name); + testutil_die(errno, "fopen: %s", name); while (fgets(buf, sizeof(buf), fp) != NULL) { for (p = buf; *p != '\0' && *p != '\n'; ++p) ; @@ -582,7 +584,7 @@ config_single(const char *s, int perm) *cp->vstr = strdup(ep); } if (*cp->vstr == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); return; } @@ -625,7 +627,7 @@ config_map_file_type(const char *s, u_int *vp) strcmp(s, "row-store") == 0) *vp = ROW; else - die(EINVAL, "illegal file type configuration: %s", s); + testutil_die(EINVAL, "illegal file type configuration: %s", s); } /* @@ -642,7 +644,7 @@ config_map_checksum(const char *s, u_int *vp) else if (strcmp(s, "uncompressed") == 0) *vp = CHECKSUM_UNCOMPRESSED; else - die(EINVAL, "illegal checksum configuration: %s", s); + testutil_die(EINVAL, "illegal checksum configuration: %s", s); } /* @@ -667,7 +669,8 @@ config_map_compression(const char *s, u_int *vp) else if (strcmp(s, "zlib-noraw") == 0) *vp = COMPRESS_ZLIB_NO_RAW; else - die(EINVAL, "illegal compression configuration: %s", s); + testutil_die(EINVAL, + "illegal compression configuration: %s", s); } /* @@ -682,7 +685,7 @@ config_map_encryption(const char *s, u_int *vp) else if (strcmp(s, "rotn-7") == 0) *vp = ENCRYPT_ROTN_7; else - die(EINVAL, "illegal encryption configuration: %s", s); + testutil_die(EINVAL, "illegal encryption configuration: %s", s); } /* @@ -701,7 +704,7 @@ config_map_isolation(const char *s, u_int *vp) else if (strcmp(s, "snapshot") == 0) *vp = ISOLATION_SNAPSHOT; else - die(EINVAL, "illegal isolation configuration: %s", s); + testutil_die(EINVAL, "illegal isolation configuration: %s", s); } /* diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 03da1a84c9c..c54fd061736 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -334,12 +334,6 @@ void wts_salvage(void); void wts_stats(void); void wts_verify(const char *); -void die(int, const char *, ...) -#if defined(__GNUC__) -__attribute__((__noreturn__)) -#endif -; - /* * mmrand -- * Return a random value between a min/max pair. diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c index b7392829d30..451d2f4fa3c 100644 --- a/src/third_party/wiredtiger/test/format/lrt.c +++ b/src/third_party/wiredtiger/test/format/lrt.c @@ -60,11 +60,9 @@ lrt(void *arg) /* Open a session and cursor. */ conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor( + session, g.uri, NULL, NULL, &cursor)); for (pinned = 0;;) { if (pinned) { @@ -73,7 +71,8 @@ lrt(void *arg) &key, saved_keyno, 1)) == WT_ROLLBACK) ; if (ret != 0) - die(ret, "read_row %" PRIu64, saved_keyno); + testutil_die(ret, + "read_row %" PRIu64, saved_keyno); /* Compare the previous value with the current one. */ if (g.type == FIX) { @@ -83,21 +82,19 @@ lrt(void *arg) } else ret = cursor->get_value(cursor, &value); if (ret != 0) - die(ret, + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); if (buf_size != value.size || memcmp(buf, value.data, value.size) != 0) - die(0, "mismatched start/stop values"); + testutil_die(0, "mismatched start/stop values"); /* End the transaction. */ - if ((ret = - session->commit_transaction(session, NULL)) != 0) - die(ret, "session.commit_transaction"); + testutil_check( + session->commit_transaction(session, NULL)); /* Reset the cursor, releasing our pin. */ - if ((ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); + testutil_check(cursor->reset(cursor)); pinned = 0; } else { /* @@ -106,9 +103,8 @@ lrt(void *arg) * positioned. As soon as the cursor loses its position * a new snapshot will be allocated. */ - if ((ret = session->begin_transaction( - session, "isolation=snapshot")) != 0) - die(ret, "session.begin_transaction"); + testutil_check(session->begin_transaction( + session, "isolation=snapshot")); /* Read a record at the end of the table. */ do { @@ -120,7 +116,8 @@ lrt(void *arg) ; } while (ret == WT_NOTFOUND); if (ret != 0) - die(ret, "read_row %" PRIu64, saved_keyno); + testutil_die(ret, + "read_row %" PRIu64, saved_keyno); /* Copy the cursor's value. */ if (g.type == FIX) { @@ -130,11 +127,11 @@ lrt(void *arg) } else ret = cursor->get_value(cursor, &value); if (ret != 0) - die(ret, + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); if (buf_len < value.size && (buf = realloc(buf, buf_len = value.size)) == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); memcpy(buf, value.data, buf_size = value.size); /* @@ -149,7 +146,7 @@ lrt(void *arg) ; } while (ret == WT_NOTFOUND); if (ret != 0) - die(ret, "read_row %" PRIu64, keyno); + testutil_die(ret, "read_row %" PRIu64, keyno); pinned = 1; } @@ -166,8 +163,7 @@ lrt(void *arg) break; } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); free(keybuf); free(buf); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 36d56df1505..5d66f4d5391 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -56,7 +56,7 @@ wts_ops(int lastrun) pthread_t backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; - int ret, running; + int running; conn = g.wts_conn; @@ -97,36 +97,32 @@ wts_ops(int lastrun) /* Open a session. */ if (g.logging != 0) { - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops start ==============="); } /* Create thread structure; start the worker threads. */ if ((tinfo = calloc((size_t)g.c_threads, sizeof(*tinfo))) == NULL) - die(errno, "calloc"); + testutil_die(errno, "calloc"); for (i = 0; i < g.c_threads; ++i) { tinfo[i].id = (int)i + 1; tinfo[i].state = TINFO_RUNNING; - if ((ret = - pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])) != 0) - die(ret, "pthread_create"); + testutil_check( + pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); } /* * If a multi-threaded run, start optional backup, compaction and * long-running reader threads. */ - if (g.c_backups && - (ret = pthread_create(&backup_tid, NULL, backup, NULL)) != 0) - die(ret, "pthread_create: backup"); - if (g.c_compact && - (ret = pthread_create(&compact_tid, NULL, compact, NULL)) != 0) - die(ret, "pthread_create: compaction"); - if (!SINGLETHREADED && g.c_long_running_txn && - (ret = pthread_create(&lrt_tid, NULL, lrt, NULL)) != 0) - die(ret, "pthread_create: long-running reader"); + if (g.c_backups) + testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); + if (g.c_compact) + testutil_check( + pthread_create(&compact_tid, NULL, compact, NULL)); + if (!SINGLETHREADED && g.c_long_running_txn) + testutil_check(pthread_create(&lrt_tid, NULL, lrt, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { @@ -192,8 +188,7 @@ wts_ops(int lastrun) if (g.logging != 0) { (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); } } @@ -234,7 +229,7 @@ ops(void *arg) uint32_t op; uint8_t *keybuf, *valbuf; u_int np; - int ckpt_available, dir, insert, intxn, notfound, readonly, ret; + int ckpt_available, dir, insert, intxn, notfound, readonly; char *ckpt_config, ckpt_name[64]; tinfo = arg; @@ -269,9 +264,8 @@ ops(void *arg) */ if (intxn && (tinfo->ops == ckpt_op || tinfo->ops == session_op)) { - if ((ret = session->commit_transaction( - session, NULL)) != 0) - die(ret, "session.commit_transaction"); + testutil_check( + session->commit_transaction(session, NULL)); ++tinfo->commit; intxn = 0; } @@ -279,13 +273,11 @@ ops(void *arg) /* Open up a new session and cursors. */ if (tinfo->ops == session_op || session == NULL || cursor == NULL) { - if (session != NULL && - (ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + if (session != NULL) + testutil_check(session->close(session, NULL)); - if ((ret = conn->open_session(conn, NULL, - ops_session_config(&tinfo->rnd), &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, + ops_session_config(&tinfo->rnd), &session)); /* * 10% of the time, perform some read-only operations @@ -300,9 +292,8 @@ ops(void *arg) */ if (!SINGLETHREADED && !DATASOURCE("lsm") && ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) { - if ((ret = session->open_cursor(session, - g.uri, NULL, ckpt_name, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor(session, + g.uri, NULL, ckpt_name, &cursor)); /* Pick the next session/cursor close/open. */ session_op += 250; @@ -323,13 +314,12 @@ ops(void *arg) * want to have to specify the record number, * which requires an append configuration. */ - if ((ret = session->open_cursor(session, g.uri, - NULL, "overwrite", &cursor)) != 0) - die(ret, "session.open_cursor"); - if ((g.type == FIX || g.type == VAR) && - (ret = session->open_cursor(session, g.uri, - NULL, "append", &cursor_insert)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor(session, + g.uri, NULL, "overwrite", &cursor)); + if (g.type == FIX || g.type == VAR) + testutil_check(session->open_cursor( + session, g.uri, + NULL, "append", &cursor_insert)); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); @@ -358,21 +348,17 @@ ops(void *arg) } /* Named checkpoints lock out backups */ - if (ckpt_config != NULL && - (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) - die(ret, - "pthread_rwlock_wrlock: backup lock"); - - if ((ret = - session->checkpoint(session, ckpt_config)) != 0) - die(ret, "session.checkpoint%s%s", - ckpt_config == NULL ? "" : ": ", - ckpt_config == NULL ? "" : ckpt_config); - - if (ckpt_config != NULL && - (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) - die(ret, - "pthread_rwlock_wrlock: backup lock"); + if (ckpt_config != NULL) + testutil_check( + pthread_rwlock_wrlock(&g.backup_lock)); + + testutil_checkfmt( + session->checkpoint(session, ckpt_config), + "%s", ckpt_config == NULL ? "" : ckpt_config); + + if (ckpt_config != NULL) + testutil_check( + pthread_rwlock_unlock(&g.backup_lock)); /* Rephrase the checkpoint name for cursor open. */ if (ckpt_config == NULL) @@ -393,8 +379,7 @@ ops(void *arg) * have to do the reset outside of a transaction. */ if (tinfo->ops > reset_op && !intxn) { - if ((ret = session->reset(session)) != 0) - die(ret, "session.reset"); + testutil_check(session->reset(session)); /* Pick the next reset operation. */ reset_op += mmrand(&tinfo->rnd, 20000, 50000); @@ -406,9 +391,8 @@ ops(void *arg) */ if (!SINGLETHREADED && !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) { - if ((ret = - session->begin_transaction(session, NULL)) != 0) - die(ret, "session.begin_transaction"); + testutil_check( + session->begin_transaction(session, NULL)); intxn = 1; } @@ -466,9 +450,8 @@ ops(void *arg) if (col_insert(tinfo, cursor_insert, &key, &value, &keyno)) goto deadlock; - if ((ret = - cursor_insert->reset(cursor_insert)) != 0) - die(ret, "cursor.reset"); + testutil_check( + cursor_insert->reset(cursor_insert)); insert = 1; break; @@ -518,8 +501,7 @@ skip_insert: if (col_update(tinfo, goto deadlock; /* Reset the cursor: there is no reason to keep pages pinned. */ - if ((ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); + testutil_check(cursor->reset(cursor)); /* * If we're in the transaction, commit 40% of the time and @@ -528,9 +510,8 @@ skip_insert: if (col_update(tinfo, if (intxn) switch (mmrand(&tinfo->rnd, 1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ - if ((ret = session->commit_transaction( - session, NULL)) != 0) - die(ret, "session.commit_transaction"); + testutil_check(session->commit_transaction( + session, NULL)); ++tinfo->commit; intxn = 0; break; @@ -538,10 +519,8 @@ skip_insert: if (col_update(tinfo, if (0) { deadlock: ++tinfo->deadlock; } - if ((ret = session->rollback_transaction( - session, NULL)) != 0) - die(ret, - "session.rollback_transaction"); + testutil_check(session->rollback_transaction( + session, NULL)); ++tinfo->rollback; intxn = 0; break; @@ -550,8 +529,8 @@ deadlock: ++tinfo->deadlock; } } - if (session != NULL && (ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + if (session != NULL) + testutil_check(session->close(session, NULL)); free(keybuf); free(valbuf); @@ -573,7 +552,6 @@ wts_read_scan(void) WT_SESSION *session; uint64_t cnt, last_cnt; uint8_t *keybuf; - int ret; conn = g.wts_conn; @@ -581,12 +559,10 @@ wts_read_scan(void) key_gen_setup(&keybuf); /* Open a session and cursor pair. */ - if ((ret = conn->open_session( - conn, NULL, ops_session_config(NULL), &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(conn->open_session( + conn, NULL, ops_session_config(NULL), &session)); + testutil_check(session->open_cursor( + session, g.uri, NULL, NULL, &cursor)); /* Check a random subset of the records using the key. */ for (last_cnt = cnt = 0; cnt < g.key_cnt;) { @@ -599,12 +575,11 @@ wts_read_scan(void) } key.data = keybuf; - if ((ret = read_row(cursor, &key, cnt, 0)) != 0) - die(ret, "read_scan"); + testutil_checkfmt( + read_row(cursor, &key, cnt, 0), "%s", "read_scan"); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); free(keybuf); } @@ -666,7 +641,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) return (WT_NOTFOUND); break; default: - die(ret, "read_row: read row %" PRIu64, keyno); + testutil_die(ret, "read_row: read row %" PRIu64, keyno); } #ifdef HAVE_BERKELEY_DB @@ -703,7 +678,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) "read_row: value mismatch %" PRIu64 ":\n", keyno); print_item("bdb", &bdb_value); print_item(" wt", &value); - die(0, NULL); + testutil_die(0, NULL); } } #endif @@ -748,7 +723,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) break; } if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "%s", which); + testutil_die(ret, "%s", which); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -777,7 +752,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) fprintf(stderr, "nextprev: %s key mismatch:\n", which); print_item("bdb-key", &bdb_key); print_item(" wt-key", &key); - die(0, NULL); + testutil_die(0, NULL); } } else { if (keyno != (uint64_t)atoll(bdb_key.data)) { @@ -787,7 +762,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) "nextprev: %s key mismatch: %.*s != %" PRIu64 "\n", which, (int)bdb_key.size, (char *)bdb_key.data, keyno); - die(0, NULL); + testutil_die(0, NULL); } } if (value.size != bdb_value.size || @@ -795,7 +770,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) fprintf(stderr, "nextprev: %s value mismatch:\n", which); print_item("bdb-value", &bdb_value); print_item(" wt-value", &value); - die(0, NULL); + testutil_die(0, NULL); } if (g.logging == LOG_OPS) @@ -851,7 +826,8 @@ row_update(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_update: update row %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_update: update row %" PRIu64 " by key", keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -905,7 +881,7 @@ col_update(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "col_update: %" PRIu64, keyno); + testutil_die(ret, "col_update: %" PRIu64, keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -937,7 +913,7 @@ table_append_init(void) free(g.append); if ((g.append = calloc(g.append_max, sizeof(uint64_t))) == NULL) - die(errno, "calloc"); + testutil_die(errno, "calloc"); } /* @@ -948,7 +924,7 @@ static void table_append(uint64_t keyno) { uint64_t *p, *ep; - int done, ret; + int done; ep = g.append + g.append_max; @@ -979,8 +955,7 @@ table_append(uint64_t keyno) * and we find a slot. */ for (done = 0;;) { - if ((ret = pthread_rwlock_wrlock(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_wrlock: append_lock"); + testutil_check(pthread_rwlock_wrlock(&g.append_lock)); /* * If this is the thread we've been waiting for, and its record @@ -1017,8 +992,7 @@ table_append(uint64_t keyno) break; } - if ((ret = pthread_rwlock_unlock(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_unlock: append_lock"); + testutil_check(pthread_rwlock_unlock(&g.append_lock)); if (done) break; @@ -1055,7 +1029,8 @@ row_insert(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_insert: insert row %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_insert: insert row %" PRIu64 " by key", keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -1094,10 +1069,9 @@ col_insert(TINFO *tinfo, if ((ret = cursor->insert(cursor)) != 0) { if (ret == WT_ROLLBACK) return (WT_ROLLBACK); - die(ret, "cursor.insert"); + testutil_die(ret, "cursor.insert"); } - if ((ret = cursor->get_key(cursor, &keyno)) != 0) - die(ret, "cursor.get_key"); + testutil_check(cursor->get_key(cursor, &keyno)); *keynop = (uint32_t)keyno; table_append(keyno); /* Extend the object. */ @@ -1157,7 +1131,8 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_remove: remove %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_remove: remove %" PRIu64 " by key", keyno); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -1200,7 +1175,8 @@ col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "col_remove: remove %" PRIu64 " by key", keyno); + testutil_die(ret, + "col_remove: remove %" PRIu64 " by key", keyno); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -1245,7 +1221,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, " not found in Berkeley DB, found in WiredTiger\n"); - die(0, NULL); + testutil_die(0, NULL); } if (wt_ret == WT_NOTFOUND) { fprintf(stderr, "%s: %s:", g.progname, f); @@ -1253,7 +1229,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, " found in Berkeley DB, not found in WiredTiger\n"); - die(0, NULL); + testutil_die(0, NULL); } return (0); } diff --git a/src/third_party/wiredtiger/test/format/rebalance.c b/src/third_party/wiredtiger/test/format/rebalance.c index 8e8fa1a371f..d35dcec1d53 100644 --- a/src/third_party/wiredtiger/test/format/rebalance.c +++ b/src/third_party/wiredtiger/test/format/rebalance.c @@ -33,7 +33,6 @@ wts_rebalance(void) { WT_CONNECTION *conn; WT_SESSION *session; - int ret; char cmd[1024]; if (g.c_rebalance == 0) @@ -45,26 +44,23 @@ wts_rebalance(void) (void)snprintf(cmd, sizeof(cmd), "../../wt -h %s dump -f %s/rebalance.orig %s", g.home, g.home, g.uri); - if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Rebalance, then verify the object. */ wts_reopen(); conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance start ==============="); - if ((ret = session->rebalance(session, g.uri, NULL)) != 0) - die(ret, "session.rebalance: %s: %s", g.uri); + testutil_checkfmt( + session->rebalance(session, g.uri, NULL), "%s", g.uri); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); wts_verify("post-rebalance verify"); wts_close(); @@ -72,13 +68,11 @@ wts_rebalance(void) (void)snprintf(cmd, sizeof(cmd), "../../wt -h %s dump -f %s/rebalance.new %s", g.home, g.home, g.uri); - if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Compare the old/new versions of the object. */ (void)snprintf(cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", g.home, g.home); - if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); } diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index d0358e998b4..526e1563390 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -42,12 +42,10 @@ salvage(void) conn = g.wts_conn; track("salvage", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->salvage(session, g.uri, "force=true")) != 0) - die(ret, "session.salvage: %s", g.uri); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_die(ret, "session.salvage: %s", g.uri); + testutil_check(session->close(session, NULL)); } /* @@ -101,37 +99,37 @@ corrupt(void) return (0); found: if (fstat(fd, &sb) == -1) - die(errno, "salvage-corrupt: fstat"); + testutil_die(errno, "salvage-corrupt: fstat"); offset = mmrand(NULL, 0, (u_int)sb.st_size); len = (size_t)(20 + (sb.st_size / 100) * 2); (void)snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home); if ((fp = fopen(buf, "w")) == NULL) - die(errno, "salvage-corrupt: open: %s", buf); + testutil_die(errno, "salvage-corrupt: open: %s", buf); (void)fprintf(fp, "salvage-corrupt: offset %" PRIuMAX ", length " SIZET_FMT "\n", (uintmax_t)offset, len); fclose_and_clear(&fp); if (lseek(fd, offset, SEEK_SET) == -1) - die(errno, "salvage-corrupt: lseek"); + testutil_die(errno, "salvage-corrupt: lseek"); memset(buf, 'z', sizeof(buf)); for (; len > 0; len -= nw) { nw = (size_t)(len > sizeof(buf) ? sizeof(buf) : len); if (write(fd, buf, nw) == -1) - die(errno, "salvage-corrupt: write"); + testutil_die(errno, "salvage-corrupt: write"); } if (close(fd) == -1) - die(errno, "salvage-corrupt: close"); + testutil_die(errno, "salvage-corrupt: close"); /* * Save a copy of the corrupted file so we can replay the salvage step * as necessary. */ if ((ret = system(copycmd)) != 0) - die(ret, "salvage corrupt copy step failed"); + testutil_die(ret, "salvage corrupt copy step failed"); return (1); } @@ -157,7 +155,7 @@ wts_salvage(void) * step as necessary. */ if ((ret = system(g.home_salvage_copy)) != 0) - die(ret, "salvage copy step failed"); + testutil_die(ret, "salvage copy step failed"); /* Salvage, then verify. */ wts_open(g.home, 1, &g.wts_conn); diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 0c0485c8bfe..28c22e23cb8 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -30,17 +30,20 @@ GLOBAL g; +static void format_die(void); static void startup(void); static void usage(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = format_die; /* Local death handler. */ + int main(int argc, char *argv[]) { time_t start; - int ch, i, onerun, reps, ret; + int ch, i, onerun, reps; const char *config, *home; config = NULL; @@ -125,9 +128,9 @@ main(int argc, char *argv[]) /* If it's a replay, use the home directory's CONFIG file. */ if (g.replay) { if (config != NULL) - die(EINVAL, "-c incompatible with -r"); + testutil_die(EINVAL, "-c incompatible with -r"); if (access(g.home_config, R_OK) != 0) - die(ENOENT, "%s", g.home_config); + testutil_die(ENOENT, "%s", g.home_config); config = g.home_config; } @@ -176,12 +179,9 @@ main(int argc, char *argv[]) * Initialize locks to single-thread named checkpoints and backups, last * last-record updates, and failures. */ - if ((ret = pthread_rwlock_init(&g.append_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: append lock"); - if ((ret = pthread_rwlock_init(&g.backup_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: backup lock"); - if ((ret = pthread_rwlock_init(&g.death_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: death lock"); + testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { @@ -273,10 +273,8 @@ main(int argc, char *argv[]) config_print(0); - if ((ret = pthread_rwlock_destroy(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_destroy: append lock"); - if ((ret = pthread_rwlock_destroy(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_destroy: backup lock"); + testutil_check(pthread_rwlock_destroy(&g.append_lock)); + testutil_check(pthread_rwlock_destroy(&g.backup_lock)); config_clear(); @@ -298,27 +296,28 @@ startup(void) /* Create or initialize the home and data-source directories. */ if ((ret = system(g.home_init)) != 0) - die(ret, "home directory initialization failed"); + testutil_die(ret, "home directory initialization failed"); /* Open/truncate the logging file. */ if (g.logging != 0 && (g.logfp = fopen(g.home_log, "w")) == NULL) - die(errno, "fopen: %s", g.home_log); + testutil_die(errno, "fopen: %s", g.home_log); /* Open/truncate the random number logging file. */ if ((g.randfp = fopen(g.home_rand, g.replay ? "r" : "w")) == NULL) - die(errno, "%s", g.home_rand); + testutil_die(errno, "%s", g.home_rand); } /* * die -- - * Report an error and quit, dumping the configuration. + * Report an error, dumping the configuration. */ -void -die(int e, const char *fmt, ...) +static void +format_die(void) { - va_list ap; - - /* Single-thread error handling. */ + /* + * Single-thread error handling, our caller exits after calling + * us - don't release the lock. + */ (void)pthread_rwlock_wrlock(&g.death_lock); /* Try and turn off tracking so it doesn't obscure the error message. */ @@ -326,15 +325,6 @@ die(int e, const char *fmt, ...) g.c_quiet = 1; fprintf(stderr, "\n"); } - if (fmt != NULL) { /* Death message. */ - fprintf(stderr, "%s: ", g.progname); - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - if (e != 0) - fprintf(stderr, ": %s", wiredtiger_strerror(e)); - fprintf(stderr, "\n"); - } /* Flush/close any logging information. */ fclose_and_clear(&g.logfp); @@ -343,8 +333,6 @@ die(int e, const char *fmt, ...) /* Display the configuration that failed. */ if (g.run_cnt) config_print(1); - - exit(EXIT_FAILURE); } /* diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 82a6de97ab6..347b2ea1db3 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -42,7 +42,7 @@ dmalloc(size_t len) void *p; if ((p = malloc(len)) == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); return (p); } @@ -56,7 +56,7 @@ dstrdup(const char *str) char *p; if ((p = strdup(str)) == NULL) - die(errno, "strdup"); + testutil_die(errno, "strdup"); return (p); } @@ -268,9 +268,9 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) lastlen = len; if (printf("%s\r", msg) < 0) - die(EIO, "printf"); + testutil_die(EIO, "printf"); if (fflush(stdout) == EOF) - die(errno, "fflush"); + testutil_die(errno, "fflush"); } /* @@ -407,7 +407,7 @@ rng(WT_RAND_STATE *rnd) "\n" "end of random number log reached\n"); exit(EXIT_SUCCESS); } - die(errno, "random number log"); + testutil_die(errno, "random number log"); } return ((uint32_t)strtoul(buf, NULL, 10)); @@ -435,6 +435,6 @@ fclose_and_clear(FILE **fpp) return; *fpp = NULL; if (fclose(fp) != 0) - die(errno, "fclose"); + testutil_die(errno, "fclose"); return; } diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 9d4d3fe5cb8..a0e57dc2bee 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -53,7 +53,7 @@ compressor(uint32_t compress_flag) default: break; } - die(EINVAL, "illegal compression flag: 0x%x", compress_flag); + testutil_die(EINVAL, "illegal compression flag: 0x%x", compress_flag); } /* @@ -71,7 +71,7 @@ encryptor(uint32_t encrypt_flag) default: break; } - die(EINVAL, "illegal encryption flag: 0x%x", encrypt_flag); + testutil_die(EINVAL, "illegal encryption flag: 0x%x", encrypt_flag); } static int @@ -222,7 +222,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) p += snprintf(p, REMAIN(p, end), ",%s", g.config_open); if (REMAIN(p, end) == 0) - die(ENOMEM, "wiredtiger_open configuration buffer too small"); + testutil_die(ENOMEM, + "wiredtiger_open configuration buffer too small"); /* * Direct I/O may not work with backups, doing copies through the buffer @@ -233,8 +234,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) if (strstr(config, "direct_io") != NULL) g.c_backups = 0; - if ((ret = wiredtiger_open(home, &event_handler, config, &conn)) != 0) - die(ret, "wiredtiger_open: %s", home); + testutil_checkfmt( + wiredtiger_open(home, &event_handler, config, &conn), "%s", home); if (set_api) g.wt_api = conn->get_extension_api(conn); @@ -247,7 +248,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) */ if (DATASOURCE("helium")) { if (g.helium_mount == NULL) - die(EINVAL, "no Helium mount point specified"); + testutil_die(EINVAL, "no Helium mount point specified"); (void)snprintf(helium_config, sizeof(helium_config), "entry=wiredtiger_extension_init,config=[" "helium_verbose=0," @@ -256,7 +257,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) g.helium_mount); if ((ret = conn->load_extension( conn, HELIUM_PATH, helium_config)) != 0) - die(ret, + testutil_die(ret, "WT_CONNECTION.load_extension: %s:%s", HELIUM_PATH, helium_config); } @@ -270,11 +271,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) void wts_reopen(void) { - int ret; - - if ((ret = wiredtiger_open(g.home, - &event_handler, g.wiredtiger_open_config, &g.wts_conn)) != 0) - die(ret, "wiredtiger_open: %s", g.home); + testutil_checkfmt(wiredtiger_open(g.home, &event_handler, + g.wiredtiger_open_config, &g.wts_conn), "%s", g.home); } /* @@ -287,7 +285,6 @@ wts_create(void) WT_CONNECTION *conn; WT_SESSION *session; uint32_t maxintlpage, maxintlkey, maxleafpage, maxleafkey, maxleafvalue; - int ret; char config[4096], *end, *p; conn = g.wts_conn; @@ -431,32 +428,28 @@ wts_create(void) } if (REMAIN(p, end) == 0) - die(ENOMEM, "WT_SESSION.create configuration buffer too small"); + testutil_die(ENOMEM, + "WT_SESSION.create configuration buffer too small"); /* * Create the underlying store. */ - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->create(session, g.uri, config)) != 0) - die(ret, "session.create: %s", g.uri); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_checkfmt(session->create(session, g.uri, config), "%s", g.uri); + testutil_check(session->close(session, NULL)); } void wts_close(void) { WT_CONNECTION *conn; - int ret; const char *config; conn = g.wts_conn; config = g.c_leak_memory ? "leak_memory" : NULL; - if ((ret = conn->close(conn, config)) != 0) - die(ret, "connection.close"); + testutil_check(conn->close(conn, config)); g.wts_conn = NULL; g.wt_api = NULL; } @@ -466,7 +459,6 @@ wts_dump(const char *tag, int dump_bdb) { #ifdef HAVE_BERKELEY_DB size_t len; - int ret; char *cmd; /* @@ -491,8 +483,7 @@ wts_dump(const char *tag, int dump_bdb) g.uri == NULL ? "" : "-n", g.uri == NULL ? "" : g.uri); - if ((ret = system(cmd)) != 0) - die(ret, "%s: dump comparison failed", tag); + testutil_checkfmt(system(cmd), "%s: dump comparison failed", tag); free(cmd); #else (void)tag; /* [-Wunused-variable] */ @@ -513,8 +504,7 @@ wts_verify(const char *tag) conn = g.wts_conn; track("verify", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify start ==============="); @@ -522,13 +512,12 @@ wts_verify(const char *tag) /* Session operations for LSM can return EBUSY. */ ret = session->verify(session, g.uri, "strict"); if (ret != 0 && !(ret == EBUSY && DATASOURCE("lsm"))) - die(ret, "session.verify: %s: %s", g.uri, tag); + testutil_die(ret, "session.verify: %s: %s", g.uri, tag); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); } /* @@ -558,49 +547,43 @@ wts_stats(void) conn = g.wts_conn; track("stat", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if ((fp = fopen(g.home_stats, "w")) == NULL) - die(errno, "fopen: %s", g.home_stats); + testutil_die(errno, "fopen: %s", g.home_stats); /* Connection statistics. */ fprintf(fp, "====== Connection statistics:\n"); - if ((ret = session->open_cursor(session, - "statistics:", NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor( + session, "statistics:", NULL, NULL, &cursor)); while ((ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - die(errno, "fprintf"); + testutil_die(errno, "fprintf"); if (ret != WT_NOTFOUND) - die(ret, "cursor.next"); - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); /* Data source statistics. */ fprintf(fp, "\n\n====== Data source statistics:\n"); stat_name = dmalloc(strlen("statistics:") + strlen(g.uri) + 1); sprintf(stat_name, "statistics:%s", g.uri); - if ((ret = session->open_cursor( - session, stat_name, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor( + session, stat_name, NULL, NULL, &cursor)); free(stat_name); while ((ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - die(errno, "fprintf"); + testutil_die(errno, "fprintf"); if (ret != WT_NOTFOUND) - die(ret, "cursor.next"); - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); fclose_and_clear(&fp); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/huge/huge.c b/src/third_party/wiredtiger/test/huge/huge.c index d09f6f375fb..ad19035ff99 100644 --- a/src/third_party/wiredtiger/test/huge/huge.c +++ b/src/third_party/wiredtiger/test/huge/huge.c @@ -167,6 +167,8 @@ run(CONFIG *cp, int bigkey, size_t bytes) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/readonly/Makefile.am b/src/third_party/wiredtiger/test/readonly/Makefile.am new file mode 100644 index 00000000000..384e197a1f8 --- /dev/null +++ b/src/third_party/wiredtiger/test/readonly/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = t +t_SOURCES = readonly.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +# Run this during a "make check" smoke test. +TESTS = smoke.sh + +clean-local: + rm -rf WiredTiger* *.core __* diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c new file mode 100644 index 00000000000..100ccbf81b7 --- /dev/null +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -0,0 +1,402 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <sys/wait.h> +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + +#include <wiredtiger.h> + +#include "test_util.i" + +#define HOME_SIZE 512 +static char home[HOME_SIZE]; /* Program working dir lock file */ +static char home_wr[HOME_SIZE]; /* Writable dir copy no lock file */ +static char home_rd[HOME_SIZE]; /* Read-only dir */ +static char home_rd2[HOME_SIZE]; /* Read-only dir no lock file */ +static const char *progname; /* Program name */ +static const char *saved_argv0; /* Program command */ +static const char * const uri = "table:main"; + +#define ENV_CONFIG \ + "create,log=(file_max=10M,archive=false,enabled)," \ + "transaction_sync=(enabled,method=none)" +#define ENV_CONFIG_RD "readonly=true" +#define ENV_CONFIG_WR "readonly=false" +#define MAX_VAL 4096 +#define MAX_KV 10000 + +#define EXPECT_ERR 1 +#define EXPECT_SUCCESS 0 + +#define OP_READ 0 +#define OP_WRITE 1 + +static void +usage(void) +{ + fprintf(stderr, "usage: %s [-h dir]\n", progname); + exit(EXIT_FAILURE); +} + +static int +run_child(const char *homedir, int op, int expect) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + int i, ret; + const char *cfg; + + /* + * We expect the read-only database will allow the second read-only + * handle to succeed because no one can create or set the lock file. + */ + if (op == OP_READ) + cfg = ENV_CONFIG_RD; + else + cfg = ENV_CONFIG_WR; + ret = wiredtiger_open(homedir, NULL, cfg, &conn); + if (expect == EXPECT_SUCCESS && ret != 0) + testutil_die(ret, "wiredtiger_open success err"); + if (expect == EXPECT_ERR) { + if (ret == 0) + testutil_die( + ret, "wiredtiger_open expected err succeeded"); + /* + * If we expect an error and got one, we're done. + */ + return (0); + } + + /* + * Make sure we can read the data. + */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); + + i = 0; + while ((ret = cursor->next(cursor)) == 0) + ++i; + if (i != MAX_KV) + testutil_die(EPERM, "cursor walk"); + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "conn_close"); + return (0); +} + +/* + * Child process opens both databases readonly. + */ +static void +open_dbs(int op, const char *dir, + const char *dir_wr, const char *dir_rd, const char *dir_rd2) +{ + int expect, ret; + + /* + * The parent has an open connection to all directories. + * We expect opening the writeable homes to return an error. + * It is a failure if the child successfully opens that. + */ + expect = EXPECT_ERR; + if ((ret = run_child(dir, op, expect)) != 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + if ((ret = run_child(dir_wr, op, expect)) != 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + + /* + * The parent must have a read-only connection open to the + * read-only databases. If the child is opening read-only + * too, we expect success. Otherwise an error if the child + * attempts to open read/write (permission error). + */ + if (op == OP_READ) + expect = EXPECT_SUCCESS; + if ((ret = run_child(dir_rd, op, expect)) != 0) + testutil_die(ret, "run child 1"); + if ((ret = run_child(dir_rd2, op, expect)) != 0) + testutil_die(ret, "run child 2"); + exit(EXIT_SUCCESS); +} + +extern int __wt_optind; +extern char *__wt_optarg; + +void (*custom_die)(void) = NULL; + +int +main(int argc, char *argv[]) +{ + WT_CONNECTION *conn, *conn2, *conn3, *conn4; + WT_CURSOR *cursor; + WT_ITEM data; + WT_SESSION *session; + uint64_t i; + int ch, status, op, ret; + bool child; + const char *working_dir; + char cmd[512]; + uint8_t buf[MAX_VAL]; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + /* + * Needed unaltered for system command later. + */ + saved_argv0 = argv[0]; + + working_dir = "WT_RD"; + child = false; + op = OP_READ; + while ((ch = __wt_getopt(progname, argc, argv, "Rh:W")) != EOF) + switch (ch) { + case 'R': + child = true; + op = OP_READ; + break; + case 'W': + child = true; + op = OP_WRITE; + break; + case 'h': + working_dir = __wt_optarg; + break; + default: + usage(); + } + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + usage(); + + memset(buf, 0, sizeof(buf)); + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, 512, working_dir); + strncpy(home_wr, home, HOME_SIZE); + strcat(home_wr, ".WRNOLOCK"); + strncpy(home_rd, home, HOME_SIZE); + strcat(home_rd, ".RD"); + strncpy(home_rd2, home, HOME_SIZE); + strcat(home_rd2, ".RDNOLOCK"); + if (!child) { + testutil_make_work_dir(home); + testutil_make_work_dir(home_wr); + testutil_make_work_dir(home_rd); + testutil_make_work_dir(home_rd2); + } else + /* + * We are a child process, we just want to call + * the open_dbs with the directories we have. + * The child function will exit. + */ + open_dbs(op, home, home_wr, home_rd, home_rd2); + + /* + * Parent creates a database and table. Then cleanly shuts down. + * Then copy database to read-only directory and chmod. + * Also copy database to read-only directory and remove the lock + * file. One read-only database will have a lock file in the + * file system and the other will not. + * Parent opens all databases with read-only configuration flag. + * Parent forks off child who tries to also open all databases + * with the read-only flag. It should error on the writeable + * directory, but allow it on the read-only directories. + * The child then confirms it can read all the data. + */ + /* + * Run in the home directory and create the table. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG, &conn)) != 0) + testutil_die(ret, "wiredtiger_open"); + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + if ((ret = session->create(session, + uri, "key_format=Q,value_format=u")) != 0) + testutil_die(ret, "WT_SESSION.create: %s", uri); + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); + + /* + * Write data into the table and then cleanly shut down connection. + */ + data.data = buf; + data.size = MAX_VAL; + for (i = 0; i < MAX_KV; ++i) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, &data); + if ((ret = cursor->insert(cursor)) != 0) + testutil_die(ret, "WT_CURSOR.insert"); + } + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + + /* + * Copy the database. Remove any lock file from one copy + * and chmod the copies to be read-only permissions. + */ + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", + home, home_wr, home_wr); + (void)system(cmd); + + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", + home, home_rd, home_rd, home_rd); + (void)system(cmd); + + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " + "chmod 0555 %s; chmod -R 0444 %s/*", + home, home_rd2, home_rd2, home_rd2, home_rd2); + (void)system(cmd); + + /* + * Run four scenarios. Sometimes expect errors, sometimes success. + * The writable database directories should always fail to allow the + * child to open due to the lock file. The read-only ones will only + * succeed when the child attempts read-only. + * + * 1. Parent has read-only handle to all databases. Child opens + * read-only also. + * 2. Parent has read-only handle to all databases. Child opens + * read-write. + * 3. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-only. + * 4. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-write. + */ + /* + * Open a connection handle to all databases. + */ + fprintf(stderr, " *** Expect several error messages from WT ***\n"); + /* + * Scenario 1. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn3)) != 0) + testutil_die(ret, "wiredtiger_open readonly"); + if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn4)) != 0) + testutil_die(ret, "wiredtiger_open readonly nolock"); + + /* + * Create a child to also open a connection handle to the databases. + * We cannot use fork here because using fork the child inherits the + * same memory image. Therefore the WT process structure is set in + * the child even though it should not be. So use 'system' to spawn + * an entirely new process. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + /* + * The child will exit with success if its test passes. + */ + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Scenario 2. Run child with writable config. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Reopen the two writable directories and rerun the child. + */ + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn2->close(conn2, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + /* + * Scenario 3. Child read-only. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Scenario 4. Run child with writable config. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Clean-up. + */ + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn2->close(conn2, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn3->close(conn3, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn4->close(conn4, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + /* + * We need to chmod the read-only databases back so that they can + * be removed by scripts. + */ + (void)snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2); + (void)system(cmd); + (void)snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", + home_rd, home_rd2); + (void)system(cmd); + printf(" *** Readonly test successful ***\n"); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/readonly/smoke.sh b/src/third_party/wiredtiger/test/readonly/smoke.sh new file mode 100755 index 00000000000..740deb5743a --- /dev/null +++ b/src/third_party/wiredtiger/test/readonly/smoke.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +trap 'chmod -R u+w WT_*; exit 0' 0 1 2 3 13 15 + +set -e + +# Smoke-test format as part of running "make check". +$TEST_WRAPPER ./t diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index ddcafbc80fd..c9cc10d2db3 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -42,7 +42,7 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ -static const char *uri = "table:main"; +static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -88,7 +88,8 @@ fill_db(void) /* * Run in the home directory so that the records file is in there too. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) @@ -109,7 +110,7 @@ fill_db(void) /* * Set to no buffering. */ - setvbuf(fp, NULL, _IONBF, 0); + (void)setvbuf(fp, NULL, _IONBF, 0); /* * Write data into the table until we are killed by the parent. @@ -135,6 +136,8 @@ fill_db(void) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { @@ -201,13 +204,15 @@ main(int argc, char *argv[]) printf("Kill child\n"); if (kill(pid, SIGKILL) != 0) testutil_die(errno, "kill"); - waitpid(pid, &status, 0); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); printf("Open database, run recovery and verify content\n"); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); @@ -239,7 +244,8 @@ main(int argc, char *argv[]) ++absent; } } - fclose(fp); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (absent) { diff --git a/src/third_party/wiredtiger/test/recovery/truncated-log.c b/src/third_party/wiredtiger/test/recovery/truncated-log.c index 4add7a61f66..23269e99d35 100644 --- a/src/third_party/wiredtiger/test/recovery/truncated-log.c +++ b/src/third_party/wiredtiger/test/recovery/truncated-log.c @@ -45,7 +45,7 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ -static const char *uri = "table:main"; +static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -54,7 +54,6 @@ static const char *uri = "table:main"; "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(recover=on)" #define LOG_FILE_1 "WiredTigerLog.0000000001" -#define MAX_VAL 4096 #define K_SIZE 16 #define V_SIZE 256 @@ -86,7 +85,8 @@ fill_db(void) /* * Run in the home directory so that the records file is in there too. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) @@ -107,7 +107,7 @@ fill_db(void) /* * Set to no buffering. */ - setvbuf(fp, NULL, _IONBF, 0); + (void)setvbuf(fp, NULL, _IONBF, 0); save_lsn.l.file = 0; /* @@ -156,7 +156,8 @@ fill_db(void) "%" PRIu32 " %" PRIu32 "\n", save_lsn.l.offset, i - 1) == -1) testutil_die(errno, "fprintf"); - fclose(fp); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); abort(); } } @@ -168,6 +169,8 @@ fill_db(void) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { @@ -218,20 +221,24 @@ main(int argc, char *argv[]) /* parent */ /* Wait for child to kill itself. */ - waitpid(pid, &status, 0); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); + printf("Open database, run recovery and verify content\n"); if ((fp = fopen(RECORDS_FILE, "r")) == NULL) testutil_die(errno, "fopen"); ret = fscanf(fp, "%" SCNu64 " %" SCNu32 "\n", &offset, &max_key); - fclose(fp); if (ret != 2) testutil_die(errno, "fscanf"); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); /* * The offset is the beginning of the last record. Truncate to * the middle of that last record (i.e. ahead of that offset). diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index c2ad6224b11..a1517d70787 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -64,6 +64,8 @@ static int verbose; /* -v flag */ extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py index 7d4503b84b7..36f1ef733a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py @@ -265,9 +265,13 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") cursor.set_key(key_populate(cursor, 10)) cursor.set_value("XXX") - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.remove()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.update()) + msg = "/not supported/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.insert(), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.remove(), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.update(), msg) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index 14781b0f050..7ad05cd2536 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -50,11 +50,12 @@ class test_compact02(wttest.WiredTigerTestCase): # being stored: compaction doesn't work on tables with many overflow items # because we don't rewrite them. Experimentally, 8KB is as small as the test # can go. Additionally, we can't set the maximum page size too large because - # there won't be enough pages to rewrite. Experimentally, 32KB (the default) - # is as large as the test can go. + # there won't be enough pages to rewrite. Experimentally, 128KB works. fileConfig = [ ('default', dict(fileConfig='')), ('8KB', dict(fileConfig='leaf_page_max=8kb')), + ('64KB', dict(fileConfig='leaf_page_max=64KB')), + ('128KB', dict(fileConfig='leaf_page_max=128KB')), ] scenarios = \ number_scenarios(multiply_scenarios('.', types, cacheSize, fileConfig)) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor06.py b/src/third_party/wiredtiger/test/suite/test_cursor06.py index ff7c1144344..d702f97c5dd 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor06.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor06.py @@ -89,10 +89,11 @@ class test_cursor06(wttest.WiredTigerTestCase): self.session.drop(uri, "force") self.populate(uri) cursor = self.session.open_cursor(uri, None, open_config) + msg = '/not supported/' if open_config == "readonly=1": self.set_kv(cursor) - self.assertRaises(wiredtiger.WiredTigerError, - lambda: cursor.update()) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.update(), msg) else: self.set_kv(cursor) cursor.update() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index 1fd30d93c11..cd91a925b0c 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -51,15 +51,21 @@ class test_cursor_random(wttest.WiredTigerTestCase): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None, self.config) - self.assertRaises( - wiredtiger.WiredTigerError, lambda: cursor.compare(cursor)) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.prev()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.remove()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.search()) - self.assertRaises( - wiredtiger.WiredTigerError, lambda: cursor.search_near()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.update()) + msg = "/not supported/" + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.compare(cursor), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.prev(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.remove(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.search(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.search_near(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.update(), msg) self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) self.assertEquals(cursor.reconfigure(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_join01.py b/src/third_party/wiredtiger/test/suite/test_join01.py index f03c7c6f06c..7630706379c 100644 --- a/src/third_party/wiredtiger/test/suite/test_join01.py +++ b/src/third_party/wiredtiger/test/suite/test_join01.py @@ -33,7 +33,6 @@ from wtscenario import check_scenarios, multiply_scenarios, number_scenarios # Join operations # Basic tests for join class test_join01(wttest.WiredTigerTestCase): - table_name1 = 'test_join01' nentries = 100 scenarios = [ @@ -342,11 +341,12 @@ class test_join01(wttest.WiredTigerTestCase): '/index cursor is being used in a join/') # Only a small number of operations allowed on a join cursor - self.assertRaises(wiredtiger.WiredTigerError, - lambda: jc.search()) + msg = "/not supported/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.search(), msg) - self.assertRaises(wiredtiger.WiredTigerError, - lambda: jc.prev()) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.prev(), msg) self.assertEquals(jc.next(), 0) self.assertEquals(jc.next(), wiredtiger.WT_NOTFOUND) @@ -390,6 +390,7 @@ class test_join01(wttest.WiredTigerTestCase): def test_cursor_close2(self): self.cursor_close_common(False) + # test statistics using the framework set up for this test def test_stats(self): bloomcfg1000 = ',strategy=bloom,count=1000' bloomcfg10 = ',strategy=bloom,count=10' @@ -399,6 +400,40 @@ class test_join01(wttest.WiredTigerTestCase): # statistics should pick up some false positives. self.join_common(bloomcfg10, bloomcfg10, False, True) + # test statistics with a simple one index join cursor + def test_simple_stats(self): + self.session.create("table:join01b", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:join01b:index", "columns=(v)") + + cursor = self.session.open_cursor("table:join01b", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:join01b:index", None, None) + cursor.set_key(11) + cursor.search() + + jcursor = self.session.open_cursor("join:table:join01b", None, None) + self.session.join(jcursor, cursor, "compare=gt") + + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + + statcur = self.session.open_cursor("statistics:join", jcursor, None) + found = False + while statcur.next() == 0: + [desc, pvalue, value] = statcur.get_values() + #self.tty(str(desc) + "=" + str(pvalue)) + found = True + self.assertEquals(found, True) + + jcursor.close() + cursor.close() + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly01.py b/src/third_party/wiredtiger/test/suite/test_readonly01.py new file mode 100644 index 00000000000..86604eb6bfb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_readonly01.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly01.py +# Readonly: Test readonly mode. +# + +import fnmatch, os, shutil, time +from suite_subprocess import suite_subprocess +from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios +import wttest + +class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_readonly01' + create = True + entries = 10000 + + # + # We want a list of directory writable or readonly. + # + basecfg_list = [ + ('basecfg', dict(basecfg='config_base=true,')), + ('no_basecfg', dict(basecfg='config_base=false,')), + ] + dir_list = [ + ('write', dict(dirchmod=False)), + ('readonly', dict(dirchmod=True)), + ] + log_list = [ + ('logging', dict(logcfg='log=(archive=false,enabled,file_max=100K),')), + ('no_logging', dict(logcfg='log=(enabled=false),')), + ] + + types = [ + ('lsm', dict(tabletype='lsm', uri='lsm', + create_params = 'key_format=i,value_format=i')), + ('file-row', dict(tabletype='row', uri='file', + create_params = 'key_format=i,value_format=i')), + ('file-var', dict(tabletype='var', uri='file', + create_params = 'key_format=r,value_format=i')), + ('file-fix', dict(tabletype='fix', uri='file', + create_params = 'key_format=r,value_format=8t')), + ('table-row', dict(tabletype='row', uri='table', + create_params = 'key_format=i,value_format=i')), + ('table-var', dict(tabletype='var', uri='table', + create_params = 'key_format=r,value_format=i')), + ('table-fix', dict(tabletype='fix', uri='table', + create_params = 'key_format=r,value_format=8t')), + ] + + scenarios = multiply_scenarios('.', + basecfg_list, dir_list, log_list, types) + + def conn_config(self, dir): + self.home = dir + params = \ + 'error_prefix="%s",' % self.shortid() + \ + '%s' % self.logcfg + \ + '%s' % self.basecfg + if self.create: + conn_params = 'create,' + params + else: + conn_params = 'readonly=true,' + params + return conn_params + + def close_reopen(self): + ''' Close the connection and reopen readonly''' + # + # close the original connection. If needed, chmod the + # database directory to readonly mode. Then reopen the + # connection with readonly. + # + self.close_conn() + if self.dirchmod: + for f in os.listdir(self.home): + if os.path.isfile(f): + os.chmod(f, 0444) + os.chmod(self.home, 0555) + self.conn = self.setUpConnectionOpen(self.home) + self.session = self.setUpSessionOpen(self.conn) + + def readonly(self): + # Here's the strategy: + # - Create a table. + # - Insert data into table. + # - Close connection. + # - Possibly chmod to readonly + # - Open connection readonly + # - Confirm we can read the data. + # + tablearg = self.uri + ':' + self.tablename + self.session.create(tablearg, self.create_params) + c = self.session.open_cursor(tablearg, None, None) + for i in range(self.entries): + c[i+1] = i % 255 + # Close the connection. Reopen readonly + self.create = False + self.close_reopen() + c = self.session.open_cursor(tablearg, None, None) + i = 0 + for key, value in c: + self.assertEqual(i+1, key) + self.assertEqual(i % 255, value) + i += 1 + self.assertEqual(i, self.entries) + self.pr('Read %d entries' % i) + c.close() + self.create = True + + def test_readonly(self): + if self.dirchmod: + with self.expectedStderrPattern('Permission'): + self.readonly() + else: + self.readonly() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly02.py b/src/third_party/wiredtiger/test/suite/test_readonly02.py new file mode 100644 index 00000000000..e94dd85857d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_readonly02.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly02.py +# Readonly: Test readonly mode with illegal config combinations +# and error checking during updates. +# + +from helper import copy_wiredtiger_home +from suite_subprocess import suite_subprocess +import os, wiredtiger, wttest + +class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'table:test_readonly02' + create = True + create_params = 'key_format=i,value_format=i' + entries = 10 + + conn_params = \ + 'create,statistics=(fast),log=(enabled,file_max=100K,zero_fill=true),' + conn_params_rd = \ + 'create,readonly=true,statistics=(fast),log=(enabled,zero_fill=false),' + conn_params_rdcfg = \ + 'create,readonly=true,statistics=(fast),log=(enabled),' + + # + # Run to make sure incompatible configuration options return an error. + # The situations that cause failures (instead of silent overrides) are: + # 1. setting readonly on a new database directory + # 2. an unclean shutdown and reopening readonly + # 3. logging with zero-fill enabled and readonly + # 4. readonly and statistics logging + # + badcfg1 = 'log=(enabled,zero_fill=true)' + badcfg2 = 'statistics_log=(wait=3)' + + def setUpConnectionOpen(self, dir): + self.home = dir + rdonlydir = dir + '.rdonly' + # + # First time through check readonly on a non-existent database. + # + if self.create: + # 1. setting readonly on a new database directory + # Setting readonly prevents creation so we should see an + # ENOENT error because the lock file does not exist. + msg = '/No such file/' + os.mkdir(rdonlydir) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open( + rdonlydir, self.conn_params_rd), msg) + + self.create = False + conn = self.wiredtiger_open(dir, self.conn_params) + return conn + + def check_unclean(self): + backup = "WT_COPYDIR" + copy_wiredtiger_home(self.home, backup, True) + msg = '/needs recovery/' + # 2. an unclean shutdown and reopening readonly + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open(backup, self.conn_params_rd), msg) + + def close_checkerror(self, cfg): + ''' Close the connection and reopen readonly''' + # + # Close the original connection. Reopen readonly and also with + # the given configuration string. + # + self.close_conn() + conn_params = self.conn_params_rd + cfg + msg = '/Invalid argument/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open(self.home, conn_params), msg) + + def test_readonly(self): + tablearg = self.tablename + self.session.create(tablearg, self.create_params) + c = self.session.open_cursor(tablearg, None, None) + for i in range(self.entries): + c[i+1] = i % 255 + # Check for an error on an unclean recovery/restart. + self.check_unclean() + + # Close the connection. Reopen readonly with other bad settings. + # 3. logging with zero-fill enabled and readonly + self.close_checkerror(self.badcfg1) + # 4. readonly and statistics logging + self.close_checkerror(self.badcfg2) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly03.py b/src/third_party/wiredtiger/test/suite/test_readonly03.py new file mode 100644 index 00000000000..981a21d51ac --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_readonly03.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly03.py +# Readonly: Test connection readonly mode with modifying methods. Confirm +# all return ENOTSUP. +# + +from helper import simple_populate +from suite_subprocess import suite_subprocess +import os, sys, wiredtiger, wttest + +class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): + uri = 'table:test_readonly03' + uri2 = 'table:test_readonly03_2' + create = True + + conn_params = 'create,log=(enabled),' + conn_params_rd = 'readonly=true' + + session_ops = [ 'create', 'compact', 'drop', 'log_flush', 'log_printf', + 'rebalance', 'rename', 'salvage', 'truncate', 'upgrade', ] + cursor_ops = [ 'insert', 'remove', 'update', ] + + def setUpConnectionOpen(self, dir): + self.home = dir + if self.create: + conn_cfg = self.conn_params + else: + conn_cfg = self.conn_params_rd + conn = self.wiredtiger_open(dir, conn_cfg) + self.create = False + return conn + + + def test_readonly(self): + create_params = 'key_format=i,value_format=i' + entries = 10 + # Create a database and a table. + simple_populate(self, self.uri, create_params, entries) + + # + # Now close and reopen. Note that the connection function + # above will reopen it readonly. + self.reopen_conn() + msg = '/not supported/' + c = self.session.open_cursor(self.uri, None, None) + for op in self.cursor_ops: + c.set_key(1) + c.set_value(1) + if op == 'insert': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.insert(), msg) + elif op == 'remove': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.remove(), msg) + elif op == 'update': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.update(), msg) + else: + self.fail('Unknown cursor operation: ' + op) + c.close() + for op in self.session_ops: + if op == 'create': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.uri2, create_params), + msg) + elif op == 'compact': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.compact(self.uri, None), msg) + elif op == 'drop': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.drop(self.uri, None), msg) + elif op == 'log_flush': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.log_flush(None), msg) + elif op == 'log_printf': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.log_printf("test"), msg) + elif op == 'rebalance': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.rebalance(self.uri, None), msg) + elif op == 'rename': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.rename(self.uri, self.uri2, None), msg) + elif op == 'salvage': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.salvage(self.uri, None), msg) + elif op == 'truncate': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.truncate(self.uri, None, None, None), + msg) + elif op == 'upgrade': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.upgrade(self.uri, None), msg) + else: + self.fail('Unknown session method: ' + op) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index b5a58d1566f..a1945b4325d 100644 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -335,6 +335,14 @@ class WiredTigerTestCase(unittest.TestCase): # always get back to original directory os.chdir(self.origcwd) + # Make sure no read-only files or directories were left behind + os.chmod(self.testdir, 0777) + for root, dirs, files in os.walk(self.testdir): + for d in dirs: + os.chmod(os.path.join(root, d), 0777) + for f in files: + os.chmod(os.path.join(root, f), 0666) + # Clean up unless there's a failure if (passed or skipped) and not WiredTigerTestCase._preserveFiles: shutil.rmtree(self.testdir, ignore_errors=True) diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c index e72b54bf62a..22334076ee1 100644 --- a/src/third_party/wiredtiger/test/thread/t.c +++ b/src/third_party/wiredtiger/test/thread/t.c @@ -51,6 +51,8 @@ static void wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/utility/test_util.i b/src/third_party/wiredtiger/test/utility/test_util.i index 3b88d375381..c5cebadcb5c 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.i +++ b/src/third_party/wiredtiger/test/utility/test_util.i @@ -42,25 +42,60 @@ #define DEFAULT_DIR "WT_TEST" #define MKDIR_COMMAND "mkdir " +/* Allow tests to add their own death handling. */ +extern void (*custom_die)(void); + +static void testutil_die(int, const char *, ...) +#if defined(__GNUC__) +__attribute__((__noreturn__)) +#endif +; + /* * die -- * Report an error and quit. */ -static inline void +static void testutil_die(int e, const char *fmt, ...) { va_list ap; + /* Allow test programs to cleanup on fatal error. */ + if (custom_die != NULL) + (*custom_die)(); + va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); if (e != 0) fprintf(stderr, ": %s", wiredtiger_strerror(e)); fprintf(stderr, "\n"); + exit(EXIT_FAILURE); } /* + * testutil_check -- + * Complain and quit if a function call fails. + */ +#define testutil_check(call) do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\ +} while (0) + +/* + * testutil_checkfmt -- + * Complain and quit if a function call fails, with additional arguments. + */ +#define testutil_checkfmt(call, fmt, ...) do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s: " fmt, \ + __func__, __LINE__, #call, __VA_ARGS__); \ +} while (0) + +/* * testutil_work_dir_from_path -- * Takes a buffer, its size and the intended work directory. * Creates the full intended work directory in buffer. diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index c35c27cb7b0..f32edce88e7 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -44,6 +44,11 @@ typedef int u_int; #define R_OK 04 #define X_OK R_OK +/* MSVC Doesn't provide __func__, it has __FUNCTION__ */ +#ifdef _MSC_VER +#define __func__ __FUNCTION__ +#endif + /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 #define snprintf _wt_snprintf diff --git a/src/third_party/wiredtiger/tools/wtstats/stat_data.py b/src/third_party/wiredtiger/tools/wtstats/stat_data.py index 7cee87e49ed..f181aeb09b4 100644 --- a/src/third_party/wiredtiger/tools/wtstats/stat_data.py +++ b/src/third_party/wiredtiger/tools/wtstats/stat_data.py @@ -3,6 +3,7 @@ no_scale_per_second_list = [ 'async: maximum work queue length', 'cache: bytes currently in the cache', + 'cache: eviction currently operating in aggressive mode', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', 'cache: pages currently held in the cache', @@ -67,6 +68,7 @@ no_scale_per_second_list = [ no_clear_list = [ 'async: maximum work queue length', 'cache: bytes currently in the cache', + 'cache: eviction currently operating in aggressive mode', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', 'cache: pages currently held in the cache', |