diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-09-29 13:29:39 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-09-29 03:44:36 +0000 |
| commit | a6bd068bc2943898b64943c70ae5760c35ba65e0 (patch) | |
| tree | 231eb4cbdd306fe888e747663e67691a2618d2a4 /src/third_party | |
| parent | f31ce895f1475e1184bafbf74f3c2032b9f1801f (diff) | |
| download | mongo-a6bd068bc2943898b64943c70ae5760c35ba65e0.tar.gz | |
Import wiredtiger: bb1cc65a63a4fb081cb5969f0728ff2b74fc7c8b from branch mongodb-4.4
ref: 8a1dce8653..bb1cc65a63
for: 4.4.2
WT-5645 Add Evergreen test that cycles through known failure test/format configs
WT-6181 Have Python Evergreen tests print standard output on failure
WT-6490 Acquire snapshot for eviction threads
WT-6654 Clean up test_backup15.py
WT-6657 Fix history store panic when inserting an update without timestamp
WT-6670 Fix uninitialized buffer
WT-6674 Remove Async API code and documentation
WT-6683 Fix logically dead code
WT-6685 Add import configuration option to WT_SESSION::create
WT-6689 Add support for file import when the exported configuration is provided
WT-6721 Add a new Evergreen task to run cyclomatic complexity
WT-6734 Add missing brace to Swig Java interface
Diffstat (limited to 'src/third_party')
72 files changed, 1782 insertions, 5972 deletions
diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore index 2b98be1a25c..9d0eeaa113a 100644 --- a/src/third_party/wiredtiger/.gitignore +++ b/src/third_party/wiredtiger/.gitignore @@ -84,7 +84,6 @@ _wiredtiger.pyd **/csuite/test_wt4891_meta_ckptlist_get_alloc **/examples/c/ex_access **/examples/c/ex_all -**/examples/c/ex_async **/examples/c/ex_backup **/examples/c/ex_backup_block **/examples/c/ex_call_center diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index ccd36511c76..5eddc4e356b 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -354,9 +354,7 @@ if enableJava and enableJava.count(",") == 1: conf.env.Append(CPPPATH=[ javaPath + '/include']) conf.env.Append(CPPPATH=[ javaPath + '/include/win32']) - swigJavaFiles = ["lang/java/src/com/wiredtiger/db/AsyncOp.java", - "lang/java/src/com/wiredtiger/db/AsyncOpType.java", - "lang/java/src/com/wiredtiger/db/Connection.java", + swigJavaFiles = ["lang/java/src/com/wiredtiger/db/Connection.java", "lang/java/src/com/wiredtiger/db/Cursor.java", "lang/java/src/com/wiredtiger/db/Modify.java", "lang/java/src/com/wiredtiger/db/SearchStatus.java", @@ -409,7 +407,6 @@ examples = [ "ex_access", # Temporarily disabled # "ex_all", - "ex_async", "ex_call_center", "ex_config_parse", "ex_cursor", diff --git a/src/third_party/wiredtiger/bench/workgen/wtperf.py b/src/third_party/wiredtiger/bench/workgen/wtperf.py index 66c009ef96e..d8522e33a68 100755 --- a/src/third_party/wiredtiger/bench/workgen/wtperf.py +++ b/src/third_party/wiredtiger/bench/workgen/wtperf.py @@ -490,25 +490,6 @@ class Translator: s += 'print("populate:")\n' s += 'pop_workload.run(conn)\n' - # If configured, compact to allow LSM merging to complete. We - # set an unlimited timeout because if we close the connection - # then any in-progress compact/merge is aborted. - if opts.compact: - if opts.async_threads == 0: - self.fatal_error('unexpected value for async_threads') - s += '\n' - if self.verbose > 0: - s += 'print("compact after populate:")\n' - s += 'import time\n' - s += 'start_time = time.time()\n' - s += 'async_callback = WtperfAsyncCallback()\n' - s += 'for i in range(0, table_count):\n' - s += ' op = conn.async_new_op(tables[i]._uri, "timeout=0", async_callback)\n' - s += ' op.compact()\n' - s += 'conn.async_flush()\n' - s += 'print("compact completed in {} seconds".format(' + \ - 'time.time() - start_time))\n' - return s def translate_inner(self): @@ -553,7 +534,6 @@ class Translator: self.get_boolean_opt('random_value', False) self.get_string_opt('transaction_config', '') self.get_boolean_opt('compact', False) - self.get_int_opt('async_threads', 0) self.get_int_opt('pareto', 0) opts = self.options if opts.range_partition and opts.random_range == 0: @@ -579,33 +559,10 @@ class Translator: if not input_as_string.endswith('\n'): s += '\n' s += '\'\'\'\n\n' - async_config = '' - if opts.compact and opts.async_threads == 0: - opts.async_threads = 2; - if opts.async_threads > 0: - # Assume the default of 1024 for the max ops, although we - # could bump that up to 4096 if needed. - async_config = ',async=(enabled=true,threads=' + \ - str(opts.async_threads) + ')' - s += '# this can be further customized\n' - s += 'class WtperfAsyncCallback(AsyncCallback):\n' - s += ' def __init__(self):\n' - s += ' pass\n' - s += ' def notify_error(self, key, value, optype, desc):\n' - s += ' print("ERROR: async notify(" + str(key) + "," + \\\n' - s += ' str(value) + "," + str(optype) + "): " + desc)\n' - s += ' def notify(self, op, op_ret, flags):\n' - s += ' if op_ret != 0:\n' - s += ' self.notify_error(op._key, op._value,\\\n' - s += ' op._optype, wiredtiger_strerror(op_ret))\n' - s += ' return op_ret\n' - s += '\n' s += 'context = Context()\n' extra_config = '' s += 'conn_config = ""\n' - if async_config != '': - s += 'conn_config += ",' + async_config + '" # async config\n' if conn_config != '': s += 'conn_config += ",' + conn_config + '" # explicitly added\n' if compression != '': diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/medium-lsm-async.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/medium-lsm-async.wtperf deleted file mode 100644 index 2661fe0b14b..00000000000 --- a/src/third_party/wiredtiger/bench/wtperf/runners/medium-lsm-async.wtperf +++ /dev/null @@ -1,12 +0,0 @@ -# wtperf options file: medium lsm configuration using async operations -conn_config="cache_size=1G,lsm_manager=(worker_thread_max=6)" -async_threads=10 -table_config="lsm=(chunk_size=100MB),type=lsm,os_cache_dirty_max=16MB" -icount=50000000 -report_interval=5 -run_time=120 -populate_threads=1 -threads=((count=5,reads=1,updates=2)) -# Add throughput/latency monitoring -max_latency=2000 -sample_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 6eefbbf373f..ce08af6e407 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -173,79 +173,6 @@ update_value_delta(WTPERF_THREAD *thread, int64_t delta) } } -static int -cb_asyncop(WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP *op, int ret, uint32_t flags) -{ - TRACK *trk; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_ASYNC_OPTYPE type; - uint32_t *tables; - int t_ret; - char *value; - - (void)cb; - (void)flags; - - wtperf = NULL; /* -Wconditional-uninitialized */ - thread = NULL; /* -Wconditional-uninitialized */ - - type = op->get_type(op); - if (type != WT_AOP_COMPACT) { - thread = (WTPERF_THREAD *)op->app_private; - wtperf = thread->wtperf; - } - - trk = NULL; - switch (type) { - case WT_AOP_COMPACT: - tables = (uint32_t *)op->app_private; - (void)__wt_atomic_add32(tables, (uint32_t)-1); - break; - case WT_AOP_INSERT: - trk = &thread->insert; - break; - case WT_AOP_SEARCH: - trk = &thread->read; - if (ret == 0 && (t_ret = op->get_value(op, &value)) != 0) { - ret = t_ret; - lprintf(wtperf, ret, 0, "get_value in read."); - goto err; - } - break; - case WT_AOP_UPDATE: - trk = &thread->update; - break; - case WT_AOP_NONE: - case WT_AOP_REMOVE: - /* We never expect this type. */ - lprintf(wtperf, ret, 0, "No type in op %" PRIu64, op->get_id(op)); - goto err; - } - - /* - * Either we have success and we track it, or failure and panic. - * - * Reads and updates can fail with WT_NOTFOUND: we may be searching in a random range, or an - * insert op might have updated the last record in the table but not yet finished the actual - * insert. - */ - if (type == WT_AOP_COMPACT) - return (0); - if (ret == 0 || (ret == WT_NOTFOUND && type != WT_AOP_INSERT)) { - if (!wtperf->in_warmup) - (void)__wt_atomic_add64(&trk->ops, 1); - return (0); - } -err: - /* Panic if error */ - lprintf(wtperf, ret, 0, "Error in op %" PRIu64, op->get_id(op)); - wtperf->error = wtperf->stop = true; - return (1); -} - -static WT_ASYNC_CALLBACK cb = {cb_asyncop}; - /* * track_operation -- * Update an operation's tracking structure with new latency information. @@ -310,119 +237,6 @@ op_name(uint8_t *op) /* NOTREACHED */ } -static WT_THREAD_RET -worker_async(void *arg) -{ - CONFIG_OPTS *opts; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_ASYNC_OP *asyncop; - WT_CONNECTION *conn; - uint64_t next_val; - uint8_t *op, *op_end; - int ret; - char *key_buf, *value_buf; - - thread = (WTPERF_THREAD *)arg; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - - key_buf = thread->key_buf; - value_buf = thread->value_buf; - - op = thread->workload->ops; - op_end = op + sizeof(thread->workload->ops); - - while (!wtperf->stop) { - /* - * Generate the next key and setup operation specific statistics tracking objects. - */ - switch (*op) { - case WORKER_INSERT: - case WORKER_INSERT_RMW: - if (opts->random_range) - next_val = wtperf_rand(thread); - else - next_val = opts->icount + get_next_incr(wtperf); - break; - case WORKER_READ: - case WORKER_UPDATE: - next_val = wtperf_rand(thread); - - /* - * If the workload is started without a populate phase we rely on at least one insert to - * get a valid item id. - */ - if (wtperf_value_range(wtperf) < next_val) - continue; - break; - default: - lprintf(wtperf, 0, 0, "invalid op!"); - goto err; /* can't happen */ - } - - generate_key(opts, key_buf, next_val); - - /* - * Spread the data out around the multiple databases. Sleep to allow workers a chance to run - * and process async ops. Then retry to get an async op. - */ - while ( - (ret = conn->async_new_op(conn, wtperf->uris[map_key_to_table(wtperf->opts, next_val)], - NULL, &cb, &asyncop)) == EBUSY) - (void)usleep(10000); - if (ret != 0) { - lprintf(wtperf, ret, 0, "failed async_new_op"); - goto err; - } - - asyncop->app_private = thread; - asyncop->set_key(asyncop, key_buf); - switch (*op) { - case WORKER_READ: - ret = asyncop->search(asyncop); - if (ret == 0) - break; - goto op_err; - case WORKER_INSERT: - if (opts->random_value) - randomize_value(thread, value_buf, 0); - asyncop->set_value(asyncop, value_buf); - if ((ret = asyncop->insert(asyncop)) == 0) - break; - goto op_err; - case WORKER_UPDATE: - if (opts->random_value) - randomize_value( - thread, value_buf, thread->workload ? thread->workload->update_delta : 0); - asyncop->set_value(asyncop, value_buf); - if ((ret = asyncop->update(asyncop)) == 0) - break; - goto op_err; - default: -op_err: - lprintf(wtperf, ret, 0, "%s failed for: %s, range: %" PRIu64, op_name(op), key_buf, - wtperf_value_range(wtperf)); - goto err; /* can't happen */ - } - - /* Schedule the next operation */ - if (++op == op_end) - op = thread->workload->ops; - } - - if (conn->async_flush(conn) != 0) - goto err; - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: - wtperf->error = wtperf->stop = true; - } - return (WT_THREAD_RET_VALUE); -} - /* * do_range_reads -- * If configured to execute a sequence of next operations after each search do them. Ensuring @@ -1194,103 +1008,6 @@ err: } static WT_THREAD_RET -populate_async(void *arg) -{ - CONFIG_OPTS *opts; - TRACK *trk; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_ASYNC_OP *asyncop; - WT_CONNECTION *conn; - WT_SESSION *session; - uint64_t op, start, stop, usecs; - int measure_latency, ret; - char *value_buf, *key_buf; - - thread = (WTPERF_THREAD *)arg; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - session = NULL; - ret = 0; - start = 0; - trk = &thread->insert; - - key_buf = thread->key_buf; - value_buf = thread->value_buf; - - if ((ret = conn->open_session(conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, "populate: WT_CONNECTION.open_session"); - goto err; - } - - /* - * Measuring latency of one async op is not meaningful. We will measure the time it takes to do - * all of them, including the time to process by workers. - */ - measure_latency = - opts->sample_interval != 0 && trk->ops != 0 && (trk->ops % opts->sample_rate == 0); - if (measure_latency) - start = __wt_clock(NULL); - - /* Populate the databases. */ - for (;;) { - op = get_next_incr(wtperf); - if (op > (uint64_t)opts->icount + (uint64_t)opts->scan_icount) - break; - /* - * Allocate an async op for whichever table. - */ - while ((ret = conn->async_new_op(conn, wtperf->uris[map_key_to_table(wtperf->opts, op)], - NULL, &cb, &asyncop)) == EBUSY) - (void)usleep(10000); - if (ret != 0) { - lprintf(wtperf, ret, 0, "Failed async_new_op"); - goto err; - } - - asyncop->app_private = thread; - generate_key(opts, key_buf, op); - asyncop->set_key(asyncop, key_buf); - if (opts->random_value) - randomize_value(thread, value_buf, 0); - asyncop->set_value(asyncop, value_buf); - if ((ret = asyncop->insert(asyncop)) != 0) { - lprintf(wtperf, ret, 0, "Failed inserting"); - goto err; - } - } - - /* - * Gather statistics. We measure the latency of inserting a single key. If there are multiple - * tables, it is the time for insertion into all of them. Note that currently every populate - * thread will call async_flush and those calls will convoy. That is not the most efficient way, - * but we want to flush before measuring latency. - */ - if (conn->async_flush(conn) != 0) { - lprintf(wtperf, ret, 0, "Failed async flush"); - goto err; - } - if (measure_latency) { - stop = __wt_clock(NULL); - ++trk->latency_ops; - usecs = WT_CLOCKDIFF_US(stop, start); - track_operation(trk, usecs); - } - if ((ret = session->close(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error closing session in populate"); - goto err; - } - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: - wtperf->error = wtperf->stop = true; - } - return (WT_THREAD_RET_VALUE); -} - -static WT_THREAD_RET monitor(void *arg) { struct timespec t; @@ -1671,15 +1388,12 @@ static int execute_populate(WTPERF *wtperf) { CONFIG_OPTS *opts; - WT_ASYNC_OP *asyncop; WTPERF_THREAD *popth; - WT_THREAD_CALLBACK (*pfunc)(void *); - size_t i; uint64_t last_ops, max_key, msecs, print_ops_sec, start, stop; - uint32_t interval, tables; + uint32_t interval; wt_thread_t idle_table_cycle_thread; double print_secs; - int elapsed, ret; + int elapsed; opts = wtperf->opts; max_key = (uint64_t)opts->icount + (uint64_t)opts->scan_icount; @@ -1693,12 +1407,7 @@ execute_populate(WTPERF *wtperf) wtperf->insert_key = 0; wtperf->popthreads = dcalloc(opts->populate_threads, sizeof(WTPERF_THREAD)); - if (wtperf->use_asyncops) { - lprintf(wtperf, 0, 1, "Starting %" PRIu32 " async thread(s)", opts->async_threads); - pfunc = populate_async; - } else - pfunc = populate_thread; - start_threads(wtperf, NULL, wtperf->popthreads, opts->populate_threads, pfunc); + start_threads(wtperf, NULL, wtperf->popthreads, opts->populate_threads, populate_thread); start = __wt_clock(NULL); for (elapsed = 0, interval = 0, last_ops = 0; wtperf->insert_key < max_key && !wtperf->error;) { @@ -1759,41 +1468,6 @@ execute_populate(WTPERF *wtperf) "load ops/sec: %" PRIu64, print_secs, print_ops_sec); - /* - * If configured, compact to allow LSM merging to complete. We set an unlimited timeout because - * if we close the connection then any in-progress compact/merge is aborted. - */ - if (opts->compact) { - assert(opts->async_threads > 0); - lprintf(wtperf, 0, 1, "Compact after populate"); - start = __wt_clock(NULL); - tables = opts->table_count; - for (i = 0; i < opts->table_count; i++) { - /* - * If no ops are available, retry. Any other error, return. - */ - while ((ret = wtperf->conn->async_new_op( - wtperf->conn, wtperf->uris[i], "timeout=0", &cb, &asyncop)) == EBUSY) - (void)usleep(10000); - if (ret != 0) - return (ret); - - asyncop->app_private = &tables; - if ((ret = asyncop->compact(asyncop)) != 0) { - lprintf(wtperf, ret, 0, "Async compact failed."); - return (ret); - } - } - if ((ret = wtperf->conn->async_flush(wtperf->conn)) != 0) { - lprintf(wtperf, ret, 0, "Populate async flush failed."); - return (ret); - } - stop = __wt_clock(NULL); - lprintf(wtperf, 0, 1, "Compact completed in %" PRIu64 " seconds", - (uint64_t)(WT_CLOCKDIFF_SEC(stop, start))); - assert(tables == 0); - } - /* Stop cycling idle tables. */ stop_idle_table_cycle(wtperf, idle_table_cycle_thread); @@ -1829,17 +1503,6 @@ close_reopen(WTPERF *wtperf) lprintf(wtperf, ret, 0, "Re-opening the connection failed"); return (ret); } - /* - * If we started async threads only for the purposes of compact, then turn it off before - * starting the workload so that those extra threads looking for work that will never arrive - * don't affect performance. - */ - if (opts->compact && !wtperf->use_asyncops) { - if ((ret = wtperf->conn->reconfigure(wtperf->conn, "async=(enabled=false)")) != 0) { - lprintf(wtperf, ret, 0, "Reconfigure async off failed"); - return (ret); - } - } return (0); } @@ -1851,7 +1514,6 @@ execute_workload(WTPERF *wtperf) WTPERF_THREAD *threads; WT_CONNECTION *conn; WT_SESSION **sessions; - WT_THREAD_CALLBACK (*pfunc)(void *); wt_thread_t idle_table_cycle_thread; uint64_t last_ckpts, last_scans; uint64_t last_inserts, last_reads, last_truncates; @@ -1882,12 +1544,6 @@ execute_workload(WTPERF *wtperf) /* Allocate memory for the worker threads. */ wtperf->workers = dcalloc((size_t)wtperf->workers_cnt, sizeof(WTPERF_THREAD)); - if (wtperf->use_asyncops) { - lprintf(wtperf, 0, 1, "Starting %" PRIu32 " async thread(s)", opts->async_threads); - pfunc = worker_async; - } else - pfunc = worker; - if (opts->session_count_idle != 0) { sessions = dcalloc((size_t)opts->session_count_idle, sizeof(WT_SESSION *)); conn = wtperf->conn; @@ -1911,7 +1567,7 @@ execute_workload(WTPERF *wtperf) goto err; /* Start the workload's threads. */ - start_threads(wtperf, workp, threads, (u_int)workp->threads, pfunc); + start_threads(wtperf, workp, threads, (u_int)workp->threads, worker); threads += workp->threads; } @@ -2174,9 +1830,6 @@ wtperf_copy(const WTPERF *src, WTPERF **retp) dest->uris[i] = dstrdup(src->uris[i]); } - if (src->async_config != NULL) - dest->async_config = dstrdup(src->async_config); - dest->ckptthreads = NULL; dest->scanthreads = NULL; dest->popthreads = NULL; @@ -2220,8 +1873,6 @@ wtperf_free(WTPERF *wtperf) free(wtperf->uris); } - free(wtperf->async_config); - free(wtperf->ckptthreads); free(wtperf->scanthreads); free(wtperf->popthreads); @@ -2615,31 +2266,6 @@ main(int argc, char *argv[]) goto err; } - wtperf->async_config = NULL; - /* - * If the user specified async_threads we use async for all ops. If the user wants compaction, - * then we also enable async for the compact operation, but not for the workloads. - */ - if (opts->async_threads > 0) { - if (F_ISSET(wtperf, CFG_TRUNCATE)) { - lprintf(wtperf, 1, 0, "Cannot run truncate and async\n"); - goto err; - } - wtperf->use_asyncops = true; - } - if (opts->compact && opts->async_threads == 0) - opts->async_threads = 2; - if (opts->async_threads > 0) { - /* - * The maximum number of async threads is two digits, so just use that to compute the space - * we need. Assume the default of 1024 for the max ops. Although we could bump that up to - * 4096 if needed. - */ - req_len = strlen(",async=(enabled=true,threads=)") + 4; - wtperf->async_config = dmalloc(req_len); - testutil_check(__wt_snprintf(wtperf->async_config, req_len, - ",async=(enabled=true,threads=%" PRIu32 ")", opts->async_threads)); - } if ((ret = config_compress(wtperf)) != 0) goto err; @@ -2660,9 +2286,8 @@ main(int argc, char *argv[]) /* Concatenate non-default configuration strings. */ if (user_cconfig != NULL || opts->session_count_idle > 0 || wtperf->compress_ext != NULL || - wtperf->async_config != NULL || opts->in_memory) { + opts->in_memory) { req_len = 20; - req_len += wtperf->async_config != NULL ? strlen(wtperf->async_config) : 0; req_len += wtperf->compress_ext != NULL ? strlen(wtperf->compress_ext) : 0; if (opts->session_count_idle > 0) { sreq_len = strlen("session_max=") + 6; @@ -2677,11 +2302,6 @@ main(int argc, char *argv[]) pos = 0; append_comma = ""; - if (wtperf->async_config != NULL && strlen(wtperf->async_config) != 0) { - testutil_check(__wt_snprintf_len_incr( - cc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, wtperf->async_config)); - append_comma = ","; - } if (wtperf->compress_ext != NULL && strlen(wtperf->compress_ext) != 0) { testutil_check(__wt_snprintf_len_incr( cc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, wtperf->compress_ext)); diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index a004e5e3997..ed8bea2234c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -128,9 +128,6 @@ struct __wtperf { /* Per-database structure */ FILE *logf; /* Logging handle */ - char *async_config; /* Config string for async */ - bool use_asyncops; /* Use async operations */ - const char *compress_ext; /* Compression extension for conn */ const char *compress_table; /* Compression arg to table create */ diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index a9066f58174..00409660dd1 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -80,7 +80,6 @@ * The difference between CONFIG_STRING and STRING is that CONFIG_STRING * options are appended to existing content, whereas STRING options overwrite. */ -DEF_OPT_AS_UINT32(async_threads, 0, "number of async worker threads") DEF_OPT_AS_UINT32( checkpoint_interval, 120, "checkpoint every interval seconds during the workload phase.") DEF_OPT_AS_UINT32(checkpoint_stress_rate, 0, diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index c2139a07202..4356af87e87 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -428,21 +428,6 @@ table_meta = format_meta + table_only_config # Connection runtime config, shared by conn.reconfigure and wiredtiger_open connection_runtime_config = [ - Config('async', '', r''' - asynchronous operations configuration options''', - type='category', subconfig=[ - Config('enabled', 'false', r''' - enable asynchronous operation''', - type='boolean'), - Config('ops_max', '1024', r''' - maximum number of expected simultaneous asynchronous - operations''', min='1', max='4096'), - Config('threads', '2', r''' - the number of worker threads to service asynchronous requests. - Each worker thread uses a session from the configured - session_max''', - min='1', max='20'), # !!! Must match WT_ASYNC_MAX_WORKERS - ]), Config('cache_size', '100MB', r''' maximum heap memory to allocate for the cache. A database should configure either \c cache_size or \c shared_cache but not both''', @@ -1172,6 +1157,18 @@ methods = { object exists, check that its settings match the specified configuration''', type='boolean'), + Config('import', '', r''' + configure import of an existing object into the currently running database''', + type='category', subconfig=[ + Config('enabled', 'false', r''' + whether to import the input URI from disk''', + type='boolean'), + Config('repair', 'false', r''' + whether to reconstruct the metadata from the raw file content''', + type='boolean'), + Config('file_metadata', '', r''' + the file configuration extracted from the metadata of the export database'''), + ]), ]), 'WT_SESSION.drop' : Method([ @@ -1592,28 +1589,6 @@ methods = { 'WT_CONNECTION.add_data_source' : Method([]), 'WT_CONNECTION.add_encryptor' : Method([]), 'WT_CONNECTION.add_extractor' : Method([]), -'WT_CONNECTION.async_new_op' : Method([ - Config('append', 'false', r''' - append the value as a new record, creating a new record - number key; valid only for operations with record number keys''', - type='boolean'), - Config('overwrite', 'true', r''' - configures whether the cursor's insert, update and remove - methods check the existing state of the record. If \c overwrite - is \c false, WT_CURSOR::insert fails with ::WT_DUPLICATE_KEY - if the record exists, WT_CURSOR::update and WT_CURSOR::remove - fail with ::WT_NOTFOUND if the record does not exist''', - type='boolean'), - Config('raw', 'false', r''' - ignore the encodings for the key and value, manage data as if - the formats were \c "u". See @ref cursor_raw for details''', - type='boolean'), - Config('timeout', '1200', r''' - maximum amount of time to allow for compact in seconds. The - actual amount of time spent in compact may exceed the configured - value. A value of zero disables the timeout''', - type='int'), -]), 'WT_CONNECTION.close' : Method([ Config('leak_memory', 'false', r''' don't free memory during close''', diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index cd12dd6826d..139347514d3 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -1,9 +1,6 @@ # filelist -- # List of source files for WiredTiger library. -src/async/async_api.c -src/async/async_op.c -src/async/async_worker.c src/block/block_addr.c src/block/block_ckpt.c src/block/block_ckpt_scan.c diff --git a/src/third_party/wiredtiger/dist/s_copyright.list b/src/third_party/wiredtiger/dist/s_copyright.list index b79fae5fc6e..1cb3e0b4744 100644 --- a/src/third_party/wiredtiger/dist/s_copyright.list +++ b/src/third_party/wiredtiger/dist/s_copyright.list @@ -19,8 +19,6 @@ skip dist/stat_data.py skip dist/style.py skip dist/wtperf_config.py skip lang/java/java_doc.i -skip lang/java/src/com/wiredtiger/db/AsyncOp.java -skip lang/java/src/com/wiredtiger/db/AsyncOpType.java skip lang/java/src/com/wiredtiger/db/Connection.java skip lang/java/src/com/wiredtiger/db/Cursor.java skip lang/java/src/com/wiredtiger/db/SearchStatus.java diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 52c2e5ee652..9faf673afcc 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -14,13 +14,11 @@ APIs ARG ARGS ASM -ASYNC Addr Ailamaki Alakuijala Alexandrescu's Alloc -Async AsyncOp Athanassoulis Athlon @@ -499,8 +497,6 @@ asm aspell assertfmt async -asyncopp -asyncops atomicity autocommit autoconf diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 03a04bca589..2669dd0c970 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -31,10 +31,6 @@ class Stat: def __cmp__(self, other): return cmp(self.desc.lower(), other.desc.lower()) -class AsyncStat(Stat): - prefix = 'async' - def __init__(self, name, desc, flags=''): - Stat.__init__(self, name, AsyncStat.prefix, desc, flags) class BlockStat(Stat): prefix = 'block-manager' def __init__(self, name, desc, flags=''): @@ -175,23 +171,6 @@ connection_stats = [ ConnStat('write_io', 'total write I/Os'), ########################################## - # Async API statistics - ########################################## - AsyncStat('async_alloc_race', 'number of allocation state races'), - AsyncStat('async_alloc_view', 'number of operation slots viewed for allocation'), - AsyncStat('async_cur_queue', 'current work queue length', 'no_scale'), - AsyncStat('async_flush', 'number of flush calls'), - AsyncStat('async_full', 'number of times operation allocation failed'), - AsyncStat('async_max_queue', 'maximum work queue length', 'no_clear,no_scale'), - AsyncStat('async_nowork', 'number of times worker found no work'), - AsyncStat('async_op_alloc', 'total allocations'), - AsyncStat('async_op_compact', 'total compact calls'), - AsyncStat('async_op_insert', 'total insert calls'), - AsyncStat('async_op_remove', 'total remove calls'), - AsyncStat('async_op_search', 'total search calls'), - AsyncStat('async_op_update', 'total update calls'), - - ########################################## # Block manager statistics ########################################## BlockStat('block_byte_map_read', 'mapped bytes read', 'size'), diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index 5e9b91843d5..dfbd32581e0 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -6,7 +6,6 @@ AM_CPPFLAGS +=-I$(top_srcdir)/test/utility noinst_PROGRAMS = \ ex_access \ ex_all \ - ex_async \ ex_backup \ ex_backup_block \ ex_call_center \ diff --git a/src/third_party/wiredtiger/examples/c/ex_async.c b/src/third_party/wiredtiger/examples/c/ex_async.c deleted file mode 100644 index b4d01d6d57f..00000000000 --- a/src/third_party/wiredtiger/examples/c/ex_async.c +++ /dev/null @@ -1,204 +0,0 @@ -/*- - * Public Domain 2014-2020 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * ex_async.c - * demonstrates how to use the asynchronous API. - */ -#include <test_util.h> - -static const char *home; - -#if defined(_lint) -#define ATOMIC_ADD(v, val) ((v) += (val), (v)) -#elif defined(_WIN32) -#define ATOMIC_ADD(v, val) (_InterlockedExchangeAdd(&(v), val) + val) -#else -#define ATOMIC_ADD(v, val) __atomic_add_fetch(&(v), val, __ATOMIC_SEQ_CST) -#endif - -static int global_error = 0; - -/*! [async example callback implementation] */ -typedef struct { - WT_ASYNC_CALLBACK iface; - uint32_t num_keys; -} ASYNC_KEYS; - -static int -async_callback(WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP *op, int wiredtiger_error, uint32_t flags) -{ - ASYNC_KEYS *asynckey = (ASYNC_KEYS *)cb; - WT_ASYNC_OPTYPE type; - WT_ITEM k, v; - const char *key, *value; - uint64_t id; - - (void)flags; /* Unused */ - - /*! [async get type] */ - /* Retrieve the operation's WT_ASYNC_OPTYPE type. */ - type = op->get_type(op); - /*! [async get type] */ - - /*! [async get identifier] */ - /* Retrieve the operation's 64-bit identifier. */ - id = op->get_id(op); - /*! [async get identifier] */ - - /* Check for a WiredTiger error. */ - if (wiredtiger_error != 0) { - fprintf(stderr, "ID %" PRIu64 " error %d: %s\n", id, wiredtiger_error, - wiredtiger_strerror(wiredtiger_error)); - global_error = wiredtiger_error; - return (1); - } - - /* If doing a search, retrieve the key/value pair. */ - if (type == WT_AOP_SEARCH) { - /*! [async get the operation's string key] */ - error_check(op->get_key(op, &k)); - key = k.data; - /*! [async get the operation's string key] */ - /*! [async get the operation's string value] */ - error_check(op->get_value(op, &v)); - value = v.data; - /*! [async get the operation's string value] */ - ATOMIC_ADD(asynckey->num_keys, 1); - printf("Id %" PRIu64 " got record: %s : %s\n", id, key, value); - } - return (0); -} -/*! [async example callback implementation] */ - -static ASYNC_KEYS ex_asynckeys = {{async_callback}, 0}; - -#define MAX_KEYS 15 - -int -main(int argc, char *argv[]) -{ - WT_ASYNC_OP *op; - WT_CONNECTION *conn; - WT_SESSION *session; - int i, ret; - char k[MAX_KEYS][16], v[MAX_KEYS][16]; - - home = example_setup(argc, argv); - - /*! [async example connection] */ - error_check(wiredtiger_open( - home, NULL, "create,cache_size=100MB,async=(enabled=true,ops_max=20,threads=2)", &conn)); - /*! [async example connection] */ - - /*! [async example table create] */ - error_check(conn->open_session(conn, NULL, NULL, &session)); - error_check(session->create(session, "table:async", "key_format=S,value_format=S")); - /*! [async example table create] */ - - /* Insert a set of keys asynchronously. */ - for (i = 0; i < MAX_KEYS; i++) { - /*! [async handle allocation] */ - while ( - (ret = conn->async_new_op(conn, "table:async", NULL, &ex_asynckeys.iface, &op)) != 0) { - /* - * If we used up all the handles, pause and retry to give the workers a chance to catch - * up. - */ - fprintf(stderr, "asynchronous operation handle not available\n"); - if (ret == EBUSY) - sleep(1); - else - return (EXIT_FAILURE); - } - /*! [async handle allocation] */ - - /*! [async insert] */ - /* - * Set the operation's string key and value, and then do an asynchronous insert. - */ - /*! [async set the operation's string key] */ - (void)snprintf(k[i], sizeof(k), "key%d", i); - op->set_key(op, k[i]); - /*! [async set the operation's string key] */ - - /*! [async set the operation's string value] */ - (void)snprintf(v[i], sizeof(v), "value%d", i); - op->set_value(op, v[i]); - /*! [async set the operation's string value] */ - - error_check(op->insert(op)); - /*! [async insert] */ - } - - /*! [async flush] */ - /* Wait for all outstanding operations to complete. */ - error_check(conn->async_flush(conn)); - /*! [async flush] */ - - /*! [async compaction] */ - /* - * Compact a table asynchronously, limiting the run-time to 5 minutes. - */ - error_check(conn->async_new_op(conn, "table:async", "timeout=300", &ex_asynckeys.iface, &op)); - error_check(op->compact(op)); - /*! [async compaction] */ - - /* Search for the keys we just inserted, asynchronously. */ - for (i = 0; i < MAX_KEYS; i++) { - while ( - (ret = conn->async_new_op(conn, "table:async", NULL, &ex_asynckeys.iface, &op)) != 0) { - /* - * If we used up all the handles, pause and retry to give the workers a chance to catch - * up. - */ - fprintf(stderr, "asynchronous operation handle not available\n"); - if (ret == EBUSY) - sleep(1); - else - return (EXIT_FAILURE); - } - - /*! [async search] */ - /* - * Set the operation's string key and value, and then do an asynchronous search. - */ - (void)snprintf(k[i], sizeof(k), "key%d", i); - op->set_key(op, k[i]); - error_check(op->search(op)); - /*! [async search] */ - } - - /* - * Connection close automatically does an async_flush so it will wait for all queued search - * operations to complete. - */ - error_check(conn->close(conn, NULL)); - - printf("Searched for %" PRIu32 " keys\n", ex_asynckeys.num_keys); - - return (EXIT_SUCCESS); -} diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_async.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_async.java deleted file mode 100644 index fd99d21ef6b..00000000000 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_async.java +++ /dev/null @@ -1,221 +0,0 @@ -/*- - * Public Domain 2014-2020 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * ex_async.java - * demonstrates how to use the asynchronous API. - */ -package com.wiredtiger.examples; -import com.wiredtiger.db.*; -import java.io.*; -import java.util.*; - -/*! [async example callback implementation] */ -class AsyncKeys implements AsyncCallback { - - public int numKeys = 0; - - public AsyncKeys() {} - - public void notifyError(String desc) { - System.err.println("ERROR: notify: " + desc); - } - - public int notify(AsyncOp op, int opReturn, int flags) { - /* - * Note: we are careful not to throw any errors here. Any - * exceptions would be swallowed by a native worker thread. - */ - int ret = 0; - try { - /*! [async get type] */ - /* Retrieve the operation's type. */ - AsyncOpType optype = op.getType(); - /*! [async get type] */ - /*! [async get identifier] */ - /* Retrieve the operation's 64-bit identifier. */ - long id = op.getId(); - /*! [async get identifier] */ - - /* If doing a search, retrieve the key/value pair. */ - if (optype == AsyncOpType.WT_AOP_SEARCH) { - /*! [async get the operation's string key] */ - String key = op.getKeyString(); - /*! [async get the operation's string key] */ - /*! [async get the operation's string value] */ - String value = op.getValueString(); - /*! [async get the operation's string value] */ - synchronized (this) { - numKeys += 1; - } - System.out.println("Id " + id + " got record: " + key + - " : " + value); - } - } - catch (Exception e) { - System.err.println("ERROR: exception in notify: " + e.toString() + - ", opreturn=" + opReturn); - ret = 1; - } - return (ret); - } -} -/*! [async example callback implementation] */ - -public class ex_async { - - public static String home; - - public static final int MAX_KEYS = 15; - - public static AsyncOp tryAsyncNewOp(Connection conn, String uri, - String config, AsyncCallback cb) throws WiredTigerException - { - WiredTigerException savedwte = null; - - for (int tries = 0; tries < 10; tries++) - try { - return conn.async_new_op(uri, config, cb); - } - catch (WiredTigerException wte) { - /* - * If we used up all the handles, pause and retry to - * give the workers a chance to catch up. - */ - System.err.println( - "asynchronous operation handle not available: " + wte); - savedwte = wte; - try { - Thread.sleep(1); - } catch (InterruptedException ie) { - /* not a big problem, continue to retry */ - } - } - - throw savedwte; - } - - public static int - asyncExample() - throws WiredTigerException - { - AsyncOp op; - Connection conn; - Session session; - int i, ret; - String k[] = new String[MAX_KEYS]; - String v[] = new String[MAX_KEYS]; - - /*! [async example callback implementation part 2] */ - AsyncKeys asynciface = new AsyncKeys(); - /*! [async example callback implementation part 2] */ - - /*! [async example connection] */ - conn = wiredtiger.open(home, "create,cache_size=100MB," + - "async=(enabled=true,ops_max=20,threads=2)"); - /*! [async example connection] */ - - /*! [async example table create] */ - session = conn.open_session(null); - ret = session.create("table:async", "key_format=S,value_format=S"); - /*! [async example table create] */ - - /* Insert a set of keys asynchronously. */ - for (i = 0; i < MAX_KEYS; i++) { - /*! [async handle allocation] */ - op = tryAsyncNewOp(conn, "table:async", null, asynciface); - /*! [async handle allocation] */ - - /*! [async insert] */ - /* - * Set the operation's string key and value, and then do - * an asynchronous insert. - */ - /*! [async set the operation's string key] */ - k[i] = "key" + i; - op.putKeyString(k[i]); - /*! [async set the operation's string key] */ - - /*! [async set the operation's string value] */ - v[i] = "value" + i; - op.putValueString(v[i]); - /*! [async set the operation's string value] */ - - ret = op.insert(); - /*! [async insert] */ - } - - /*! [async flush] */ - /* Wait for all outstanding operations to complete. */ - ret = conn.async_flush(); - /*! [async flush] */ - - /*! [async compaction] */ - /* - * Compact a table asynchronously, limiting the run-time to 5 minutes. - */ - op = tryAsyncNewOp(conn, "table:async", "timeout=300", asynciface); - ret = op.compact(); - /*! [async compaction] */ - - /* Search for the keys we just inserted, asynchronously. */ - for (i = 0; i < MAX_KEYS; i++) { - op = tryAsyncNewOp(conn, "table:async", null, asynciface); - /*! [async search] */ - /* - * Set the operation's string key and value, and then do - * an asynchronous search. - */ - k[i] = "key" + i; - op.putKeyString(k[i]); - ret = op.search(); - /*! [async search] */ - } - - /* - * Connection close automatically does an async_flush so it will wait - * for all queued search operations to complete. - */ - ret = conn.close(null); - - System.out.println("Searched for " + asynciface.numKeys + " keys"); - - return (ret); - } - - public static void - main(String[] argv) - { - try { - System.exit(asyncExample()); - } - catch (WiredTigerException wte) { - System.err.println("Exception: " + wte); - wte.printStackTrace(); - System.exit(1); - } - } -} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 94814e3695d..fa00c3ab98d 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "8a1dce8653b409762cbdaab5538e4d4c8ebf5403" + "commit": "bb1cc65a63a4fb081cb5969f0728ff2b74fc7c8b" } diff --git a/src/third_party/wiredtiger/lang/java/Makefile.am b/src/third_party/wiredtiger/lang/java/Makefile.am index 71515c430fd..221c08070e3 100644 --- a/src/third_party/wiredtiger/lang/java/Makefile.am +++ b/src/third_party/wiredtiger/lang/java/Makefile.am @@ -13,9 +13,6 @@ JDOCDIR = $(top_srcdir)/docs/java javadir = $(datadir)/java/$(PACKAGE)-$(PACKAGE_VERSION) JAVA_SRC = \ - $(JAVADESTFULL)/AsyncCallback.java \ - $(JAVADESTFULL)/AsyncOp.java \ - $(JAVADESTFULL)/AsyncOpType.java \ $(JAVADESTFULL)/Connection.java \ $(JAVADESTFULL)/Cursor.java \ $(JAVADESTFULL)/Modify.java \ @@ -35,7 +32,6 @@ JAVA_SRC = \ $(JAVADESTFULL)/WT_MODIFY_LIST.java \ $(JAVAEXAMPLES)/ex_access.java \ $(JAVAEXAMPLES)/ex_all.java \ - $(JAVAEXAMPLES)/ex_async.java \ $(JAVAEXAMPLES)/ex_call_center.java \ $(JAVAEXAMPLES)/ex_cursor.java \ $(JAVAEXAMPLES)/ex_log.java \ @@ -45,7 +41,6 @@ JAVA_SRC = \ JAVA_JUNIT = \ $(JAVATEST)/AutoCloseTest.java \ - $(JAVATEST)/AsyncTest.java \ $(JAVATEST)/BackupCursorTest.java \ $(JAVATEST)/ConfigTest.java \ $(JAVATEST)/ConcurrentCloseTest.java \ diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i index f5fb035c863..df9bc4fb609 100644 --- a/src/third_party/wiredtiger/lang/java/java_doc.i +++ b/src/third_party/wiredtiger/lang/java/java_doc.i @@ -20,17 +20,6 @@ COPYDOC(__wt_cursor, WT_CURSOR, close) COPYDOC(__wt_cursor, WT_CURSOR, reconfigure) COPYDOC(__wt_cursor, WT_CURSOR, cache) COPYDOC(__wt_cursor, WT_CURSOR, reopen) -COPYDOC(__wt_async_op, WT_ASYNC_OP, get_key) -COPYDOC(__wt_async_op, WT_ASYNC_OP, get_value) -COPYDOC(__wt_async_op, WT_ASYNC_OP, set_key) -COPYDOC(__wt_async_op, WT_ASYNC_OP, set_value) -COPYDOC(__wt_async_op, WT_ASYNC_OP, search) -COPYDOC(__wt_async_op, WT_ASYNC_OP, insert) -COPYDOC(__wt_async_op, WT_ASYNC_OP, update) -COPYDOC(__wt_async_op, WT_ASYNC_OP, remove) -COPYDOC(__wt_async_op, WT_ASYNC_OP, compact) -COPYDOC(__wt_async_op, WT_ASYNC_OP, get_id) -COPYDOC(__wt_async_op, WT_ASYNC_OP, get_type) COPYDOC(__wt_session, WT_SESSION, close) COPYDOC(__wt_session, WT_SESSION, reconfigure) COPYDOC(__wt_session, WT_SESSION, open_cursor) @@ -59,8 +48,6 @@ COPYDOC(__wt_session, WT_SESSION, checkpoint) COPYDOC(__wt_session, WT_SESSION, transaction_pinned_range) COPYDOC(__wt_session, WT_SESSION, transaction_sync) COPYDOC(__wt_session, WT_SESSION, breakpoint) -COPYDOC(__wt_connection, WT_CONNECTION, async_flush) -COPYDOC(__wt_connection, WT_CONNECTION, async_new_op) COPYDOC(__wt_connection, WT_CONNECTION, close) COPYDOC(__wt_connection, WT_CONNECTION, debug_info) COPYDOC(__wt_connection, WT_CONNECTION, reconfigure) diff --git a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/AsyncCallback.java b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/AsyncCallback.java deleted file mode 100644 index f065e280b37..00000000000 --- a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/AsyncCallback.java +++ /dev/null @@ -1,43 +0,0 @@ -/*- - * Public Domain 2014-2020 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -package com.wiredtiger.db; - -/** - * An interface that must be implemented to receive notifications - * from asynchronous operations. - */ -public interface AsyncCallback { - /** - * Notify when an asynchronous operation completes. - * - * \param op The operation that completed - * \param opReturn The return value of the operation - * \param flags Flags (currently 0). - */ - public int notify(AsyncOp op, int opReturn, int flags); -} diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i index e544951909e..1e6c1770b89 100644 --- a/src/third_party/wiredtiger/lang/java/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i @@ -78,17 +78,11 @@ * swigCPtr is 0. */ typedef struct { - JavaVM *javavm; /* Used in async threads to craft a jnienv */ JNIEnv *jnienv; /* jni env that created the Session/Cursor */ WT_SESSION_IMPL *session; /* session used for alloc/free */ bool cursor_raw; /* is the cursor opened raw? */ - jobject jobj; /* the java Session/Cursor/AsyncOp object */ - jobject jcallback; /* callback object for async ops */ + jobject jobj; /* the java Session/Cursor object */ jfieldID cptr_fid; /* cached Cursor.swigCPtr field id in session */ - jfieldID asynccptr_fid; /* cached AsyncOp.swigCptr fid in conn */ - jfieldID kunp_fid; /* cached AsyncOp.keyUnpacker fid in conn */ - jfieldID vunp_fid; /* cached AsyncOp.valueUnpacker fid in conn */ - jmethodID notify_mid; /* cached AsyncCallback.notify mid in conn */ } JAVA_CALLBACK; static void throwWiredTigerException(JNIEnv *jenv, int err) { @@ -306,7 +300,6 @@ WT_CLASS_WITH_CLOSE_HANDLER(struct __wt_session, WT_SESSION, session, closeHandler, $1, ((WT_SESSION_IMPL *)$1)->lang_private) WT_CLASS_WITH_CLOSE_HANDLER(struct __wt_cursor, WT_CURSOR, cursor, cursorCloseHandler, $1->session, ((WT_CURSOR *)$1)->lang_private) -WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %define COPYDOC(SIGNATURE_CLASS, CLASS, METHOD) %javamethodmodifiers SIGNATURE_CLASS::METHOD " @@ -318,23 +311,6 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %include "java_doc.i" -/* WT_ASYNC_OP customization. */ -/* First, replace the varargs get / set methods with Java equivalents. */ -%ignore __wt_async_op::get_key; -%ignore __wt_async_op::get_value; -%ignore __wt_async_op::set_key; -%ignore __wt_async_op::set_value; -%ignore __wt_async_op::insert; -%ignore __wt_async_op::remove; -%ignore __wt_async_op::search; -%ignore __wt_async_op::update; -%immutable __wt_async_op::connection; -%immutable __wt_async_op::key_format; -%immutable __wt_async_op::value_format; - -%javamethodmodifiers __wt_async_op::key_format "protected"; -%javamethodmodifiers __wt_async_op::value_format "protected"; - /* WT_CURSOR customization. */ /* First, replace the varargs get / set methods with Java equivalents. */ %ignore __wt_cursor::get_key; @@ -363,10 +339,6 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %rename (compare_wrap) __wt_cursor::compare; %ignore __wt_cursor::equals(WT_CURSOR *, WT_CURSOR *, int *); %rename (equals_wrap) __wt_cursor::equals; -%rename (AsyncOpType) WT_ASYNC_OPTYPE; -%rename (getKeyFormat) __wt_async_op::getKey_format; -%rename (getValueFormat) __wt_async_op::getValue_format; -%rename (getType) __wt_async_op::get_type; /* * Special cases: override the out typemap, return checking is done in the @@ -471,710 +443,6 @@ javaCloseHandler(WT_EVENT_HANDLER *handler, WT_SESSION *session, } WT_EVENT_HANDLER javaApiEventHandler = {NULL, NULL, NULL, javaCloseHandler}; - -static int -javaAsyncHandler(WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP *asyncop, int opret, - uint32_t flags) -{ - int ret, envret; - JAVA_CALLBACK *jcb, *conn_jcb; - JavaVM *javavm; - jclass cls; - jfieldID fid; - jmethodID mid; - jobject jcallback; - JNIEnv *jenv; - WT_ASYNC_OP_IMPL *op; - WT_SESSION_IMPL *session; - - WT_UNUSED(cb); - WT_UNUSED(flags); - op = (WT_ASYNC_OP_IMPL *)asyncop; - session = O2S(op); - jcb = (JAVA_CALLBACK *)asyncop->c.lang_private; - conn_jcb = (JAVA_CALLBACK *)S2C(session)->lang_private; - asyncop->c.lang_private = NULL; - jcallback = jcb->jcallback; - - /* - * We rely on the fact that the async machinery uses a pool of - * threads. Here we attach the current native (POSIX) - * thread to a Java thread and never detach it. If the native - * thread was previously seen by this callback, it will be - * attached to the same Java thread as before without - * incurring the cost of the thread initialization. - * Marking the Java thread as a daemon means its existence - * won't keep an application from exiting. - */ - javavm = jcb->javavm; - envret = (*javavm)->GetEnv(javavm, (void **)&jenv, JNI_VERSION_1_6); - if (envret == JNI_EDETACHED) { - if ((*javavm)->AttachCurrentThreadAsDaemon(javavm, - (void **)&jenv, NULL) != 0) { - ret = EBUSY; - goto err; - } - } else if (envret != JNI_OK) { - ret = EBUSY; - goto err; - } - - /* - * Look up any needed field and method ids, and cache them - * in the connection's lang_private. fid and mids are - * stable. - */ - if (conn_jcb->notify_mid == NULL) { - /* Any JNI error until the actual callback is unexpected. */ - ret = EINVAL; - - cls = (*jenv)->GetObjectClass(jenv, jcb->jobj); - if (cls == NULL) - goto err; - fid = (*jenv)->GetFieldID(jenv, cls, - "keyUnpacker", "Lcom/wiredtiger/db/PackInputStream;"); - if (fid == NULL) - goto err; - conn_jcb->kunp_fid = fid; - - fid = (*jenv)->GetFieldID(jenv, cls, - "valueUnpacker", "Lcom/wiredtiger/db/PackInputStream;"); - if (fid == NULL) - goto err; - conn_jcb->vunp_fid = fid; - - cls = (*jenv)->GetObjectClass(jenv, jcallback); - if (cls == NULL) - goto err; - mid = (*jenv)->GetMethodID(jenv, cls, "notify", - "(Lcom/wiredtiger/db/AsyncOp;II)I"); - if (mid == NULL) - goto err; - conn_jcb->notify_mid = mid; - } - - /* - * Invalidate the unpackers so any calls to op.getKey() - * and op.getValue get fresh results. - */ - (*jenv)->SetObjectField(jenv, jcb->jobj, conn_jcb->kunp_fid, NULL); - (*jenv)->SetObjectField(jenv, jcb->jobj, conn_jcb->vunp_fid, NULL); - - /* Call the registered callback. */ - ret = (*jenv)->CallIntMethod(jenv, jcallback, conn_jcb->notify_mid, - jcb->jobj, opret, flags); - - if ((*jenv)->ExceptionOccurred(jenv)) { - (*jenv)->ExceptionDescribe(jenv); - (*jenv)->ExceptionClear(jenv); - } - if (0) { -err: __wt_err(session, ret, "Java async callback error"); - } - - /* Invalidate the AsyncOp, further use throws NullPointerException. */ - ret = javaClose(jenv, NULL, jcb, &conn_jcb->asynccptr_fid); - - (*jenv)->DeleteGlobalRef(jenv, jcallback); - - if (ret == 0 && (opret == 0 || opret == WT_NOTFOUND)) - return (0); - else - return (1); -} - -WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; -%} - -%extend __wt_async_op { - %javamethodmodifiers get_key_wrap "protected"; - WT_ITEM get_key_wrap(JNIEnv *jenv) { - WT_ITEM k; - int ret; - k.data = NULL; - if ((ret = $self->get_key($self, &k)) != 0) - throwWiredTigerException(jenv, ret); - return (k); - } - - %javamethodmodifiers get_value_wrap "protected"; - WT_ITEM get_value_wrap(JNIEnv *jenv) { - WT_ITEM v; - int ret; - v.data = NULL; - if ((ret = $self->get_value($self, &v)) != 0) - throwWiredTigerException(jenv, ret); - return (v); - } - - %javamethodmodifiers insert_wrap "protected"; - int insert_wrap(WT_ITEM *k, WT_ITEM *v) { - $self->set_key($self, k); - $self->set_value($self, v); - return ($self->insert($self)); - } - - %javamethodmodifiers remove_wrap "protected"; - int remove_wrap(WT_ITEM *k) { - $self->set_key($self, k); - return ($self->remove($self)); - } - - %javamethodmodifiers search_wrap "protected"; - int search_wrap(WT_ITEM *k) { - $self->set_key($self, k); - return ($self->search($self)); - } - - %javamethodmodifiers update_wrap "protected"; - int update_wrap(WT_ITEM *k, WT_ITEM *v) { - $self->set_key($self, k); - $self->set_value($self, v); - return ($self->update($self)); - } - - %javamethodmodifiers _java_raw "protected"; - bool _java_raw(JNIEnv *jenv) { - (void)jenv; - JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->c.lang_private; - return (jcb->cursor_raw); - } - - %javamethodmodifiers _java_init "protected"; - int _java_init(jobject jasyncop) { - JAVA_CALLBACK *jcb = - (JAVA_CALLBACK *)$self->c.lang_private; - jcb->jobj = JCALL1(NewGlobalRef, jcb->jnienv, jasyncop); - JCALL1(DeleteLocalRef, jcb->jnienv, jasyncop); - return (0); - } -} - -/* Cache key/value formats in Async_op */ -%typemap(javabody) struct __wt_async_op %{ - private long swigCPtr; - protected boolean swigCMemOwn; - protected boolean javaRaw; - protected String keyFormat; - protected String valueFormat; - protected PackOutputStream keyPacker; - protected PackOutputStream valuePacker; - protected PackInputStream keyUnpacker; - protected PackInputStream valueUnpacker; - - protected $javaclassname(long cPtr, boolean cMemoryOwn) { - swigCMemOwn = cMemoryOwn; - swigCPtr = cPtr; - keyFormat = getKey_format(); - valueFormat = getValue_format(); - javaRaw = _java_raw(); - keyPacker = new PackOutputStream(keyFormat, javaRaw); - valuePacker = new PackOutputStream(valueFormat, javaRaw); - wiredtigerJNI.AsyncOp__java_init(swigCPtr, this, this); - } - - protected static long getCPtr($javaclassname obj) { - return ((obj == null) ? 0 : obj.swigCPtr); - } -%} - -%typemap(javacode) struct __wt_async_op %{ - - /** - * Retrieve the format string for this async_op's key. - */ - public String getKeyFormat() { - return keyFormat; - } - - /** - * Retrieve the format string for this async_op's value. - */ - public String getValueFormat() { - return valueFormat; - } - - /** - * Append a byte to the async_op's key. - * - * \param value The value to append. - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyByte(byte value) - throws WiredTigerPackingException { - keyUnpacker = null; - keyPacker.addByte(value); - return this; - } - - /** - * Append a byte array to the async_op's key. - * - * \param value The value to append. - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyByteArray(byte[] value) - throws WiredTigerPackingException { - this.putKeyByteArray(value, 0, value.length); - return this; - } - - /** - * Append a byte array to the async_op's key. - * - * \param value The value to append. - * \param off The offset into value at which to start. - * \param len The length of the byte array. - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyByteArray(byte[] value, int off, int len) - throws WiredTigerPackingException { - keyUnpacker = null; - keyPacker.addByteArray(value, off, len); - return this; - } - - /** - * Append an integer to the async_op's key. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyInt(int value) - throws WiredTigerPackingException { - keyUnpacker = null; - keyPacker.addInt(value); - return this; - } - - /** - * Append a long to the async_op's key. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyLong(long value) - throws WiredTigerPackingException { - keyUnpacker = null; - keyPacker.addLong(value); - return this; - } - - /** - * Append a record number to the async_op's key. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyRecord(long value) - throws WiredTigerPackingException { - keyUnpacker = null; - keyPacker.addRecord(value); - return this; - } - - /** - * Append a short integer to the async_op's key. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyShort(short value) - throws WiredTigerPackingException { - keyUnpacker = null; - keyPacker.addShort(value); - return this; - } - - /** - * Append a string to the async_op's key. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putKeyString(String value) - throws WiredTigerPackingException { - keyUnpacker = null; - keyPacker.addString(value); - return this; - } - - /** - * Append a byte to the async_op's value. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueByte(byte value) - throws WiredTigerPackingException { - valueUnpacker = null; - valuePacker.addByte(value); - return this; - } - - /** - * Append a byte array to the async_op's value. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueByteArray(byte[] value) - throws WiredTigerPackingException { - this.putValueByteArray(value, 0, value.length); - return this; - } - - /** - * Append a byte array to the async_op's value. - * - * \param value The value to append - * \param off The offset into value at which to start. - * \param len The length of the byte array. - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueByteArray(byte[] value, int off, int len) - throws WiredTigerPackingException { - valueUnpacker = null; - valuePacker.addByteArray(value, off, len); - return this; - } - - /** - * Append an integer to the async_op's value. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueInt(int value) - throws WiredTigerPackingException { - valueUnpacker = null; - valuePacker.addInt(value); - return this; - } - - /** - * Append a long to the async_op's value. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueLong(long value) - throws WiredTigerPackingException { - valueUnpacker = null; - valuePacker.addLong(value); - return this; - } - - /** - * Append a record number to the async_op's value. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueRecord(long value) - throws WiredTigerPackingException { - valueUnpacker = null; - valuePacker.addRecord(value); - return this; - } - - /** - * Append a short integer to the async_op's value. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueShort(short value) - throws WiredTigerPackingException { - valueUnpacker = null; - valuePacker.addShort(value); - return this; - } - - /** - * Append a string to the async_op's value. - * - * \param value The value to append - * \return This async_op object, so put calls can be chained. - */ - public AsyncOp putValueString(String value) - throws WiredTigerPackingException { - valueUnpacker = null; - valuePacker.addString(value); - return this; - } - - /** - * Retrieve a byte from the async_op's key. - * - * \return The requested value. - */ - public byte getKeyByte() - throws WiredTigerPackingException { - return getKeyUnpacker().getByte(); - } - - /** - * Retrieve a byte array from the async_op's key. - * - * \param output The byte array where the returned value will be stored. - * The array should be large enough to store the entire - * data item, if not a truncated value will be returned. - */ - public void getKeyByteArray(byte[] output) - throws WiredTigerPackingException { - this.getKeyByteArray(output, 0, output.length); - } - - /** - * Retrieve a byte array from the async_op's key. - * - * \param output The byte array where the returned value will be stored. - * \param off Offset into the destination buffer to start copying into. - * \param len The length should be large enough to store the entire - * data item, if not a truncated value will be returned. - */ - public void getKeyByteArray(byte[] output, int off, int len) - throws WiredTigerPackingException { - getKeyUnpacker().getByteArray(output, off, len); - } - - /** - * Retrieve a byte array from the async_op's key. - * - * \return The requested value. - */ - public byte[] getKeyByteArray() - throws WiredTigerPackingException { - return getKeyUnpacker().getByteArray(); - } - - /** - * Retrieve an integer from the async_op's key. - * - * \return The requested value. - */ - public int getKeyInt() - throws WiredTigerPackingException { - return getKeyUnpacker().getInt(); - } - - /** - * Retrieve a long from the async_op's key. - * - * \return The requested value. - */ - public long getKeyLong() - throws WiredTigerPackingException { - return getKeyUnpacker().getLong(); - } - - /** - * Retrieve a record number from the async_op's key. - * - * \return The requested value. - */ - public long getKeyRecord() - throws WiredTigerPackingException { - return getKeyUnpacker().getRecord(); - } - - /** - * Retrieve a short integer from the async_op's key. - * - * \return The requested value. - */ - public short getKeyShort() - throws WiredTigerPackingException { - return getKeyUnpacker().getShort(); - } - - /** - * Retrieve a string from the async_op's key. - * - * \return The requested value. - */ - public String getKeyString() - throws WiredTigerPackingException { - return getKeyUnpacker().getString(); - } - - /** - * Retrieve a byte from the async_op's value. - * - * \return The requested value. - */ - public byte getValueByte() - throws WiredTigerPackingException { - return getValueUnpacker().getByte(); - } - - /** - * Retrieve a byte array from the async_op's value. - * - * \param output The byte array where the returned value will be stored. - * The array should be large enough to store the entire - * data item, if not a truncated value will be returned. - */ - public void getValueByteArray(byte[] output) - throws WiredTigerPackingException { - this.getValueByteArray(output, 0, output.length); - } - - /** - * Retrieve a byte array from the async_op's value. - * - * \param output The byte array where the returned value will be stored. - * \param off Offset into the destination buffer to start copying into. - * \param len The length should be large enough to store the entire - * data item, if not a truncated value will be returned. - */ - public void getValueByteArray(byte[] output, int off, int len) - throws WiredTigerPackingException { - getValueUnpacker().getByteArray(output, off, len); - } - - /** - * Retrieve a byte array from the async_op's value. - * - * \return The requested value. - */ - public byte[] getValueByteArray() - throws WiredTigerPackingException { - return getValueUnpacker().getByteArray(); - } - - /** - * Retrieve an integer from the async_op's value. - * - * \return The requested value. - */ - public int getValueInt() - throws WiredTigerPackingException { - return getValueUnpacker().getInt(); - } - - /** - * Retrieve a long from the async_op's value. - * - * \return The requested value. - */ - public long getValueLong() - throws WiredTigerPackingException { - return getValueUnpacker().getLong(); - } - - /** - * Retrieve a record number from the async_op's value. - * - * \return The requested value. - */ - public long getValueRecord() - throws WiredTigerPackingException { - return getValueUnpacker().getRecord(); - } - - /** - * Retrieve a short integer from the async_op's value. - * - * \return The requested value. - */ - public short getValueShort() - throws WiredTigerPackingException { - return getValueUnpacker().getShort(); - } - - /** - * Retrieve a string from the async_op's value. - * - * \return The requested value. - */ - public String getValueString() - throws WiredTigerPackingException { - return getValueUnpacker().getString(); - } - - /** - * Insert the async_op's current key/value into the table. - * - * \return The status of the operation. - */ - public int insert() - throws WiredTigerException { - byte[] key = keyPacker.getValue(); - byte[] value = valuePacker.getValue(); - keyPacker.reset(); - valuePacker.reset(); - return insert_wrap(key, value); - } - - /** - * Update the async_op's current key/value into the table. - * - * \return The status of the operation. - */ - public int update() - throws WiredTigerException { - byte[] key = keyPacker.getValue(); - byte[] value = valuePacker.getValue(); - keyPacker.reset(); - valuePacker.reset(); - return update_wrap(key, value); - } - - /** - * Remove the async_op's current key/value into the table. - * - * \return The status of the operation. - */ - public int remove() - throws WiredTigerException { - byte[] key = keyPacker.getValue(); - keyPacker.reset(); - return remove_wrap(key); - } - - /** - * Search for an item in the table. - * - * \return The result of the comparison. - */ - public int search() - throws WiredTigerException { - int ret = search_wrap(keyPacker.getValue()); - keyPacker.reset(); - valuePacker.reset(); - return ret; - } - - /** - * Set up the key unpacker or return previously cached value. - * - * \return The key unpacker. - */ - private PackInputStream getKeyUnpacker() - throws WiredTigerPackingException { - if (keyUnpacker == null) - keyUnpacker = - new PackInputStream(keyFormat, get_key_wrap(), - javaRaw); - return keyUnpacker; - } - - /** - * Set up the value unpacker or return previously cached value. - * - * \return The value unpacker. - */ - private PackInputStream getValueUnpacker() - throws WiredTigerPackingException { - if (valueUnpacker == null) - valueUnpacker = - new PackInputStream(valueFormat, get_value_wrap(), - javaRaw); - return valueUnpacker; - } - %} %extend __wt_cursor { @@ -2061,12 +1329,6 @@ static void modify_impl_release(WT_MODIFY_IMPL *impl) { */ %javaexception("com.wiredtiger.db.WiredTigerException") { $action; } %javaexception("") wiredtiger_strerror { $action; } -%javaexception("") __wt_async_op::_java_raw { $action; } -%javaexception("") __wt_async_op::connection { $action; } -%javaexception("") __wt_async_op::get_type { $action; } -%javaexception("") __wt_async_op::get_id { $action; } -%javaexception("") __wt_async_op::key_format { $action; } -%javaexception("") __wt_async_op::value_format { $action; } %javaexception("") __wt_connection::_java_init { $action; } %javaexception("") __wt_connection::get_home { $action; } %javaexception("") __wt_connection::is_new { $action; } @@ -2124,16 +1386,12 @@ static void modify_impl_release(WT_MODIFY_IMPL *impl) { %enddef REQUIRE_WRAP(::wiredtiger_open, wiredtiger_open, open) -REQUIRE_WRAP(WT_CONNECTION::async_new_op, - __wt_connection::async_new_op, async_new_op) REQUIRE_WRAP(WT_CONNECTION::open_session, __wt_connection::open_session, open_session) REQUIRE_WRAP(WT_SESSION::transaction_pinned_range, __wt_session::transaction_pinned_range, transaction_pinned_range) REQUIRE_WRAP(WT_SESSION::open_cursor, __wt_session::open_cursor, open_cursor) -REQUIRE_WRAP(WT_ASYNC_OP::get_id, __wt_async_op::get_id,getId) -%rename(AsyncOp) __wt_async_op; %rename(Cursor) __wt_cursor; %rename(Modify) __wt_modify; %rename(Session) __wt_session; @@ -2202,36 +1460,6 @@ err: if (ret != 0) } %extend __wt_connection { - WT_ASYNC_OP *async_new_op_wrap(JNIEnv *jenv, const char *uri, - const char *config, jobject callbackObject) { - extern WT_ASYNC_CALLBACK javaApiAsyncHandler; - WT_ASYNC_OP *asyncop = NULL; - WT_CONNECTION_IMPL *connimpl; - JAVA_CALLBACK *jcb; - int ret; - - if ((ret = $self->async_new_op($self, uri, config, &javaApiAsyncHandler, &asyncop)) != 0) - goto err; - - connimpl = (WT_CONNECTION_IMPL *)$self; - if ((ret = __wt_calloc_def(connimpl->default_session, 1, &jcb)) != 0) - goto err; - - jcb->jnienv = jenv; - jcb->session = connimpl->default_session; - (*jenv)->GetJavaVM(jenv, &jcb->javavm); - jcb->jcallback = JCALL1(NewGlobalRef, jenv, callbackObject); - JCALL1(DeleteLocalRef, jenv, callbackObject); - asyncop->c.lang_private = jcb; - asyncop->c.flags |= WT_CURSTD_RAW; - -err: if (ret != 0) - throwWiredTigerException(jenv, ret); - return (asyncop); - } -} - -%extend __wt_connection { WT_SESSION *open_session_wrap(JNIEnv *jenv, const char *config) { extern WT_EVENT_HANDLER javaApiEventHandler; WT_SESSION *session = NULL; @@ -2282,13 +1510,6 @@ err: if (ret != 0) } } -%extend __wt_async_op { - long get_id_wrap(JNIEnv *jenv) { - WT_UNUSED(jenv); - return ((long)self->get_id(self)); - } -} - %extend __wt_session { long transaction_pinned_range_wrap(JNIEnv *jenv) { int ret; diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index 249cde15837..122057cca12 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -62,18 +62,10 @@ from packing import pack, unpack %typemap(in, numinputs=0) WT_SESSION ** (WT_SESSION *temp = NULL) { $1 = &temp; } -%typemap(in, numinputs=0) WT_ASYNC_OP ** (WT_ASYNC_OP *temp = NULL) { - $1 = &temp; -} %typemap(in, numinputs=0) WT_CURSOR ** (WT_CURSOR *temp = NULL) { $1 = &temp; } -%typemap(in) WT_ASYNC_CALLBACK * (PyObject *callback_obj = NULL) %{ - callback_obj = $input; - $1 = &pyApiAsyncCallback; -%} - %typemap(in, numinputs=0) WT_EVENT_HANDLER * %{ $1 = &pyApiEventHandler; %} @@ -98,33 +90,6 @@ from packing import pack, unpack } } } -%typemap(argout) WT_ASYNC_OP ** { - $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), - SWIGTYPE_p___wt_async_op, 0); - if (*$1 != NULL) { - PY_CALLBACK *pcb; - - (*$1)->c.flags |= WT_CURSTD_RAW; - PyObject_SetAttrString($result, "is_column", - PyBool_FromLong(strcmp((*$1)->key_format, "r") == 0)); - PyObject_SetAttrString($result, "key_format", - PyString_InternFromString((*$1)->key_format)); - PyObject_SetAttrString($result, "value_format", - PyString_InternFromString((*$1)->value_format)); - - if (__wt_calloc_def(NULL, 1, &pcb) != 0) - SWIG_exception_fail(SWIG_MemoryError, "WT calloc failed"); - else { - pcb->pyobj = $result; - Py_XINCREF(pcb->pyobj); - /* XXX Is there a way to avoid SWIG's numbering? */ - pcb->pyasynccb = callback_obj5; - Py_XINCREF(pcb->pyasynccb); - (*$1)->c.lang_private = pcb; - } - } -} - %typemap(argout) WT_CURSOR ** { $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), SWIGTYPE_p___wt_cursor, 0); @@ -365,15 +330,13 @@ DESTRUCTOR(__wt_session, close) * asttribute to None, and free the PY_CALLBACK. */ typedef struct { - PyObject *pyobj; /* the python Session/Cursor/AsyncOp object */ - PyObject *pyasynccb; /* the callback to use for AsyncOp */ + PyObject *pyobj; /* the python Session/Cursor object */ } PY_CALLBACK; static PyObject *wtError; static int sessionFreeHandler(WT_SESSION *session_arg); static int cursorFreeHandler(WT_CURSOR *cursor_arg); -static int asyncopFreeHandler(WT_ASYNC_OP *asyncop_arg); static int unpackBytesOrString(PyObject *obj, void **data, size_t *size); #define WT_GETATTR(var, parent, name) \ @@ -427,14 +390,6 @@ class IterableCursor: return self.__next__() ## @endcond -# An abstract class, which must be subclassed with notify() overridden. -class AsyncCallback: - def __init__(self): - raise NotImplementedError - - def notify(self, op, op_ret, flags): - raise NotImplementedError - def wiredtiger_calc_modify(session, oldv, newv, maxdiff, nmod): return _wiredtiger_calc_modify(session, oldv, newv, maxdiff, nmod) @@ -488,7 +443,6 @@ def wiredtiger_calc_modify_string(session, oldv, newv, maxdiff, nmod): %enddef SELFHELPER(struct __wt_connection, connection) -SELFHELPER(struct __wt_async_op, op) SELFHELPER(struct __wt_session, session) SELFHELPER(struct __wt_cursor, cursor) @@ -513,22 +467,6 @@ do { SWIG_ERROR_IF_NOT_SET(result); } -/* Async operations can return EBUSY when no ops are available. */ -%define EBUSY_OK(m) -%exception m { -retry: - $action - if (result != 0 && result != EBUSY) - SWIG_ERROR_IF_NOT_SET(result); - else if (result == EBUSY) { - SWIG_PYTHON_THREAD_BEGIN_ALLOW; - __wt_sleep(0, 10000); - SWIG_PYTHON_THREAD_END_ALLOW; - goto retry; - } -} -%enddef - /* An API that returns a value that shouldn't be checked uses this. */ %define ANY_OK(m) %exception m { @@ -563,8 +501,6 @@ retry: } %enddef -EBUSY_OK(__wt_connection::async_new_op) -ANY_OK(__wt_async_op::get_type) NOTFOUND_OK(__wt_cursor::next) NOTFOUND_OK(__wt_cursor::prev) NOTFOUND_OK(__wt_cursor::remove) @@ -582,8 +518,6 @@ COMPARE_NOTFOUND_OK(__wt_cursor::_search_near) %exception __wt_connection::get_home; %exception __wt_connection::is_new; %exception __wt_connection::search_near; -%exception __wt_async_op::_set_key; -%exception __wt_async_op::_set_value; %exception __wt_cursor::_set_key; %exception __wt_cursor::_set_key_str; %exception __wt_cursor::_set_value; @@ -592,14 +526,6 @@ COMPARE_NOTFOUND_OK(__wt_cursor::_search_near) %exception wiredtiger_version; %exception diagnostic_build; -/* WT_ASYNC_OP customization. */ -/* First, replace the varargs get / set methods with Python equivalents. */ -%ignore __wt_async_op::get_key; -%ignore __wt_async_op::get_value; -%ignore __wt_async_op::set_key; -%ignore __wt_async_op::set_value; -%immutable __wt_async_op::connection; - /* WT_CURSOR customization. */ /* First, replace the varargs get / set methods with Python equivalents. */ %ignore __wt_cursor::get_key; @@ -660,147 +586,6 @@ typedef int int_void; typedef int int_void; %typemap(out) int_void { $result = VOID_Object; } -%extend __wt_async_op { - /* Get / set keys and values */ - void _set_key(void *data, int size) { - WT_ITEM k; - k.data = data; - k.size = (uint32_t)size; - $self->set_key($self, &k); - } - - int_void _set_recno(uint64_t recno) { - WT_ITEM k; - uint8_t recno_buf[20]; - size_t size; - int ret; - if ((ret = wiredtiger_struct_size(NULL, - &size, "r", recno)) != 0 || - (ret = wiredtiger_struct_pack(NULL, - recno_buf, sizeof (recno_buf), "r", recno)) != 0) - return (ret); - - k.data = recno_buf; - k.size = (uint32_t)size; - $self->set_key($self, &k); - return (ret); - } - - void _set_value(void *data, int size) { - WT_ITEM v; - v.data = data; - v.size = (uint32_t)size; - $self->set_value($self, &v); - } - - /* Don't return values, just throw exceptions on failure. */ - int_void _get_key(char **datap, int *sizep) { - WT_ITEM k; - int ret = $self->get_key($self, &k); - if (ret == 0) { - *datap = (char *)k.data; - *sizep = (int)k.size; - } - return (ret); - } - - int_void _get_recno(uint64_t *recnop) { - WT_ITEM k; - int ret = $self->get_key($self, &k); - if (ret == 0) - ret = wiredtiger_struct_unpack(NULL, - k.data, k.size, "q", recnop); - return (ret); - } - - int_void _get_value(char **datap, int *sizep) { - WT_ITEM v; - int ret = $self->get_value($self, &v); - if (ret == 0) { - *datap = (char *)v.data; - *sizep = (int)v.size; - } - return (ret); - } - - int _freecb() { - return (asyncopFreeHandler($self)); - } - -%pythoncode %{ - def get_key(self): - '''get_key(self) -> object - - @copydoc WT_ASYNC_OP::get_key - Returns only the first column.''' - k = self.get_keys() - if len(k) == 1: - return k[0] - return k - - def get_keys(self): - '''get_keys(self) -> (object, ...) - - @copydoc WT_ASYNC_OP::get_key''' - if self.is_column: - return [self._get_recno(),] - else: - return unpack(self.key_format, self._get_key()) - - def get_value(self): - '''get_value(self) -> object - - @copydoc WT_ASYNC_OP::get_value - Returns only the first column.''' - v = self.get_values() - if len(v) == 1: - return v[0] - return v - - def get_values(self): - '''get_values(self) -> (object, ...) - - @copydoc WT_ASYNC_OP::get_value''' - return unpack(self.value_format, self._get_value()) - - def set_key(self, *args): - '''set_key(self) -> None - - @copydoc WT_ASYNC_OP::set_key''' - if len(args) == 1 and type(args[0]) == tuple: - args = args[0] - if self.is_column: - self._set_recno(_wt_recno(args[0])) - else: - # Keep the Python string pinned - self._key = pack(self.key_format, *args) - self._set_key(self._key) - - def set_value(self, *args): - '''set_value(self) -> None - - @copydoc WT_ASYNC_OP::set_value''' - if len(args) == 1 and type(args[0]) == tuple: - args = args[0] - # Keep the Python string pinned - self._value = pack(self.value_format, *args) - self._set_value(self._value) - - def __getitem__(self, key): - '''Python convenience for searching''' - self.set_key(key) - if self.search() != 0: - raise KeyError - return self.get_value() - - def __setitem__(self, key, value): - '''Python convenience for inserting''' - self.set_key(key) - self.set_key(value) - self.insert() -%} -}; - %extend __wt_cursor { /* Get / set keys and values */ void _set_key(void *data, int size) { @@ -1111,13 +896,7 @@ int diagnostic_build(); %ignore __wt_cursor::key_format; %ignore __wt_cursor::value_format; %immutable __wt_session::connection; -%immutable __wt_async_op::connection; -%immutable __wt_async_op::uri; -%immutable __wt_async_op::config; -%ignore __wt_async_op::key_format; -%ignore __wt_async_op::value_format; -%ignore __wt_async_callback; %ignore __wt_collator; %ignore __wt_compressor; %ignore __wt_config_item; @@ -1150,7 +929,6 @@ OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg)) /* Convert 'int *' to output args for wiredtiger_version */ %apply int *OUTPUT { int * }; -%rename(AsyncOp) __wt_async_op; %rename(Cursor) __wt_cursor; %rename(Modify) __wt_modify; %rename(Session) __wt_session; @@ -1309,7 +1087,6 @@ pythonClose(PY_CALLBACK *pcb) ret = EINVAL; /* any non-zero value will do. */ } Py_XDECREF(pcb->pyobj); - Py_XDECREF(pcb->pyasynccb); SWIG_PYTHON_THREAD_END_BLOCK; @@ -1378,18 +1155,6 @@ cursorFreeHandler(WT_CURSOR *cursor) return (0); } -/* Async Op specific close handler. */ -static int -asyncopFreeHandler(WT_ASYNC_OP *asyncop) -{ - PY_CALLBACK *pcb; - - pcb = (PY_CALLBACK *)asyncop->c.lang_private; - asyncop->c.lang_private = NULL; - __wt_free(NULL, pcb); - return (0); -} - static int pythonCloseCallback(WT_EVENT_HANDLER *handler, WT_SESSION *session, WT_CURSOR *cursor) @@ -1410,70 +1175,6 @@ static WT_EVENT_HANDLER pyApiEventHandler = { }; %} -/* Add async callback support. */ -%{ - -static int -pythonAsyncCallback(WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP *asyncop, int opret, - uint32_t flags) -{ - int ret, t_ret; - PY_CALLBACK *pcb; - PyObject *arglist, *notify_method, *pyresult; - WT_ASYNC_OP_IMPL *op; - WT_SESSION_IMPL *session; - - /* - * Ensure the global interpreter lock is held since we'll be - * making Python calls now. - */ - SWIG_PYTHON_THREAD_BEGIN_BLOCK; - - op = (WT_ASYNC_OP_IMPL *)asyncop; - session = O2S(op); - pcb = (PY_CALLBACK *)asyncop->c.lang_private; - asyncop->c.lang_private = NULL; - ret = 0; - - if (pcb->pyasynccb == NULL) - goto err; - if ((arglist = Py_BuildValue("(Oii)", pcb->pyobj, - opret, flags)) == NULL) - goto err; - if ((notify_method = PyObject_GetAttrString(pcb->pyasynccb, - "notify")) == NULL) - goto err; - - pyresult = PyEval_CallObject(notify_method, arglist); - if (pyresult == NULL || !PyArg_Parse(pyresult, "i", &ret)) - goto err; - - if (0) { - if (ret == 0) - ret = EINVAL; -err: __wt_err(session, ret, "python async callback error"); - } - Py_XDECREF(pyresult); - Py_XDECREF(notify_method); - Py_XDECREF(arglist); - - SWIG_PYTHON_THREAD_END_BLOCK; - - if (pcb != NULL) { - if ((t_ret = pythonClose(pcb) != 0) && ret == 0) - ret = t_ret; - } - __wt_free(session, pcb); - - if (ret == 0 && (opret == 0 || opret == WT_NOTFOUND)) - return (0); - else - return (1); -} - -static WT_ASYNC_CALLBACK pyApiAsyncCallback = { pythonAsyncCallback }; -%} - %pythoncode %{ class stat: '''keys for statistics cursors''' diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c deleted file mode 100644 index 06f8291b08b..00000000000 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ /dev/null @@ -1,583 +0,0 @@ -/*- - * Copyright (c) 2014-2020 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -/* - * __async_get_format -- - * Find or allocate the uri/config/format structure. - */ -static int -__async_get_format( - WT_CONNECTION_IMPL *conn, const char *uri, const char *config, WT_ASYNC_OP_IMPL *op) -{ - WT_ASYNC *async; - WT_ASYNC_FORMAT *af; - WT_CURSOR *c; - WT_DECL_RET; - WT_SESSION *wt_session; - WT_SESSION_IMPL *session; - uint64_t cfg_hash, uri_hash; - - async = conn->async; - c = NULL; - op->format = NULL; - - if (uri != NULL) - uri_hash = __wt_hash_city64(uri, strlen(uri)); - else - uri_hash = 0; - if (config != NULL) - cfg_hash = __wt_hash_city64(config, strlen(config)); - else - cfg_hash = 0; - - /* - * We don't need to hold a lock around this walk. The list is permanent and always valid. We - * might race an insert and there is a possibility a duplicate entry might be inserted, but that - * is not harmful. - */ - TAILQ_FOREACH (af, &async->formatqh, q) { - if (af->uri_hash == uri_hash && af->cfg_hash == cfg_hash) - goto setup; - } - /* - * We didn't find one in the cache. Allocate and initialize one. Insert it at the head expecting - * LRU usage. We need a real session for the cursor. - */ - WT_RET(__wt_open_internal_session(conn, "async-cursor", true, 0, &session)); - __wt_spin_lock(session, &async->ops_lock); - WT_ERR(__wt_calloc_one(session, &af)); - WT_ERR(__wt_strdup(session, uri, &af->uri)); - WT_ERR(__wt_strdup(session, config, &af->config)); - af->uri_hash = uri_hash; - af->cfg_hash = cfg_hash; - /* - * Get the key_format and value_format for this URI and store it in the structure so that - * async->set_key/value work. - */ - wt_session = &session->iface; - WT_ERR(wt_session->open_cursor(wt_session, uri, NULL, NULL, &c)); - WT_ERR(__wt_strdup(session, c->key_format, &af->key_format)); - WT_ERR(__wt_strdup(session, c->value_format, &af->value_format)); - WT_ERR(c->close(c)); - c = NULL; - - TAILQ_INSERT_HEAD(&async->formatqh, af, q); - __wt_spin_unlock(session, &async->ops_lock); - WT_ERR(__wt_session_close_internal(session)); - -setup: - op->format = af; - /* - * Copy the pointers for the formats. Items in the async format queue remain there until the - * connection is closed. We must initialize the format fields in the async_op, which are - * publicly visible, and its internal cursor used by internal key/value functions. - */ - op->iface.c.key_format = op->iface.key_format = af->key_format; - op->iface.c.value_format = op->iface.value_format = af->value_format; - return (0); - -err: - if (c != NULL) - WT_TRET(c->close(c)); - __wt_free(session, af->uri); - __wt_free(session, af->config); - __wt_free(session, af->key_format); - __wt_free(session, af->value_format); - __wt_free(session, af); - return (ret); -} - -/* - * __async_new_op_alloc -- - * Find and allocate the next available async op handle. - */ -static int -__async_new_op_alloc( - WT_SESSION_IMPL *session, const char *uri, const char *config, WT_ASYNC_OP_IMPL **opp) -{ - WT_ASYNC *async; - WT_ASYNC_OP_IMPL *op; - WT_CONNECTION_IMPL *conn; - uint32_t i, save_i, view; - - *opp = NULL; - - conn = S2C(session); - async = conn->async; - WT_STAT_CONN_INCR(session, async_op_alloc); - -retry: - op = NULL; - WT_ORDERED_READ(save_i, async->ops_index); - /* - * Look after the last one allocated for a free one. We'd expect ops to be freed mostly FIFO so - * we should quickly find one. - */ - for (view = 1, i = save_i; i < conn->async_size; i++, view++) { - op = &async->async_ops[i]; - if (op->state == WT_ASYNCOP_FREE) - break; - } - - /* - * Loop around back to the beginning if we need to. - */ - if (op == NULL || op->state != WT_ASYNCOP_FREE) - for (i = 0; i < save_i; i++, view++) { - op = &async->async_ops[i]; - if (op->state == WT_ASYNCOP_FREE) - break; - } - - /* - * We still haven't found one. Return an error. - */ - if (op == NULL || op->state != WT_ASYNCOP_FREE) { - WT_STAT_CONN_INCR(session, async_full); - return (__wt_set_return(session, EBUSY)); - } - /* - * Set the state of this op handle as READY for the user to use. If we can set the state then - * the op entry is ours. Start the next search at the next entry after this one. - */ - if (!__wt_atomic_cas32(&op->state, WT_ASYNCOP_FREE, WT_ASYNCOP_READY)) { - WT_STAT_CONN_INCR(session, async_alloc_race); - goto retry; - } - WT_STAT_CONN_INCRV(session, async_alloc_view, view); - WT_RET(__async_get_format(conn, uri, config, op)); - op->unique_id = __wt_atomic_add64(&async->op_id, 1); - op->optype = WT_AOP_NONE; - async->ops_index = (i + 1) % conn->async_size; - *opp = op; - return (0); -} - -/* - * __async_config -- - * Parse and setup the async API options. - */ -static int -__async_config(WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *conn, const char **cfg, bool *runp) -{ - WT_CONFIG_ITEM cval; - - /* - * The async configuration is off by default. - */ - WT_RET(__wt_config_gets(session, cfg, "async.enabled", &cval)); - *runp = cval.val != 0; - - /* - * Even if async is turned off, we want to parse and store the default values so that - * reconfigure can just enable them. - * - * Bound the minimum maximum operations at 10. - */ - WT_RET(__wt_config_gets(session, cfg, "async.ops_max", &cval)); - conn->async_size = (uint32_t)WT_MAX(cval.val, 10); - - WT_RET(__wt_config_gets(session, cfg, "async.threads", &cval)); - conn->async_workers = (uint32_t)cval.val; - /* Sanity check that api_data.py is in sync with async.h */ - WT_ASSERT(session, conn->async_workers <= WT_ASYNC_MAX_WORKERS); - - return (0); -} - -/* - * __wt_async_stats_update -- - * Update the async stats for return to the application. - */ -void -__wt_async_stats_update(WT_SESSION_IMPL *session) -{ - WT_ASYNC *async; - WT_CONNECTION_IMPL *conn; - WT_CONNECTION_STATS **stats; - - conn = S2C(session); - async = conn->async; - if (async == NULL) - return; - stats = conn->stats; - WT_STAT_SET(session, stats, async_cur_queue, async->cur_queue); - WT_STAT_SET(session, stats, async_max_queue, async->max_queue); -} - -/* - * __async_start -- - * Start the async subsystem. All configuration processing has already been done by the caller. - */ -static int -__async_start(WT_SESSION_IMPL *session) -{ - WT_ASYNC *async; - WT_CONNECTION_IMPL *conn; - uint32_t i, session_flags; - - conn = S2C(session); - conn->async_cfg = true; - /* - * Async is on, allocate the WT_ASYNC structure and initialize the ops. - */ - WT_RET(__wt_calloc_one(session, &conn->async)); - async = conn->async; - TAILQ_INIT(&async->formatqh); - WT_RET(__wt_spin_init(session, &async->ops_lock, "ops")); - WT_RET(__wt_cond_alloc(session, "async flush", &async->flush_cond)); - WT_RET(__wt_async_op_init(session)); - - /* - * Start up the worker threads. - */ - F_SET(conn, WT_CONN_SERVER_ASYNC); - for (i = 0; i < conn->async_workers; i++) { - /* - * Each worker has its own session. We set both a general server flag in the connection and - * an individual flag in the session. The user may reconfigure the number of workers and we - * may want to selectively stop some workers while leaving the rest running. - */ - session_flags = WT_SESSION_SERVER_ASYNC; - WT_RET(__wt_open_internal_session( - conn, "async-worker", true, session_flags, &async->worker_sessions[i])); - } - for (i = 0; i < conn->async_workers; i++) { - /* - * Start the threads. - */ - WT_RET(__wt_thread_create( - session, &async->worker_tids[i], __wt_async_worker, async->worker_sessions[i])); - } - __wt_async_stats_update(session); - return (0); -} - -/* - * __wt_async_create -- - * Start the async subsystem and worker threads. - */ -int -__wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) -{ - WT_CONNECTION_IMPL *conn; - bool run; - - conn = S2C(session); - - /* Handle configuration. */ - run = false; - WT_RET(__async_config(session, conn, cfg, &run)); - - /* If async is not configured, we're done. */ - if (!run) - return (0); - return (__async_start(session)); -} - -/* - * __wt_async_reconfig -- - * Start the async subsystem and worker threads. - */ -int -__wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) -{ - WT_ASYNC *async; - WT_CONNECTION_IMPL *conn, tmp_conn; - WT_DECL_RET; - uint32_t i, session_flags; - bool run; - - conn = S2C(session); - async = conn->async; - memset(&tmp_conn, 0, sizeof(tmp_conn)); - tmp_conn.async_cfg = conn->async_cfg; - tmp_conn.async_workers = conn->async_workers; - tmp_conn.async_size = conn->async_size; - - /* Handle configuration. */ - run = conn->async_cfg; - WT_RET(__async_config(session, &tmp_conn, cfg, &run)); - - /* - * There are some restrictions on the live reconfiguration of async. Unlike other subsystems - * where we simply destroy anything existing and restart with the new configuration, async is - * not so easy. If the user is just changing the number of workers, we want to allow the - * existing op handles and other information to remain in existence. So we must handle various - * combinations of changes individually. - * - * One restriction is that if async is currently on, the user cannot change the number of async - * op handles available. The user can try but we do nothing with it. However we must allow the - * ops_max config string so that a user can completely start async via reconfigure. - */ - - /* - * Easy cases: - * 1. If async is on and the user wants it off, shut it down. - * 2. If async is off, and the user wants it on, start it. - * 3. If not a toggle and async is off, we're done. - */ - if (conn->async_cfg && !run) { /* Case 1 */ - WT_TRET(__wt_async_flush(session)); - ret = __wt_async_destroy(session); - conn->async_cfg = false; - return (ret); - } - if (!conn->async_cfg && run) /* Case 2 */ - return (__wt_async_create(session, cfg)); - if (!conn->async_cfg) /* Case 3 */ - return (0); - - /* - * Running async worker modification cases: - * 4. If number of workers didn't change, we're done. - * 5. If more workers, start new ones. - * 6. If fewer workers, kill some. - */ - if (conn->async_workers == tmp_conn.async_workers) - /* No change in the number of workers. */ - return (0); - if (conn->async_workers < tmp_conn.async_workers) { - /* Case 5 */ - /* - * The worker_sessions array is allocated for the maximum allowed number of workers, so - * starting more is easy. - */ - for (i = conn->async_workers; i < tmp_conn.async_workers; i++) { - /* - * Each worker has its own session. - */ - session_flags = WT_SESSION_SERVER_ASYNC; - WT_RET(__wt_open_internal_session( - conn, "async-worker", true, session_flags, &async->worker_sessions[i])); - } - for (i = conn->async_workers; i < tmp_conn.async_workers; i++) { - /* - * Start the threads. - */ - WT_RET(__wt_thread_create( - session, &async->worker_tids[i], __wt_async_worker, async->worker_sessions[i])); - } - conn->async_workers = tmp_conn.async_workers; - } - if (conn->async_workers > tmp_conn.async_workers) { - /* Case 6 */ - /* - * Stopping an individual async worker is the most complex case. We clear the session async - * flag on the targeted worker thread so that only that thread stops, and the others keep - * running. - */ - for (i = conn->async_workers - 1; i >= tmp_conn.async_workers; i--) { - /* - * Join any worker we're stopping. After the thread is stopped, close its session. - */ - WT_ASSERT(session, async->worker_tids[i].created); - WT_ASSERT(session, async->worker_sessions[i] != NULL); - F_CLR(async->worker_sessions[i], WT_SESSION_SERVER_ASYNC); - WT_TRET(__wt_thread_join(session, &async->worker_tids[i])); - WT_TRET(__wt_session_close_internal(async->worker_sessions[i])); - async->worker_sessions[i] = NULL; - } - conn->async_workers = tmp_conn.async_workers; - } - - return (0); -} - -/* - * __wt_async_destroy -- - * Destroy the async worker threads and async subsystem. - */ -int -__wt_async_destroy(WT_SESSION_IMPL *session) -{ - WT_ASYNC *async; - WT_ASYNC_FORMAT *af; - WT_ASYNC_OP *op; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - uint32_t i; - - conn = S2C(session); - async = conn->async; - - if (!conn->async_cfg) - return (0); - - F_CLR(conn, WT_CONN_SERVER_ASYNC); - for (i = 0; i < conn->async_workers; i++) - WT_TRET(__wt_thread_join(session, &async->worker_tids[i])); - __wt_cond_destroy(session, &async->flush_cond); - - /* Close the server threads' sessions. */ - for (i = 0; i < conn->async_workers; i++) - if (async->worker_sessions[i] != NULL) { - WT_TRET(__wt_session_close_internal(async->worker_sessions[i])); - async->worker_sessions[i] = NULL; - } - /* Free any op key/value buffers. */ - for (i = 0; i < conn->async_size; i++) { - op = (WT_ASYNC_OP *)&async->async_ops[i]; - if (op->c.key.data != NULL) - __wt_buf_free(session, &op->c.key); - if (op->c.value.data != NULL) - __wt_buf_free(session, &op->c.value); - } - - /* Free format resources */ - while ((af = TAILQ_FIRST(&async->formatqh)) != NULL) { - TAILQ_REMOVE(&async->formatqh, af, q); - __wt_free(session, af->uri); - __wt_free(session, af->config); - __wt_free(session, af->key_format); - __wt_free(session, af->value_format); - __wt_free(session, af); - } - __wt_free(session, async->async_queue); - __wt_free(session, async->async_ops); - __wt_spin_destroy(session, &async->ops_lock); - __wt_free(session, conn->async); - - return (ret); -} - -/* - * __wt_async_flush -- - * Implementation of the WT_CONN->async_flush method. - */ -int -__wt_async_flush(WT_SESSION_IMPL *session) -{ - WT_ASYNC *async; - WT_CONNECTION_IMPL *conn; - uint32_t i, workers; - - conn = S2C(session); - if (!conn->async_cfg) - return (0); - - async = conn->async; - /* - * Only add a flush operation if there are workers who can process it. Otherwise we will wait - * forever. - */ - workers = 0; - for (i = 0; i < conn->async_workers; ++i) - if (async->worker_tids[i].created) - ++workers; - if (workers == 0) - return (0); - - WT_STAT_CONN_INCR(session, async_flush); -/* - * We have to do several things. First we have to prevent other callers from racing with us so that - * only one flush is happening at a time. Next we have to wait for the worker threads to notice the - * flush and indicate that the flush is complete on their side. Then we clear the flush flags and - * return. - */ -retry: - while (async->flush_state != WT_ASYNC_FLUSH_NONE) - /* - * We're racing an in-progress flush. We need to wait our turn to start our own. We need to - * convoy the racing calls because a later call may be waiting for specific enqueued ops to - * be complete before this returns. - */ - __wt_sleep(0, 100000); - - if (!__wt_atomic_cas32(&async->flush_state, WT_ASYNC_FLUSH_NONE, WT_ASYNC_FLUSH_IN_PROGRESS)) - goto retry; - /* - * We're the owner of this flush operation. Set the WT_ASYNC_FLUSH_IN_PROGRESS to block other - * callers. We're also preventing all worker threads from taking things off the work queue with - * the lock. - */ - async->flush_count = 0; - (void)__wt_atomic_add64(&async->flush_gen, 1); - WT_ASSERT(session, async->flush_op.state == WT_ASYNCOP_FREE); - async->flush_op.state = WT_ASYNCOP_READY; - WT_RET(__wt_async_op_enqueue(session, &async->flush_op)); - while (async->flush_state != WT_ASYNC_FLUSH_COMPLETE) - __wt_cond_wait(session, async->flush_cond, 100000, NULL); - /* - * Flush is done. Clear the flags. - */ - async->flush_op.state = WT_ASYNCOP_FREE; - WT_PUBLISH(async->flush_state, WT_ASYNC_FLUSH_NONE); - return (0); -} - -/* - * __async_runtime_config -- - * Configure runtime fields at allocation. - */ -static int -__async_runtime_config(WT_ASYNC_OP_IMPL *op, const char *cfg[]) -{ - WT_ASYNC_OP *asyncop; - WT_CONFIG_ITEM cval; - WT_SESSION_IMPL *session; - - session = O2S(op); - asyncop = (WT_ASYNC_OP *)op; - WT_RET(__wt_config_gets_def(session, cfg, "append", 0, &cval)); - if (cval.val) - F_SET(&asyncop->c, WT_CURSTD_APPEND); - else - F_CLR(&asyncop->c, WT_CURSTD_APPEND); - WT_RET(__wt_config_gets_def(session, cfg, "overwrite", 1, &cval)); - if (cval.val) - F_SET(&asyncop->c, WT_CURSTD_OVERWRITE); - else - F_CLR(&asyncop->c, WT_CURSTD_OVERWRITE); - WT_RET(__wt_config_gets_def(session, cfg, "raw", 0, &cval)); - if (cval.val) - F_SET(&asyncop->c, WT_CURSTD_RAW); - else - F_CLR(&asyncop->c, WT_CURSTD_RAW); - return (0); -} - -/* - * __wt_async_new_op -- - * Implementation of the WT_CONN->async_new_op method. - */ -int -__wt_async_new_op(WT_SESSION_IMPL *session, const char *uri, const char *config, - WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP_IMPL **opp) -{ - WT_ASYNC_OP_IMPL *op; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - const char *cfg[] = {S2C(session)->cfg, NULL, NULL}; - - *opp = NULL; - - conn = S2C(session); - if (!conn->async_cfg) - WT_RET(__wt_async_create(session, cfg)); - if (!conn->async_cfg) - WT_RET_MSG(session, ENOTSUP, "Asynchronous operations not configured"); - - op = NULL; - WT_ERR(__async_new_op_alloc(session, uri, config, &op)); - cfg[1] = config; - WT_ERR(__async_runtime_config(op, cfg)); - op->cb = cb; - *opp = op; - return (0); - -err: - /* - * If we get an error after allocating op, set its state to free. - */ - if (op != NULL) - op->state = WT_ASYNCOP_FREE; - return (ret); -} diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c deleted file mode 100644 index 3faa53808c8..00000000000 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ /dev/null @@ -1,354 +0,0 @@ -/*- - * Copyright (c) 2014-2020 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -/* - * __async_get_key -- - * WT_ASYNC_OP->get_key implementation for op handles. - */ -static int -__async_get_key(WT_ASYNC_OP *asyncop, ...) -{ - WT_DECL_RET; - va_list ap; - - va_start(ap, asyncop); - ret = __wt_cursor_get_keyv(&asyncop->c, asyncop->c.flags, ap); - va_end(ap); - return (ret); -} - -/* - * __async_set_key -- - * WT_ASYNC_OP->set_key implementation for op handles. - */ -static void -__async_set_key(WT_ASYNC_OP *asyncop, ...) -{ - WT_CURSOR *c; - va_list ap; - - c = &asyncop->c; - va_start(ap, asyncop); - WT_IGNORE_RET(__wt_cursor_set_keyv(c, c->flags, ap)); - if (!WT_DATA_IN_ITEM(&c->key) && !WT_CURSOR_RECNO(c)) - c->saved_err = - __wt_buf_set(O2S((WT_ASYNC_OP_IMPL *)asyncop), &c->key, c->key.data, c->key.size); - va_end(ap); -} - -/* - * __async_get_value -- - * WT_ASYNC_OP->get_value implementation for op handles. - */ -static int -__async_get_value(WT_ASYNC_OP *asyncop, ...) -{ - WT_DECL_RET; - va_list ap; - - va_start(ap, asyncop); - ret = __wt_cursor_get_valuev(&asyncop->c, ap); - va_end(ap); - return (ret); -} - -/* - * __async_set_value -- - * WT_ASYNC_OP->set_value implementation for op handles. - */ -static void -__async_set_value(WT_ASYNC_OP *asyncop, ...) -{ - WT_CURSOR *c; - va_list ap; - - c = &asyncop->c; - va_start(ap, asyncop); - WT_IGNORE_RET(__wt_cursor_set_valuev(c, ap)); - /* Copy the data, if it is pointing at data elsewhere. */ - if (!WT_DATA_IN_ITEM(&c->value)) - c->saved_err = - __wt_buf_set(O2S((WT_ASYNC_OP_IMPL *)asyncop), &c->value, c->value.data, c->value.size); - va_end(ap); -} - -/* - * __async_op_wrap -- - * Common wrapper for all async operations. - */ -static int -__async_op_wrap(WT_ASYNC_OP_IMPL *op, WT_ASYNC_OPTYPE type) -{ - op->optype = type; - return (__wt_async_op_enqueue(O2S(op), op)); -} - -/* - * __async_search -- - * WT_ASYNC_OP->search implementation for op handles. - */ -static int -__async_search(WT_ASYNC_OP *asyncop) -{ - WT_ASYNC_OP_IMPL *op; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - op = (WT_ASYNC_OP_IMPL *)asyncop; - ASYNCOP_API_CALL(O2C(op), session, search); - WT_STAT_CONN_INCR(O2S(op), async_op_search); - WT_ERR(__async_op_wrap(op, WT_AOP_SEARCH)); -err: - API_END_RET(session, ret); -} - -/* - * __async_insert -- - * WT_ASYNC_OP->insert implementation for op handles. - */ -static int -__async_insert(WT_ASYNC_OP *asyncop) -{ - WT_ASYNC_OP_IMPL *op; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - op = (WT_ASYNC_OP_IMPL *)asyncop; - ASYNCOP_API_CALL(O2C(op), session, insert); - WT_STAT_CONN_INCR(O2S(op), async_op_insert); - WT_ERR(__async_op_wrap(op, WT_AOP_INSERT)); -err: - API_END_RET(session, ret); -} - -/* - * __async_update -- - * WT_ASYNC_OP->update implementation for op handles. - */ -static int -__async_update(WT_ASYNC_OP *asyncop) -{ - WT_ASYNC_OP_IMPL *op; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - op = (WT_ASYNC_OP_IMPL *)asyncop; - ASYNCOP_API_CALL(O2C(op), session, update); - WT_STAT_CONN_INCR(O2S(op), async_op_update); - WT_ERR(__async_op_wrap(op, WT_AOP_UPDATE)); -err: - API_END_RET(session, ret); -} - -/* - * __async_remove -- - * WT_ASYNC_OP->remove implementation for op handles. - */ -static int -__async_remove(WT_ASYNC_OP *asyncop) -{ - WT_ASYNC_OP_IMPL *op; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - op = (WT_ASYNC_OP_IMPL *)asyncop; - ASYNCOP_API_CALL(O2C(op), session, remove); - WT_STAT_CONN_INCR(O2S(op), async_op_remove); - WT_ERR(__async_op_wrap(op, WT_AOP_REMOVE)); -err: - API_END_RET(session, ret); -} - -/* - * __async_compact -- - * WT_ASYNC_OP->compact implementation for op handles. - */ -static int -__async_compact(WT_ASYNC_OP *asyncop) -{ - WT_ASYNC_OP_IMPL *op; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - op = (WT_ASYNC_OP_IMPL *)asyncop; - ASYNCOP_API_CALL(O2C(op), session, compact); - WT_STAT_CONN_INCR(O2S(op), async_op_compact); - WT_ERR(__async_op_wrap(op, WT_AOP_COMPACT)); -err: - API_END_RET(session, ret); -} - -/* - * __async_get_id -- - * WT_ASYNC_OP->get_id implementation for op handles. - */ -static uint64_t -__async_get_id(WT_ASYNC_OP *asyncop) -{ - return (((WT_ASYNC_OP_IMPL *)asyncop)->unique_id); -} - -/* - * __async_get_type -- - * WT_ASYNC_OP->get_type implementation for op handles. - */ -static WT_ASYNC_OPTYPE -__async_get_type(WT_ASYNC_OP *asyncop) -{ - return (((WT_ASYNC_OP_IMPL *)asyncop)->optype); -} - -/* - * __async_op_init -- - * Initialize all the op handle fields. - */ -static void -__async_op_init(WT_CONNECTION_IMPL *conn, WT_ASYNC_OP_IMPL *op, uint32_t id) -{ - WT_ASYNC_OP *asyncop; - - asyncop = (WT_ASYNC_OP *)op; - asyncop->connection = (WT_CONNECTION *)conn; - asyncop->key_format = asyncop->value_format = NULL; - asyncop->c.key_format = asyncop->c.value_format = NULL; - asyncop->get_key = __async_get_key; - asyncop->get_value = __async_get_value; - asyncop->set_key = __async_set_key; - asyncop->set_value = __async_set_value; - asyncop->search = __async_search; - asyncop->insert = __async_insert; - asyncop->update = __async_update; - asyncop->remove = __async_remove; - asyncop->compact = __async_compact; - asyncop->get_id = __async_get_id; - asyncop->get_type = __async_get_type; - /* - * The cursor needs to have the get/set key/value functions initialized. It also needs the - * key/value related fields set up. - */ - asyncop->c.get_key = __wt_cursor_get_key; - asyncop->c.set_key = __wt_cursor_set_key; - asyncop->c.get_value = __wt_cursor_get_value; - asyncop->c.set_value = __wt_cursor_set_value; - asyncop->c.recno = WT_RECNO_OOB; - memset(asyncop->c.raw_recno_buf, 0, sizeof(asyncop->c.raw_recno_buf)); - memset(&asyncop->c.key, 0, sizeof(asyncop->c.key)); - memset(&asyncop->c.value, 0, sizeof(asyncop->c.value)); - asyncop->c.session = (WT_SESSION *)conn->default_session; - asyncop->c.saved_err = 0; - asyncop->c.flags = 0; - - op->internal_id = id; - op->state = WT_ASYNCOP_FREE; -} - -/* - * __wt_async_op_enqueue -- - * Enqueue an operation onto the work queue. - */ -int -__wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) -{ - WT_ASYNC *async; - WT_CONNECTION_IMPL *conn; - uint64_t cur_head, cur_tail, my_alloc, my_slot; -#ifdef HAVE_DIAGNOSTIC - WT_ASYNC_OP_IMPL *my_op; -#endif - - conn = S2C(session); - async = conn->async; - - /* - * If an application re-uses a WT_ASYNC_OP, we end up here with an invalid object. - */ - if (op->state != WT_ASYNCOP_READY) - WT_RET_MSG(session, EINVAL, "application error: WT_ASYNC_OP already in use"); - - /* - * Enqueue op at the tail of the work queue. We get our slot in the ring buffer to use. - */ - my_alloc = __wt_atomic_add64(&async->alloc_head, 1); - my_slot = my_alloc % async->async_qsize; - - /* - * Make sure we haven't wrapped around the queue. If so, wait for the tail to advance off this - * slot. - */ - WT_ORDERED_READ(cur_tail, async->tail_slot); - while (cur_tail == my_slot) { - __wt_yield(); - WT_ORDERED_READ(cur_tail, async->tail_slot); - } - -#ifdef HAVE_DIAGNOSTIC - WT_ORDERED_READ(my_op, async->async_queue[my_slot]); - if (my_op != NULL) - return (__wt_panic(session, WT_PANIC, "async failure")); -#endif - WT_PUBLISH(async->async_queue[my_slot], op); - op->state = WT_ASYNCOP_ENQUEUED; - if (__wt_atomic_add32(&async->cur_queue, 1) > async->max_queue) - WT_PUBLISH(async->max_queue, async->cur_queue); - /* - * Multiple threads may be adding ops to the queue. We need to wait our turn to make our slot - * visible to workers. - */ - WT_ORDERED_READ(cur_head, async->head); - while (cur_head != (my_alloc - 1)) { - __wt_yield(); - WT_ORDERED_READ(cur_head, async->head); - } - WT_PUBLISH(async->head, my_alloc); - return (0); -} - -/* - * __wt_async_op_init -- - * Initialize all the op handles. - */ -int -__wt_async_op_init(WT_SESSION_IMPL *session) -{ - WT_ASYNC *async; - WT_ASYNC_OP_IMPL *op; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - uint32_t i; - - conn = S2C(session); - async = conn->async; - - /* - * Initialize the flush op structure. - */ - __async_op_init(conn, &async->flush_op, OPS_INVALID_INDEX); - - /* - * Allocate and initialize the work queue. This is sized so that the ring buffer is known to be - * big enough such that the head can never overlap the tail. Include extra for the flush op. - */ - async->async_qsize = conn->async_size + 2; - WT_RET(__wt_calloc_def(session, async->async_qsize, &async->async_queue)); - /* - * Allocate and initialize all the user ops. - */ - WT_ERR(__wt_calloc_def(session, conn->async_size, &async->async_ops)); - for (i = 0; i < conn->async_size; i++) { - op = &async->async_ops[i]; - __async_op_init(conn, op, i); - } - return (0); - -err: - __wt_free(session, async->async_ops); - __wt_free(session, async->async_queue); - return (ret); -} diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c deleted file mode 100644 index eb301b2cdfa..00000000000 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ /dev/null @@ -1,323 +0,0 @@ -/*- - * Copyright (c) 2014-2020 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -/* - * __async_op_dequeue -- - * Wait for work to be available. Then atomically take it off the work queue. - */ -static int -__async_op_dequeue(WT_CONNECTION_IMPL *conn, WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL **op) -{ - WT_ASYNC *async; - uint64_t cur_tail, last_consume, my_consume, my_slot, prev_slot; - uint64_t sleep_usec; - uint32_t tries; - - *op = NULL; - - async = conn->async; -/* - * Wait for work to do. Work is available when async->head moves. Then grab the slot containing the - * work. If we lose, try again. - */ -retry: - tries = 0; - sleep_usec = 100; - WT_ORDERED_READ(last_consume, async->alloc_tail); - /* - * We stay in this loop until there is work to do. - */ - while (last_consume == async->head && async->flush_state != WT_ASYNC_FLUSHING) { - WT_STAT_CONN_INCR(session, async_nowork); - if (++tries < MAX_ASYNC_YIELD) - /* - * Initially when we find no work, allow other threads to run. - */ - __wt_yield(); - else { - /* - * If we haven't found work in a while, start sleeping to wait for work to arrive - * instead of spinning. - */ - __wt_sleep(0, sleep_usec); - sleep_usec = WT_MIN(sleep_usec * 2, MAX_ASYNC_SLEEP_USECS); - } - if (!F_ISSET(session, WT_SESSION_SERVER_ASYNC)) - return (0); - if (!F_ISSET(conn, WT_CONN_SERVER_ASYNC)) - return (0); - WT_ORDERED_READ(last_consume, async->alloc_tail); - } - if (async->flush_state == WT_ASYNC_FLUSHING) - return (0); - /* - * Try to increment the tail to claim this slot. If we lose a race, try again. - */ - my_consume = last_consume + 1; - if (!__wt_atomic_cas64(&async->alloc_tail, last_consume, my_consume)) - goto retry; - /* - * This item of work is ours to process. Clear it out of the queue and return. - */ - my_slot = my_consume % async->async_qsize; - prev_slot = last_consume % async->async_qsize; - *op = async->async_queue[my_slot]; - async->async_queue[my_slot] = NULL; - - WT_ASSERT(session, async->cur_queue > 0); - WT_ASSERT(session, *op != NULL); - WT_ASSERT(session, (*op)->state == WT_ASYNCOP_ENQUEUED); - (void)__wt_atomic_sub32(&async->cur_queue, 1); - (*op)->state = WT_ASYNCOP_WORKING; - - if (*op == &async->flush_op) - /* - * We're the worker to take the flush op off the queue. - */ - WT_PUBLISH(async->flush_state, WT_ASYNC_FLUSHING); - WT_ORDERED_READ(cur_tail, async->tail_slot); - while (cur_tail != prev_slot) { - __wt_yield(); - WT_ORDERED_READ(cur_tail, async->tail_slot); - } - WT_PUBLISH(async->tail_slot, my_slot); - return (0); -} - -/* - * __async_flush_wait -- - * Wait for the final worker to finish flushing. - */ -static void -__async_flush_wait(WT_SESSION_IMPL *session, WT_ASYNC *async, uint64_t my_gen) -{ - while (async->flush_state == WT_ASYNC_FLUSHING && async->flush_gen == my_gen) { - __wt_cond_wait(session, async->flush_cond, 10000, NULL); - WT_BARRIER(); - } -} - -/* - * __async_worker_cursor -- - * Return a cursor for the worker thread to use for its op. The worker thread caches cursors. So - * first search for one with the same config/uri signature. Otherwise open a new cursor and - * cache it. - */ -static int -__async_worker_cursor(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op, WT_ASYNC_WORKER_STATE *worker, - WT_CURSOR **cursorp) -{ - WT_ASYNC_CURSOR *ac; - WT_CURSOR *c; - WT_DECL_RET; - WT_SESSION *wt_session; - - *cursorp = NULL; - - wt_session = (WT_SESSION *)session; - /* - * Compact doesn't need a cursor. - */ - if (op->optype == WT_AOP_COMPACT) - return (0); - WT_ASSERT(session, op->format != NULL); - TAILQ_FOREACH (ac, &worker->cursorqh, q) { - if (op->format->cfg_hash == ac->cfg_hash && op->format->uri_hash == ac->uri_hash) { - /* - * If one of our cached cursors has a matching signature, use it and we're done. - */ - *cursorp = ac->c; - return (0); - } - } - /* - * We didn't find one in our cache. Open one and cache it. Insert it at the head expecting LRU - * usage. - */ - WT_RET(__wt_calloc_one(session, &ac)); - WT_ERR(wt_session->open_cursor(wt_session, op->format->uri, NULL, op->format->config, &c)); - ac->cfg_hash = op->format->cfg_hash; - ac->uri_hash = op->format->uri_hash; - ac->c = c; - TAILQ_INSERT_HEAD(&worker->cursorqh, ac, q); - worker->num_cursors++; - *cursorp = c; - return (0); - -err: - __wt_free(session, ac); - return (ret); -} - -/* - * __async_worker_execop -- - * A worker thread executes an individual op with a cursor. - */ -static int -__async_worker_execop(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op, WT_CURSOR *cursor) -{ - WT_ASYNC_OP *asyncop; - WT_ITEM val; - WT_SESSION *wt_session; - - asyncop = (WT_ASYNC_OP *)op; - /* - * Set the key of our local cursor from the async op handle. If needed, also set the value. - */ - if (op->optype != WT_AOP_COMPACT) { - WT_RET(__wt_cursor_get_raw_key(&asyncop->c, &val)); - __wt_cursor_set_raw_key(cursor, &val); - if (op->optype == WT_AOP_INSERT || op->optype == WT_AOP_UPDATE) { - WT_RET(__wt_cursor_get_raw_value(&asyncop->c, &val)); - __wt_cursor_set_raw_value(cursor, &val); - } - } - switch (op->optype) { - case WT_AOP_COMPACT: - wt_session = &session->iface; - WT_RET(wt_session->compact(wt_session, op->format->uri, op->format->config)); - break; - case WT_AOP_INSERT: - WT_RET(cursor->insert(cursor)); - break; - case WT_AOP_UPDATE: - WT_RET(cursor->update(cursor)); - break; - case WT_AOP_REMOVE: - WT_RET(cursor->remove(cursor)); - break; - case WT_AOP_SEARCH: - WT_RET(cursor->search(cursor)); - /* - * Get the value from the cursor and put it into the op for op->get_value. - */ - WT_RET(__wt_cursor_get_raw_value(cursor, &val)); - __wt_cursor_set_raw_value(&asyncop->c, &val); - break; - case WT_AOP_NONE: - WT_RET_MSG(session, EINVAL, "Unknown async optype %d", (int)op->optype); - } - return (0); -} - -/* - * __async_worker_op -- - * A worker thread handles an individual op. - */ -static int -__async_worker_op(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op, WT_ASYNC_WORKER_STATE *worker) -{ - WT_ASYNC_OP *asyncop; - WT_CURSOR *cursor; - WT_DECL_RET; - WT_SESSION *wt_session; - int cb_ret; - - asyncop = (WT_ASYNC_OP *)op; - - cb_ret = 0; - - wt_session = &session->iface; - if (op->optype != WT_AOP_COMPACT) - WT_RET(wt_session->begin_transaction(wt_session, NULL)); - WT_ASSERT(session, op->state == WT_ASYNCOP_WORKING); - WT_RET(__async_worker_cursor(session, op, worker, &cursor)); - /* - * Perform op and invoke the callback. - */ - ret = __async_worker_execop(session, op, cursor); - if (op->cb != NULL && op->cb->notify != NULL) - cb_ret = op->cb->notify(op->cb, asyncop, ret, 0); - - /* - * If the operation succeeded and the user callback returned zero then commit. Otherwise - * rollback. - */ - if (op->optype != WT_AOP_COMPACT) { - if ((ret == 0 || ret == WT_NOTFOUND) && cb_ret == 0) - WT_TRET(wt_session->commit_transaction(wt_session, NULL)); - else - WT_TRET(wt_session->rollback_transaction(wt_session, NULL)); - F_CLR(&asyncop->c, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - WT_TRET(cursor->reset(cursor)); - } - /* - * After the callback returns, and the transaction resolved release the op back to the free - * pool. We do this regardless of success or failure. - */ - WT_PUBLISH(op->state, WT_ASYNCOP_FREE); - return (ret); -} - -/* - * __wt_async_worker -- - * The async worker threads. - */ -WT_THREAD_RET -__wt_async_worker(void *arg) -{ - WT_ASYNC *async; - WT_ASYNC_CURSOR *ac; - WT_ASYNC_OP_IMPL *op; - WT_ASYNC_WORKER_STATE worker; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_SESSION_IMPL *session; - uint64_t flush_gen; - - session = arg; - conn = S2C(session); - async = conn->async; - - worker.num_cursors = 0; - TAILQ_INIT(&worker.cursorqh); - while (F_ISSET(conn, WT_CONN_SERVER_ASYNC) && F_ISSET(session, WT_SESSION_SERVER_ASYNC)) { - WT_ERR(__async_op_dequeue(conn, session, &op)); - if (op != NULL && op != &async->flush_op) { - /* - * Operation failure doesn't cause the worker thread to exit. - */ - (void)__async_worker_op(session, op, &worker); - } else if (async->flush_state == WT_ASYNC_FLUSHING) { - /* - * Worker flushing going on. Last worker to the party needs to clear the FLUSHING flag - * and signal the cond. If FLUSHING is going on, we do not take anything off the queue. - */ - WT_ORDERED_READ(flush_gen, async->flush_gen); - if (__wt_atomic_add32(&async->flush_count, 1) == conn->async_workers) { - /* - * We're last. All workers accounted for so signal the condition and clear the - * FLUSHING flag to release the other worker threads. Set the FLUSH_COMPLETE flag so - * that the caller can return to the application. - */ - WT_PUBLISH(async->flush_state, WT_ASYNC_FLUSH_COMPLETE); - __wt_cond_signal(session, async->flush_cond); - } else - /* - * We need to wait for the last worker to signal the condition. - */ - __async_flush_wait(session, async, flush_gen); - } - } - - if (0) { -err: - WT_IGNORE_RET(__wt_panic(session, ret, "async worker error")); - } - /* - * Worker thread cleanup, close our cached cursors and free all the WT_ASYNC_CURSOR structures. - */ - while ((ac = TAILQ_FIRST(&worker.cursorqh)) != NULL) { - TAILQ_REMOVE(&worker.cursorqh, ac, q); - WT_TRET(ac->c->close(ac->c)); - __wt_free(session, ac); - } - return (WT_THREAD_RET_VALUE); -} diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 6c7931360fb..e72009b98da 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -2,11 +2,6 @@ #include "wt_internal.h" -static const WT_CONFIG_CHECK confchk_WT_CONNECTION_async_new_op[] = { - {"append", "boolean", NULL, NULL, NULL, 0}, {"overwrite", "boolean", NULL, NULL, NULL, 0}, - {"raw", "boolean", NULL, NULL, NULL, 0}, {"timeout", "int", NULL, NULL, NULL, 0}, - {NULL, NULL, NULL, NULL, NULL, 0}}; - static const WT_CONFIG_CHECK confchk_WT_CONNECTION_close[] = { {"leak_memory", "boolean", NULL, NULL, NULL, 0}, {"use_timestamp", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -38,10 +33,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_query_timestamp[] = { NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; -static const WT_CONFIG_CHECK confchk_wiredtiger_open_async_subconfigs[] = { - {"enabled", "boolean", NULL, NULL, NULL, 0}, {"ops_max", "int", NULL, "min=1,max=4096", NULL, 0}, - {"threads", "int", NULL, "min=1,max=20", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; - static const WT_CONFIG_CHECK confchk_wiredtiger_open_checkpoint_subconfigs[] = { {"log_size", "int", NULL, "min=0,max=2GB", NULL, 0}, {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -99,7 +90,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_statistics_log_su {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { - {"async", "category", NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3}, {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, {"cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0}, @@ -223,6 +213,10 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create_encryption_subconfigs[] = {"keyid", "string", NULL, NULL, NULL, 0}, {"name", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_WT_SESSION_create_import_subconfigs[] = { + {"enabled", "boolean", NULL, NULL, NULL, 0}, {"file_metadata", "string", NULL, NULL, NULL, 0}, + {"repair", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_create_merge_custom_subconfigs[] = { {"prefix", "string", NULL, NULL, NULL, 0}, {"start_generation", "int", NULL, "min=0,max=10", NULL, 0}, @@ -258,6 +252,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"huffman_key", "string", NULL, NULL, NULL, 0}, {"huffman_value", "string", NULL, NULL, NULL, 0}, {"ignore_in_memory_cache_size", "boolean", NULL, NULL, NULL, 0}, {"immutable", "boolean", NULL, NULL, NULL, 0}, + {"import", "category", NULL, NULL, confchk_WT_SESSION_create_import_subconfigs, 3}, {"internal_item_max", "int", NULL, "min=0", NULL, 0}, {"internal_key_max", "int", NULL, "min=0", NULL, 0}, {"internal_key_truncate", "boolean", NULL, NULL, NULL, 0}, @@ -535,7 +530,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_transaction_sync_subconfigs {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { - {"async", "category", NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -612,7 +606,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { - {"async", "category", NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -689,7 +682,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { - {"async", "category", NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -761,7 +753,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { - {"async", "category", NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3}, {"buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0}, {"builtin_extension_config", "string", NULL, NULL, NULL, 0}, {"cache_cursors", "boolean", NULL, NULL, NULL, 0}, @@ -835,8 +826,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "", NULL, 0}, {"WT_CONNECTION.add_compressor", "", NULL, 0}, {"WT_CONNECTION.add_data_source", "", NULL, 0}, {"WT_CONNECTION.add_encryptor", "", NULL, 0}, {"WT_CONNECTION.add_extractor", "", NULL, 0}, - {"WT_CONNECTION.async_new_op", "append=false,overwrite=true,raw=false,timeout=1200", - confchk_WT_CONNECTION_async_new_op, 4}, {"WT_CONNECTION.close", "leak_memory=false,use_timestamp=true", confchk_WT_CONNECTION_close, 2}, {"WT_CONNECTION.debug_info", "cache=false,cursors=false,handles=false,log=false,sessions=false" @@ -852,9 +841,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", confchk_WT_CONNECTION_open_session, 3}, {"WT_CONNECTION.query_timestamp", "get=all_durable", confchk_WT_CONNECTION_query_timestamp, 1}, {"WT_CONNECTION.reconfigure", - "async=(enabled=false,ops_max=1024,threads=2),cache_max_wait_ms=0" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),compatibility=(release=)," + "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),compatibility=(release=)," "debug_mode=(checkpoint_retention=0,cursor_copy=false," "eviction=false,log_retention=0,realloc_exact=false," "rollback_error=0,slow_checkpoint=false,table_logging=false)," @@ -872,7 +860,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,verbose=", - confchk_WT_CONNECTION_reconfigure, 28}, + confchk_WT_CONNECTION_reconfigure, 27}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "commit_timestamp=,durable_timestamp=,force=false," @@ -908,6 +896,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "columns=,dictionary=0,encryption=(keyid=,name=),exclusive=false," "extractor=,format=btree,huffman_key=,huffman_value=," "ignore_in_memory_cache_size=false,immutable=false," + "import=(enabled=false,file_metadata=,repair=false)," "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," @@ -920,7 +909,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "prefix_compression=false,prefix_compression_min=4,source=," "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," "type=file,value_format=u", - confchk_WT_SESSION_create, 44}, + confchk_WT_SESSION_create, 45}, {"WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," "remove_files=true", @@ -1029,9 +1018,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "value_format=u", confchk_table_meta, 6}, {"wiredtiger_open", - "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" + ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," "config_base=true,create=false,debug_mode=(checkpoint_retention=0" @@ -1059,11 +1047,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,verify_metadata=false,write_through=", - confchk_wiredtiger_open, 56}, + confchk_wiredtiger_open, 55}, {"wiredtiger_open_all", - "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" + ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," "config_base=true,create=false,debug_mode=(checkpoint_retention=0" @@ -1092,11 +1079,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",method=fsync),use_environment=true,use_environment_priv=false," "verbose=,verify_metadata=false,version=(major=0,minor=0)," "write_through=", - confchk_wiredtiger_open_all, 57}, + confchk_wiredtiger_open_all, 56}, {"wiredtiger_open_basecfg", - "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" + ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," "debug_mode=(checkpoint_retention=0,cursor_copy=false," @@ -1124,11 +1110,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),verbose=,verify_metadata=false,version=(major=0," "minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 51}, + confchk_wiredtiger_open_basecfg, 50}, {"wiredtiger_open_usercfg", - "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",builtin_extension_config=,cache_cursors=true," - "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," + "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" + ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," "debug_mode=(checkpoint_retention=0,cursor_copy=false," @@ -1155,7 +1140,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),verbose=,verify_metadata=false,write_through=", - confchk_wiredtiger_open_usercfg, 50}, + confchk_wiredtiger_open_usercfg, 49}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 38bce98ca93..4fe1086f339 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -664,50 +664,6 @@ __wt_conn_remove_extractor(WT_SESSION_IMPL *session) } /* - * __conn_async_flush -- - * WT_CONNECTION.async_flush method. - */ -static int -__conn_async_flush(WT_CONNECTION *wt_conn) -{ - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - conn = (WT_CONNECTION_IMPL *)wt_conn; - CONNECTION_API_CALL_NOCONF(conn, session, async_flush); - WT_ERR(__wt_async_flush(session)); - -err: - API_END_RET_NOTFOUND_MAP(session, ret); -} - -/* - * __conn_async_new_op -- - * WT_CONNECTION.async_new_op method. - */ -static int -__conn_async_new_op(WT_CONNECTION *wt_conn, const char *uri, const char *config, - WT_ASYNC_CALLBACK *callback, WT_ASYNC_OP **asyncopp) -{ - WT_ASYNC_OP_IMPL *op; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - conn = (WT_CONNECTION_IMPL *)wt_conn; - CONNECTION_API_CALL(conn, session, async_new_op, config, cfg); - WT_UNUSED(cfg); - - WT_ERR(__wt_async_new_op(session, uri, config, callback, &op)); - - *asyncopp = &op->iface; - -err: - API_END_RET_NOTFOUND_MAP(session, ret); -} - -/* * __conn_get_extension_api -- * WT_CONNECTION.get_extension_api method. */ @@ -1055,20 +1011,13 @@ err: */ session->txn->isolation = WT_ISO_READ_UNCOMMITTED; - /* - * Clear any pending async operations and shut down the async worker threads and system before - * closing LSM. - */ - WT_TRET(__wt_async_flush(session)); - WT_TRET(__wt_async_destroy(session)); - WT_TRET(__wt_lsm_manager_destroy(session)); /* - * After the async and LSM threads have exited, we won't open more files for the application. - * However, the sweep server is still running and it can close file handles at the same time the - * final checkpoint is reviewing open data handles (forcing checkpoint to reopen handles). Shut - * down the sweep server. + * After the LSM threads have exited, we won't open more files for the application. However, the + * sweep server is still running and it can close file handles at the same time the final + * checkpoint is reviewing open data handles (forcing checkpoint to reopen handles). Shut down + * the sweep server. */ WT_TRET(__wt_sweep_destroy(session)); @@ -2228,9 +2177,6 @@ __conn_session_size(WT_SESSION_IMPL *session, const char *cfg[], uint32_t *vp) v = WT_EXTRA_INTERNAL_SESSIONS; /* Then, add in the thread counts applications can configure. */ - WT_RET(__wt_config_gets(session, cfg, "async.threads", &cval)); - v += cval.val; - WT_RET(__wt_config_gets(session, cfg, "eviction.threads_max", &cval)); v += cval.val; @@ -2361,12 +2307,11 @@ int wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *config, WT_CONNECTION **connectionp) { - static const WT_CONNECTION stdc = {__conn_async_flush, __conn_async_new_op, __conn_close, - __conn_debug_info, __conn_reconfigure, __conn_get_home, __conn_configure_method, - __conn_is_new, __conn_open_session, __conn_query_timestamp, __conn_set_timestamp, - __conn_rollback_to_stable, __conn_load_extension, __conn_add_data_source, __conn_add_collator, - __conn_add_compressor, __conn_add_encryptor, __conn_add_extractor, __conn_set_file_system, - __conn_get_extension_api}; + static const WT_CONNECTION stdc = {__conn_close, __conn_debug_info, __conn_reconfigure, + __conn_get_home, __conn_configure_method, __conn_is_new, __conn_open_session, + __conn_query_timestamp, __conn_set_timestamp, __conn_rollback_to_stable, + __conn_load_extension, __conn_add_data_source, __conn_add_collator, __conn_add_compressor, + __conn_add_encryptor, __conn_add_extractor, __conn_set_file_system, __conn_get_extension_api}; static const WT_NAME_FLAG file_types[] = {{"checkpoint", WT_DIRECT_IO_CHECKPOINT}, {"data", WT_DIRECT_IO_DATA}, {"log", WT_DIRECT_IO_LOG}, {NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 2840c6dc2e8..3c0ee7fe74c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -74,11 +74,10 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) session = conn->default_session; /* - * The LSM and async services are not shut down in this path (which is called when - * wiredtiger_open hits an error (as well as during normal shutdown). Assert they're not - * running. + * The LSM services are not shut down in this path (which is called when wiredtiger_open hits an + * error (as well as during normal shutdown). Assert they're not running. */ - WT_ASSERT(session, !F_ISSET(conn, WT_CONN_SERVER_ASYNC | WT_CONN_SERVER_LSM)); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_SERVER_LSM)); /* Shut down the subsystems, ensuring workers see the state change. */ F_SET(conn, WT_CONN_CLOSING); diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index 18c974e61a3..fe8c3953014 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -418,7 +418,6 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_ERR(ret); WT_ERR(__wt_conn_optrack_setup(session, cfg, true)); WT_ERR(__wt_conn_statistics_config(session, cfg)); - WT_ERR(__wt_async_reconfig(session, cfg)); WT_ERR(__wt_cache_config(session, true, cfg)); WT_ERR(__wt_capacity_server_create(session, cfg)); WT_ERR(__wt_checkpoint_server_create(session, cfg)); diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index cf54572f70a..69989939c43 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -73,7 +73,6 @@ __wt_conn_stat_init(WT_SESSION_IMPL *session) conn = S2C(session); stats = conn->stats; - __wt_async_stats_update(session); __wt_cache_stats_update(session); __wt_txn_stats_update(session); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index a05136d4f0c..146e794c025 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -1570,8 +1570,6 @@ INCLUDE_FILE_PATTERNS = # instead of the = operator. PREDEFINED = DOXYGEN \ - __wt_async_callback:=WT_ASYNC_CALLBACK \ - __wt_async_op:=WT_ASYNC_OP \ __wt_collator:=WT_COLLATOR \ __wt_compressor:=WT_COMPRESSOR \ __wt_config_arg:=WT_CONFIG_ARG \ diff --git a/src/third_party/wiredtiger/src/docs/async.dox b/src/third_party/wiredtiger/src/docs/async.dox deleted file mode 100644 index fcc0e590369..00000000000 --- a/src/third_party/wiredtiger/src/docs/async.dox +++ /dev/null @@ -1,135 +0,0 @@ -/*! @m_page{{c,java},async,Asynchronous operations} - -WiredTiger supports asynchronous operations; as an example of where this -can be useful, a server application handling requests from a network as -fast as possible may want its worker threads to initiate a unit of work -and then immediately respond to the next request, rather than waiting -for the results of the first request. - -WiredTiger supports asynchronous operations through the WT_ASYNC_OP -handle. The work unit represented by the WT_ASYNC_OP handle is queued -by the application and performed by an internal WiredTiger worker -thread. When the work unit completes, the WiredTiger thread makes a -callback to notify the application the operation is finished, along with -providing any results and error values. - -The asynchronous operation handle operates in a manner similar to a -WT_CURSOR handle. An asynchronous operation includes: - -- getter/setters for key and value fields -- encoding of fields to store in the data source -- methods to modify and retrieve specific data (for example, insert and update) -- a method to compact a table - -The WT_ASYNC_OP handle does not survive after the callback function -returns into the WiredTiger library. When the application callback -returns the handle is returned to the system pool. The application -callback must copy out any key, value or other information that it needs -before the callback function returns. - -@section async_config Configuring asynchronous operations - -To perform asynchronous operations, the application must first include -the \c async configuration option when ::wiredtiger_open is called. -Additional configuration parameters include the number of WiredTiger -worker threads created to handle the incoming queue of operations and -the maximum number of simultaneous asynchronous operations that are -expected. - -For example, the following configures an application for asynchronous -operations, with a maximum of 10 asynchronous operations and 2 supporting -threads: - -@snippet ex_async.c async example connection - -If the number of requests exceeds the configured maximum number, a -WT_ASYNC_OP handle won't immediately be available and an error will be -returned to the application when it attempts to allocate a handle. If -the number of configured worker threads are unable to keep up with the -requests, requests will be forced to wait for worker threads to become -available. - -@section async_alloc Allocating an asynchronous operations handle - -A WT_ASYNC_OP handle is allocated using the WT_CONNECTION::async_new_op -method. This method takes an existing object URI and a callback. For -example: - -@snippet ex_async.c async handle allocation - -To aid the application in matching up an asynchronous operation with a -subsequent call to the callback function, every handle contains a unique -\c uint64_t identifier and ::WT_ASYNC_OPTYPE type. The \c identifier is -assigned when the handle is allocated and the \c type is assigned when -the asynchronous operation is queued. - -To retrieve the id, use the WT_ASYNC_OP::get_id method: - -@snippet ex_async.c async get identifier - -To retrieve the ::WT_ASYNC_OPTYPE type, use the WT_ASYNC_OP::get_type method: - -@snippet ex_async.c async get type - -WiredTiger only allows a limited number of method calls back into the -library using the WT_ASYNC_OP handle, while in the callback function. -The application is allowed to retrieve than handle's key, value, -identifier and the operation type from the WT_ASYNC_OP handle. - -Here is a complete example callback function implementation, from -the example program @ex_ref{ex_async.c}: - -@snippet ex_async.c async example callback implementation - -@m_if{java} -@snippet ex_async.c async example callback implementation part 2 -@m_endif - -@section async_operations Executing asynchronous operations - -The WT_ASYNC_OP handle behaves similarly to the WT_CURSOR handle, that -is, the key and value are initialized and then an operation is -performed. - -For example, the following code does an asynchronous insert into the -table: - -@snippet ex_async.c async insert - -For example, the following code does an asynchronous search of the -table: - -@snippet ex_async.c async search - -When a database contains multiple tables, it may be desired to compact -several tables in parallel without having to manage separate threads -to each call WT_SESSION::compact. Alternatively, compacting several -tables serially may take much longer. The WT_ASYNC_OP::compact method -allows the application to compact multiple objects asynchronously. - -@snippet ex_async.c async compaction - -@section async_flush Waiting for outstanding operations to complete - -The WT_CONNECTION::async_flush method can be used to wait for all -previous operations to complete. When that call returns, all previously -queued operations are guaranteed to have been completed and their -callback functions have returned. - -@snippet ex_async.c async flush - -Because WT_CONNECTION::close implicitly does a WT_CONNECTION::async_flush, -the call is not required in all applications. - -@section async_transactions Asynchronous operations and transactions - -Each asynchronous worker thread operates in its own session, executing -a single asynchronous operation with the context of the session's -transaction. Therefore, there is no way to combine multiple, related -updates into a single transaction when using asynchronous operations. - -The transaction is committed if the operation was successful and the -application callback returns success, otherwise the transaction is -rolled back. - -*/ diff --git a/src/third_party/wiredtiger/src/docs/examples.dox b/src/third_party/wiredtiger/src/docs/examples.dox index c5a106a00c9..7d80894014e 100644 --- a/src/third_party/wiredtiger/src/docs/examples.dox +++ b/src/third_party/wiredtiger/src/docs/examples.dox @@ -2,9 +2,6 @@ @example ex_access.c Create, insert and access a simple table. -@example ex_async.c -Show how to configure and use asynchronous operations. - @example ex_call_center.c A more complex schema based on a call center example, showing how to map some SQL constructs onto the WiredTiger API. diff --git a/src/third_party/wiredtiger/src/docs/programming.dox b/src/third_party/wiredtiger/src/docs/programming.dox index 449e89eb231..d2b9e244b79 100644 --- a/src/third_party/wiredtiger/src/docs/programming.dox +++ b/src/third_party/wiredtiger/src/docs/programming.dox @@ -36,7 +36,6 @@ each of which is ordered by one or more columns. @m_endif <h2>Advanced topics</h2> -- @subpage async - @subpage backup - @subpage compact - @subpage checkpoint diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index d229012b626..af8fbff10a8 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -128,7 +128,6 @@ apiflags ar archiver arg -async atomicity autoconf autogen diff --git a/src/third_party/wiredtiger/src/docs/threads.dox b/src/third_party/wiredtiger/src/docs/threads.dox index 16b0d7940da..1021a1df81e 100644 --- a/src/third_party/wiredtiger/src/docs/threads.dox +++ b/src/third_party/wiredtiger/src/docs/threads.dox @@ -22,11 +22,6 @@ state in WiredTiger, and no built-in synchronization of session or cursor handles, either, so if multiple threads access a session or cursor handle, access must be serialized by the application. -WT_ASYNC_OP methods are not thread-safe, and must be accessed by only a -single thread at a time. WT_ASYNC_OP methods may be accessed by -different threads serially (and that is expected to happen when the -asynchronous callback function runs). - @section threads_example Code samples The code below is taken from the complete example program diff --git a/src/third_party/wiredtiger/src/docs/tools/doxfilter.py b/src/third_party/wiredtiger/src/docs/tools/doxfilter.py index b822c53c770..26c8e07c691 100755 --- a/src/third_party/wiredtiger/src/docs/tools/doxfilter.py +++ b/src/third_party/wiredtiger/src/docs/tools/doxfilter.py @@ -48,9 +48,7 @@ def java_post_substitutions(source): result = source for datatype in [['WT_CONNECTION', 'Connection'], ['WT_CURSOR', 'Cursor'], - ['WT_SESSION', 'Session'], - ['WT_ASYNC_OPTYPE', 'AsyncOpType'], - ['WT_ASYNC_OP', 'AsyncOp']]: + ['WT_SESSION', 'Session']]: fromdt = datatype[0] todt = datatype[1] diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index feacbea9f9c..5b072c319fd 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -4,6 +4,11 @@ @section version_322 Upgrading to Version 3.2.2 <dl> +<dt>Asynchronous API</dt> +<dd> +The asynchronous API has been removed from WiredTiger. +</dd> + <dt>Named snapshots</dt> <dd> Named snapshot functionality has been removed from WiredTiger as timestamps offer a better solution diff --git a/src/third_party/wiredtiger/src/docs/wtperf.dox b/src/third_party/wiredtiger/src/docs/wtperf.dox index d11a606dbdc..9c2aac3aae1 100644 --- a/src/third_party/wiredtiger/src/docs/wtperf.dox +++ b/src/third_party/wiredtiger/src/docs/wtperf.dox @@ -128,8 +128,6 @@ configuration options: DO NOT EDIT: THIS PART OF THE FILE IS GENERATED BY dist/s_docs. \endif -@par async_threads (unsigned int, default=0) -number of async worker threads @par checkpoint_interval (unsigned int, default=120) checkpoint every interval seconds during the workload phase. @par checkpoint_stress_rate (unsigned int, default=0) diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 749050edf6e..bf5b10c6bb6 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -2224,6 +2224,7 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) WT_REF *ref; WT_TRACK_OP_DECL; uint64_t time_start, time_stop; + uint32_t flags; uint8_t previous_state; bool app_timer; @@ -2236,6 +2237,8 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) cache = S2C(session)->cache; time_start = time_stop = 0; + flags = 0; + /* * An internal session flags either the server itself or an eviction worker thread. */ @@ -2254,6 +2257,10 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) } } + /* Set a flag to indicate that either eviction server or worker thread is evicting the page. */ + if (F_ISSET(session, WT_SESSION_INTERNAL)) + LF_SET(WT_REC_EVICTION_THREAD); + /* * In case something goes wrong, don't pick the same set of pages every time. * @@ -2263,7 +2270,7 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) */ __wt_cache_read_gen_bump(session, ref->page); - WT_WITH_BTREE(session, btree, ret = __wt_evict(session, ref, previous_state, 0)); + WT_WITH_BTREE(session, btree, ret = __wt_evict(session, ref, previous_state, flags)); (void)__wt_atomic_subv32(&btree->evict_busy, 1); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 7b432308b5c..c6fbfcb860d 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -546,6 +546,7 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool WT_PAGE *page; uint32_t flags; bool closing, modified; + bool snapshot_acquired; *inmem_splitp = false; @@ -553,8 +554,7 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool page = ref->page; flags = WT_REC_EVICT; closing = FLD_ISSET(evict_flags, WT_EVICT_CALL_CLOSING); - if (!WT_SESSION_BTREE_SYNC(session)) - LF_SET(WT_REC_VISIBLE_ALL); + snapshot_acquired = false; /* * Fail if an internal has active children, the children must be evicted first. The test is @@ -675,12 +675,64 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool LF_SET(WT_REC_SCRUB); } - /* Reconcile the page. */ + /* Acquire a snapshot if coming through eviction thread route. */ + + /* + * TODO: We are deliberately not using a snapshot when checkpoint is active. This will ensure + * that point-in-time checkpoints have a consistent version of data. Remove this condition once + * fuzzy transaction ID based checkpoints work is merged. + */ + if (FLD_ISSET(evict_flags, WT_REC_EVICTION_THREAD) && !WT_IS_HS(S2BT(session)) && + !conn->txn_global.checkpoint_running) { + /* + * We rely on the fact that the eviction threads are created with read committed isolation + * by default. If this fact doesn't hold anymore in future, we have to force isolation + * level. + */ + WT_ASSERT(session, session->txn->isolation == WT_ISO_READ_COMMITTED); + + /* + * Eviction threads do not need to pin anything in the cache. We have a exclusive lock for + * the page being evicted so we are sure that the page will always be there while it is + * being processed. Therefore, we use snapshot API that doesn't publish shared IDs to the + * outside world. + */ + __wt_txn_bump_snapshot(session); + + /* + * Make sure once more that there is no checkpoint running. A new checkpoint might have + * started between previous check and acquiring snapshot. If there is a checkpoint running, + * release the checkpoint and fallback to global visibility checks. + */ + if (conn->txn_global.checkpoint_running) { + __wt_txn_release_snapshot(session); + LF_SET(WT_REC_VISIBLE_ALL); + } else { + /* + * If we acquired a snapshot for eviction, force the isolation to ensure the snapshot + * isn't released when history store cursors are closed. + */ + ++session->txn->forced_iso; + snapshot_acquired = true; + } + } else if (!WT_SESSION_BTREE_SYNC(session)) + LF_SET(WT_REC_VISIBLE_ALL); + + WT_ASSERT(session, LF_ISSET(WT_REC_VISIBLE_ALL) || F_ISSET(session->txn, WT_TXN_HAS_SNAPSHOT)); + + /* + * Reconcile the page. + */ ret = __wt_reconcile(session, ref, NULL, flags); if (ret != 0) WT_STAT_CONN_INCR(session, cache_eviction_fail_in_reconciliation); + if (snapshot_acquired) { + --session->txn->forced_iso; + __wt_txn_release_snapshot(session); + } + WT_RET(ret); /* diff --git a/src/third_party/wiredtiger/src/history/hs.c b/src/third_party/wiredtiger/src/history/hs.c index a7ac8003228..f7588ca35c0 100644 --- a/src/third_party/wiredtiger/src/history/hs.c +++ b/src/third_party/wiredtiger/src/history/hs.c @@ -898,15 +898,19 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) continue; } - /* Skip updates that are already in the history store or are obsolete. */ - if (F_ISSET(upd, WT_UPDATE_HS | WT_UPDATE_OBSOLETE)) { + /* Skip updates that are already in the history store. */ + if (F_ISSET(upd, WT_UPDATE_HS)) { if (hs_inserted) WT_ERR_PANIC(session, WT_PANIC, - "Inserting updates older than obsolete updates or updates that are already " - "in the history store to the history store may corrupt the data."); + "Reinserting updates to the history store may corrupt the data as it may " + "clear the history store data newer than it."); continue; } + /* Skip updates that are obsolete. */ + if (F_ISSET(upd, WT_UPDATE_OBSOLETE)) + continue; + /* * If the time points are out of order (which can happen if the application performs * updates with out-of-order timestamps), so this value can never be seen, don't bother diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 0acb3c4097f..ed27d2854d0 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -256,7 +256,3 @@ if ((ret) == WT_PREPARE_CONFLICT) \ (ret) = WT_ROLLBACK; \ TXN_API_END(s, ret) - -#define ASYNCOP_API_CALL(conn, s, n) \ - s = (conn)->default_session; \ - API_CALL_NOCONF(s, asyncop, n, NULL) diff --git a/src/third_party/wiredtiger/src/include/async.h b/src/third_party/wiredtiger/src/include/async.h deleted file mode 100644 index 083de4423f4..00000000000 --- a/src/third_party/wiredtiger/src/include/async.h +++ /dev/null @@ -1,125 +0,0 @@ -/*- - * Copyright (c) 2014-2020 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#define MAX_ASYNC_SLEEP_USECS 100000 /* Maximum sleep waiting for work */ -#define MAX_ASYNC_YIELD 200 /* Maximum number of yields for work */ - -#define O2C(op) ((WT_CONNECTION_IMPL *)(op)->iface.connection) -#define O2S(op) (((WT_CONNECTION_IMPL *)(op)->iface.connection)->default_session) -/* - * WT_ASYNC_FORMAT -- - * The URI/config/format cache. - */ -struct __wt_async_format { - TAILQ_ENTRY(__wt_async_format) q; - const char *config; - uint64_t cfg_hash; /* Config hash */ - const char *uri; - uint64_t uri_hash; /* URI hash */ - const char *key_format; - const char *value_format; -}; - -/* - * WT_ASYNC_OP_IMPL -- - * Implementation of the WT_ASYNC_OP. - */ -struct __wt_async_op_impl { - WT_ASYNC_OP iface; - - WT_ASYNC_CALLBACK *cb; - - uint32_t internal_id; /* Array position id. */ - uint64_t unique_id; /* Unique identifier. */ - - WT_ASYNC_FORMAT *format; /* Format structure */ - -#define WT_ASYNCOP_ENQUEUED 0 /* Placed on the work queue */ -#define WT_ASYNCOP_FREE 1 /* Able to be allocated to user */ -#define WT_ASYNCOP_READY 2 /* Allocated, ready for user to use */ -#define WT_ASYNCOP_WORKING 3 /* Operation in progress by worker */ - uint32_t state; - - WT_ASYNC_OPTYPE optype; /* Operation type */ -}; - -/* - * Definition of the async subsystem. - */ -struct __wt_async { - /* - * Ops array protected by the ops_lock. - */ - WT_SPINLOCK ops_lock; /* Locked: ops array */ - WT_ASYNC_OP_IMPL *async_ops; /* Async ops */ -#define OPS_INVALID_INDEX 0xffffffff - uint32_t ops_index; /* Active slot index */ - uint64_t op_id; /* Unique ID counter */ - WT_ASYNC_OP_IMPL **async_queue; /* Async ops work queue */ - uint32_t async_qsize; /* Async work queue size */ - /* - * We need to have two head and tail values. All but one is - * maintained as an ever increasing value to ease wrap around. - * - * alloc_head: the next one to allocate for producers. head: the - * current head visible to consumers. head is always <= - * alloc_head. alloc_tail: the next slot for consumers to - * dequeue. alloc_tail is always <= head. tail_slot: the last - * slot consumed. A producer may need wait for tail_slot to - * advance. - */ - uint64_t alloc_head; /* Next slot to enqueue */ - uint64_t head; /* Head visible to worker */ - uint64_t alloc_tail; /* Next slot to dequeue */ - uint64_t tail_slot; /* Worker slot consumed */ - - TAILQ_HEAD(__wt_async_format_qh, __wt_async_format) formatqh; - uint32_t cur_queue; /* Currently enqueued */ - uint32_t max_queue; /* Maximum enqueued */ - -#define WT_ASYNC_FLUSH_NONE 0 /* No flush in progress */ -#define WT_ASYNC_FLUSH_COMPLETE 1 /* Notify flush caller done */ -#define WT_ASYNC_FLUSH_IN_PROGRESS 2 /* Prevent other callers */ -#define WT_ASYNC_FLUSHING 3 /* Notify workers */ - uint32_t flush_state; - - /* Notify any waiting threads when flushing is done. */ - WT_CONDVAR *flush_cond; - WT_ASYNC_OP_IMPL flush_op; /* Special flush op */ - uint32_t flush_count; /* Worker count */ - uint64_t flush_gen; /* Flush generation number */ - -#define WT_ASYNC_MAX_WORKERS 20 - WT_SESSION_IMPL *worker_sessions[WT_ASYNC_MAX_WORKERS]; - /* Async worker threads */ - wt_thread_t worker_tids[WT_ASYNC_MAX_WORKERS]; - - uint32_t flags; /* Currently unused. */ -}; - -/* - * WT_ASYNC_CURSOR -- - * Async container for a cursor. Each async worker thread - * has a cache of async cursors to reuse for operations. - */ -struct __wt_async_cursor { - TAILQ_ENTRY(__wt_async_cursor) q; /* Worker cache */ - uint64_t cfg_hash; /* Config hash */ - uint64_t uri_hash; /* URI hash */ - WT_CURSOR *c; /* WT cursor */ -}; - -/* - * WT_ASYNC_WORKER_STATE -- - * State for an async worker thread. - */ -struct __wt_async_worker_state { - uint32_t id; - TAILQ_HEAD(__wt_cursor_qh, __wt_async_cursor) cursorqh; - uint32_t num_cursors; -}; diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index baa8fbbdb50..49586073665 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -29,11 +29,12 @@ #define WT_REC_CHECKPOINT 0x002u #define WT_REC_CLEAN_AFTER_REC 0x004u #define WT_REC_EVICT 0x008u -#define WT_REC_HS 0x010u -#define WT_REC_IN_MEMORY 0x020u -#define WT_REC_SCRUB 0x040u -#define WT_REC_VISIBILITY_ERR 0x080u -#define WT_REC_VISIBLE_ALL 0x100u +#define WT_REC_EVICTION_THREAD 0x010u +#define WT_REC_HS 0x020u +#define WT_REC_IN_MEMORY 0x040u +#define WT_REC_SCRUB 0x080u +#define WT_REC_VISIBILITY_ERR 0x100u +#define WT_REC_VISIBLE_ALL 0x200u /* AUTOMATIC FLAG VALUE GENERATION STOP */ /* diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 0be38097dba..a297ea6eb4c 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -56,55 +56,54 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_CONNECTION_add_data_source 2 #define WT_CONFIG_ENTRY_WT_CONNECTION_add_encryptor 3 #define WT_CONFIG_ENTRY_WT_CONNECTION_add_extractor 4 -#define WT_CONFIG_ENTRY_WT_CONNECTION_async_new_op 5 -#define WT_CONFIG_ENTRY_WT_CONNECTION_close 6 -#define WT_CONFIG_ENTRY_WT_CONNECTION_debug_info 7 -#define WT_CONFIG_ENTRY_WT_CONNECTION_load_extension 8 -#define WT_CONFIG_ENTRY_WT_CONNECTION_open_session 9 -#define WT_CONFIG_ENTRY_WT_CONNECTION_query_timestamp 10 -#define WT_CONFIG_ENTRY_WT_CONNECTION_reconfigure 11 -#define WT_CONFIG_ENTRY_WT_CONNECTION_rollback_to_stable 12 -#define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 13 -#define WT_CONFIG_ENTRY_WT_CONNECTION_set_timestamp 14 -#define WT_CONFIG_ENTRY_WT_CURSOR_close 15 -#define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 16 -#define WT_CONFIG_ENTRY_WT_SESSION_alter 17 -#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 18 -#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 19 -#define WT_CONFIG_ENTRY_WT_SESSION_close 20 -#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 21 -#define WT_CONFIG_ENTRY_WT_SESSION_compact 22 -#define WT_CONFIG_ENTRY_WT_SESSION_create 23 -#define WT_CONFIG_ENTRY_WT_SESSION_drop 24 -#define WT_CONFIG_ENTRY_WT_SESSION_import 25 -#define WT_CONFIG_ENTRY_WT_SESSION_join 26 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 27 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 28 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 29 -#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 30 -#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 31 -#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 32 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 33 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 34 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 35 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 36 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 37 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 38 -#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 39 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 40 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 41 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 42 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 43 -#define WT_CONFIG_ENTRY_colgroup_meta 44 -#define WT_CONFIG_ENTRY_file_config 45 -#define WT_CONFIG_ENTRY_file_meta 46 -#define WT_CONFIG_ENTRY_index_meta 47 -#define WT_CONFIG_ENTRY_lsm_meta 48 -#define WT_CONFIG_ENTRY_table_meta 49 -#define WT_CONFIG_ENTRY_wiredtiger_open 50 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 51 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 52 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 53 +#define WT_CONFIG_ENTRY_WT_CONNECTION_close 5 +#define WT_CONFIG_ENTRY_WT_CONNECTION_debug_info 6 +#define WT_CONFIG_ENTRY_WT_CONNECTION_load_extension 7 +#define WT_CONFIG_ENTRY_WT_CONNECTION_open_session 8 +#define WT_CONFIG_ENTRY_WT_CONNECTION_query_timestamp 9 +#define WT_CONFIG_ENTRY_WT_CONNECTION_reconfigure 10 +#define WT_CONFIG_ENTRY_WT_CONNECTION_rollback_to_stable 11 +#define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 12 +#define WT_CONFIG_ENTRY_WT_CONNECTION_set_timestamp 13 +#define WT_CONFIG_ENTRY_WT_CURSOR_close 14 +#define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 15 +#define WT_CONFIG_ENTRY_WT_SESSION_alter 16 +#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 17 +#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 18 +#define WT_CONFIG_ENTRY_WT_SESSION_close 19 +#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 20 +#define WT_CONFIG_ENTRY_WT_SESSION_compact 21 +#define WT_CONFIG_ENTRY_WT_SESSION_create 22 +#define WT_CONFIG_ENTRY_WT_SESSION_drop 23 +#define WT_CONFIG_ENTRY_WT_SESSION_import 24 +#define WT_CONFIG_ENTRY_WT_SESSION_join 25 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 26 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 27 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 28 +#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 29 +#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 30 +#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 31 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 32 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 33 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 34 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 35 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 36 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 37 +#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 38 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 39 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 40 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 41 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 42 +#define WT_CONFIG_ENTRY_colgroup_meta 43 +#define WT_CONFIG_ENTRY_file_config 44 +#define WT_CONFIG_ENTRY_file_meta 45 +#define WT_CONFIG_ENTRY_index_meta 46 +#define WT_CONFIG_ENTRY_lsm_meta 47 +#define WT_CONFIG_ENTRY_table_meta 48 +#define WT_CONFIG_ENTRY_wiredtiger_open 49 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 50 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 51 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 52 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 48b7784d23f..a4befa30322 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -318,11 +318,6 @@ struct __wt_connection_impl { WT_CONNECTION_STATS *stats[WT_COUNTER_SLOTS]; WT_CONNECTION_STATS *stat_array; - WT_ASYNC *async; /* Async structure */ - bool async_cfg; /* Global async configuration */ - uint32_t async_size; /* Async op array size */ - uint32_t async_workers; /* Number of async workers */ - WT_CAPACITY capacity; /* Capacity structure */ WT_SESSION_IMPL *capacity_session; /* Capacity thread session */ wt_thread_t capacity_tid; /* Capacity thread */ @@ -535,35 +530,34 @@ struct __wt_connection_impl { WT_FILE_SYSTEM *file_system; /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CONN_CACHE_CURSORS 0x00000001u -#define WT_CONN_CACHE_POOL 0x00000002u -#define WT_CONN_CKPT_SYNC 0x00000004u -#define WT_CONN_CLOSING 0x00000008u -#define WT_CONN_CLOSING_NO_MORE_OPENS 0x00000010u -#define WT_CONN_CLOSING_TIMESTAMP 0x00000020u -#define WT_CONN_COMPATIBILITY 0x00000040u -#define WT_CONN_DATA_CORRUPTION 0x00000080u -#define WT_CONN_EVICTION_RUN 0x00000100u -#define WT_CONN_FILE_CLOSE_SYNC 0x00000200u -#define WT_CONN_HS_OPEN 0x00000400u -#define WT_CONN_INCR_BACKUP 0x00000800u -#define WT_CONN_IN_MEMORY 0x00001000u -#define WT_CONN_LEAK_MEMORY 0x00002000u -#define WT_CONN_LSM_MERGE 0x00004000u -#define WT_CONN_OPTRACK 0x00008000u -#define WT_CONN_PANIC 0x00010000u -#define WT_CONN_READONLY 0x00020000u -#define WT_CONN_RECONFIGURING 0x00040000u -#define WT_CONN_RECOVERING 0x00080000u -#define WT_CONN_SALVAGE 0x00100000u -#define WT_CONN_SERVER_ASYNC 0x00200000u -#define WT_CONN_SERVER_CAPACITY 0x00400000u -#define WT_CONN_SERVER_CHECKPOINT 0x00800000u -#define WT_CONN_SERVER_LOG 0x01000000u -#define WT_CONN_SERVER_LSM 0x02000000u -#define WT_CONN_SERVER_STATISTICS 0x04000000u -#define WT_CONN_SERVER_SWEEP 0x08000000u -#define WT_CONN_WAS_BACKUP 0x10000000u +#define WT_CONN_CACHE_CURSORS 0x0000001u +#define WT_CONN_CACHE_POOL 0x0000002u +#define WT_CONN_CKPT_SYNC 0x0000004u +#define WT_CONN_CLOSING 0x0000008u +#define WT_CONN_CLOSING_NO_MORE_OPENS 0x0000010u +#define WT_CONN_CLOSING_TIMESTAMP 0x0000020u +#define WT_CONN_COMPATIBILITY 0x0000040u +#define WT_CONN_DATA_CORRUPTION 0x0000080u +#define WT_CONN_EVICTION_RUN 0x0000100u +#define WT_CONN_FILE_CLOSE_SYNC 0x0000200u +#define WT_CONN_HS_OPEN 0x0000400u +#define WT_CONN_INCR_BACKUP 0x0000800u +#define WT_CONN_IN_MEMORY 0x0001000u +#define WT_CONN_LEAK_MEMORY 0x0002000u +#define WT_CONN_LSM_MERGE 0x0004000u +#define WT_CONN_OPTRACK 0x0008000u +#define WT_CONN_PANIC 0x0010000u +#define WT_CONN_READONLY 0x0020000u +#define WT_CONN_RECONFIGURING 0x0040000u +#define WT_CONN_RECOVERING 0x0080000u +#define WT_CONN_SALVAGE 0x0100000u +#define WT_CONN_SERVER_CAPACITY 0x0200000u +#define WT_CONN_SERVER_CHECKPOINT 0x0400000u +#define WT_CONN_SERVER_LOG 0x0800000u +#define WT_CONN_SERVER_LSM 0x1000000u +#define WT_CONN_SERVER_STATISTICS 0x2000000u +#define WT_CONN_SERVER_SWEEP 0x4000000u +#define WT_CONN_WAS_BACKUP 0x8000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 4939b3d3952..9e61d1ad53d 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -2,7 +2,6 @@ extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const ch WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern WT_THREAD_RET __wt_async_worker(void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_THREAD_RET __wt_cache_pool_server(void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_UPDATE *__wt_update_obsolete_check(WT_SESSION_IMPL *session, WT_PAGE *page, @@ -72,20 +71,6 @@ extern const char *__wt_wiredtiger_error(int error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, WT_CURSOR *cur, WT_CURSOR_TABLE *ctable, int (*f)(WT_CURSOR *)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_async_destroy(WT_SESSION_IMPL *session) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_async_flush(WT_SESSION_IMPL *session) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_async_new_op(WT_SESSION_IMPL *session, const char *uri, const char *config, - WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP_IMPL **opp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_async_op_init(WT_SESSION_IMPL *session) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_backup_file_remove(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_backup_load_incr(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *blkcfg, @@ -1607,7 +1592,6 @@ extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base) WT_GCC_F extern void *__wt_ext_scr_alloc(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t size); extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_async_stats_update(WT_SESSION_IMPL *session); extern void __wt_backup_destroy(WT_SESSION_IMPL *session); extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci); extern void __wt_block_configure_first_fit(WT_BLOCK *block, bool on); diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 355cc5f1343..8fb161bd655 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -194,7 +194,6 @@ struct __wt_session_impl { #define WT_SESSION_RESOLVING_TXN 0x04000000u #define WT_SESSION_ROLLBACK_TO_STABLE 0x08000000u #define WT_SESSION_SCHEMA_TXN 0x10000000u -#define WT_SESSION_SERVER_ASYNC 0x20000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 9217a31c310..2b80c498fa8 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -305,19 +305,6 @@ struct __wt_connection_stats { int64_t lsm_work_units_done; int64_t lsm_work_units_created; int64_t lsm_work_queue_max; - int64_t async_cur_queue; - int64_t async_max_queue; - int64_t async_alloc_race; - int64_t async_flush; - int64_t async_alloc_view; - int64_t async_full; - int64_t async_nowork; - int64_t async_op_alloc; - int64_t async_op_compact; - int64_t async_op_insert; - int64_t async_op_remove; - int64_t async_op_search; - int64_t async_op_update; int64_t block_preload; int64_t block_read; int64_t block_write; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index c3e7a0e9dc9..087b2a9f0c6 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -61,9 +61,6 @@ extern "C" { /******************************************* * Public forward structure declarations *******************************************/ -struct __wt_async_callback; - typedef struct __wt_async_callback WT_ASYNC_CALLBACK; -struct __wt_async_op; typedef struct __wt_async_op WT_ASYNC_OP; struct __wt_collator; typedef struct __wt_collator WT_COLLATOR; struct __wt_compressor; typedef struct __wt_compressor WT_COMPRESSOR; struct __wt_config_item; typedef struct __wt_config_item WT_CONFIG_ITEM; @@ -733,199 +730,6 @@ struct __wt_cursor { #endif }; -/*! Asynchronous operation types. */ -typedef enum { - WT_AOP_NONE=0, /*!< No operation type set */ - WT_AOP_COMPACT, /*!< WT_ASYNC_OP::compact */ - WT_AOP_INSERT, /*!< WT_ASYNC_OP::insert */ - WT_AOP_REMOVE, /*!< WT_ASYNC_OP::remove */ - WT_AOP_SEARCH, /*!< WT_ASYNC_OP::search */ - WT_AOP_UPDATE /*!< WT_ASYNC_OP::update */ -} WT_ASYNC_OPTYPE; - -/*! - * A WT_ASYNC_OP handle is the interface to an asynchronous operation. - * - * An asynchronous operation describes a data manipulation to be performed - * asynchronously by a WiredTiger worker thread. These operations implement - * the CRUD (create, read, update and delete) operations. Each operation - * is a self-contained work unit. The operation will be performed in the - * context of the worker thread's session. Each operation is performed - * within the context of a transaction. The application is notified of its - * completion with a callback. The transaction is resolved once the callback - * returns. - * - * The table referenced in an operation must already exist. - * - * Raw data is represented by key/value pairs of WT_ITEM structures, but - * operations can also provide access to fields within the key and value if - * the formats are described in the WT_SESSION::create method. - * - * <b>Thread safety:</b> A WT_ASYNC_OP handle may not be shared between - * threads, see @ref threads for more information. - */ -struct __wt_async_op { - /*! The connection for this operation. */ - WT_CONNECTION *connection; - - /*! - * The format of the data packed into key items. See @ref packing for - * details. If not set, a default value of "u" is assumed, and - * applications must use WT_ITEM structures to manipulate untyped byte - * arrays. - */ - const char *key_format; - - /*! - * The format of the data packed into value items. See @ref packing - * for details. If not set, a default value of "u" is assumed, and - * applications must use WT_ITEM structures to manipulate untyped byte - * arrays. - */ - const char *value_format; - - /* - * Don't expose app_private to non-C language bindings - they have - * their own way to attach data to an operation. - */ -#if !defined(SWIG) - /*! - * A location for applications to store information that will be - * available in the callback from an async operation. - */ - void *app_private; -#endif - - /*! - * @name Data access - * @{ - */ - /*! - * Invoke the underlying WT_CURSOR::get_key method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - * @returns as described for WT_CURSOR::get_key - */ - int __F(get_key)(WT_ASYNC_OP *op, ...); - - /*! - * Invoke the underlying WT_CURSOR::get_value method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - * @returns as described for WT_CURSOR::get_value - */ - int __F(get_value)(WT_ASYNC_OP *op, ...); - - /*! - * Invoke the underlying WT_CURSOR::set_key method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - */ - void __F(set_key)(WT_ASYNC_OP *op, ...); - - /*! - * Invoke the underlying WT_CURSOR::set_value method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - */ - void __F(set_value)(WT_ASYNC_OP *op, ...); - /*! @} */ - - /*! - * @name Positioning - * @{ - */ - /*! - * Invoke the underlying WT_CURSOR::search method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - * @returns via the callback as described for WT_CURSOR::search - */ - int __F(search)(WT_ASYNC_OP *op); - /*! @} */ - - /*! - * @name Data modification - * @{ - */ - /*! - * Invoke the underlying WT_CURSOR::insert method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - * @returns via the callback as described for WT_CURSOR::insert - */ - int __F(insert)(WT_ASYNC_OP *op); - - /*! - * Invoke the underlying WT_CURSOR::update method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - * @returns via the callback as described for WT_CURSOR::update - */ - int __F(update)(WT_ASYNC_OP *op); - - /*! - * Invoke the underlying WT_CURSOR::remove method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - * @returns via the callback as described for WT_CURSOR::remove - */ - int __F(remove)(WT_ASYNC_OP *op); - /*! @} */ - - /*! - * @name Table operations - * @{ - */ - /*! - * Invoke the underlying WT_SESSION::compact method; see that method - * for configuration, return and error values. - * - * @param op the operation handle - * @returns via the callback as described for WT_SESSION::compact - */ - int __F(compact)(WT_ASYNC_OP *op); - /*! @} */ - - /*! - * Get the unique identifier for this operation. - * - * @snippet ex_async.c async get identifier - * - * @param op the operation handle - * @returns the id of the operation - */ - uint64_t __F(get_id)(WT_ASYNC_OP *op); - - /*! - * Get the type for this operation. - * - * @snippet ex_async.c async get type - * - * @param op the operation handle - * @returns the ::WT_ASYNC_OPTYPE of the operation - */ - WT_ASYNC_OPTYPE __F(get_type)(WT_ASYNC_OP *op); - - /* - * Protected fields, only to be used by internal implementation. - * Everything we need for maintaining the key/value is part of - * a cursor. So, include one here so that we can use the cursor - * functions to manage them. - */ -#if !defined(SWIG) && !defined(DOXYGEN) - WT_CURSOR c; -#endif -}; - /*! * All data operations are performed in the context of a WT_SESSION. This * encapsulates the thread and transactional context of the operation. @@ -1283,6 +1087,16 @@ struct __wt_session { * the configured cache limit., a boolean flag; default \c false.} * @config{immutable, configure the index to be immutable - that is an index is not changed * by any update to a record in the table., a boolean flag; default \c false.} + * @config{import = (, configure import of an existing object into the currently running + * database., a set of related configuration options defined below.} + * @config{ enabled, whether to import the input URI from disk., a + * boolean flag; default \c false.} + * @config{ file_metadata, the file + * configuration extracted from the metadata of the export database., a string; default + * empty.} + * @config{ repair, whether to reconstruct the metadata from + * the raw file content., a boolean flag; default \c false.} + * @config{ ),,} * @config{internal_key_max, the largest key stored in an internal node\, in bytes. If * set\, keys larger than the specified size are stored as overflow items (which may require * additional I/O to access). The default and the maximum allowed value are both one-tenth @@ -2048,51 +1862,6 @@ struct __wt_session { */ struct __wt_connection { /*! - * @name Async operation handles - * @{ - */ - /*! - * Wait for all outstanding operations to complete. - * - * @snippet ex_async.c async flush - * - * @param connection the connection handle - * @errors - */ - int __F(async_flush)(WT_CONNECTION *connection); - - /*! - * Return an async operation handle - * - * @snippet ex_async.c async handle allocation - * - * @param connection the connection handle - * @param uri the connection handle - * @configstart{WT_CONNECTION.async_new_op, see dist/api_data.py} - * @config{append, append the value as a new record\, creating a new record number key; - * valid only for operations with record number keys., a boolean flag; default \c false.} - * @config{overwrite, configures whether the cursor's insert\, update and remove methods - * check the existing state of the record. If \c overwrite is \c false\, WT_CURSOR::insert - * fails with ::WT_DUPLICATE_KEY if the record exists\, WT_CURSOR::update and - * WT_CURSOR::remove fail with ::WT_NOTFOUND if the record does not exist., a boolean flag; - * default \c true.} - * @config{raw, ignore the encodings for the key and value\, manage data as if the formats - * were \c "u". See @ref cursor_raw for details., a boolean flag; default \c false.} - * @config{timeout, maximum amount of time to allow for compact in seconds. The actual - * amount of time spent in compact may exceed the configured value. A value of zero - * disables the timeout., an integer; default \c 1200.} - * @configend - * @param callback the operation callback - * @param[out] asyncopp the new op handle - * @errors - * If there are no available handles, \c EBUSY is returned. - */ - int __F(async_new_op)(WT_CONNECTION *connection, - const char *uri, const char *config, WT_ASYNC_CALLBACK *callback, - WT_ASYNC_OP **asyncopp); - /*! @} */ - - /*! * Close a connection. * * Any open sessions will be closed. This will release the resources @@ -2143,16 +1912,6 @@ struct __wt_connection { * * @param connection the connection handle * @configstart{WT_CONNECTION.reconfigure, see dist/api_data.py} - * @config{async = (, asynchronous operations configuration options., a set of related - * configuration options defined below.} - * @config{ enabled, enable - * asynchronous operation., a boolean flag; default \c false.} - * @config{ ops_max, maximum number of expected simultaneous - * asynchronous operations., an integer between 1 and 4096; default \c 1024.} - * @config{ threads, the number of worker threads to service - * asynchronous requests. Each worker thread uses a session from the configured - * session_max., an integer between 1 and 20; default \c 2.} - * @config{ ),,} * @config{cache_max_wait_ms, the maximum number of milliseconds an application thread will * wait for space to be available in cache before giving up. Default will wait forever., an * integer greater than or equal to 0; default \c 0.} @@ -2747,16 +2506,6 @@ struct __wt_connection { * event handler is installed that writes error messages to stderr. See * @ref event_message_handling for more information. * @configstart{wiredtiger_open, see dist/api_data.py} - * @config{async = (, asynchronous operations configuration options., a set of related configuration - * options defined below.} - * @config{ enabled, enable asynchronous operation., - * a boolean flag; default \c false.} - * @config{ ops_max, maximum number of - * expected simultaneous asynchronous operations., an integer between 1 and 4096; default \c 1024.} - * @config{ threads, the number of worker threads to service asynchronous - * requests. Each worker thread uses a session from the configured session_max., an integer between - * 1 and 20; default \c 2.} - * @config{ ),,} * @config{buffer_alignment, in-memory alignment (in bytes) for buffers used for I/O. The default * value of -1 indicates a platform-specific alignment value should be used (4KB on Linux systems * when direct I/O is configured\, zero elsewhere)., an integer between -1 and 1MB; default \c -1.} @@ -3156,32 +2905,6 @@ int wiredtiger_open(const char *home, */ const char *wiredtiger_strerror(int error) WT_ATTRIBUTE_LIBRARY_VISIBLE; -#if !defined(SWIG) -/*! - * The interface implemented by applications to accept notifications - * of the completion of asynchronous operations. - * - * Applications register their implementation with WiredTiger by calling - * WT_CONNECTION::async_new_op. - * - * @snippet ex_async.c async handle allocation - */ -struct __wt_async_callback { - /*! - * Callback to receive completion notification. - * - * @param[in] op the operation handle - * @param[in] op_ret the result of the async operation - * @param[in] flags currently unused - * @returns zero for success, non-zero to indicate an error. - * - * @snippet ex_async.c async example callback implementation - */ - int (*notify)(WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP *op, - int op_ret, uint32_t flags); -}; -#endif - /*! * The interface implemented by applications to handle error, informational and * progress messages. Entries set to NULL are ignored and the default handlers @@ -4988,1140 +4711,1114 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1008 /*! LSM: tree queue hit maximum */ #define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1009 -/*! async: current work queue length */ -#define WT_STAT_CONN_ASYNC_CUR_QUEUE 1010 -/*! async: maximum work queue length */ -#define WT_STAT_CONN_ASYNC_MAX_QUEUE 1011 -/*! async: number of allocation state races */ -#define WT_STAT_CONN_ASYNC_ALLOC_RACE 1012 -/*! async: number of flush calls */ -#define WT_STAT_CONN_ASYNC_FLUSH 1013 -/*! async: number of operation slots viewed for allocation */ -#define WT_STAT_CONN_ASYNC_ALLOC_VIEW 1014 -/*! async: number of times operation allocation failed */ -#define WT_STAT_CONN_ASYNC_FULL 1015 -/*! async: number of times worker found no work */ -#define WT_STAT_CONN_ASYNC_NOWORK 1016 -/*! async: total allocations */ -#define WT_STAT_CONN_ASYNC_OP_ALLOC 1017 -/*! async: total compact calls */ -#define WT_STAT_CONN_ASYNC_OP_COMPACT 1018 -/*! async: total insert calls */ -#define WT_STAT_CONN_ASYNC_OP_INSERT 1019 -/*! async: total remove calls */ -#define WT_STAT_CONN_ASYNC_OP_REMOVE 1020 -/*! async: total search calls */ -#define WT_STAT_CONN_ASYNC_OP_SEARCH 1021 -/*! async: total update calls */ -#define WT_STAT_CONN_ASYNC_OP_UPDATE 1022 /*! block-manager: blocks pre-loaded */ -#define WT_STAT_CONN_BLOCK_PRELOAD 1023 +#define WT_STAT_CONN_BLOCK_PRELOAD 1010 /*! block-manager: blocks read */ -#define WT_STAT_CONN_BLOCK_READ 1024 +#define WT_STAT_CONN_BLOCK_READ 1011 /*! block-manager: blocks written */ -#define WT_STAT_CONN_BLOCK_WRITE 1025 +#define WT_STAT_CONN_BLOCK_WRITE 1012 /*! block-manager: bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_READ 1026 +#define WT_STAT_CONN_BLOCK_BYTE_READ 1013 /*! block-manager: bytes read via memory map API */ -#define WT_STAT_CONN_BLOCK_BYTE_READ_MMAP 1027 +#define WT_STAT_CONN_BLOCK_BYTE_READ_MMAP 1014 /*! block-manager: bytes read via system call API */ -#define WT_STAT_CONN_BLOCK_BYTE_READ_SYSCALL 1028 +#define WT_STAT_CONN_BLOCK_BYTE_READ_SYSCALL 1015 /*! block-manager: bytes written */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1029 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1016 /*! block-manager: bytes written for checkpoint */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE_CHECKPOINT 1030 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE_CHECKPOINT 1017 /*! block-manager: bytes written via memory map API */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE_MMAP 1031 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE_MMAP 1018 /*! block-manager: bytes written via system call API */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE_SYSCALL 1032 +#define WT_STAT_CONN_BLOCK_BYTE_WRITE_SYSCALL 1019 /*! block-manager: mapped blocks read */ -#define WT_STAT_CONN_BLOCK_MAP_READ 1033 +#define WT_STAT_CONN_BLOCK_MAP_READ 1020 /*! block-manager: mapped bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1034 +#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1021 /*! * block-manager: number of times the file was remapped because it * changed size via fallocate or truncate */ -#define WT_STAT_CONN_BLOCK_REMAP_FILE_RESIZE 1035 +#define WT_STAT_CONN_BLOCK_REMAP_FILE_RESIZE 1022 /*! block-manager: number of times the region was remapped via write */ -#define WT_STAT_CONN_BLOCK_REMAP_FILE_WRITE 1036 +#define WT_STAT_CONN_BLOCK_REMAP_FILE_WRITE 1023 /*! cache: application threads page read from disk to cache count */ -#define WT_STAT_CONN_CACHE_READ_APP_COUNT 1037 +#define WT_STAT_CONN_CACHE_READ_APP_COUNT 1024 /*! cache: application threads page read from disk to cache time (usecs) */ -#define WT_STAT_CONN_CACHE_READ_APP_TIME 1038 +#define WT_STAT_CONN_CACHE_READ_APP_TIME 1025 /*! cache: application threads page write from cache to disk count */ -#define WT_STAT_CONN_CACHE_WRITE_APP_COUNT 1039 +#define WT_STAT_CONN_CACHE_WRITE_APP_COUNT 1026 /*! cache: application threads page write from cache to disk time (usecs) */ -#define WT_STAT_CONN_CACHE_WRITE_APP_TIME 1040 +#define WT_STAT_CONN_CACHE_WRITE_APP_TIME 1027 /*! cache: bytes allocated for updates */ -#define WT_STAT_CONN_CACHE_BYTES_UPDATES 1041 +#define WT_STAT_CONN_CACHE_BYTES_UPDATES 1028 /*! cache: bytes belonging to page images in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_IMAGE 1042 +#define WT_STAT_CONN_CACHE_BYTES_IMAGE 1029 /*! cache: bytes belonging to the history store table in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_HS 1043 +#define WT_STAT_CONN_CACHE_BYTES_HS 1030 /*! cache: bytes currently in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INUSE 1044 +#define WT_STAT_CONN_CACHE_BYTES_INUSE 1031 /*! cache: bytes dirty in the cache cumulative */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1045 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1032 /*! cache: bytes not belonging to page images in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_OTHER 1046 +#define WT_STAT_CONN_CACHE_BYTES_OTHER 1033 /*! cache: bytes read into cache */ -#define WT_STAT_CONN_CACHE_BYTES_READ 1047 +#define WT_STAT_CONN_CACHE_BYTES_READ 1034 /*! cache: bytes written from cache */ -#define WT_STAT_CONN_CACHE_BYTES_WRITE 1048 +#define WT_STAT_CONN_CACHE_BYTES_WRITE 1035 /*! cache: cache overflow score */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_SCORE 1049 +#define WT_STAT_CONN_CACHE_LOOKASIDE_SCORE 1036 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1050 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1037 /*! cache: eviction calls to get a page */ -#define WT_STAT_CONN_CACHE_EVICTION_GET_REF 1051 +#define WT_STAT_CONN_CACHE_EVICTION_GET_REF 1038 /*! cache: eviction calls to get a page found queue empty */ -#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY 1052 +#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY 1039 /*! cache: eviction calls to get a page found queue empty after locking */ -#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY2 1053 +#define WT_STAT_CONN_CACHE_EVICTION_GET_REF_EMPTY2 1040 /*! cache: eviction currently operating in aggressive mode */ -#define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1054 +#define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1041 /*! cache: eviction empty score */ -#define WT_STAT_CONN_CACHE_EVICTION_EMPTY_SCORE 1055 +#define WT_STAT_CONN_CACHE_EVICTION_EMPTY_SCORE 1042 /*! cache: eviction passes of a file */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_PASSES 1056 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_PASSES 1043 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1057 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1044 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1058 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1045 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1059 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1046 /*! * cache: eviction server slept, because we did not make progress with * eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1060 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1047 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1061 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1048 /*! cache: eviction server waiting for a leaf page */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_LEAF_NOTFOUND 1062 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_LEAF_NOTFOUND 1049 /*! cache: eviction state */ -#define WT_STAT_CONN_CACHE_EVICTION_STATE 1063 +#define WT_STAT_CONN_CACHE_EVICTION_STATE 1050 /*! cache: eviction walk target pages histogram - 0-9 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1064 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1051 /*! cache: eviction walk target pages histogram - 10-31 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1065 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1052 /*! cache: eviction walk target pages histogram - 128 and higher */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1066 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1053 /*! cache: eviction walk target pages histogram - 32-63 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1067 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1054 /*! cache: eviction walk target pages histogram - 64-128 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1068 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1055 /*! cache: eviction walk target strategy both clean and dirty pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_BOTH_CLEAN_AND_DIRTY 1069 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_BOTH_CLEAN_AND_DIRTY 1056 /*! cache: eviction walk target strategy only clean pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_CLEAN 1070 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_CLEAN 1057 /*! cache: eviction walk target strategy only dirty pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_DIRTY 1071 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_DIRTY 1058 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1072 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1059 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1073 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1060 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1074 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1061 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1075 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1062 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1076 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1063 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1077 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1064 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1078 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1065 /*! cache: eviction worker thread active */ -#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1079 +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1066 /*! cache: eviction worker thread created */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1080 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1067 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1081 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1068 /*! cache: eviction worker thread removed */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1082 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1069 /*! cache: eviction worker thread stable number */ -#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1083 +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1070 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1084 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1071 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1085 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1072 /*! cache: force re-tuning of eviction workers once in a while */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1086 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1073 /*! * cache: forced eviction - history store pages failed to evict while * session has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_FAIL 1087 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_FAIL 1074 /*! * cache: forced eviction - history store pages selected while session * has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS 1088 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS 1075 /*! * cache: forced eviction - history store pages successfully evicted * while session has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_SUCCESS 1089 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_SUCCESS 1076 /*! cache: forced eviction - pages evicted that were clean count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1090 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1077 /*! cache: forced eviction - pages evicted that were clean time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1091 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1078 /*! cache: forced eviction - pages evicted that were dirty count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1092 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1079 /*! cache: forced eviction - pages evicted that were dirty time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1093 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1080 /*! * cache: forced eviction - pages selected because of too many deleted * items count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1094 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1081 /*! cache: forced eviction - pages selected count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1095 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1082 /*! cache: forced eviction - pages selected unable to be evicted count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1096 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1083 /*! cache: forced eviction - pages selected unable to be evicted time */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1097 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1084 /*! * cache: forced eviction - session returned rollback error while force * evicting due to being oldest */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_ROLLBACK 1098 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_ROLLBACK 1085 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1099 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1086 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1100 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1087 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1101 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1088 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1102 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1089 /*! cache: history store score */ -#define WT_STAT_CONN_CACHE_HS_SCORE 1103 +#define WT_STAT_CONN_CACHE_HS_SCORE 1090 /*! cache: history store table insert calls */ -#define WT_STAT_CONN_CACHE_HS_INSERT 1104 +#define WT_STAT_CONN_CACHE_HS_INSERT 1091 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1105 +#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1092 /*! cache: history store table max on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1106 +#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1093 /*! cache: history store table on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK 1107 +#define WT_STAT_CONN_CACHE_HS_ONDISK 1094 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1108 +#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1095 /*! * cache: history store table out-of-order updates that were fixed up by * moving existing records */ -#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_MOVE 1109 +#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_MOVE 1096 /*! * cache: history store table out-of-order updates that were fixed up * during insertion */ -#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_INSERT 1110 +#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_INSERT 1097 /*! cache: history store table reads */ -#define WT_STAT_CONN_CACHE_HS_READ 1111 +#define WT_STAT_CONN_CACHE_HS_READ 1098 /*! cache: history store table reads missed */ -#define WT_STAT_CONN_CACHE_HS_READ_MISS 1112 +#define WT_STAT_CONN_CACHE_HS_READ_MISS 1099 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1113 +#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1100 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1114 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1101 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1115 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1102 /*! * cache: history store table truncation due to mixed timestamps that * returned restart */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_MIX_TS_RESTART 1116 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_MIX_TS_RESTART 1103 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1117 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1104 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1118 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1105 /*! * cache: history store table truncation to remove range of updates due * to mixed timestamps */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_MIX_TS 1119 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_MIX_TS 1106 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1120 +#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1107 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1121 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1108 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1122 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1109 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1123 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1110 /*! cache: internal pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1124 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1111 /*! cache: internal pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1125 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1112 /*! cache: internal pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1126 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1113 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1127 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1114 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1128 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1115 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1129 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1116 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1130 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1117 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1131 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1118 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1132 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1119 /*! cache: operations timed out waiting for space in cache */ -#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1133 +#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1120 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1134 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1121 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1135 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1122 /*! cache: page written requiring history store records */ -#define WT_STAT_CONN_CACHE_WRITE_HS 1136 +#define WT_STAT_CONN_CACHE_WRITE_HS 1123 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1137 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1124 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1138 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1125 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1139 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1126 /*! cache: pages queued for eviction post lru sorting */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1140 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1127 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1141 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1128 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1142 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1129 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1143 +#define WT_STAT_CONN_CACHE_READ 1130 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1144 +#define WT_STAT_CONN_CACHE_READ_DELETED 1131 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1145 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1132 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1146 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1133 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1147 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1134 /*! cache: pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1148 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1135 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1149 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1136 /*! * cache: pages selected for eviction unable to be evicted as the parent * page has overflow items */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1150 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1137 /*! * cache: pages selected for eviction unable to be evicted because of * active children on an internal page */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1151 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1138 /*! * cache: pages selected for eviction unable to be evicted because of * failure in reconciliation */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1152 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1139 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1153 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1140 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1154 +#define WT_STAT_CONN_CACHE_WRITE 1141 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1155 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1142 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1156 +#define WT_STAT_CONN_CACHE_OVERHEAD 1143 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1157 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1144 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1158 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1145 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1159 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1146 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1160 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1147 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1161 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1148 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1162 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1149 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1163 +#define WT_STAT_CONN_FSYNC_ALL_FH 1150 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1164 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1151 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1165 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1152 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1166 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1153 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1167 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1154 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1168 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1155 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1169 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1156 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1170 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1157 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1171 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1158 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1172 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1159 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1173 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1160 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1174 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1161 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1175 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1162 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_CONN_CC_PAGES_EVICT 1176 +#define WT_STAT_CONN_CC_PAGES_EVICT 1163 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_CONN_CC_PAGES_REMOVED 1177 +#define WT_STAT_CONN_CC_PAGES_REMOVED 1164 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1178 +#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1165 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_CONN_CC_PAGES_VISITED 1179 +#define WT_STAT_CONN_CC_PAGES_VISITED 1166 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1180 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1167 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1181 +#define WT_STAT_CONN_COND_AUTO_WAIT 1168 /*! * connection: auto adjusting condition wait raced to update timeout and * skipped updating */ -#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1182 +#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1169 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1183 +#define WT_STAT_CONN_TIME_TRAVEL 1170 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1184 +#define WT_STAT_CONN_FILE_OPEN 1171 /*! connection: hash bucket array size for data handles */ -#define WT_STAT_CONN_BUCKETS_DH 1185 +#define WT_STAT_CONN_BUCKETS_DH 1172 /*! connection: hash bucket array size general */ -#define WT_STAT_CONN_BUCKETS 1186 +#define WT_STAT_CONN_BUCKETS 1173 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1187 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1174 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1188 +#define WT_STAT_CONN_MEMORY_FREE 1175 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1189 +#define WT_STAT_CONN_MEMORY_GROW 1176 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1190 +#define WT_STAT_CONN_COND_WAIT 1177 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1191 +#define WT_STAT_CONN_RWLOCK_READ 1178 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1192 +#define WT_STAT_CONN_RWLOCK_WRITE 1179 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1193 +#define WT_STAT_CONN_FSYNC_IO 1180 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1194 +#define WT_STAT_CONN_READ_IO 1181 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1195 +#define WT_STAT_CONN_WRITE_IO 1182 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1196 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1183 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1197 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1184 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1198 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1185 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1199 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1186 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1200 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1187 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1201 +#define WT_STAT_CONN_CURSOR_CACHE 1188 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1202 +#define WT_STAT_CONN_CURSOR_CREATE 1189 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1203 +#define WT_STAT_CONN_CURSOR_INSERT 1190 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1204 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1191 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1205 +#define WT_STAT_CONN_CURSOR_MODIFY 1192 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1206 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1193 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1207 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1194 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1208 +#define WT_STAT_CONN_CURSOR_NEXT 1195 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1209 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1196 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone in rollback to stable */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE_RTS 1210 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE_RTS 1197 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1211 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1198 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1212 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1199 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1213 +#define WT_STAT_CONN_CURSOR_RESTART 1200 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1214 +#define WT_STAT_CONN_CURSOR_PREV 1201 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1215 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1202 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone in rollback to stable */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1216 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1203 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1217 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1204 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1218 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1205 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1219 +#define WT_STAT_CONN_CURSOR_REMOVE 1206 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1220 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1207 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1221 +#define WT_STAT_CONN_CURSOR_RESERVE 1208 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1222 +#define WT_STAT_CONN_CURSOR_RESET 1209 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1223 +#define WT_STAT_CONN_CURSOR_SEARCH 1210 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1224 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1211 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1225 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1212 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1226 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1213 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1227 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1214 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1228 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1215 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1229 +#define WT_STAT_CONN_CURSOR_SWEEP 1216 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1230 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1217 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1231 +#define WT_STAT_CONN_CURSOR_UPDATE 1218 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1232 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1219 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1233 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1220 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1234 +#define WT_STAT_CONN_CURSOR_REOPEN 1221 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1235 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1222 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1236 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1223 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1237 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1224 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1238 +#define WT_STAT_CONN_DH_SWEEP_REF 1225 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1239 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1226 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1240 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1227 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1241 +#define WT_STAT_CONN_DH_SWEEP_TOD 1228 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1242 +#define WT_STAT_CONN_DH_SWEEPS 1229 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1243 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1230 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1244 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1231 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1245 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1232 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1246 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1233 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1247 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1234 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1248 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1235 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1249 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1236 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1250 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1237 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1251 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1238 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1252 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1239 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1253 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1240 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1254 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1241 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1255 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1242 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1256 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1243 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1257 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1244 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1258 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1245 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1259 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1246 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1260 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1247 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1261 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1248 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1262 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1249 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1263 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1250 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1264 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1251 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1265 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1252 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1266 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1253 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1267 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1254 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1268 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1255 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1269 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1256 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1270 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1257 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1271 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1258 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1272 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1259 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1273 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1260 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1274 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1261 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1275 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1262 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1276 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1263 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1277 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1264 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1278 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1265 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1279 +#define WT_STAT_CONN_LOG_FLUSH 1266 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1280 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1267 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1281 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1268 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1282 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1269 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1283 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1270 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1284 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1271 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1285 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1272 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1286 +#define WT_STAT_CONN_LOG_SCANS 1273 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1287 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1274 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1288 +#define WT_STAT_CONN_LOG_WRITE_LSN 1275 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1289 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1276 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1290 +#define WT_STAT_CONN_LOG_SYNC 1277 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1291 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1278 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1292 +#define WT_STAT_CONN_LOG_SYNC_DIR 1279 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1293 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1280 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1294 +#define WT_STAT_CONN_LOG_WRITES 1281 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1295 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1282 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1296 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1283 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1297 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1284 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1298 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1285 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1299 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1286 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1300 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1287 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1301 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1288 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1302 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1289 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1303 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1290 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1304 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1291 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1305 +#define WT_STAT_CONN_LOG_SLOT_RACES 1292 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1306 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1293 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1307 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1294 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1308 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1295 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1309 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1296 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1310 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1297 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1311 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1298 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1312 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1299 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1313 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1300 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1314 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1301 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1315 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1302 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1316 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1303 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1317 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1304 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1318 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1305 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1319 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1306 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1320 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1307 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1321 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1308 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1322 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1309 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1323 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1310 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1324 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1311 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1325 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1312 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1326 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1313 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1327 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1314 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1328 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1315 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1329 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1316 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1330 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1317 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1331 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1318 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1332 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1319 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1333 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1320 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1334 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1321 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1335 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1322 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1336 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1323 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1337 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1324 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1338 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1325 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1339 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1326 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1340 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1327 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1341 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1328 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1342 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1329 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1343 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1330 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1344 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1331 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1345 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1332 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1346 +#define WT_STAT_CONN_REC_PAGES 1333 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1347 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1334 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1348 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1335 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1349 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1336 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1350 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1337 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1351 +#define WT_STAT_CONN_REC_PAGE_DELETE 1338 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1352 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1339 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1353 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1340 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1354 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1341 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1355 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1342 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1356 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1343 /*! * reconciliation: pages written including an aggregated oldest start * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TXN 1357 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TXN 1344 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1358 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1345 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1359 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1346 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1360 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1347 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1361 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1348 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1362 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1349 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1363 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1350 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1364 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1351 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1365 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1352 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1366 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1353 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1367 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1354 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1368 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1355 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1369 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1356 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1370 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1357 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1371 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1358 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1372 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1359 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1373 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1360 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1374 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1361 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1375 +#define WT_STAT_CONN_SESSION_OPEN 1362 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1376 +#define WT_STAT_CONN_SESSION_QUERY_TS 1363 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1377 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1364 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1378 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1365 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1379 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1366 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1380 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1367 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1381 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1368 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1382 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1369 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1383 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1370 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1384 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1371 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1385 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1372 /*! session: table import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1386 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1373 /*! session: table import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1387 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1374 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1388 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1375 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1389 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1376 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1390 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1377 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1391 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1378 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1392 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1379 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1393 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1380 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1394 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1381 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1395 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1382 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1396 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1383 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1397 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1384 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1398 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1385 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1399 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1386 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1400 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1387 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1401 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1388 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1402 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1389 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1403 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1390 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1404 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1391 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1405 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1392 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1406 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1393 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1407 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1394 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1408 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1395 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1409 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1396 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1410 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1397 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1411 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1398 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1412 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1399 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1413 +#define WT_STAT_CONN_PAGE_SLEEP 1400 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1414 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1401 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1415 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1402 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1416 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1403 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1417 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1404 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1418 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1405 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1419 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1406 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1420 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1407 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1421 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1408 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1422 +#define WT_STAT_CONN_TXN_PREPARE 1409 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1423 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1410 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1424 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1411 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1425 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1412 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1426 +#define WT_STAT_CONN_TXN_QUERY_TS 1413 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1427 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1414 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1428 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1415 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1429 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1416 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1430 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1417 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1431 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1418 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1432 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1419 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1433 +#define WT_STAT_CONN_TXN_RTS 1420 /*! * transaction: rollback to stable hs records with stop timestamps older * than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1434 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1421 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1435 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1422 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1436 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1423 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1437 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1424 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1438 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1425 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1439 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1426 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1440 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1427 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1441 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1428 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1442 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1429 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1443 +#define WT_STAT_CONN_TXN_SET_TS 1430 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1444 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1431 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1445 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1432 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1446 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1433 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1447 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1434 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1448 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1435 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1449 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1436 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1450 +#define WT_STAT_CONN_TXN_BEGIN 1437 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1451 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1438 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1452 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1439 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1453 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1440 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1454 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1441 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1442 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1456 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1443 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1444 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1445 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1446 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1447 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1461 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1448 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1449 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1463 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1450 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1451 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1452 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1466 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1453 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1467 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1454 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1468 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1455 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1469 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1456 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1470 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1457 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1471 +#define WT_STAT_CONN_TXN_CHECKPOINT 1458 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1472 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1459 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1473 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1460 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1474 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1461 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1475 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1462 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1476 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1463 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1477 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1464 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1478 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1465 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1479 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1466 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1480 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1467 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1481 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1468 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1482 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1469 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1483 +#define WT_STAT_CONN_TXN_SYNC 1470 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1484 +#define WT_STAT_CONN_TXN_COMMIT 1471 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1485 +#define WT_STAT_CONN_TXN_ROLLBACK 1472 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1486 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1473 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 113113f9e93..f45dc8a8d10 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -69,16 +69,6 @@ struct __wt_addr; typedef struct __wt_addr WT_ADDR; struct __wt_addr_copy; typedef struct __wt_addr_copy WT_ADDR_COPY; -struct __wt_async; -typedef struct __wt_async WT_ASYNC; -struct __wt_async_cursor; -typedef struct __wt_async_cursor WT_ASYNC_CURSOR; -struct __wt_async_format; -typedef struct __wt_async_format WT_ASYNC_FORMAT; -struct __wt_async_op_impl; -typedef struct __wt_async_op_impl WT_ASYNC_OP_IMPL; -struct __wt_async_worker_state; -typedef struct __wt_async_worker_state WT_ASYNC_WORKER_STATE; struct __wt_blkincr; typedef struct __wt_blkincr WT_BLKINCR; struct __wt_block; @@ -388,7 +378,6 @@ typedef uint64_t wt_timestamp_t; #include "timestamp.h" /* required by reconcile.h */ #include "api.h" -#include "async.h" #include "block.h" #include "bloom.h" #include "btmem.h" diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index 093d072a611..c3d7417ff14 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -51,16 +51,19 @@ __wt_direct_io_size_check( static int __create_file(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const char *config) { + WT_CONFIG_ITEM cval; WT_DECL_ITEM(val); WT_DECL_RET; const char *filename, **p, - *filecfg[] = {WT_CONFIG_BASE(session, file_meta), config, NULL, NULL}; + *filecfg[] = {WT_CONFIG_BASE(session, file_meta), config, NULL, NULL, NULL}; char *fileconf; uint32_t allocsize; - bool exists, is_metadata; + bool exists, import, import_repair, is_metadata; fileconf = NULL; + import = __wt_config_getones(session, config, "import.enabled", &cval) == 0 && cval.val != 0; + import_repair = false; is_metadata = strcmp(uri, WT_METAFILE_URI) == 0; filename = uri; @@ -68,7 +71,11 @@ __create_file(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const c /* Check if the file already exists. */ if (!is_metadata && (ret = __wt_metadata_search(session, uri, &fileconf)) != WT_NOTFOUND) { - if (exclusive) + /* + * Regardless of the 'exclusive' flag, we should raise an error if we try to import an + * existing URI rather than just silently returning. + */ + if (exclusive || import) WT_TRET(EEXIST); goto err; } @@ -90,8 +97,51 @@ __create_file(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const c /* Sanity check the allocation size. */ WT_ERR(__wt_direct_io_size_check(session, filecfg, "allocation_size", &allocsize)); - /* Create the file. */ - WT_ERR(__wt_block_manager_create(session, filename, allocsize)); + /* + * If we are importing an existing object rather than creating a new one, there are two possible + * scenarios. Either (1) the file configuration string from the source database metadata is + * specified in the input config string, or (2) the import.repair option is set and we need to + * reconstruct the configuration metadata from the file. + */ + if (import) { + import_repair = + __wt_config_getones(session, config, "import.repair", &cval) == 0 && cval.val != 0; + if (!import_repair) { + if (__wt_config_getones(session, config, "import.file_metadata", &cval) == 0 && + cval.len != 0) { + /* + * The string may be enclosed by delimiters (e.g. braces, quotes, parentheses) to + * avoid configuration string characters acting as separators. Discard the first and + * last characters in this case. + */ + if (cval.type == WT_CONFIG_ITEM_STRUCT) { + cval.str++; + cval.len -= 2; + } + WT_ERR(__wt_strndup(session, cval.str, cval.len, &filecfg[2])); + } else { + /* + * If there is no file metadata provided, the user should be specifying a "repair". + * To prevent mistakes with API usage, we should return an error here rather than + * inferring a repair. + */ + WT_ERR_MSG(session, EINVAL, + "%s: import requires that 'file_metadata' is specified or the 'repair' option is " + "provided", + uri); + } + } + } + + if (import) { + WT_IGNORE_RET(__wt_fs_exist(session, filename, &exists)); + if (!exists) + WT_ERR_MSG(session, ENOENT, "%s: attempted to import file that does not exist", uri); + } else { + /* Create the file. */ + WT_ERR(__wt_block_manager_create(session, filename, allocsize)); + } + if (WT_META_TRACKING(session)) WT_ERR(__wt_meta_track_fileop(session, NULL, uri)); @@ -100,14 +150,19 @@ __create_file(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const c * configuration and insert the resulting configuration into the metadata. */ if (!is_metadata) { - WT_ERR(__wt_scr_alloc(session, 0, &val)); - WT_ERR(__wt_buf_fmt(session, val, "id=%" PRIu32 ",version=(major=%d,minor=%d)", - ++S2C(session)->next_file_id, WT_BTREE_MAJOR_VERSION_MAX, WT_BTREE_MINOR_VERSION_MAX)); - for (p = filecfg; *p != NULL; ++p) - ; - *p = val->data; - WT_ERR(__wt_config_collapse(session, filecfg, &fileconf)); - WT_ERR(__wt_metadata_insert(session, uri, fileconf)); + if (!import_repair) { + WT_ERR(__wt_scr_alloc(session, 0, &val)); + WT_ERR(__wt_buf_fmt(session, val, "id=%" PRIu32 ",version=(major=%d,minor=%d)", + ++S2C(session)->next_file_id, WT_BTREE_MAJOR_VERSION_MAX, + WT_BTREE_MINOR_VERSION_MAX)); + for (p = filecfg; *p != NULL; ++p) + ; + *p = val->data; + WT_ERR(__wt_config_collapse(session, filecfg, &fileconf)); + WT_ERR(__wt_metadata_insert(session, uri, fileconf)); + } else { + /* TO-DO: WT-6691 */ + } } /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 2520fdfc4ef..bda39f03b3c 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -845,19 +845,6 @@ static const char *const __stats_connection_desc[] = { "LSM: tree maintenance operations executed", "LSM: tree maintenance operations scheduled", "LSM: tree queue hit maximum", - "async: current work queue length", - "async: maximum work queue length", - "async: number of allocation state races", - "async: number of flush calls", - "async: number of operation slots viewed for allocation", - "async: number of times operation allocation failed", - "async: number of times worker found no work", - "async: total allocations", - "async: total compact calls", - "async: total insert calls", - "async: total remove calls", - "async: total search calls", - "async: total update calls", "block-manager: blocks pre-loaded", "block-manager: blocks read", "block-manager: blocks written", @@ -1383,19 +1370,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lsm_work_units_done = 0; stats->lsm_work_units_created = 0; stats->lsm_work_queue_max = 0; - stats->async_cur_queue = 0; - /* not clearing async_max_queue */ - stats->async_alloc_race = 0; - stats->async_flush = 0; - stats->async_alloc_view = 0; - stats->async_full = 0; - stats->async_nowork = 0; - stats->async_op_alloc = 0; - stats->async_op_compact = 0; - stats->async_op_insert = 0; - stats->async_op_remove = 0; - stats->async_op_search = 0; - stats->async_op_update = 0; stats->block_preload = 0; stats->block_read = 0; stats->block_write = 0; @@ -1886,19 +1860,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->lsm_work_units_done += WT_STAT_READ(from, lsm_work_units_done); to->lsm_work_units_created += WT_STAT_READ(from, lsm_work_units_created); to->lsm_work_queue_max += WT_STAT_READ(from, lsm_work_queue_max); - to->async_cur_queue += WT_STAT_READ(from, async_cur_queue); - to->async_max_queue += WT_STAT_READ(from, async_max_queue); - to->async_alloc_race += WT_STAT_READ(from, async_alloc_race); - to->async_flush += WT_STAT_READ(from, async_flush); - to->async_alloc_view += WT_STAT_READ(from, async_alloc_view); - to->async_full += WT_STAT_READ(from, async_full); - to->async_nowork += WT_STAT_READ(from, async_nowork); - to->async_op_alloc += WT_STAT_READ(from, async_op_alloc); - to->async_op_compact += WT_STAT_READ(from, async_op_compact); - to->async_op_insert += WT_STAT_READ(from, async_op_insert); - to->async_op_remove += WT_STAT_READ(from, async_op_remove); - to->async_op_search += WT_STAT_READ(from, async_op_search); - to->async_op_update += WT_STAT_READ(from, async_op_update); to->block_preload += WT_STAT_READ(from, block_preload); to->block_read += WT_STAT_READ(from, block_read); to->block_write += WT_STAT_READ(from, block_write); diff --git a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c index 27c46afb25b..d10869263c0 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c @@ -34,11 +34,7 @@ * reconfiguration. */ -static const char *const list[] = {",async=(enabled=0)", ",async=(enabled=1)", - ",async=(ops_max=2048)", ",async=(ops_max=2348)", ",async=(ops_max=1790)", ",async=(threads=10)", - ",async=(threads=7)", ",async=(threads=17)", - - ",cache_overhead=13", ",cache_overhead=27", ",cache_overhead=8", +static const char *const list[] = {",cache_overhead=13", ",cache_overhead=27", ",cache_overhead=8", ",cache_size=75MB", ",cache_size=214MB", ",cache_size=37MB", diff --git a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c b/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c index f9de69e5332..2044a47e9b9 100644 --- a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c @@ -139,6 +139,7 @@ thread_run(void *arg) char kname[64], tscfg[64]; conn = (WT_CONNECTION *)arg; + memset(kname, 0, sizeof(kname)); testutil_check(conn->open_session(conn, NULL, "isolation=snapshot", &session)); testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); @@ -263,7 +264,7 @@ main(int argc, char *argv[]) (void)testutil_set_progname(argv); - rand_time = false; + rand_time = true; timeout = MIN_TIME; working_dir = "WT_TEST.wt6616-checkpoint-oldest-ts"; @@ -285,6 +286,8 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(home, sizeof(home), working_dir); testutil_make_work_dir(home); + + __wt_random_init_seed(NULL, &rnd); if (rand_time) { timeout = __wt_random(&rnd) % MAX_TIME; if (timeout < MIN_TIME) diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index e64a66623a6..b62cbabff56 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -277,6 +277,17 @@ functions: rm -rf "wiredtiger" rm -rf "wiredtiger.tgz" + "dump stderr/stdout": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + + # Dump stderr/stdout contents generated by the C libraries onto console for Python tests + find "WT_TEST" -name "std*.txt" ! -empty -printf "\nContents from '%p':\n\n" -exec cat {} \; + "checkpoint test": command: shell.exec params: @@ -403,6 +414,7 @@ post: - func: "upload artifact" vars: postfix: -${execution} + - func: "dump stderr/stdout" - func: "cleanup" tasks: @@ -1415,14 +1427,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 0/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 0/11 - name: unit-test-bucket01 tags: ["pull_request", "unit_test"] @@ -1430,14 +1437,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 1/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 1/11 - name: unit-test-bucket02 tags: ["pull_request", "unit_test"] @@ -1445,14 +1447,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 2/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 2/11 - name: unit-test-bucket03 tags: ["pull_request", "unit_test"] @@ -1460,14 +1457,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 3/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 3/11 - name: unit-test-bucket04 tags: ["pull_request", "unit_test"] @@ -1475,14 +1467,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 4/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 4/11 - name: unit-test-bucket05 tags: ["pull_request", "unit_test"] @@ -1490,14 +1477,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 5/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 5/11 - name: unit-test-bucket06 tags: ["pull_request", "unit_test"] @@ -1505,14 +1487,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 6/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 6/11 - name: unit-test-bucket07 tags: ["pull_request", "unit_test"] @@ -1520,14 +1497,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 7/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 7/11 - name: unit-test-bucket08 tags: ["pull_request", "unit_test"] @@ -1535,14 +1507,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 8/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 8/11 - name: unit-test-bucket09 tags: ["pull_request", "unit_test"] @@ -1550,14 +1517,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 9/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - func: "unit test" + vars: + unit_test_args: -v 2 -b 9/11 - name: unit-test-bucket10 tags: ["pull_request", "unit_test"] @@ -1565,15 +1527,9 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 10/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - + - func: "unit test" + vars: + unit_test_args: -v 2 -b 10/11 # End of Python unit test tasks - name: s-all @@ -2193,6 +2149,20 @@ tasks: cp ../../../bench/wtperf/split_heavy.wtperf . ./wtperf -O ./split_heavy.wtperf -o verbose=2 + - name: format-failure-configs-test + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/test/evergreen" + script: | + set -o errexit + set -o verbose + + ./run_format_configs.sh + - name: static-wt-build-test commands: - func: "get project" @@ -2393,6 +2363,28 @@ tasks: - <<: *recovery-stress-test name: recovery-stress-test-3 tags: ["stress-test-3", "stress-test-zseries-3"] + + - name: cyclomatic-complexity + tags: ["pull_request"] + commands: + - func: "get project" + - command: shell.exec + params: + working_dir: "wiredtiger/src" + shell: bash + script: | + set -o verbose + + git clone https://github.com/metrixplusplus/metrixplusplus metrixplusplus + python "metrixplusplus/metrix++.py" collect --std.code.lines.code --std.code.complexity.cyclomatic + python "metrixplusplus/metrix++.py" view + + # Set the cyclomatic complexity limit to 20 + python "metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:20 + + # Fail if there are functions with cyclomatic complexity larger than 92 + set -o errexit + python "metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:92 buildvariants: @@ -2426,6 +2418,8 @@ buildvariants: - name: ftruncate-test - name: long-test - name: static-wt-build-test + - name: format-failure-configs-test + - name: cyclomatic-complexity - name: ubuntu1804-compilers display_name: "! Ubuntu 18.04 Compilers" diff --git a/src/third_party/wiredtiger/test/evergreen/run_format_configs.sh b/src/third_party/wiredtiger/test/evergreen/run_format_configs.sh new file mode 100755 index 00000000000..4362e00af72 --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/run_format_configs.sh @@ -0,0 +1,39 @@ +#! /usr/bin/env bash +# +# Cycle through a list of test/format configurations that failed previously, +# and run format test against each of those configurations to capture issues. +# + +set -u + +# Switch to the Git repo toplevel directory +cd $(git rev-parse --show-toplevel) + +# Walk into the test/format directory +cd build_posix/test/format + +# Check the existence of 't' binary +if [ ! -x "t" ]; then + echo "'t' binary does not exist, exiting ..." + exit 1 +fi + +success=0 +failure=0 + +# Cycle through format CONFIGs recorded under the "failure_configs" directory +for config in $(find ../../../test/format/failure_configs/ -name CONFIG.* | sort) +do + echo -e "\nTesting CONFIG $config ...\n" + if (./t -c $config); then + let "success++" + else + let "failure++" + [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG + fi +done + +echo -e "\nSummary of '$(basename $0)': $success successful CONFIG(s), $failure failed CONFIG(s)\n" + +[[ $failure -ne 0 ]] && exit 1 +exit 0 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 new file mode 100644 index 00000000000..32f0baa246b --- /dev/null +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 @@ -0,0 +1,103 @@ +############################################ +# RUN PARAMETERS: V2 +############################################ +assert.commit_timestamp=0 +assert.read_timestamp=0 +backup=0 +backup.incremental=off +backup.incr_granularity=3973 +btree.bitcnt=4 +btree.compression=none +btree.dictionary=0 +btree.huffman_key=0 +btree.huffman_value=0 +btree.internal_key_truncation=1 +btree.internal_page_max=9 +btree.key_gap=3 +btree.key_max=35 +btree.key_min=32 +btree.leaf_page_max=12 +btree.memory_page_max=7 +btree.prefix_compression=1 +btree.prefix_compression_min=6 +btree.repeat_data_pct=68 +btree.reverse=0 +btree.split_pct=60 +btree.value_max=1087 +btree.value_min=8 +cache=70 +cache.evict_max=1 +cache.minimum=0 +checkpoint=on +checkpoint.log_size=52 +checkpoint.wait=27 +disk.checksum=uncompressed +disk.data_extend=0 +disk.direct_io=0 +disk.encryption=none +disk.firstfit=0 +disk.mmap=1 +disk.mmap_all=0 +format.abort=0 +format.independent_thread_rng=1 +format.major_timeout=0 +logging=1 +logging.archive=0 +logging.compression=none +logging.file_max=403863 +logging.prealloc=1 +lsm.auto_throttle=1 +lsm.bloom=1 +lsm.bloom_bit_count=38 +lsm.bloom_hash_count=9 +lsm.bloom_oldest=0 +lsm.chunk_size=7 +lsm.merge_max=9 +lsm.worker_threads=4 +ops.alter=0 +ops.compaction=0 +ops.hs_cursor=1 +ops.pct.delete=2 +ops.pct.insert=3 +ops.pct.modify=50 +ops.pct.read=34 +ops.pct.write=11 +ops.prepare=0 +ops.random_cursor=0 +ops.rebalance=0 +ops.salvage=0 +ops.truncate=1 +ops.verify=1 +quiet=1 +runs=1 +runs.in_memory=0 +runs.ops=100000 +runs.rows=100000 +runs.source=table +runs.threads=5 +runs.timer=360 +runs.type=row-store +runs.verify_failure_dump=0 +statistics=1 +statistics.server=0 +stress.aggressive_sweep=0 +stress.checkpoint=0 +stress.checkpoint_prepare=0 +stress.hs_checkpoint_delay=0 +stress.hs_sweep=0 +stress.split_1=0 +stress.split_2=0 +stress.split_3=0 +stress.split_4=0 +stress.split_5=0 +stress.split_6=0 +stress.split_7=0 +stress.split_8=0 +transaction.frequency=100 +transaction.isolation=snapshot +transaction.rollback_to_stable=0 +transaction.timestamps=1 +wiredtiger.config=verbose=(temporary) +wiredtiger.rwlock=1 +wiredtiger.leak_memory=0 +############################################ diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 new file mode 100644 index 00000000000..554d86c6c13 --- /dev/null +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 @@ -0,0 +1,98 @@ +############################################ +# RUN PARAMETERS: V2 +############################################ +assert.commit_timestamp=0 +assert.read_timestamp=0 +backup=1 +backup.incremental=block +backup.incr_granularity=16 +btree.bitcnt=7 +btree.compression=snappy +btree.dictionary=0 +btree.huffman_key=0 +btree.huffman_value=0 +btree.internal_key_truncation=1 +btree.internal_page_max=12 +btree.key_gap=17 +btree.key_max=103 +btree.key_min=29 +btree.leaf_page_max=12 +btree.memory_page_max=6 +btree.prefix_compression=0 +btree.prefix_compression_min=0 +btree.repeat_data_pct=20 +btree.reverse=0 +btree.split_pct=90 +btree.value_max=848 +btree.value_min=12 +cache=80 +cache.evict_max=3 +cache.minimum=0 +checkpoint=1 +checkpoint.log_size=64 +checkpoint.wait=34 +disk.checksum=uncompressed +disk.data_extend=0 +disk.direct_io=0 +disk.encryption=none +disk.firstfit=0 +disk.mmap=1 +disk.mmap_all=0 +format.abort=0 +format.independent_thread_rng=1 +format.major_timeout=0 +logging=1 +logging.archive=1 +logging.compression=snappy +logging.file_max=72264 +logging.prealloc=1 +lsm.auto_throttle=1 +lsm.bloom=1 +lsm.bloom_bit_count=28 +lsm.bloom_hash_count=31 +lsm.bloom_oldest=0 +lsm.chunk_size=8 +lsm.merge_max=12 +lsm.worker_threads=3 +ops.alter=0 +ops.compaction=0 +ops.pct.delete=98 +ops.pct.insert=2 +ops.pct.modify=0 +ops.pct.read=0 +ops.pct.write=0 +ops.prepare=0 +ops.random_cursor=0 +ops.rebalance=0 +ops.salvage=0 +ops.truncate=1 +ops.verify=0 +quiet=1 +runs=1 +runs.in_memory=0 +runs.ops=0 +runs.rows=1000000 +runs.source=table +runs.threads=16 +runs.timer=4 +runs.type=row-store +statistics=0 +statistics.server=0 +stress.aggressive_sweep=0 +stress.checkpoint=0 +stress.hs_sweep=0 +stress.split_1=0 +stress.split_2=0 +stress.split_3=0 +stress.split_4=0 +stress.split_5=0 +stress.split_6=0 +stress.split_7=0 +stress.split_8=0 +transaction.frequency=100 +transaction.isolation=snapshot +transaction.timestamps=1 +wiredtiger.config= +wiredtiger.rwlock=1 +wiredtiger.leak_memory=1 +############################################ diff --git a/src/third_party/wiredtiger/test/format/failure_configs/README b/src/third_party/wiredtiger/test/format/failure_configs/README new file mode 100644 index 00000000000..b766d5139ab --- /dev/null +++ b/src/third_party/wiredtiger/test/format/failure_configs/README @@ -0,0 +1,2 @@ +This folder contains a list of test/format configurations that generated failures in the past. +The ticket number is included as the filename postfix to indicate the source of each failure. diff --git a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/AsyncTest.java b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/AsyncTest.java deleted file mode 100644 index 3b0db812d94..00000000000 --- a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/AsyncTest.java +++ /dev/null @@ -1,504 +0,0 @@ -/*- - * Public Domain 2014-2020 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -package com.wiredtiger.test; - -import com.wiredtiger.db.AsyncCallback; -import com.wiredtiger.db.AsyncOpType; -import com.wiredtiger.db.AsyncOp; -import com.wiredtiger.db.Connection; -import com.wiredtiger.db.Cursor; -import com.wiredtiger.db.Session; -import com.wiredtiger.db.WiredTigerException; -import com.wiredtiger.db.WiredTigerPackingException; -import com.wiredtiger.db.wiredtiger; - -import static org.junit.Assert.assertEquals; - -import org.junit.After; -import org.junit.Test; -import org.junit.Assert; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.util.HashMap; - -abstract class Callback<KeyType, ValueType> implements AsyncCallback { - - /*package*/ int ninsert = 0; - /*package*/ int nsearch = 0; - /*package*/ int nerror = 0; - - HashMap<KeyType, ValueType> current = null; - Object syncobj = null; // used for locking - - public Callback(HashMap<KeyType, ValueType> current, Object syncobj) { - this.current = current; - this.syncobj = syncobj; - } - - public void notifyError(String desc) { - System.err.println("ERROR: notify: " + desc); - } - - public abstract KeyType opToKey(AsyncOp op) - throws WiredTigerPackingException; - public abstract ValueType opToValue(AsyncOp op) - throws WiredTigerPackingException; - - public int notify(AsyncOp op, int opReturn, int flags) { - /* - * Note: we are careful not to throw any errors here. Any - * exceptions would be swallowed by a native worker thread. - */ - int ret = 0; - int ntotal = 0; - try { - synchronized (syncobj) { - ntotal = ninsert + nsearch + nerror; - } - KeyType key = opToKey(op); - ValueType value = opToValue(op); - AsyncOpType optype = op.getType(); - - if (optype == AsyncOpType.WT_AOP_INSERT) { - synchronized (syncobj) { - ninsert += 1; - } - } - else if (optype == AsyncOpType.WT_AOP_SEARCH) { - synchronized (syncobj) { - nsearch += 1; - } - } - else { - notifyError("unexpected optype"); - ret = 1; - synchronized (syncobj) { - nerror += 1; - } - } - - if (!current.get(key).equals(value)) { - notifyError("unexpected value: wanted \"" + current.get(key) + "\", and got \"" + value + "\""); - ret = 1; - synchronized (syncobj) { - nerror += 1; - } - } - } - catch (Exception e) { - System.err.println("ERROR: exception in notify: " + e.toString() + - ", opreturn=" + opReturn + - ", ninsert=" + ninsert + - ", nsearch=" + nsearch + - ", nerror=" + nerror + - ", ntotal=" + ntotal); - ret = 1; - } - return (ret); - } -} - -interface TypeAdapter { - /** - * puts the key value pair into the async op. - */ - void putkv(Session s, AsyncOp op, int i) - throws WiredTigerPackingException; - - /** - * puts a key value pair into the the reference HashMap. - */ - void putref(Session s, HashMap map, int i); -} - -// SS = String String -class CallbackSS extends Callback<String, String> implements TypeAdapter { - public CallbackSS(HashMap<String, String> current, Object syncobj) { - super(current, syncobj); - } - - public String opToKey(AsyncOp op) - throws WiredTigerPackingException { - return op.getKeyString(); - } - - public String opToValue(AsyncOp op) - throws WiredTigerPackingException { - return op.getValueString(); - } - - private String genkey(Session s, int i) { - return "key" + i; - } - - private String genvalue(Session s, int i) { - return "value" + i; - } - - public void putkv(Session s, AsyncOp op, int i) - throws WiredTigerPackingException { - op.putKeyString(genkey(s, i)); - op.putValueString(genvalue(s, i)); - } - - public void putref(Session s, HashMap map, int i) { - current.put(genkey(s, i), genvalue(s, i)); - } -} - -// RS = recno String -class CallbackRS extends Callback<Long, String> implements TypeAdapter { - public CallbackRS(HashMap<Long, String> current, Object syncobj) { - super(current, syncobj); - } - - public Long opToKey(AsyncOp op) - throws WiredTigerPackingException { - return op.getKeyLong(); - } - - public String opToValue(AsyncOp op) - throws WiredTigerPackingException { - return op.getValueString(); - } - - private long genkey(Session s, int i) { - return i + 1; - } - - private String genvalue(Session s, int i) { - return "value" + i; - } - - public void putkv(Session s, AsyncOp op, int i) - throws WiredTigerPackingException { - op.putKeyLong(genkey(s, i)); - op.putValueString(genvalue(s, i)); - } - - public void putref(Session s, HashMap map, int i) { - current.put(genkey(s, i), genvalue(s, i)); - } -} - -// RI = recno 8t -class CallbackRI extends Callback<Long, Integer> implements TypeAdapter { - public CallbackRI(HashMap<Long, Integer> current, Object syncobj) { - super(current, syncobj); - } - - public Long opToKey(AsyncOp op) - throws WiredTigerPackingException { - return op.getKeyLong(); - } - - public Integer opToValue(AsyncOp op) - throws WiredTigerPackingException { - return op.getValueInt(); - } - - private long genkey(Session s, int i) { - return i + 1; - } - - private int genvalue(Session s, int i) { - return i % 0xff; - } - - public void putkv(Session s, AsyncOp op, int i) - throws WiredTigerPackingException { - op.putKeyLong(genkey(s, i)); - op.putValueInt(genvalue(s, i)); - } - - public void putref(Session s, HashMap map, int i) { - current.put(genkey(s, i), genvalue(s, i)); - } -} - - -public class AsyncTest { - - /* - * Connvalid tells us that we really closed the connection. - * That allows teardown to reliably clean up so that - * a single failure in one test does not cascade. - */ - Connection conn; - boolean connvalid = false; - - public static final int N_ENTRIES = 1000; - public static final int N_OPS_MAX = N_ENTRIES / 2; - public static final int N_ASYNC_THREADS = 10; - - public static final int MAX_RETRIES = 10; - - private Session sessionSetup(String name, - String keyFormat, String valueFormat, - int opsMax, int asyncThreads) { - conn = wiredtiger.open("WT_HOME", "create," + - "async=(enabled=true,ops_max=" + - opsMax + ",threads=" + asyncThreads + ")"); - connvalid = true; - Session s = conn.open_session(null); - s.create("table:" + name, - "key_format=" + keyFormat + ",value_format=" + valueFormat); - return s; - } - - private void sleepMillis(int n) - { - try { - Thread.sleep(n); - } - catch (InterruptedException ie) { - // ignore - } - } - - private void asyncTester(String name, - String keyFormat, String valueFormat, - int entries, int opsMax, int asyncThreads, int opsBatch, - int milliSleep, AsyncCallback callback, TypeAdapter adapter, - HashMap current) - throws WiredTigerException { - Session s = sessionSetup(name, keyFormat, valueFormat, - opsMax, asyncThreads); - - for (int i = 0; i < entries; i++) { - // adapter call does equivalent of: - // current.put(genkey(s, i), genvalue(s, i)); - adapter.putref(s, current, i); - } - - for (int i = 0; i < entries; i++) { - for (int retry = 1; retry <= MAX_RETRIES; retry++) { - try { - AsyncOp op = conn.async_new_op("table:" + name, null, - callback); - - // adapter call does equivalent of: - // op.putKeyString(genkey(s, i)); - // op.putValueString(genvalue(s, i)); - adapter.putkv(s, op, i); - - op.insert(); - break; - } - catch (WiredTigerException wte) { - if (retry == MAX_RETRIES) - throw wte; - sleepMillis(1 << retry); - } - } - - // Introduce a little delay time, - // otherwise the workers will not get a chance - // to run before we fill them up. - // In a real application, we might catch - // 'Cannot allocate memory' exceptions from - // the async_new_op call, and use that as - // an indicator to throttle back. - if ((i + 1) % opsBatch == 0) - sleepMillis(milliSleep); - } - - // Wait for all outstanding async ops to finish. - conn.async_flush(); - - for (int i = 0; i < entries; i++) { - for (int retry = 1; retry <= MAX_RETRIES; retry++) { - try { - AsyncOp op = conn.async_new_op("table:" + name, null, - callback); - - // adapter call does equivalent of: - // op.putKeyString(genkey(s, i)); - // op.putValueString(genvalue(s, i)); - adapter.putkv(s, op, i); - - op.search(); - break; - } - catch (WiredTigerException wte) { - if (retry == MAX_RETRIES) - throw wte; - sleepMillis(retry); - } - } - - if ((i + 1) % opsBatch == 0) - sleepMillis(milliSleep); - } - - // Wait for all outstanding async ops to finish. - conn.async_flush(); - - s.close(""); - - } - - @Test - public void async01() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<String, String> current = new HashMap<String, String>(); - CallbackSS callback = new CallbackSS(current, syncobj); - - asyncTester("async01", "S", "S", N_ENTRIES, N_OPS_MAX, - N_ASYNC_THREADS, 100, 1, callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(N_ENTRIES, callback.nsearch); - assertEquals(N_ENTRIES, callback.ninsert); - } - - @Test - public void async02() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<String, String> current = new HashMap<String, String>(); - CallbackSS callback = new CallbackSS(current, syncobj); - - asyncTester("async02", "S", "S", 100, 50, 3, 10, 1, - callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(100, callback.nsearch); - assertEquals(100, callback.ninsert); - } - - @Test - public void async03() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<String, String> current = new HashMap<String, String>(); - CallbackSS callback = new CallbackSS(current, syncobj); - - asyncTester("async03", "S", "S", 100, 10, 3, 10, 1, - callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(100, callback.nsearch); - assertEquals(100, callback.ninsert); - } - - @Test - public void async04() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<Long, String> current = new HashMap<Long, String>(); - CallbackRS callback = new CallbackRS(current, syncobj); - - asyncTester("async04", "q", "S", N_ENTRIES, N_OPS_MAX, - N_ASYNC_THREADS, 100, 1, callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(N_ENTRIES, callback.nsearch); - assertEquals(N_ENTRIES, callback.ninsert); - } - - @Test - public void async05() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<Long, Integer> current = new HashMap<Long, Integer>(); - CallbackRI callback = new CallbackRI(current, syncobj); - - asyncTester("async05", "q", "i", N_ENTRIES, N_OPS_MAX, - N_ASYNC_THREADS, 100, 1, callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(N_ENTRIES, callback.nsearch); - assertEquals(N_ENTRIES, callback.ninsert); - } - - @Test - public void async06() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<Long, Integer> current = new HashMap<Long, Integer>(); - CallbackRI callback = new CallbackRI(current, syncobj); - - asyncTester("async06", "q", "i", 1000000, 4000, - 3, 1000000, 0, callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(1000000, callback.nsearch); - assertEquals(1000000, callback.ninsert); - } - - @Test - public void asyncManyConnections01() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<String, String> current = new HashMap<String, String>(); - - for (int i = 0; i < 100; i++) { - CallbackSS callback = new CallbackSS(current, syncobj); - asyncTester("asyncMany01", "S", "S", 100, 10, 3, 10, 1, - callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(100, callback.nsearch); - assertEquals(100, callback.ninsert); - conn.close(""); - connvalid = false; - } - } - - @Test - public void asyncManyConnections02() - throws WiredTigerException { - Object syncobj = new Object(); - HashMap<Long, Integer> current = new HashMap<Long, Integer>(); - - for (int i = 0; i < 10; i++) { - // These are each long tests, so give some additional feedback - System.err.print(","); - CallbackRI callback = new CallbackRI(current, syncobj); - asyncTester("asyncMany02", "q", "i", 1000000, 4000, - 3, 1000000, 0, callback, callback, current); - - assertEquals(0, callback.nerror); - assertEquals(1000000, callback.nsearch); - assertEquals(1000000, callback.ninsert); - conn.close(""); - connvalid = false; - } - } - - @After - public void teardown() { - if (connvalid) { - conn.close(""); - } - } - -} diff --git a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java index 1b1c7eaab79..07c1fd49d0d 100644 --- a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java +++ b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java @@ -32,7 +32,6 @@ import org.junit.runners.Suite; @RunWith(Suite.class) @Suite.SuiteClasses( { - AsyncTest.class, AutoCloseTest.class, BackupCursorTest.class, ConfigTest.class, diff --git a/src/third_party/wiredtiger/test/suite/test_async01.py b/src/third_party/wiredtiger/test/suite/test_async01.py deleted file mode 100755 index 8e404064333..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_async01.py +++ /dev/null @@ -1,264 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2020 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import sys, threading, wiredtiger, wttest -from suite_subprocess import suite_subprocess -from wiredtiger import WiredTigerError -from wtscenario import make_scenarios - -# TODO - tmp code -def tty_pr(s): - o = open('/dev/tty', 'w') - o.write(s + '\n') - o.close() - -class Callback(wiredtiger.AsyncCallback): - def __init__(self, current): - self.current = current - self.ncompact = 0 - self.ninsert = 0 - self.nremove = 0 - self.nsearch = 0 - self.nupdate = 0 - self.nerror = 0 - self.lock = threading.RLock() - - def notify_error(self, key, value, optype, desc): - tty_pr('ERROR: notify(' + str(key) + ',' + str(value) + ',' + - str(optype) + '): ' + desc) - - def notify(self, op, op_ret, flags): - - # Note: we are careful not to throw any errors here. Any - # exceptions would be swallowed by a non-python worker thread. - try: - optype = op.get_type() - if optype != wiredtiger.WT_AOP_COMPACT: - key = op.get_key() - # - # Remove does not set a value. Just set it from the - # reference list. - # - if optype != wiredtiger.WT_AOP_REMOVE: - value = op.get_value() - else: - value = self.current[key] - - if optype == wiredtiger.WT_AOP_INSERT: - self.lock.acquire() - self.ninsert += 1 - self.lock.release() - elif optype == wiredtiger.WT_AOP_COMPACT: - self.lock.acquire() - self.ncompact += 1 - self.lock.release() - # Skip checking key/value. - return 0 - elif optype == wiredtiger.WT_AOP_REMOVE: - self.lock.acquire() - self.nremove += 1 - self.lock.release() - elif optype == wiredtiger.WT_AOP_SEARCH: - self.lock.acquire() - self.nsearch += 1 - self.lock.release() - elif optype == wiredtiger.WT_AOP_UPDATE: - self.lock.acquire() - self.nupdate += 1 - self.lock.release() - else: - self.notify_error(key, value, optype, 'unexpected optype') - self.lock.acquire() - self.nerror += 1 - self.lock.release() - if self.current[key] != value: - self.notify_error(key, value, optype, 'unexpected value') - self.lock.acquire() - self.nerror += 1 - self.lock.release() - except (BaseException) as err: - tty_pr('ERROR: exception in notify: ' + str(err)) - raise - - return 0 - -# test_async01.py -# Async operations -# Basic smoke-test of file and table async ops: tests get/set key, insert -# update, and remove. -class test_async01(wttest.WiredTigerTestCase, suite_subprocess): - """ - Test basic operations - """ - table_name1 = 'test_async01' - nentries = 100 - async_ops = nentries // 2 - async_threads = 3 - current = {} - - scenarios = make_scenarios([ - ('file-col', dict(tablekind='col',uri='file')), - ('file-fix', dict(tablekind='fix',uri='file')), - ('file-row', dict(tablekind='row',uri='file')), - ('lsm-row', dict(tablekind='row',uri='lsm')), - ('table-col', dict(tablekind='col',uri='table')), - ('table-fix', dict(tablekind='fix',uri='table')), - ('table-row', dict(tablekind='row',uri='table')), - ]) - - # Enable async for this test. - def conn_config(self): - return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ - 'threads=%s)' % self.async_threads - - def genkey(self, i): - if self.tablekind == 'row': - return 'key' + str(i) - else: - return self.recno(i+1) - - def genvalue(self, i): - if self.tablekind == 'fix': - return int(i & 0xff) - else: - return 'value' + str(i) - - # Create and populate the object. - def create_session(self, tablearg): - if self.tablekind == 'row': - keyformat = 'key_format=S' - else: - keyformat = 'key_format=r' # record format - if self.tablekind == 'fix': - valformat = 'value_format=8t' - else: - valformat = 'value_format=S' - create_args = keyformat + ',' + valformat - - self.pr('creating session: ' + create_args) - self.session.create(tablearg, create_args) - - def test_ops(self): - tablearg = self.uri + ':' + self.table_name1 - ncompact = 0 - self.create_session(tablearg) - - # Populate our reference table first, so we don't need to - # use locks to reference it. - self.current = {} - for i in range(0, self.nentries): - k = self.genkey(i) - v = self.genvalue(i) - self.current[k] = v - - # Populate table with async inserts, callback checks - # to ensure key/value is correct. - callback = Callback(self.current) - for i in range(0, self.nentries): - self.pr('creating async op') - op = self.conn.async_new_op(tablearg, None, callback) - k = self.genkey(i) - v = self.genvalue(i) - op.set_key(k) - op.set_value(v) - op.insert() - - # Compact after insert. - # Wait for all outstanding async ops to finish. - op = self.conn.async_new_op(tablearg, 'timeout=0', callback) - op.compact() - ncompact += 1 - self.conn.async_flush() - self.pr('flushed') - - # Now test async search and check key/value in the callback. - for i in range(0, self.nentries): - op = self.conn.async_new_op(tablearg, None, callback) - k = self.genkey(i) - op.set_key(k) - op.search() - - # Wait for all outstanding async ops to finish. - self.conn.async_flush() - self.pr('flushed') - - # Reset the reference table for the update. - for i in range(0, self.nentries): - k = self.genkey(i) - v = self.genvalue(i+10) - self.current[k] = v - - for i in range(0, self.nentries): - self.pr('creating async op') - op = self.conn.async_new_op(tablearg, None, callback) - k = self.genkey(i) - v = self.genvalue(i+10) - op.set_key(k) - op.set_value(v) - op.update() - - # Compact after update. - # Wait for all outstanding async ops to finish. - op = self.conn.async_new_op(tablearg, None, callback) - op.compact() - ncompact += 1 - self.conn.async_flush() - self.pr('flushed') - - # Now test async search and check key/value in the callback. - for i in range(0, self.nentries): - op = self.conn.async_new_op(tablearg, None, callback) - k = self.genkey(i) - op.set_key(k) - op.search() - - # Wait for all outstanding async ops to finish. - self.conn.async_flush() - self.pr('flushed') - - for i in range(0, self.nentries): - self.pr('creating async op') - op = self.conn.async_new_op(tablearg, None, callback) - k = self.genkey(i) - op.set_key(k) - op.remove() - - # Wait for all outstanding async ops to finish. - self.conn.async_flush() - self.pr('flushed') - - # Make sure all callbacks went according to plan. - self.assertTrue(callback.ncompact == ncompact) - self.assertTrue(callback.ninsert == self.nentries) - self.assertTrue(callback.nremove == self.nentries) - self.assertTrue(callback.nsearch == self.nentries * 2) - self.assertTrue(callback.nupdate == self.nentries) - self.assertTrue(callback.nerror == 0) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_async02.py b/src/third_party/wiredtiger/test/suite/test_async02.py deleted file mode 100755 index 265e36fbf2a..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_async02.py +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2020 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import sys, threading, wiredtiger, wttest -from suite_subprocess import suite_subprocess -from wiredtiger import WiredTigerError -from wtscenario import make_scenarios - -class Callback(wiredtiger.AsyncCallback): - def __init__(self, current): - self.current = current - self.ncompact = 0 - self.ninsert = 0 - self.nremove = 0 - self.nsearch = 0 - self.nupdate = 0 - self.nerror = 0 - self.lock = threading.RLock() - - def notify_error(self, key, value, optype, exp, desc): - self.tty('ERROR: notify(' + str(key) + ',' + str(value) + ',' + - str(optype) + '): ' + 'Expected: ' + str(exp) + ' ' + desc) - - def notify(self, op, op_ret, flags): - - # Note: we are careful not to throw any errors here. Any - # exceptions would be swallowed by a non-python worker thread. - try: - optype = op.get_type() - if optype != wiredtiger.WT_AOP_COMPACT: - key = op.get_key() - # - # Remove does not set a value. Just set it from the - # reference list. If an error is expected, the - # reference list should list it as the expected value. - # - if op_ret != 0: - value = op_ret - elif optype != wiredtiger.WT_AOP_REMOVE: - value = op.get_value() - else: - value = self.current[key] - - if optype == wiredtiger.WT_AOP_INSERT: - self.lock.acquire() - self.ninsert += 1 - self.lock.release() - elif optype == wiredtiger.WT_AOP_COMPACT: - self.lock.acquire() - self.ncompact += 1 - self.lock.release() - # Skip checking key/value. - return 0 - elif optype == wiredtiger.WT_AOP_REMOVE: - self.lock.acquire() - self.nremove += 1 - self.lock.release() - elif optype == wiredtiger.WT_AOP_SEARCH: - self.lock.acquire() - self.nsearch += 1 - self.lock.release() - elif optype == wiredtiger.WT_AOP_UPDATE: - self.lock.acquire() - self.nupdate += 1 - self.lock.release() - else: - self.notify_error(key, value, optype, 0, 'unexpected optype') - self.lock.acquire() - self.nerror += 1 - self.lock.release() - if self.current[key] != value: - self.notify_error(key, value, optype, self.current[key], 'unexpected value') - self.lock.acquire() - self.nerror += 1 - self.lock.release() - except (BaseException) as err: - self.tty('ERROR: exception in notify: ' + str(err)) - raise - - return 0 - -# test_async02.py -# Async operations -# Basic smoke-test of file and table async ops: tests get/set key, insert -# update, and remove. -class test_async02(wttest.WiredTigerTestCase, suite_subprocess): - """ - Test basic operations - """ - table_name1 = 'test_async02' - nentries = 100 - async_ops = nentries // 2 - async_threads = 3 - current = {} - - scenarios = make_scenarios([ - ('file-col', dict(tablekind='col',uri='file')), - ('file-fix', dict(tablekind='fix',uri='file')), - ('file-row', dict(tablekind='row',uri='file')), - ('lsm-row', dict(tablekind='row',uri='lsm')), - ('table-col', dict(tablekind='col',uri='table')), - ('table-fix', dict(tablekind='fix',uri='table')), - ('table-row', dict(tablekind='row',uri='table')), - ]) - - # Enable async for this test. - def conn_config(self): - return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ - 'threads=%s)' % self.async_threads - - def genkey(self, i): - if self.tablekind == 'row': - return 'key' + str(i) - else: - return self.recno(i+1) - - def genvalue(self, i): - if self.tablekind == 'fix': - return int(i & 0xff) - else: - return 'value' + str(i) - - # Create and populate the object. - def create_session(self, tablearg): - if self.tablekind == 'row': - keyformat = 'key_format=S' - else: - keyformat = 'key_format=r' # record format - if self.tablekind == 'fix': - valformat = 'value_format=8t' - else: - valformat = 'value_format=S' - create_args = keyformat + ',' + valformat - - self.pr('creating session: ' + create_args) - self.session.create(tablearg, create_args) - - def test_ops(self): - tablearg = self.uri + ':' + self.table_name1 - self.create_session(tablearg) - - # Populate our reference table first, so we don't need to - # use locks to reference it. - self.current = {} - for i in range(0, self.nentries): - k = self.genkey(i) - v = self.genvalue(i) - self.current[k] = v - - # Populate table with async inserts, callback checks - # to ensure key/value is correct. - callback = Callback(self.current) - for i in range(0, self.nentries): - self.pr('creating async op') - op = self.conn.async_new_op(tablearg, None, callback) - k = self.genkey(i) - v = self.genvalue(i) - op.set_key(k) - op.set_value(v) - op.insert() - - self.conn.async_flush() - self.pr('flushed') - - k = self.genkey(self.nentries // 2) - v = self.genvalue(self.nentries // 2) - k1 = self.genkey(self.nentries + 1) - v1 = self.genvalue(self.nentries + 1) - self.current[k] = wiredtiger.WT_DUPLICATE_KEY - self.current[k1] = wiredtiger.WT_NOTFOUND - # - # Error cases: - # Check inserting an existing record - WT_DUPLICATE_KEY. - # Check updating a non-existent record - WT_NOTFOUND. - # Check removing a non-existent record - WT_NOTFOUND. - # - op = self.conn.async_new_op(tablearg, 'overwrite=false', callback) - op.set_key(k) - op.set_value(v) - op.insert() - op = self.conn.async_new_op(tablearg, 'overwrite=false', callback) - op.set_key(k1) - op.set_value(v1) - op.update() - op = self.conn.async_new_op(tablearg, 'overwrite=false', callback) - op.set_key(k1) - op.remove() - - # Wait for all outstanding async ops to finish. - self.conn.async_flush() - self.pr('flushed') - - # Make sure all callbacks went according to plan. - self.assertTrue(callback.nerror == 0) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_async03.py b/src/third_party/wiredtiger/test/suite/test_async03.py deleted file mode 100644 index 957f180c172..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_async03.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2020 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -from wiredtiger import WiredTigerError -import sys, threading, wiredtiger, wttest - -class Callback(wiredtiger.AsyncCallback): - def __init__(self): - pass - - def notify(self, op, op_ret, flags): - raise AssertionError('callback should not be called in this test') - -# test_async03.py -# Async operations -# Try to run async code with an incorrect connection config. -# We expect an operation not supported error. -class test_async03(wttest.WiredTigerTestCase): - """ - Test basic operations - """ - table_name1 = 'test_async03' - - # Note: this usage is intentionally wrong, it is missing - # async=(enabled=true,... - conn_config = 'async=(ops_max=50,threads=3)' - - def test_ops(self): - tablearg = 'table:' + self.table_name1 - self.session.create(tablearg, 'key_format=S,value_format=S') - - callback = Callback() - - msg = '/Asynchronous operations not configured/' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.async_new_op(tablearg, None, callback), msg) - - self.conn.async_flush() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup15.py b/src/third_party/wiredtiger/test/suite/test_backup15.py index 08ddf177ea6..0327abe9838 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup15.py +++ b/src/third_party/wiredtiger/test/suite/test_backup15.py @@ -41,7 +41,7 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): counter=0 conn_config='cache_size=1G,log=(enabled,file_max=100K)' logmax="100K" - max_iteration=7 + max_iteration=5 mult=0 nops=100000 savefirst=0 @@ -70,20 +70,34 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): # def setup_directories(self): for i in range(0, self.max_iteration): - remove_dir = self.home_incr + '.' + str(i) - - create_dir = self.home_incr + '.' + str(i) + '/' + self.logpath - if os.path.exists(remove_dir): - os.remove(remove_dir) - os.makedirs(create_dir) - - if i == 0: - continue - remove_dir = self.home_full + '.' + str(i) - create_dir = self.home_full + '.' + str(i) + '/' + self.logpath - if os.path.exists(remove_dir): - os.remove(remove_dir) - os.makedirs(create_dir) + # The log directory is a subdirectory of the home directory, + # creating that will make the home directory also. + log_dir = self.home_incr + '.' + str(i) + '/' + self.logpath + os.makedirs(log_dir) + if i != 0: + log_dir = self.home_full + '.' + str(i) + '/' + self.logpath + os.makedirs(log_dir) + + def range_copy(self, filename, offset, size): + read_from = filename + old_to = self.home_incr + '.' + str(self.counter - 1) + '/' + filename + write_to = self.home_incr + '.' + str(self.counter) + '/' + filename + rfp = open(read_from, "r+b") + self.pr('RANGE CHECK file ' + old_to + ' offset ' + str(offset) + ' len ' + str(size)) + rfp2 = open(old_to, "r+b") + rfp.seek(offset, 0) + rfp2.seek(offset, 0) + buf = rfp.read(size) + buf2 = rfp2.read(size) + # This assertion tests that the offset range we're given actually changed + # from the previous backup. + self.assertNotEqual(buf, buf2) + wfp = open(write_to, "w+b") + wfp.seek(offset, 0) + wfp.write(buf) + rfp.close() + rfp2.close() + wfp.close() def take_full_backup(self): if self.counter != 0: @@ -99,12 +113,15 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): if self.initial_backup == True: buf = 'incremental=(granularity=1M,enabled=true,this_id=ID0)' - cursor = self.session.open_cursor('backup:', None, buf) + bkup_c = self.session.open_cursor('backup:', None, buf) + # We cannot use 'for newfile in bkup_c:' usage because backup cursors don't have + # values and adding in get_values returns ENOTSUP and causes the usage to fail. + # If that changes then this, and the use of the duplicate below can change. while True: - ret = cursor.next() + ret = bkup_c.next() if ret != 0: break - newfile = cursor.get_key() + newfile = bkup_c.get_key() if self.counter == 0: # Take a full bakcup into each incremental directory @@ -126,13 +143,18 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): shutil.copy(copy_from, copy_to) self.assertEqual(ret, wiredtiger.WT_NOTFOUND) - cursor.close() + bkup_c.close() def take_incr_backup(self): + self.assertTrue(self.counter > 0) # Open the backup data source for incremental backup. - buf = 'incremental=(src_id="ID' + str(self.counter-1) + '",this_id="ID' + str(self.counter) + '")' + buf = 'incremental=(src_id="ID' + str(self.counter - 1) + '",this_id="ID' + str(self.counter) + '")' self.pr(buf) bkup_c = self.session.open_cursor('backup:', None, buf) + + # We cannot use 'for newfile in bkup_c:' usage because backup cursors don't have + # values and adding in get_values returns ENOTSUP and causes the usage to fail. + # If that changes then this, and the use of the duplicate below can change. while True: ret = bkup_c.next() if ret != 0: @@ -150,9 +172,12 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): first = True config = 'incremental=(file=' + newfile + ')' dup_cnt = 0 + # For each file listed, open a duplicate backup cursor and copy the blocks. incr_c = self.session.open_cursor(None, bkup_c, config) - # For each file listed, open a duplicate backup cursor and copy the blocks. + # We cannot use 'for newfile in incr_c:' usage because backup cursors don't have + # values and adding in get_values returns ENOTSUP and causes the usage to fail. + # If that changes then this, and the use of the duplicate below can change. while True: ret = incr_c.next() if ret != 0: @@ -161,10 +186,9 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): offset = incrlist[0] size = incrlist[1] curtype = incrlist[2] - # 1 is WT_BACKUP_FILE - # 2 is WT_BACKUP_RANGE - self.assertTrue(curtype == 1 or curtype == 2) - if curtype == 1: + self.assertTrue(curtype == wiredtiger.WT_BACKUP_FILE or curtype == wiredtiger.WT_BACKUP_RANGE) + if curtype == wiredtiger.WT_BACKUP_FILE: + # Copy the whole file. if first == True: h = self.home_incr + '.' + str(self.counter) first = False @@ -176,33 +200,14 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): copy_to = h shutil.copy(copy_from, copy_to) else: + # Copy the block range. self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) - read_from = newfile - if self.counter > 0: - old_to = self.home_incr + '.' + str(self.counter - 1) + '/' + newfile - else: - old_to = newfile - write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile - write_to = self.home_incr + '.' + str(self.counter) + '/' + newfile - rfp = open(read_from, "r+b") - self.pr('RANGE CHECK file ' + old_to + ' offset ' + str(offset) + ' len ' + str(size)) - rfp2 = open(old_to, "r+b") - rfp.seek(offset, 0) - rfp2.seek(offset, 0) - buf = rfp.read(size) - buf2 = rfp2.read(size) - self.assertNotEqual(buf, buf2) - wfp = open(write_to, "w+b") - wfp.seek(offset, 0) - wfp.write(buf) - rfp.close() - rfp2.close() - wfp.close() + self.range_copy(newfile, offset, size) dup_cnt += 1 self.assertEqual(ret, wiredtiger.WT_NOTFOUND) incr_c.close() - # For each file, we want to copy the file into each of the later incremental directories + # For each file, we want to copy it into each of the later incremental directories. for i in range(self.counter, self.max_iteration): h = self.home_incr + '.' + str(i) copy_from = newfile @@ -215,18 +220,14 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): bkup_c.close() def compare_backups(self, t_uri): - # - # Run wt dump on full backup directory - # + # Run wt dump on full backup directory. full_backup_out = self.full_out + '.' + str(self.counter) home_dir = self.home_full + '.' + str(self.counter) if self.counter == 0: home_dir = self.home - self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=full_backup_out) - # - # Run wt dump on incremental backup directory - # + + # Run wt dump on incremental backup directory. incr_backup_out = self.incr_out + '.' + str(self.counter) home_dir = self.home_incr + '.' + str(self.counter) self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=incr_backup_out) @@ -264,11 +265,14 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): if self.initial_backup == False: self.counter += 1 - # - # This function will add records to the table (table:main), take incremental/full backups and - # validate the backups. - # - def add_data_validate_backups(self): + def test_backup15(self): + os.mkdir(self.bkp_home) + self.home = self.bkp_home + self.session.create(self.uri, "key_format=S,value_format=S") + + self.setup_directories() + + self.pr('*** Add data, checkpoint, take backups and validate ***') self.pr('Adding initial data') self.initial_backup = True self.add_data(self.uri) @@ -278,38 +282,18 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): # Each call now to take a full backup will make a copy into a full directory. Then # each incremental will take an incremental backup and we can compare them. - self.add_data(self.uri) - self.session.checkpoint() - self.take_full_backup() - self.take_incr_backup() - self.compare_backups(self.uri) - - self.add_data(self.uri) - self.session.checkpoint() - self.take_incr_backup() - self.take_full_backup() - self.compare_backups(self.uri) - - self.add_data(self.uri) - self.session.checkpoint() - self.take_full_backup() - self.take_incr_backup() - self.compare_backups(self.uri) - - self.add_data(self.uri) - self.take_incr_backup() - self.take_full_backup() - self.compare_backups(self.uri) - - def test_backup15(self): - os.mkdir(self.bkp_home) - self.home = self.bkp_home - self.session.create(self.uri, "key_format=S,value_format=S") - - self.setup_directories() - - self.pr('*** Add data, checkpoint, take backups and validate ***') - self.add_data_validate_backups() + for i in range(1, self.max_iteration): + self.add_data(self.uri) + self.session.checkpoint() + # Swap the order of the full and incremental backups. It should not matter. They + # should not interfere with each other. + if i % 2 == 0: + self.take_full_backup() + self.take_incr_backup() + else: + self.take_incr_backup() + self.take_full_backup() + self.compare_backups(self.uri) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs16.py b/src/third_party/wiredtiger/test/suite/test_hs16.py new file mode 100644 index 00000000000..1e5da4a19a1 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs16.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import time, wiredtiger, wttest + +def timestamp_str(t): + return '%x' % t + +# test_hs16.py +# Ensure that we don't panic when inserting an update without timestamp to the history store. +class test_hs16(wttest.WiredTigerTestCase): + conn_config = 'cache_size=5MB' + session_config = 'isolation=snapshot' + + def test_hs16(self): + uri = 'table:test_hs16' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + + # Insert an update without timestamp + self.session.begin_transaction() + cursor[str(0)] = 'a' + self.session.commit_transaction() + + # Update an update at timestamp 1 + self.session.begin_transaction() + cursor[str(0)] = 'b' + self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + + # Open anther session to make the next update without timestamp non-globally visible + session2 = self.setUpSessionOpen(self.conn) + cursor2 = session2.open_cursor(uri) + session2.begin_transaction() + cursor[str(1)] = 'a' + + # Update an update without timestamp + self.session.begin_transaction() + cursor[str(0)] = 'c' + self.session.commit_transaction() + + # Update an update at timestamp 2 + self.session.begin_transaction() + cursor[str(0)] = 'd' + self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + + # Do a checkpoint, it should not panic + self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_import01.py b/src/third_party/wiredtiger/test/suite/test_import01.py new file mode 100644 index 00000000000..f6c67a82c3b --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_import01.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_import01.py +# Import a file into a running database. + +import os, re, shutil +import wiredtiger, wttest + +def timestamp_str(t): + return '%x' % t + +class test_import01(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + session_config = 'isolation=snapshot' + + def update(self, uri, key, value, commit_ts): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[key] = value + self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + cursor.close() + + def check(self, uri, key, value, read_ts): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + cursor.set_key(key) + self.assertEqual(0, cursor.search()) + self.assertEqual(value, cursor.get_value()) + self.session.rollback_transaction() + cursor.close() + + # We know the ID can be different between configs, so just remove it from comparison. + # Everything else should be the same. + def config_compare(self, aconf, bconf): + a = re.sub('id=\d+,?', '', aconf) + a = (re.sub('\w+=\(.*?\)+,?', '', a).strip(',').split(',') + + re.findall('\w+=\(.*?\)+', a)) + b = re.sub('id=\d+,?', '', bconf) + b = (re.sub('\w+=\(.*?\)+,?', '', b).strip(',').split(',') + + re.findall('\w+=\(.*?\)+', b)) + self.assertTrue(a.sort() == b.sort()) + + # Helper for populating a database to simulate importing files into an existing database. + def populate(self): + # Create file:test_import01_[1-100]. + for fileno in range(1, 100): + uri = 'file:test_import01_{}'.format(fileno) + self.session.create(uri, 'key_format=i,value_format=S') + cursor = self.session.open_cursor(uri) + # Insert keys [1-100] with value 'foo'. + for key in range(1, 100): + cursor[key] = 'foo' + cursor.close() + + def copy_file(self, file_name, old_dir, new_dir): + old_path = os.path.join(old_dir, file_name) + if os.path.isfile(old_path) and "WiredTiger.lock" not in file_name and \ + "Tmplog" not in file_name and "Preplog" not in file_name: + shutil.copy(old_path, new_dir) + + def test_file_import(self): + original_db_file = 'original_db_file' + uri = 'file:' + original_db_file + + create_config = 'allocation_size=512,key_format=u,log=(enabled=true),value_format=u' + self.session.create(uri, create_config) + + key1 = b'1' + key2 = b'2' + key3 = b'3' + key4 = b'4' + value1 = b'\x01\x02aaa\x03\x04' + value2 = b'\x01\x02bbb\x03\x04' + value3 = b'\x01\x02ccc\x03\x04' + value4 = b'\x01\x02ddd\x03\x04' + + # Add some data. + self.update(uri, key1, value1, 10) + self.update(uri, key2, value2, 20) + + # Perform a checkpoint. + self.session.checkpoint() + + # Add more data. + self.update(uri, key3, value3, 30) + self.update(uri, key4, value4, 40) + + # Perform a checkpoint. + self.session.checkpoint() + + # Export the metadata for the table. + c = self.session.open_cursor('metadata:', None, None) + original_db_file_config = c[uri] + c.close() + + self.printVerbose(3, '\nFILE CONFIG\n' + original_db_file_config) + + # Close the connection. + self.close_conn() + + # Create a new database and connect to it. + newdir = 'IMPORT_DB' + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + self.conn = self.setUpConnectionOpen(newdir) + self.session = self.setUpSessionOpen(self.conn) + + # Make a bunch of files and fill them with data. + self.populate() + + # Copy over the datafiles for the object we want to import. + self.copy_file(original_db_file, '.', newdir) + + # Contruct the config string. + import_config = 'import=(enabled,repair=false,file_metadata=(' + \ + original_db_file_config + '))' + + # Import the file. + self.session.create(uri, import_config) + + # Verify object. + self.session.verify(uri) + + # Check that the previously inserted values survived the import. + self.check(uri, key1, value1, 10) + self.check(uri, key2, value2, 20) + self.check(uri, key3, value3, 30) + self.check(uri, key4, value4, 40) + + # Compare configuration metadata. + c = self.session.open_cursor('metadata:', None, None) + current_db_file_config = c[uri] + c.close() + self.config_compare(original_db_file_config, current_db_file_config) + + key5 = b'5' + key6 = b'6' + value5 = b'\x01\x02eee\x03\x04' + value6 = b'\x01\x02fff\x03\x04' + + # Add some data and check that the file operates as usual after importing. + self.update(uri, key5, value5, 50) + self.update(uri, key6, value6, 60) + + self.check(uri, key5, value5, 50) + self.check(uri, key6, value6, 60) + + # Perform a checkpoint. + self.session.checkpoint() + + def test_file_import_dropped_file(self): + original_db_file = 'original_db_file' + uri = 'file:' + original_db_file + + create_config = 'allocation_size=512,key_format=u,log=(enabled=true),value_format=u' + self.session.create(uri, create_config) + + key1 = b'1' + key2 = b'2' + value1 = b'\x01\x02aaa\x03\x04' + value2 = b'\x01\x02bbb\x03\x04' + + # Add some data. + self.update(uri, key1, value1, 10) + self.update(uri, key2, value2, 20) + + # Perform a checkpoint. + self.session.checkpoint() + + # Export the metadata for the table. + c = self.session.open_cursor('metadata:', None, None) + original_db_file_config = c[uri] + c.close() + + self.printVerbose(3, '\nFILE CONFIG\n' + original_db_file_config) + + # Make a bunch of files and fill them with data. + self.populate() + + # Make a copy of the data file that we're about to drop. + backup_dir = 'BACKUP' + shutil.rmtree(backup_dir, ignore_errors=True) + os.mkdir(backup_dir) + self.copy_file(original_db_file, '.', backup_dir) + + # Drop the table. + # We'll be importing it back into our database shortly. + self.session.drop(uri) + + # Now copy it back to our database directory. + self.copy_file(original_db_file, backup_dir, '.') + + # Contruct the config string. + import_config = 'import=(enabled,repair=false,file_metadata=(' + \ + original_db_file_config + '))' + + # Import the file. + self.session.create(uri, import_config) + + # Verify object. + self.session.verify(uri) + + # Check that the previously inserted values survived the import. + self.check(uri, key1, value1, 10) + self.check(uri, key2, value2, 20) + + # Compare configuration metadata. + c = self.session.open_cursor('metadata:', None, None) + current_db_file_config = c[uri] + c.close() + self.config_compare(original_db_file_config, current_db_file_config) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_import02.py b/src/third_party/wiredtiger/test/suite/test_import02.py new file mode 100644 index 00000000000..6fda30d8899 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_import02.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_import02.py +# Error conditions when trying to import files. + +import os, shutil +import wiredtiger, wttest + +def timestamp_str(t): + return '%x' % t + +class test_import02(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + session_config = 'isolation=snapshot' + + def update(self, uri, key, value, commit_ts): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[key] = value + self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + cursor.close() + + # Helper for populating a database to simulate importing files into an existing database. + def populate(self): + # Create file:test_import02_[1-100]. + for fileno in range(1, 100): + uri = 'file:test_import02_{}'.format(fileno) + self.session.create(uri, 'key_format=i,value_format=S') + cursor = self.session.open_cursor(uri) + # Insert keys [1-100] with value 'foo'. + for key in range(1, 100): + cursor[key] = 'foo' + cursor.close() + + def copy_file(self, file_name, old_dir, new_dir): + old_path = os.path.join(old_dir, file_name) + if os.path.isfile(old_path) and "WiredTiger.lock" not in file_name and \ + "Tmplog" not in file_name and "Preplog" not in file_name: + shutil.copy(old_path, new_dir) + + # The cases where 'file_metadata' is empty or the config option itself is missing entirely are + # almost identical. Let's capture this in a helper and call them from each test. + def no_metadata_helper(self, import_config): + original_db_file = 'original_db_file' + uri = 'file:' + original_db_file + + create_config = 'allocation_size=512,key_format=u,log=(enabled=true),value_format=u' + self.session.create(uri, create_config) + + key1 = b'1' + key2 = b'2' + value1 = b'\x01\x02aaa\x03\x04' + value2 = b'\x01\x02bbb\x03\x04' + + # Add some data. + self.update(uri, key1, value1, 10) + self.update(uri, key2, value2, 20) + + # Perform a checkpoint. + self.session.checkpoint() + + # Close the connection. + self.close_conn() + + # Create a new database and connect to it. + newdir = 'IMPORT_DB' + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + self.conn = self.setUpConnectionOpen(newdir) + self.session = self.setUpSessionOpen(self.conn) + + # Copy over the datafiles for the object we want to import. + self.copy_file(original_db_file, '.', newdir) + + # Import the file. + # Since we need "file_metadata" without the "repair" option, we should expect an error here. + with self.expectedStderrPattern( + 'file:original_db_file: import requires that \'file_metadata\' is specified'): + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.session.create(uri, import_config)) + + def test_file_import_empty_metadata(self): + self.no_metadata_helper('import=(enabled,repair=false,file_metadata="")') + + def test_file_import_no_metadata(self): + self.no_metadata_helper('import=(enabled,repair=false)') + + def test_file_import_existing_uri(self): + original_db_file = 'original_db_file' + uri = 'file:' + original_db_file + + create_config = 'allocation_size=512,key_format=u,log=(enabled=true),value_format=u' + self.session.create(uri, create_config) + + key1 = b'1' + key2 = b'2' + + value1 = b'\x01\x02aaa\x03\x04' + value2 = b'\x01\x02bbb\x03\x04' + + # Add some data. + self.update(uri, key1, value1, 10) + self.update(uri, key2, value2, 20) + + # Perform a checkpoint. + self.session.checkpoint() + + # Export the metadata for the table. + c = self.session.open_cursor('metadata:', None, None) + original_db_file_config = c[uri] + c.close() + + self.printVerbose(3, '\nFILE CONFIG\n' + original_db_file_config) + + # Make a bunch of files and fill them with data. + self.populate() + + # Contruct the config string. + import_config = 'import=(enabled,repair=false,file_metadata=(' + \ + original_db_file_config + '))' + + # Try to import the file even though it already exists in our database. + # We should get an error back. + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.session.create(uri, import_config)) + + def test_import_file_missing_file(self): + original_db_file = 'original_db_file' + uri = 'file:' + original_db_file + + # Make a bunch of files and fill them with data. + self.populate() + + self.session.checkpoint() + + # Export the metadata for one of the files we made. + # We just need an example of what a file configuration would typically look like. + c = self.session.open_cursor('metadata:', None, None) + example_db_file_config = c['file:test_import02_1'] + c.close() + + self.printVerbose(3, '\nFILE CONFIG\n' + example_db_file_config) + + # Contruct the config string. + import_config = 'import=(enabled,repair=false,file_metadata=(' + \ + example_db_file_config + '))' + + # Try to import a file that doesn't exist on disk. + # We should get an error back. + with self.expectedStderrPattern( + 'file:original_db_file: attempted to import file that does not exist'): + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.session.create(uri, import_config)) diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig01.py b/src/third_party/wiredtiger/test/suite/test_reconfig01.py index 1e7050f2a4c..7a75419af52 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig01.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig01.py @@ -37,33 +37,6 @@ class test_reconfig01(wttest.WiredTigerTestCase): def test_reconfig_shared_cache(self): self.conn.reconfigure("shared_cache=(name=pool,size=300M)") - def test_reconfig_async(self): - # Async starts off. Reconfigure through all the various cases, each - # building from the previous state. - # Async is off, and remains off. - self.conn.reconfigure("async=(enabled=false)") - # Async is off, turn it on. - self.conn.reconfigure("async=(enabled=true)") - # Async is on, and remains on. - self.conn.reconfigure("async=(enabled=true)") - # Async is on, turn it off. - self.conn.reconfigure("async=(enabled=false)") - # Async is off, turn it on with ops_max and threads. - self.conn.reconfigure("async=(enabled=true,ops_max=512,threads=10)") - # Reconfigure and use same thread count. (no-op) - self.conn.reconfigure("async=(threads=10)") - # Reconfigure more threads. - self.conn.reconfigure("async=(threads=14)") - # Reconfigure fewer threads. - self.conn.reconfigure("async=(threads=8)") - # Reconfigure illegal ops_max (ignored). - self.conn.reconfigure("async=(ops_max=1024)") - # Turn async off. - self.conn.reconfigure("async=(enabled=false)") - # Async is off, turn it on. Should end up with the - # same ops_max of 512 and thread of 8. - self.conn.reconfigure("async=(enabled=true)") - def test_reconfig_eviction(self): # Increase the max number of running threads (default 8). self.conn.reconfigure("eviction=(threads_max=10)") diff --git a/src/third_party/wiredtiger/test/suite/test_txn24.py b/src/third_party/wiredtiger/test/suite/test_txn24.py new file mode 100644 index 00000000000..0e239ec77c6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn24.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn24.py +# Transactions and eviction: Test if using snapshot isolation for eviction threads helps with +# cache stuck issue. +# + +import wiredtiger, wttest +import time + +class test_txn24(wttest.WiredTigerTestCase): + + session_config = 'isolation=snapshot' + + def conn_config(self): + # We want to either eliminate or keep the application thread role in eviction to minimum. + # This will ensure that the dedicated eviction threads are doing the heavy lifting. + return 'cache_size=100MB,eviction_target=80,eviction_dirty_target=5,eviction_trigger=100,\ + eviction_updates_target=5,eviction_dirty_trigger=99,eviction_updates_trigger=100,\ + eviction=(threads_max=4)' + + def test_snapshot_isolation_and_eviction(self): + + # Create and populate a table. + uri = "table:test_txn24" + table_params = 'key_format=i,value_format=S' + default_val = 'ABCD' * 60 + new_val = 'YYYY' * 60 + final_val = 'ZZZZ' * 60 + n_rows = 480000 + + self.session.create(uri, table_params) + cursor = self.session.open_cursor(uri, None) + for i in range(0, n_rows): + cursor[i] = default_val + cursor.close() + + # Perform a checkpoint. There should be no dirty content in the cache after this. + self.session.checkpoint() + + # Start a transaction, make an update and keep it running. + cursor = self.session.open_cursor(uri, None) + self.session.begin_transaction('isolation=snapshot') + cursor[0] = new_val + + # Start few sessions and transactions, make updates and try committing them. + session2 = self.setUpSessionOpen(self.conn) + cursor2 = session2.open_cursor(uri) + start_row = int(n_rows/4) + for i in range(0, 120): + session2.begin_transaction('isolation=snapshot') + for j in range(0,1000): + cursor2[start_row] = new_val + start_row += 1 + session2.commit_transaction() + + session3 = self.setUpSessionOpen(self.conn) + cursor3 = session3.open_cursor(uri) + start_row = int(n_rows/2) + for i in range(0, 120): + session3.begin_transaction('isolation=snapshot') + for j in range(0,1000): + cursor3[start_row] = new_val + start_row += 1 + session3.commit_transaction() + + # At this point in time, we have made roughly 90% cache dirty. If we are not using + # snaphsots for eviction threads, the cache state will remain like this forever and we may + # never reach this part of code. We might get a rollback error by now or WT will panic with + # cache stuck error. + # + # Even if we dont get an error by now and if we try to insert new data at this point in + # time, dirty cache usage will exceed 100% if eviction threads are not using snapshot + # isolation. In that case, we will eventually get a rollback error for sure. + + session4 = self.setUpSessionOpen(self.conn) + cursor4 = session4.open_cursor(uri) + start_row = 1 + for i in range(0, 120): + session4.begin_transaction('isolation=snapshot') + for j in range(0,1000): + cursor4[start_row] = new_val + start_row += 1 + session4.commit_transaction() + + # If we have done all operations error free so far, eviction threads have been successful. + + self.session.commit_transaction() + cursor.close() + self.session.close() + + cursor2.close() + session2.close() + + cursor3.close() + session3.close() + + cursor4.close() + session4.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 617a8326582..32d52694103 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -449,10 +449,8 @@ class WiredTigerTestCase(unittest.TestCase): try: self.fdTearDown() - # Only check for unexpected output if the test passed - if passed: - self.captureout.check(self) - self.captureerr.check(self) + self.captureout.check(self) + self.captureerr.check(self) finally: # always get back to original directory os.chdir(self.origcwd) |