diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-08-28 00:23:10 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-08-28 00:23:10 +0000 |
commit | 33621eab05f101b161ae8834cb8efc3980e8f17f (patch) | |
tree | c417ac8ca14a888e7974e97146dab164e7744afd /src | |
parent | 60d8961a7db72f5e4a31c8017c4b7442f8e04c54 (diff) | |
download | mongo-33621eab05f101b161ae8834cb8efc3980e8f17f.tar.gz |
Import wiredtiger: 956384325e5ff99567ea7b63b1b87b7bb62c76b8 from branch mongodb-4.4
ref: 7dfd939186..956384325e
for: 4.3.1
WT-4831 Add option to python tests to not fail if stdout is not empty
WT-4935 Add a perf test to find out the wiredtiger_calc_modify overhead
WT-5062 Adjust the record size to consume less size
WT-5063 Return proper error message for cursor modify operation for not supported cursor types
Diffstat (limited to 'src')
15 files changed, 413 insertions, 142 deletions
diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 73816d980ac..e9a03d44250 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -220,6 +220,33 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) goto err; continue; } + if (STRING_MATCH("modify", k.str, k.len)) { + if ((workp->modify = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("modify_delta", k.str, k.len)) { + if (v.type == WT_CONFIG_ITEM_ID || v.type == WT_CONFIG_ITEM_STRING) { + if (strncmp(v.str, "rand", 4) != 0) + goto err; + /* Special random value */ + workp->modify_delta = INT64_MAX; + F_SET(wtperf, CFG_GROW); + } else { + workp->modify_delta = v.val; + if (v.val > 0) + F_SET(wtperf, CFG_GROW); + if (v.val < 0) + F_SET(wtperf, CFG_SHRINK); + } + continue; + } + if (STRING_MATCH("modify_force_update", k.str, k.len)) { + if (v.type != WT_CONFIG_ITEM_BOOL) + goto err; + workp->modify_force_update = v.val; + continue; + } if (STRING_MATCH("ops_per_txn", k.str, k.len)) { if ((workp->ops_per_txn = v.val) < 0) goto err; @@ -302,7 +329,8 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) scan = NULL; if (ret != 0) goto err; - if (workp->insert == 0 && workp->read == 0 && workp->update == 0 && workp->truncate == 0) + if (workp->insert == 0 && workp->modify == 0 && workp->read == 0 && workp->truncate == 0 && + workp->update == 0) goto err; /* Why run with truncate if we don't want any truncation. */ if (workp->truncate != 0 && workp->truncate_pct == 0 && workp->truncate_count == 0) @@ -312,7 +340,8 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) /* Truncate should have its own exclusive thread. */ if (workp->truncate != 0 && workp->threads > 1) goto err; - if (workp->truncate != 0 && (workp->insert > 0 || workp->read > 0 || workp->update > 0)) + if (workp->truncate != 0 && + (workp->insert > 0 || workp->modify > 0 || workp->read > 0 || workp->update > 0)) goto err; wtperf->workers_cnt += (u_int)workp->threads; } @@ -765,11 +794,11 @@ config_sanity(WTPERF *wtperf) if (wtperf->workload != NULL) for (i = 0, workp = wtperf->workload; i < wtperf->workload_cnt; ++i, ++workp) { - if (opts->readonly && - (workp->insert != 0 || workp->update != 0 || workp->truncate != 0)) { + if (opts->readonly && (workp->insert != 0 || workp->modify != 0 || + workp->truncate != 0 || workp->update != 0)) { fprintf(stderr, - "Invalid workload: insert, update or " - "truncate specified with readonly\n"); + "Invalid workload: insert, modify, truncate or update specified with " + "readonly\n"); return (EINVAL); } if (workp->insert != 0 && workp->table_index != INT32_MAX) { @@ -785,6 +814,12 @@ config_sanity(WTPERF *wtperf) workp->table_index, opts->table_count); return (EINVAL); } + if (workp->modify != 0 && workp->ops_per_txn == 0) { + fprintf(stderr, + "Modify operations must be executed with explicit transaction, specify " + "ops_per_txn."); + return (EINVAL); + } } return (0); } diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/modify-force-update-large-record-btree.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/modify-force-update-large-record-btree.wtperf new file mode 100644 index 00000000000..ced0ef90ac1 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/modify-force-update-large-record-btree.wtperf @@ -0,0 +1,14 @@ +# wtperf options file: btree with updates. The goal here is to have a workload of +# large documents that uses a lot of transaction IDs for modify with forced update +# to ensure that update operation happened instead of modify. +conn_config="cache_size=20G" +sess_config="isolation=snapshot" +table_config="type=file" +key_sz=40 +value_sz=10000 +icount=500000 +report_interval=5 +random_value=true +run_time=120 +populate_threads=1 +threads=((count=4,modify=1,ops_per_txn=1,modify_force_update=true)) diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/modify-large-record-btree.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/modify-large-record-btree.wtperf new file mode 100644 index 00000000000..1141766481f --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/modify-large-record-btree.wtperf @@ -0,0 +1,13 @@ +# wtperf options file: btree with updates. The goal here is to have a workload +# of large documents that uses a lot of transaction IDs for modify operations +conn_config="cache_size=20G" +sess_config="isolation=snapshot" +table_config="type=file" +key_sz=40 +value_sz=10000 +icount=500000 +report_interval=5 +random_value=true +run_time=120 +populate_threads=1 +threads=((count=4,modify=1,ops_per_txn=1)) diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh index 881ca7c7865..f339034faf2 100755 --- a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh +++ b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh @@ -39,18 +39,18 @@ rm -f $outfile echo "Parsed $# args: test: $wttest runmax: $runmax args: $wtarg" >> $outfile # Each of these has an entry for each op in ops below. -avg=(0 0 0 0) -max=(0 0 0 0) -min=(0 0 0 0) -sum=(0 0 0 0) +avg=(0 0 0 0 0) +max=(0 0 0 0 0) +min=(0 0 0 0 0) +sum=(0 0 0 0 0) # Load needs floating point and bc, handle separately. -loadindex=5 +loadindex=6 avg[$loadindex]=0 max[$loadindex]=0 min[$loadindex]=0 sum[$loadindex]=0 -ops=(read insert update truncate) -outp=("Read count:" "Insert count:" "Update count:" "Truncate count:") +ops=(insert modify read truncate update) +outp=("Insert count:" "Modify count:" "Read count:" "Truncate count:" "Update count:" ) outp[$loadindex]="Load time:" # getval min/max val cur diff --git a/src/third_party/wiredtiger/bench/wtperf/track.c b/src/third_party/wiredtiger/bench/wtperf/track.c index cf0e98061ff..7e1f1e7e321 100644 --- a/src/third_party/wiredtiger/bench/wtperf/track.c +++ b/src/third_party/wiredtiger/bench/wtperf/track.c @@ -118,6 +118,11 @@ sum_insert_ops(WTPERF *wtperf) return (sum_ops(wtperf, offsetof(WTPERF_THREAD, insert))); } uint64_t +sum_modify_ops(WTPERF *wtperf) +{ + return (sum_ops(wtperf, offsetof(WTPERF_THREAD, modify))); +} +uint64_t sum_read_ops(WTPERF *wtperf) { return (sum_ops(wtperf, offsetof(WTPERF_THREAD, read))); @@ -185,11 +190,11 @@ latency_op(WTPERF *wtperf, size_t field_offset, uint32_t *avgp, uint32_t *minp, } } void -latency_read(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) +latency_insert(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) { static uint32_t last_avg = 0, last_max = 0, last_min = 0; - latency_op(wtperf, offsetof(WTPERF_THREAD, read), avgp, minp, maxp); + latency_op(wtperf, offsetof(WTPERF_THREAD, insert), avgp, minp, maxp); /* * If nothing happened, graph the average, minimum and maximum as they were the last time, it @@ -206,11 +211,32 @@ latency_read(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) } } void -latency_insert(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) +latency_modify(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) { static uint32_t last_avg = 0, last_max = 0, last_min = 0; - latency_op(wtperf, offsetof(WTPERF_THREAD, insert), avgp, minp, maxp); + latency_op(wtperf, offsetof(WTPERF_THREAD, modify), avgp, minp, maxp); + + /* + * If nothing happened, graph the average, minimum and maximum as they were the last time, it + * keeps the graphs from having discontinuities. + */ + if (*minp == 0) { + *avgp = last_avg; + *minp = last_min; + *maxp = last_max; + } else { + last_avg = *avgp; + last_min = *minp; + last_max = *maxp; + } +} +void +latency_read(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) +{ + static uint32_t last_avg = 0, last_max = 0, last_min = 0; + + latency_op(wtperf, offsetof(WTPERF_THREAD, read), avgp, minp, maxp); /* * If nothing happened, graph the average, minimum and maximum as they were the last time, it @@ -286,6 +312,11 @@ sum_insert_latency(WTPERF *wtperf, TRACK *total) sum_latency(wtperf, offsetof(WTPERF_THREAD, insert), total); } static void +sum_modify_latency(WTPERF *wtperf, TRACK *total) +{ + sum_latency(wtperf, offsetof(WTPERF_THREAD, modify), total); +} +static void sum_read_latency(WTPERF *wtperf, TRACK *total) { sum_latency(wtperf, offsetof(WTPERF_THREAD, read), total); @@ -344,6 +375,8 @@ latency_print(WTPERF *wtperf) sum_insert_latency(wtperf, &total); latency_print_single(wtperf, &total, "insert"); + sum_modify_latency(wtperf, &total); + latency_print_single(wtperf, &total, "modify"); sum_read_latency(wtperf, &total); latency_print_single(wtperf, &total, "read"); sum_update_latency(wtperf, &total); diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 3c13304f1c1..f6c92a12ffe 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -39,7 +39,7 @@ static int execute_workload(WTPERF *); static int find_table_count(WTPERF *); static WT_THREAD_RET monitor(void *); static WT_THREAD_RET populate_thread(void *); -static void randomize_value(WTPERF_THREAD *, char *); +static void randomize_value(WTPERF_THREAD *, char *, int64_t); static void recreate_dir(const char *); static WT_THREAD_RET scan_worker(void *); static int start_all_runs(WTPERF *); @@ -48,7 +48,7 @@ static void start_threads( WTPERF *, WORKLOAD *, WTPERF_THREAD *, u_int, WT_THREAD_CALLBACK (*)(void *)); static void stop_threads(u_int, WTPERF_THREAD *); static WT_THREAD_RET thread_run_wtperf(void *); -static void update_value_delta(WTPERF_THREAD *); +static void update_value_delta(WTPERF_THREAD *, int64_t); static WT_THREAD_RET worker(void *); static uint64_t wtperf_rand(WTPERF_THREAD *); @@ -65,10 +65,11 @@ get_next_incr(WTPERF *wtperf) /* * Each time this function is called we will overwrite the first and one other element in the value - * buffer. + * buffer. The delta can be zero for insert operations and may have a value for update or modify + * workloads. */ static void -randomize_value(WTPERF_THREAD *thread, char *value_buf) +randomize_value(WTPERF_THREAD *thread, char *value_buf, int64_t delta) { CONFIG_OPTS *opts; uint8_t *vb; @@ -78,13 +79,13 @@ randomize_value(WTPERF_THREAD *thread, char *value_buf) /* * Limit how much of the buffer we validate for length, this means that only threads that do - * growing updates will ever make changes to values outside of the initial value size, but - * that's a fair trade off for avoiding figuring out how long the value is more accurately in - * this performance sensitive function. + * growing updates/modifies will ever make changes to values outside of the initial value size, + * but that's a fair trade off for avoiding figuring out how long the value is more accurately + * in this performance sensitive function. */ - if (thread->workload == NULL || thread->workload->update_delta == 0) + if (delta == 0) max_range = opts->value_sz; - else if (thread->workload->update_delta > 0) + else if (delta > 0) max_range = opts->value_sz_max; else max_range = opts->value_sz_min; @@ -134,21 +135,20 @@ map_key_to_table(CONFIG_OPTS *opts, uint64_t k) } /* - * Figure out and extend the size of the value string, used for growing updates. We know that the - * value to be updated is in the threads value scratch buffer. + * Figure out and extend the size of the value string, used for growing updates/modifies. Delta is + * the value to be updated according to the thread operation. */ static inline void -update_value_delta(WTPERF_THREAD *thread) +update_value_delta(WTPERF_THREAD *thread, int64_t delta) { CONFIG_OPTS *opts; WTPERF *wtperf; char *value; - int64_t delta, len, new_len; + int64_t len, new_len; wtperf = thread->wtperf; opts = wtperf->opts; value = thread->value_buf; - delta = thread->workload->update_delta; len = (int64_t)strlen(value); if (delta == INT64_MAX) @@ -296,6 +296,8 @@ op_name(uint8_t *op) return ("insert"); case WORKER_INSERT_RMW: return ("insert_rmw"); + case WORKER_MODIFY: + return ("modify"); case WORKER_READ: return ("read"); case WORKER_TRUNCATE: @@ -385,14 +387,15 @@ worker_async(void *arg) goto op_err; case WORKER_INSERT: if (opts->random_value) - randomize_value(thread, value_buf); + randomize_value(thread, value_buf, 0); asyncop->set_value(asyncop, value_buf); if ((ret = asyncop->insert(asyncop)) == 0) break; goto op_err; case WORKER_UPDATE: if (opts->random_value) - randomize_value(thread, value_buf); + randomize_value( + thread, value_buf, thread->workload ? thread->workload->update_delta : 0); asyncop->set_value(asyncop, value_buf); if ((ret = asyncop->update(asyncop)) == 0) break; @@ -504,14 +507,16 @@ worker(void *arg) WTPERF_THREAD *thread; WT_CONNECTION *conn; WT_CURSOR **cursors, *cursor, *log_table_cursor, *tmp_cursor; + WT_ITEM newv, oldv; + WT_MODIFY entries[MAX_MODIFY_NUM]; WT_SESSION *session; - size_t i; - uint32_t total_table_count; - int64_t ops, ops_per_txn; + size_t i, iter, modify_offset, modify_size, total_modify_size, value_len; + int64_t delta, ops, ops_per_txn; uint64_t log_id, next_val, usecs; + uint32_t rand_val, total_table_count; uint8_t *op, *op_end; - int measure_latency, ret, truncated; - char *value_buf, *key_buf, *value; + int measure_latency, nmodify, ret, truncated; + char *key_buf, *value, *value_buf; char buf[512]; thread = (WTPERF_THREAD *)arg; @@ -607,12 +612,17 @@ worker(void *arg) else next_val = opts->icount + get_next_incr(wtperf); break; + case WORKER_MODIFY: + trk = &thread->modify; + /* FALLTHROUGH */ case WORKER_READ: - trk = &thread->read; + if (*op == WORKER_READ) + trk = &thread->read; /* FALLTHROUGH */ case WORKER_UPDATE: if (*op == WORKER_UPDATE) trk = &thread->update; + next_val = wtperf_rand(thread); /* @@ -681,7 +691,7 @@ worker(void *arg) /* FALLTHROUGH */ case WORKER_INSERT: if (opts->random_value) - randomize_value(thread, value_buf); + randomize_value(thread, value_buf, 0); cursor->set_value(cursor, value_buf); if ((ret = cursor->insert(cursor)) == 0) break; @@ -697,6 +707,7 @@ worker(void *arg) break; } goto op_err; + case WORKER_MODIFY: case WORKER_UPDATE: if ((ret = cursor->search(cursor)) == 0) { if ((ret = cursor->get_value(cursor, &value)) != 0) { @@ -707,24 +718,104 @@ worker(void *arg) * Copy as much of the previous value as is safe, and be sure to NUL-terminate. */ strncpy(value_buf, value, opts->value_sz_max - 1); - if (workload->update_delta != 0) - update_value_delta(thread); + + if (*op == WORKER_MODIFY) + delta = workload->modify_delta; + else + delta = workload->update_delta; + + if (delta != 0) + update_value_delta(thread, delta); if (value_buf[0] == 'a') value_buf[0] = 'b'; else value_buf[0] = 'a'; if (opts->random_value) - randomize_value(thread, value_buf); - cursor->set_value(cursor, value_buf); - if ((ret = cursor->update(cursor)) == 0) - break; + randomize_value(thread, value_buf, delta); + + if (*op == WORKER_UPDATE) { + cursor->set_value(cursor, value_buf); + if ((ret = cursor->update(cursor)) == 0) + break; + goto op_err; + } + + /* + * Distribute the modifications across the whole document. We randomly choose up to + * the maximum number of modifications and modify up to the maximum percent of the + * record size. + */ + nmodify = MAX_MODIFY_NUM; + + /* + * The maximum that will be modified is a fixed percentage of the total record size. + */ + value_len = strlen(value); + total_modify_size = value_len / MAX_MODIFY_PCT; + + /* + * Randomize the maximum modification size and offset per modify. + */ + rand_val = __wt_random(&thread->rnd); + if ((total_modify_size / (size_t)nmodify) != 0) + modify_size = rand_val % (total_modify_size / (size_t)nmodify); + else + modify_size = 0; + + /* + * Offset location difference between modifications + */ + if ((value_len / (size_t)nmodify) != 0) + modify_offset = (size_t)rand_val % (value_len / (size_t)nmodify); + else + modify_offset = 0; + + /* + * Make sure the offset is more than size, otherwise modifications don't spread + * properly. + */ + if (modify_offset < modify_size) + modify_offset = modify_size + 1; + + for (iter = (size_t)nmodify; iter > 0; iter--) + memmove(&value_buf[(iter * modify_offset) - modify_offset], + &value_buf[(iter * modify_offset) - modify_size], modify_size); + + /* + * Increase the number of modifications, so that normal modify operations succeeded. + */ + if (!workload->modify_force_update) + nmodify++; + + oldv.data = value; + oldv.size = value_len; + + newv.data = value_buf; + newv.size = strlen(value_buf); + + /* + * Pass the old and new data to find out the modify vectors, according to the passed + * input. This function may fail when the modifications count reaches the maximum + * number of modifications that are allowed for the modify operation. + */ + ret = wiredtiger_calc_modify( + session, &oldv, &newv, total_modify_size, entries, &nmodify); + + if (ret == WT_NOTFOUND || workload->modify_force_update) { + cursor->set_value(cursor, value_buf); + if ((ret = cursor->update(cursor)) == 0) + break; + } else { + if ((ret = cursor->modify(cursor, entries, nmodify)) == 0) + break; + } goto op_err; } /* * Reads can fail with WT_NOTFOUND: we may be searching in a random range, or an insert * thread might have updated the last record in the table but not yet finished the - * actual insert. Count failed search in a random range as a "read". + * actual insert. Count a failed search in a random range as a "read". */ if (ret == WT_NOTFOUND) break; @@ -896,7 +987,7 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) opts = wtperf->opts; if (workp->truncate != 0) { - if (workp->insert != 0 || workp->read != 0 || workp->update != 0) { + if (workp->insert != 0 || workp->modify != 0 || workp->read != 0 || workp->update != 0) { lprintf(wtperf, EINVAL, 0, "Can't configure truncate in a mixed workload"); return (EINVAL); } @@ -904,23 +995,27 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) return (0); } - /* Confirm reads, inserts and updates cannot all be zero. */ - if (workp->insert == 0 && workp->read == 0 && workp->update == 0) { + /* Confirm inserts, modifies, reads and updates cannot all be zero. */ + if (workp->insert == 0 && workp->modify == 0 && workp->read == 0 && workp->update == 0) { lprintf(wtperf, EINVAL, 0, "no operations scheduled"); return (EINVAL); } /* - * Check for a simple case where the thread is only doing insert or update operations (because - * the default operation for a job-mix is read, the subsequent code works fine if only reads are - * specified). + * Check for a simple case where the thread is only doing insert or modify or update operations + * (because the default operation for a job-mix is read, the subsequent code works fine if only + * reads are specified). */ - if (workp->insert != 0 && workp->read == 0 && workp->update == 0) { + if (workp->insert != 0 && workp->modify == 0 && workp->read == 0 && workp->update == 0) { memset( workp->ops, opts->insert_rmw ? WORKER_INSERT_RMW : WORKER_INSERT, sizeof(workp->ops)); return (0); } - if (workp->insert == 0 && workp->read == 0 && workp->update != 0) { + if (workp->insert == 0 && workp->modify != 0 && workp->read == 0 && workp->update == 0) { + memset(workp->ops, WORKER_MODIFY, sizeof(workp->ops)); + return (0); + } + if (workp->insert == 0 && workp->modify == 0 && workp->read == 0 && workp->update != 0) { memset(workp->ops, WORKER_UPDATE, sizeof(workp->ops)); return (0); } @@ -944,10 +1039,13 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) */ memset(workp->ops, WORKER_READ, sizeof(workp->ops)); - pct = (workp->insert * 100) / (workp->insert + workp->read + workp->update); + pct = (workp->insert * 100) / (workp->insert + workp->modify + workp->read + workp->update); if (pct != 0) run_mix_schedule_op(workp, opts->insert_rmw ? WORKER_INSERT_RMW : WORKER_INSERT, pct); - pct = (workp->update * 100) / (workp->insert + workp->read + workp->update); + pct = (workp->modify * 100) / (workp->insert + workp->modify + workp->read + workp->update); + if (pct != 0) + run_mix_schedule_op(workp, WORKER_MODIFY, pct); + pct = (workp->update * 100) / (workp->insert + workp->modify + workp->read + workp->update); if (pct != 0) run_mix_schedule_op(workp, WORKER_UPDATE, pct); return (0); @@ -1025,7 +1123,7 @@ populate_thread(void *arg) __wt_epoch(NULL, &start); cursor->set_key(cursor, key_buf); if (opts->random_value) - randomize_value(thread, value_buf); + randomize_value(thread, value_buf, 0); cursor->set_value(cursor, value_buf); if ((ret = cursor->insert(cursor)) == WT_ROLLBACK) { lprintf(wtperf, ret, 0, "insert retrying"); @@ -1149,7 +1247,7 @@ populate_async(void *arg) generate_key(opts, key_buf, op); asyncop->set_key(asyncop, key_buf); if (opts->random_value) - randomize_value(thread, value_buf); + randomize_value(thread, value_buf, 0); asyncop->set_value(asyncop, value_buf); if ((ret = asyncop->insert(asyncop)) != 0) { lprintf(wtperf, ret, 0, "Failed inserting"); @@ -1195,13 +1293,15 @@ monitor(void *arg) FILE *fp, *jfp; WTPERF *wtperf; size_t len; - uint64_t min_thr, reads, inserts, updates; - uint64_t cur_reads, cur_inserts, cur_updates; - uint64_t last_reads, last_inserts, last_updates; - uint32_t read_avg, read_min, read_max; - uint32_t insert_avg, insert_min, insert_max; - uint32_t update_avg, update_min, update_max; + uint64_t min_thr; + uint64_t inserts, modifies, reads, updates; + uint64_t cur_inserts, cur_modifies, cur_reads, cur_updates; + uint64_t last_inserts, last_modifies, last_reads, last_updates; uint32_t latency_max, level; + uint32_t insert_avg, insert_max, insert_min; + uint32_t modify_avg, modify_max, modify_min; + uint32_t read_avg, read_max, read_min; + uint32_t update_avg, update_max, update_min; u_int i; size_t buf_size; int msg_err; @@ -1239,22 +1339,26 @@ monitor(void *arg) fprintf(fp, "#time," "totalsec," - "read ops per second," "insert ops per second," + "modify ops per second," + "read ops per second," "update ops per second," "checkpoints," "scans," - "read average latency(uS)," - "read minimum latency(uS)," - "read maximum latency(uS)," "insert average latency(uS)," "insert min latency(uS)," "insert maximum latency(uS)," + "modify average latency(uS)," + "modify min latency(uS)," + "modify maximum latency(uS)" + "read average latency(uS)," + "read minimum latency(uS)," + "read maximum latency(uS)," "update average latency(uS)," "update min latency(uS)," "update maximum latency(uS)" "\n"); - last_reads = last_inserts = last_updates = 0; + last_inserts = last_modifies = last_reads = last_updates = 0; while (!wtperf->stop) { for (i = 0; i < opts->sample_interval; i++) { sleep(1); @@ -1271,15 +1375,15 @@ monitor(void *arg) testutil_check(__wt_localtime(NULL, &t.tv_sec, &localt)); testutil_assert(strftime(buf, sizeof(buf), "%b %d %H:%M:%S", &localt) != 0); - reads = sum_read_ops(wtperf); inserts = sum_insert_ops(wtperf); + modifies = sum_modify_ops(wtperf); + reads = sum_read_ops(wtperf); updates = sum_update_ops(wtperf); - latency_read(wtperf, &read_avg, &read_min, &read_max); latency_insert(wtperf, &insert_avg, &insert_min, &insert_max); + latency_modify(wtperf, &modify_avg, &modify_min, &modify_max); + latency_read(wtperf, &read_avg, &read_min, &read_max); latency_update(wtperf, &update_avg, &update_min, &update_max); - cur_reads = (reads - last_reads) / opts->sample_interval; - cur_updates = (updates - last_updates) / opts->sample_interval; /* * For now the only item we need to worry about changing is inserts when we transition from * the populate phase to workload phase. @@ -1289,13 +1393,19 @@ monitor(void *arg) else cur_inserts = (inserts - last_inserts) / opts->sample_interval; - (void)fprintf(fp, "%s,%" PRIu32 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 - ",%c,%c" - ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 - ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 "\n", - buf, wtperf->totalsec, cur_reads, cur_inserts, cur_updates, wtperf->ckpt ? 'Y' : 'N', - wtperf->scan ? 'Y' : 'N', read_avg, read_min, read_max, insert_avg, insert_min, - insert_max, update_avg, update_min, update_max); + cur_modifies = (modifies - last_modifies) / opts->sample_interval; + cur_reads = (reads - last_reads) / opts->sample_interval; + cur_updates = (updates - last_updates) / opts->sample_interval; + + (void)fprintf(fp, + "%s,%" PRIu32 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 + ",%c,%c" + ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 + ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 "\n", + buf, wtperf->totalsec, cur_inserts, cur_modifies, cur_reads, cur_updates, + wtperf->ckpt ? 'Y' : 'N', wtperf->scan ? 'Y' : 'N', insert_avg, insert_min, insert_max, + modify_avg, modify_min, modify_max, read_avg, read_min, read_max, update_avg, update_min, + update_max); if (jfp != NULL) { buf_size = strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &localt); testutil_assert(buf_size != 0); @@ -1307,23 +1417,27 @@ monitor(void *arg) first = false; } (void)fprintf(jfp, "\"localTime\":\"%s\",\"wtperf\":{", buf); - /* Note does not have initial comma before "read" */ - (void)fprintf(jfp, "\"read\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 - ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", - cur_reads, read_avg, read_min, read_max); + /* Note does not have initial comma before "insert" */ (void)fprintf(jfp, - ",\"insert\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 + "\"insert\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", cur_inserts, insert_avg, insert_min, insert_max); (void)fprintf(jfp, + ",\"modify\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 + ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", + cur_modifies, modify_avg, modify_min, modify_max); + (void)fprintf(jfp, ",\"read\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 + ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", + cur_reads, read_avg, read_min, read_max); + (void)fprintf(jfp, ",\"update\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", cur_updates, update_avg, update_min, update_max); fprintf(jfp, "}}\n"); } - if (latency_max != 0 && - (read_max > latency_max || insert_max > latency_max || update_max > latency_max)) { + if (latency_max != 0 && (insert_max > latency_max || modify_max > latency_max || + read_max > latency_max || update_max > latency_max)) { if (opts->max_latency_fatal) { level = 1; msg_err = WT_PANIC; @@ -1334,13 +1448,14 @@ monitor(void *arg) str = "WARNING"; } lprintf(wtperf, msg_err, level, - "%s: max latency exceeded: threshold %" PRIu32 " read max %" PRIu32 - " insert max %" PRIu32 " update max %" PRIu32, - str, latency_max, read_max, insert_max, update_max); + "%s: max latency exceeded: threshold %" PRIu32 " insert max %" PRIu32 + " modify max %" PRIu32 " read max %" PRIu32 " update max %" PRIu32, + str, latency_max, insert_max, modify_max, read_max, update_max); } - if (min_thr != 0 && - ((cur_reads != 0 && cur_reads < min_thr) || (cur_inserts != 0 && cur_inserts < min_thr) || - (cur_updates != 0 && cur_updates < min_thr))) { + if (min_thr != 0 && ((cur_inserts != 0 && cur_inserts < min_thr) || + (cur_modifies != 0 && cur_modifies < min_thr) || + (cur_reads != 0 && cur_reads < min_thr) || + (cur_updates != 0 && cur_updates < min_thr))) { if (opts->min_throughput_fatal) { level = 1; msg_err = WT_PANIC; @@ -1351,12 +1466,13 @@ monitor(void *arg) str = "WARNING"; } lprintf(wtperf, msg_err, level, - "%s: minimum throughput not met: threshold %" PRIu64 " reads %" PRIu64 - " inserts %" PRIu64 " updates %" PRIu64, - str, min_thr, cur_reads, cur_inserts, cur_updates); + "%s: minimum throughput not met: threshold %" PRIu64 " inserts %" PRIu64 + " modifies %" PRIu64 " reads %" PRIu64 " updates %" PRIu64, + str, min_thr, cur_inserts, cur_modifies, cur_reads, cur_updates); } - last_reads = reads; last_inserts = inserts; + last_modifies = modifies; + last_reads = reads; last_updates = updates; } @@ -1742,7 +1858,8 @@ execute_workload(WTPERF *wtperf) WT_THREAD_CALLBACK (*pfunc)(void *); wt_thread_t idle_table_cycle_thread; uint64_t last_ckpts, last_scans; - uint64_t last_inserts, last_reads, last_truncates, last_updates; + uint64_t last_inserts, last_reads, last_truncates; + uint64_t last_modifies, last_updates; uint32_t interval, run_ops, run_time; u_int i; int ret; @@ -1751,10 +1868,11 @@ execute_workload(WTPERF *wtperf) wtperf->insert_key = 0; wtperf->insert_ops = wtperf->read_ops = wtperf->truncate_ops = 0; - wtperf->update_ops = 0; + wtperf->modify_ops = wtperf->update_ops = 0; last_ckpts = last_scans = 0; - last_inserts = last_reads = last_truncates = last_updates = 0; + last_inserts = last_reads = last_truncates = 0; + last_modifies = last_updates = 0; ret = 0; sessions = NULL; @@ -1787,10 +1905,10 @@ execute_workload(WTPERF *wtperf) for (threads = wtperf->workers, i = 0, workp = wtperf->workload; i < wtperf->workload_cnt; ++i, ++workp) { lprintf(wtperf, 0, 1, - "Starting workload #%u: %" PRId64 " threads, inserts=%" PRId64 ", reads=%" PRId64 - ", updates=%" PRId64 ", truncate=%" PRId64 ", throttle=%" PRIu64, - i + 1, workp->threads, workp->insert, workp->read, workp->update, workp->truncate, - workp->throttle); + "Starting workload #%u: %" PRId64 " threads, inserts=%" PRId64 ", modifies=%" PRId64 + ", reads=%" PRId64 ", truncate=%" PRId64 ", updates=%" PRId64 ", throttle=%" PRIu64, + i + 1, workp->threads, workp->insert, workp->modify, workp->read, workp->truncate, + workp->update, workp->throttle); /* Figure out the workload's schedule. */ if ((ret = run_mix_schedule(wtperf, workp)) != 0) @@ -1825,12 +1943,15 @@ execute_workload(WTPERF *wtperf) wtperf->ckpt_ops = sum_ckpt_ops(wtperf); wtperf->scan_ops = sum_scan_ops(wtperf); wtperf->insert_ops = sum_insert_ops(wtperf); + wtperf->modify_ops = sum_modify_ops(wtperf); wtperf->read_ops = sum_read_ops(wtperf); wtperf->update_ops = sum_update_ops(wtperf); wtperf->truncate_ops = sum_truncate_ops(wtperf); /* If we're checking total operations, see if we're done. */ - if (run_ops != 0 && run_ops <= wtperf->insert_ops + wtperf->read_ops + wtperf->update_ops) + if (run_ops != 0 && + run_ops <= + wtperf->insert_ops + wtperf->modify_ops + wtperf->read_ops + wtperf->update_ops) break; /* If writing out throughput information, see if it's time. */ @@ -1839,17 +1960,19 @@ execute_workload(WTPERF *wtperf) interval = opts->report_interval; wtperf->totalsec += opts->report_interval; - lprintf(wtperf, 0, 1, "%" PRIu64 " reads, %" PRIu64 " inserts, %" PRIu64 - " updates, %" PRIu64 " truncates, %" PRIu64 " checkpoints, %" PRIu64 - " scans in %" PRIu32 " secs (%" PRIu32 " total secs)", - wtperf->read_ops - last_reads, wtperf->insert_ops - last_inserts, - wtperf->update_ops - last_updates, wtperf->truncate_ops - last_truncates, - wtperf->ckpt_ops - last_ckpts, wtperf->scan_ops - last_scans, opts->report_interval, - wtperf->totalsec); - last_reads = wtperf->read_ops; + lprintf(wtperf, 0, 1, + "%" PRIu64 " inserts, %" PRIu64 " modifies, %" PRIu64 " reads, %" PRIu64 + " truncates, %" PRIu64 "updates, %" PRIu64 " checkpoints, %" PRIu64 " scans in %" PRIu32 + " secs (%" PRIu32 " total secs)", + wtperf->insert_ops - last_inserts, wtperf->modify_ops - last_modifies, + wtperf->read_ops - last_reads, wtperf->truncate_ops - last_truncates, + wtperf->update_ops - last_updates, wtperf->ckpt_ops - last_ckpts, + wtperf->scan_ops - last_scans, opts->report_interval, wtperf->totalsec); last_inserts = wtperf->insert_ops; - last_updates = wtperf->update_ops; + last_modifies = wtperf->modify_ops; + last_reads = wtperf->read_ops; last_truncates = wtperf->truncate_ops; + last_updates = wtperf->update_ops; last_ckpts = wtperf->ckpt_ops; last_scans = wtperf->scan_ops; } @@ -2304,23 +2427,28 @@ start_run(WTPERF *wtperf) goto err; /* One final summation of the operations we've completed. */ - wtperf->read_ops = sum_read_ops(wtperf); wtperf->insert_ops = sum_insert_ops(wtperf); + wtperf->modify_ops = sum_modify_ops(wtperf); + wtperf->read_ops = sum_read_ops(wtperf); wtperf->truncate_ops = sum_truncate_ops(wtperf); wtperf->update_ops = sum_update_ops(wtperf); wtperf->ckpt_ops = sum_ckpt_ops(wtperf); wtperf->scan_ops = sum_scan_ops(wtperf); - total_ops = wtperf->read_ops + wtperf->insert_ops + wtperf->update_ops; + total_ops = wtperf->insert_ops + wtperf->modify_ops + wtperf->read_ops + wtperf->update_ops; run_time = opts->run_time == 0 ? 1 : opts->run_time; lprintf(wtperf, 0, 1, - "Executed %" PRIu64 " read operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", - wtperf->read_ops, (wtperf->read_ops * 100) / total_ops, wtperf->read_ops / run_time); - lprintf(wtperf, 0, 1, "Executed %" PRIu64 " insert operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", wtperf->insert_ops, (wtperf->insert_ops * 100) / total_ops, wtperf->insert_ops / run_time); lprintf(wtperf, 0, 1, + "Executed %" PRIu64 " modify operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", + wtperf->modify_ops, (wtperf->modify_ops * 100) / total_ops, + wtperf->modify_ops / run_time); + lprintf(wtperf, 0, 1, + "Executed %" PRIu64 " read operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", + wtperf->read_ops, (wtperf->read_ops * 100) / total_ops, wtperf->read_ops / run_time); + lprintf(wtperf, 0, 1, "Executed %" PRIu64 " truncate operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", wtperf->truncate_ops, (wtperf->truncate_ops * 100) / total_ops, wtperf->truncate_ops / run_time); @@ -2706,16 +2834,17 @@ start_threads(WTPERF *wtperf, WORKLOAD *workp, WTPERF_THREAD *base, u_int num, */ memset(thread->value_buf, 'a', opts->value_sz - 1); if (opts->random_value) - randomize_value(thread, thread->value_buf); + randomize_value(thread, thread->value_buf, 0); /* * Every thread gets tracking information and is initialized for latency measurements, for * the same reason. */ thread->ckpt.min_latency = thread->scan.min_latency = thread->insert.min_latency = - thread->read.min_latency = thread->update.min_latency = UINT32_MAX; + thread->modify.min_latency = thread->read.min_latency = thread->update.min_latency = + UINT32_MAX; thread->ckpt.max_latency = thread->scan.max_latency = thread->insert.max_latency = - thread->read.max_latency = thread->update.max_latency = 0; + thread->modify.max_latency = thread->read.max_latency = thread->update.max_latency = 0; } /* Start the threads. */ diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index de36109309d..0f54d03e3c9 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -54,27 +54,34 @@ typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY; #define ZSTD_BLK BLKCMP_PFX "zstd" #define ZSTD_EXT EXT_PFX EXTPATH "zstd/.libs/libwiredtiger_zstd.so" EXT_SFX +#define MAX_MODIFY_PCT 10 +#define MAX_MODIFY_NUM 16 + typedef struct { int64_t threads; /* Thread count */ int64_t insert; /* Insert ratio */ + int64_t modify; /* Modify ratio */ int64_t read; /* Read ratio */ int64_t update; /* Update ratio */ uint64_t throttle; /* Maximum operations/second */ /* Number of operations per transaction. Zero for autocommit */ int64_t ops_per_txn; - int64_t pause; /* Time between scans */ - int64_t read_range; /* Range of reads */ - int32_t table_index; /* Table to focus ops on */ - int64_t truncate; /* Truncate ratio */ - uint64_t truncate_pct; /* Truncate Percent */ - uint64_t truncate_count; /* Truncate Count */ - int64_t update_delta; /* Value size change on update */ + int64_t pause; /* Time between scans */ + int64_t read_range; /* Range of reads */ + int32_t table_index; /* Table to focus ops on */ + int64_t truncate; /* Truncate ratio */ + uint64_t truncate_pct; /* Truncate Percent */ + uint64_t truncate_count; /* Truncate Count */ + int64_t modify_delta; /* Value size change on modify */ + int64_t update_delta; /* Value size change on update */ + bool modify_force_update; /* Do force update instead of modify */ #define WORKER_INSERT 1 /* Insert */ #define WORKER_INSERT_RMW 2 /* Insert with read-modify-write */ -#define WORKER_READ 3 /* Read */ -#define WORKER_TRUNCATE 4 /* Truncate */ -#define WORKER_UPDATE 5 /* Update */ +#define WORKER_MODIFY 3 /* Modify */ +#define WORKER_READ 4 /* Read */ +#define WORKER_TRUNCATE 5 /* Truncate */ +#define WORKER_UPDATE 6 /* Update */ uint8_t ops[100]; /* Operation schedule */ } WORKLOAD; @@ -140,6 +147,7 @@ struct __wtperf { /* Per-database structure */ uint64_t ckpt_ops; /* checkpoint operations */ uint64_t scan_ops; /* scan operations */ uint64_t insert_ops; /* insert operations */ + uint64_t modify_ops; /* modify operations */ uint64_t read_ops; /* read operations */ uint64_t truncate_ops; /* truncate operations */ uint64_t update_ops; /* update operations */ @@ -242,11 +250,12 @@ struct __wtperf_thread { /* Per-thread structure */ TRACK ckpt; /* Checkpoint operations */ TRACK insert; /* Insert operations */ + TRACK modify; /* Modify operations */ TRACK read; /* Read operations */ TRACK scan; /* Scan operations */ - TRACK update; /* Update operations */ TRACK truncate; /* Truncate operations */ TRACK truncate_sleep; /* Truncate sleep operations */ + TRACK update; /* Update operations */ }; void cleanup_truncate_config(WTPERF *); @@ -260,6 +269,7 @@ int config_opt_str(WTPERF *, const char *); void config_opt_usage(void); int config_sanity(WTPERF *); void latency_insert(WTPERF *, uint32_t *, uint32_t *, uint32_t *); +void latency_modify(WTPERF *, uint32_t *, uint32_t *, uint32_t *); void latency_print(WTPERF *); void latency_read(WTPERF *, uint32_t *, uint32_t *, uint32_t *); void latency_update(WTPERF *, uint32_t *, uint32_t *, uint32_t *); @@ -273,6 +283,7 @@ void worker_throttle(WTPERF_THREAD *); uint64_t sum_ckpt_ops(WTPERF *); uint64_t sum_scan_ops(WTPERF *); uint64_t sum_insert_ops(WTPERF *); +uint64_t sum_modify_ops(WTPERF *); uint64_t sum_pop_ops(WTPERF *); uint64_t sum_read_ops(WTPERF *); uint64_t sum_truncate_ops(WTPERF *); diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index dc4352ef6c4..27870976654 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "7dfd9391862bc9a6d84868c4dc51689c45a3aacf", + "commit": "956384325e5ff99567ea7b63b1b87b7bb62c76b8", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.4" diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index feac9932cb4..bf90ad7238e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -492,7 +492,7 @@ __wt_curds_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, con __curds_search, /* search */ __curds_search_near, /* search-near */ __curds_insert, /* insert */ - __wt_cursor_modify_notsup, /* modify */ + __wt_cursor_modify_value_format_notsup, /* modify */ __curds_update, /* update */ __curds_remove, /* remove */ __curds_reserve, /* reserve */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index e73820baad7..30fe571bdbc 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -617,7 +617,7 @@ __curfile_create(WT_SESSION_IMPL *session, WT_CURSOR *owner, const char *cfg[], __curfile_search, /* search */ __curfile_search_near, /* search-near */ __curfile_insert, /* insert */ - __wt_cursor_modify_notsup, /* modify */ + __wt_cursor_modify_value_format_notsup, /* modify */ __curfile_update, /* update */ __curfile_remove, /* remove */ __curfile_reserve, /* reserve */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 78d90a2bcf8..fa2b52d254d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -103,12 +103,39 @@ __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) } /* + * Cursor modify operation is supported only in limited cursor types and also the modify + * operation is supported only with 'S' and 'u' value formats of the cursors. Because of + * the conditional support of cursor modify operation, To provide a better error description + * to the application whenever the cursor modify is used based on the cursor types, two + * default not supported functions are used. + * + * __wt_cursor_modify_notsup - Default function for cursor types where the modify operation + * is not supported. + * + * __wt_cursor_modify_value_format_notsup - Default function for cursor types where the modify + * operation is supported with specific value formats of the cursor. + */ + +/* * __wt_cursor_modify_notsup -- * Unsupported cursor modify. */ int __wt_cursor_modify_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) { + WT_UNUSED(entries); + WT_UNUSED(nentries); + + return (__wt_cursor_notsup(cursor)); +} + +/* + * __wt_cursor_modify_value_format_notsup -- + * Unsupported value format for cursor modify. + */ +int +__wt_cursor_modify_value_format_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) +{ WT_SESSION_IMPL *session; WT_UNUSED(entries); @@ -1071,7 +1098,7 @@ __wt_cursor_init( * initialized (file cursors have a faster implementation). */ if ((WT_STREQ(cursor->value_format, "S") || WT_STREQ(cursor->value_format, "u")) && - cursor->modify == __wt_cursor_modify_notsup) + cursor->modify == __wt_cursor_modify_value_format_notsup) cursor->modify = __cursor_modify; /* diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 2b2b089a18c..32c35c8bc0a 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -538,6 +538,8 @@ extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_modify_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_modify_value_format_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, + int nentries) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_noop(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_notsup(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index de9b35cdc17..bc860952baf 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1698,7 +1698,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, cons __clsm_search, /* search */ __clsm_search_near, /* search-near */ __clsm_insert, /* insert */ - __wt_cursor_modify_notsup, /* modify */ + __wt_cursor_modify_value_format_notsup, /* modify */ __clsm_update, /* update */ __clsm_remove, /* remove */ __clsm_reserve, /* reserve */ diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 3de273c8ebf..0f86d5b4560 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -123,6 +123,7 @@ Options:\n\ -s N | --scenario N use scenario N (N can be number or symbolic)\n\ -t | --timestamp name WT_TEST according to timestamp\n\ -v N | --verbose N set verboseness to N (0<=N<=3, default=1)\n\ + -i | --ignore-stdout dont fail on unexpected stdout or stderr\n\ \n\ Tests:\n\ may be a file name in test/suite: (e.g. test_base01.py)\n\ @@ -269,7 +270,7 @@ if __name__ == '__main__': tests = unittest.TestSuite() # Turn numbers and ranges into test module names - preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = False + preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = ignoreStdout = False parallel = 0 random_sample = 0 configfile = None @@ -347,6 +348,9 @@ if __name__ == '__main__': if verbose < 0: verbose = 0 continue + if option == '--ignore-stdout' or option == 'i': + ignoreStdout = True + continue if option == '-config' or option == 'c': if configfile != None or len(args) == 0: usage() @@ -369,7 +373,7 @@ if __name__ == '__main__': # That way, verbose printing can be done at the class definition level. wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, lldbSub, verbose, wt_builddir, dirarg, - longtest) + longtest, ignoreStdout) # Without any tests listed as arguments, do discovery if len(testargs) == 0: diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 5fbcecac105..b2fec97cb17 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -95,6 +95,8 @@ class CapturedFd(object): file. If there is, raise it as a test failure. This is generally called after 'release' is called. """ + if WiredTigerTestCase._ignoreStdout: + return if self.file != None: self.file.flush() filesize = os.path.getsize(self.filename) @@ -184,7 +186,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def globalSetup(preserveFiles = False, useTimestamp = False, gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None, - longtest = False): + longtest = False, ignoreStdout = False): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg if useTimestamp: @@ -200,6 +202,7 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._lldbSubprocess = lldbSub WiredTigerTestCase._longtest = longtest WiredTigerTestCase._verbose = verbose + WiredTigerTestCase._ignoreStdout = ignoreStdout WiredTigerTestCase._dupout = os.dup(sys.stdout.fileno()) WiredTigerTestCase._stdout = sys.stdout WiredTigerTestCase._stderr = sys.stderr |