diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-08-21 05:23:37 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-08-21 05:23:37 +0000 |
commit | ac41c65f6355f83aac70136324c98561ac79daa1 (patch) | |
tree | a7c3f7ef090b59c6a06838a02c96bd1d49e1c729 /src/third_party/wiredtiger/bench/wtperf | |
parent | f54709196711c63a429b71f47c584661286d675f (diff) | |
download | mongo-ac41c65f6355f83aac70136324c98561ac79daa1.tar.gz |
Import wiredtiger: 7dfd9391862bc9a6d84868c4dc51689c45a3aacf from branch mongodb-4.4
ref: c809757d8b..7dfd939186
for: 4.3.1
WT-4658 Apply Clang Format
WT-4810 Adding WT_ERR_ASSERT and WT_RET_ASSERT macros
WT-5046 Prepared transactions aren't properly cleared from global table with WT_CONN_LOG_DEBUG_MODE enabled
Diffstat (limited to 'src/third_party/wiredtiger/bench/wtperf')
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/config.c | 1751 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/config_opt.h | 28 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c | 218 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/misc.c | 106 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/track.c | 453 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/wtperf.c | 5438 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/wtperf.h | 384 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i | 257 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/wtperf_throttle.c | 121 | ||||
-rw-r--r-- | src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c | 361 |
10 files changed, 4375 insertions, 4742 deletions
diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 18522e3f7e7..73816d980ac 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -28,1018 +28,995 @@ #include "wtperf.h" -static CONFIG_OPT config_opts_desc[] = { /* Option descriptions */ -#define OPT_DEFINE_DESC +static CONFIG_OPT config_opts_desc[] = { +/* Option descriptions */ +#define OPT_DEFINE_DESC #include "wtperf_opt.i" #undef OPT_DEFINE_DESC }; -static CONFIG_OPTS config_opts_default = { /* Option defaults */ -#define OPT_DEFINE_DEFAULT +static CONFIG_OPTS config_opts_default = { +/* Option defaults */ +#define OPT_DEFINE_DEFAULT #include "wtperf_opt.i" #undef OPT_DEFINE_DEFAULT - { NULL, NULL } /* config_head */ + {NULL, NULL} /* config_head */ }; /* * STRING_MATCH -- * Return if a string matches a bytestring of a specified length. */ -#undef STRING_MATCH -#define STRING_MATCH(str, bytes, len) \ - (strncmp(str, bytes, len) == 0 && (str)[(len)] == '\0') +#undef STRING_MATCH +#define STRING_MATCH(str, bytes, len) (strncmp(str, bytes, len) == 0 && (str)[(len)] == '\0') /* * config_opt_init -- - * Initialize the global configuration options. + * Initialize the global configuration options. */ void config_opt_init(CONFIG_OPTS **retp) { - CONFIG_OPT *desc; - CONFIG_OPTS *opts; - size_t i; - char **strp; - void *valueloc; - - opts = dmalloc(sizeof(CONFIG_OPTS)); - *opts = config_opts_default; - - TAILQ_INIT(&opts->config_head); - - /* - * Option strings come-and-go as we configure them, so allocate copies - * of the default strings now so that we can always free the string as - * we allocate new versions. - */ - for (i = 0, desc = config_opts_desc; - i < WT_ELEMENTS(config_opts_desc); i++, ++desc) - if (desc->type == CONFIG_STRING_TYPE || - desc->type == STRING_TYPE) { - valueloc = ((uint8_t *)opts + desc->offset); - strp = (char **)valueloc; - *strp = dstrdup(*strp); - } - - *retp = opts; + CONFIG_OPT *desc; + CONFIG_OPTS *opts; + size_t i; + char **strp; + void *valueloc; + + opts = dmalloc(sizeof(CONFIG_OPTS)); + *opts = config_opts_default; + + TAILQ_INIT(&opts->config_head); + + /* + * Option strings come-and-go as we configure them, so allocate copies of the default strings + * now so that we can always free the string as we allocate new versions. + */ + for (i = 0, desc = config_opts_desc; i < WT_ELEMENTS(config_opts_desc); i++, ++desc) + if (desc->type == CONFIG_STRING_TYPE || desc->type == STRING_TYPE) { + valueloc = ((uint8_t *)opts + desc->offset); + strp = (char **)valueloc; + *strp = dstrdup(*strp); + } + + *retp = opts; } /* * config_opt_cleanup -- - * Clean up the global configuration options. + * Clean up the global configuration options. */ void config_opt_cleanup(CONFIG_OPTS *opts) { - CONFIG_OPT *desc; - CONFIG_QUEUE_ENTRY *config_line; - size_t i; - char **strp; - void *valueloc; - - for (i = 0, desc = config_opts_desc; - i < WT_ELEMENTS(config_opts_desc); i++, ++desc) - if (desc->type == CONFIG_STRING_TYPE || - desc->type == STRING_TYPE) { - valueloc = ((uint8_t *)opts + desc->offset); - strp = (char **)valueloc; - free(*strp); - } - - while ((config_line = TAILQ_FIRST(&opts->config_head)) != NULL) { - TAILQ_REMOVE(&opts->config_head, config_line, q); - free(config_line->string); - free(config_line); - } - - free(opts); + CONFIG_OPT *desc; + CONFIG_QUEUE_ENTRY *config_line; + size_t i; + char **strp; + void *valueloc; + + for (i = 0, desc = config_opts_desc; i < WT_ELEMENTS(config_opts_desc); i++, ++desc) + if (desc->type == CONFIG_STRING_TYPE || desc->type == STRING_TYPE) { + valueloc = ((uint8_t *)opts + desc->offset); + strp = (char **)valueloc; + free(*strp); + } + + while ((config_line = TAILQ_FIRST(&opts->config_head)) != NULL) { + TAILQ_REMOVE(&opts->config_head, config_line, q); + free(config_line->string); + free(config_line); + } + + free(opts); } /* * config_unescape -- - * Modify a string in place, replacing any backslash escape sequences. - * The modified string is always shorter. + * Modify a string in place, replacing any backslash escape sequences. The modified string is + * always shorter. */ static int config_unescape(char *orig) { - char ch, *dst, *s; - - for (dst = s = orig; *s != '\0';) { - if ((ch = *s++) == '\\') { - ch = *s++; - switch (ch) { - case 'b': - *dst++ = '\b'; - break; - case 'f': - *dst++ = '\f'; - break; - case 'n': - *dst++ = '\n'; - break; - case 'r': - *dst++ = '\r'; - break; - case 't': - *dst++ = '\t'; - break; - case '\\': - case '/': - case '\"': /* Backslash needed for spell check. */ - *dst++ = ch; - break; - default: - /* Note: Unicode (\u) not implemented. */ - fprintf(stderr, - "invalid escape in string: %s\n", orig); - return (EINVAL); - } - } else - *dst++ = ch; - } - *dst = '\0'; - return (0); + char ch, *dst, *s; + + for (dst = s = orig; *s != '\0';) { + if ((ch = *s++) == '\\') { + ch = *s++; + switch (ch) { + case 'b': + *dst++ = '\b'; + break; + case 'f': + *dst++ = '\f'; + break; + case 'n': + *dst++ = '\n'; + break; + case 'r': + *dst++ = '\r'; + break; + case 't': + *dst++ = '\t'; + break; + case '\\': + case '/': + case '\"': /* Backslash needed for spell check. */ + *dst++ = ch; + break; + default: + /* Note: Unicode (\u) not implemented. */ + fprintf(stderr, "invalid escape in string: %s\n", orig); + return (EINVAL); + } + } else + *dst++ = ch; + } + *dst = '\0'; + return (0); } /* * config_threads -- - * Parse the thread configuration. + * Parse the thread configuration. */ static int config_threads(WTPERF *wtperf, const char *config, size_t len) { - WORKLOAD *workp; - WT_CONFIG_ITEM groupk, groupv, k, v; - WT_CONFIG_PARSER *group, *scan; - int ret; - - group = scan = NULL; - if (wtperf->workload != NULL) { - /* - * This call overrides an earlier call. Free and - * reset everything. - */ - free(wtperf->workload); - wtperf->workload = NULL; - wtperf->workload_cnt = 0; - wtperf->workers_cnt = 0; - } - /* Allocate the workload array. */ - wtperf->workload = dcalloc(WORKLOAD_MAX, sizeof(WORKLOAD)); - wtperf->workload_cnt = 0; - - /* - * The thread configuration may be in multiple groups, that is, we have - * to handle configurations like: - * threads=((count=2,reads=1),(count=8,inserts=2,updates=1)) - * - * Start a scan on the original string, then do scans on each string - * returned from the original string. - */ - if ((ret = - wiredtiger_config_parser_open(NULL, config, len, &group)) != 0) - goto err; - while ((ret = group->next(group, &groupk, &groupv)) == 0) { - if ((ret = wiredtiger_config_parser_open( - NULL, groupk.str, groupk.len, &scan)) != 0) - goto err; - - /* Move to the next workload slot. */ - if (wtperf->workload_cnt == WORKLOAD_MAX) { - fprintf(stderr, - "too many workloads configured, only %d workloads " - "supported\n", - WORKLOAD_MAX); - return (EINVAL); - } - workp = &wtperf->workload[wtperf->workload_cnt++]; - workp->table_index = INT32_MAX; - - while ((ret = scan->next(scan, &k, &v)) == 0) { - if (STRING_MATCH("count", k.str, k.len)) { - if ((workp->threads = v.val) <= 0) - goto err; - continue; - } - if (STRING_MATCH("insert", k.str, k.len) || - STRING_MATCH("inserts", k.str, k.len)) { - if ((workp->insert = v.val) < 0) - goto err; - continue; - } - if (STRING_MATCH("ops_per_txn", k.str, k.len)) { - if ((workp->ops_per_txn = v.val) < 0) - goto err; - continue; - } - if (STRING_MATCH("pause", k.str, k.len)) { - if ((workp->pause = v.val) < 0) - goto err; - continue; - } - if (STRING_MATCH("read", k.str, k.len) || - STRING_MATCH("reads", k.str, k.len)) { - if ((workp->read = v.val) < 0) - goto err; - continue; - } - if (STRING_MATCH("read_range", k.str, k.len)) { - if ((workp->read_range = v.val) < 0) - goto err; - continue; - } - if (STRING_MATCH("table", k.str, k.len)) { - if (v.val <= 0) - goto err; - workp->table_index = (int32_t)v.val - 1; - continue; - } - if (STRING_MATCH("throttle", k.str, k.len)) { - workp->throttle = (uint64_t)v.val; - continue; - } - if (STRING_MATCH("truncate", k.str, k.len)) { - if ((workp->truncate = v.val) != 1) - goto err; - /* There can only be one Truncate thread. */ - if (F_ISSET(wtperf, CFG_TRUNCATE)) - goto err; - F_SET(wtperf, CFG_TRUNCATE); - continue; - } - if (STRING_MATCH("truncate_pct", k.str, k.len)) { - if (v.val <= 0) - goto err; - workp->truncate_pct = (uint64_t)v.val; - continue; - } - if (STRING_MATCH("truncate_count", k.str, k.len)) { - if (v.val <= 0) - goto err; - workp->truncate_count = (uint64_t)v.val; - continue; - } - if (STRING_MATCH("update", k.str, k.len) || - STRING_MATCH("updates", k.str, k.len)) { - if ((workp->update = v.val) < 0) - goto err; - continue; - } - if (STRING_MATCH("update_delta", k.str, k.len)) { - if (v.type == WT_CONFIG_ITEM_STRING || - v.type == WT_CONFIG_ITEM_ID) { - if (strncmp(v.str, "rand", 4) != 0) - goto err; - /* Special random value */ - workp->update_delta = INT64_MAX; - F_SET(wtperf, CFG_GROW); - } else { - workp->update_delta = v.val; - if (v.val > 0) - F_SET(wtperf, CFG_GROW); - if (v.val < 0) - F_SET(wtperf, CFG_SHRINK); - } - continue; - } - goto err; - } - if (ret == WT_NOTFOUND) - ret = 0; - if (ret != 0 ) - goto err; - ret = scan->close(scan); - scan = NULL; - if (ret != 0) - goto err; - if (workp->insert == 0 && workp->read == 0 && - workp->update == 0 && workp->truncate == 0) - goto err; - /* Why run with truncate if we don't want any truncation. */ - if (workp->truncate != 0 && - workp->truncate_pct == 0 && workp->truncate_count == 0) - goto err; - if (workp->truncate != 0 && - (workp->truncate_pct < 1 || workp->truncate_pct > 99)) - goto err; - /* Truncate should have its own exclusive thread. */ - if (workp->truncate != 0 && workp->threads > 1) - goto err; - if (workp->truncate != 0 && - (workp->insert > 0 || workp->read > 0 || workp->update > 0)) - goto err; - wtperf->workers_cnt += (u_int)workp->threads; - } - - ret = group->close(group); - group = NULL; - if (ret != 0) - goto err; - - return (0); - -err: if (group != NULL) - testutil_check(group->close(group)); - if (scan != NULL) - testutil_check(scan->close(scan)); - - fprintf(stderr, - "invalid thread configuration or scan error: %.*s\n", - (int)len, config); - return (EINVAL); + WORKLOAD *workp; + WT_CONFIG_ITEM groupk, groupv, k, v; + WT_CONFIG_PARSER *group, *scan; + int ret; + + group = scan = NULL; + if (wtperf->workload != NULL) { + /* + * This call overrides an earlier call. Free and reset everything. + */ + free(wtperf->workload); + wtperf->workload = NULL; + wtperf->workload_cnt = 0; + wtperf->workers_cnt = 0; + } + /* Allocate the workload array. */ + wtperf->workload = dcalloc(WORKLOAD_MAX, sizeof(WORKLOAD)); + wtperf->workload_cnt = 0; + + /* + * The thread configuration may be in multiple groups, that is, we have + * to handle configurations like: + * threads=((count=2,reads=1),(count=8,inserts=2,updates=1)) + * + * Start a scan on the original string, then do scans on each string + * returned from the original string. + */ + if ((ret = wiredtiger_config_parser_open(NULL, config, len, &group)) != 0) + goto err; + while ((ret = group->next(group, &groupk, &groupv)) == 0) { + if ((ret = wiredtiger_config_parser_open(NULL, groupk.str, groupk.len, &scan)) != 0) + goto err; + + /* Move to the next workload slot. */ + if (wtperf->workload_cnt == WORKLOAD_MAX) { + fprintf(stderr, + "too many workloads configured, only %d workloads " + "supported\n", + WORKLOAD_MAX); + return (EINVAL); + } + workp = &wtperf->workload[wtperf->workload_cnt++]; + workp->table_index = INT32_MAX; + + while ((ret = scan->next(scan, &k, &v)) == 0) { + if (STRING_MATCH("count", k.str, k.len)) { + if ((workp->threads = v.val) <= 0) + goto err; + continue; + } + if (STRING_MATCH("insert", k.str, k.len) || STRING_MATCH("inserts", k.str, k.len)) { + if ((workp->insert = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("ops_per_txn", k.str, k.len)) { + if ((workp->ops_per_txn = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("pause", k.str, k.len)) { + if ((workp->pause = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("read", k.str, k.len) || STRING_MATCH("reads", k.str, k.len)) { + if ((workp->read = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("read_range", k.str, k.len)) { + if ((workp->read_range = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("table", k.str, k.len)) { + if (v.val <= 0) + goto err; + workp->table_index = (int32_t)v.val - 1; + continue; + } + if (STRING_MATCH("throttle", k.str, k.len)) { + workp->throttle = (uint64_t)v.val; + continue; + } + if (STRING_MATCH("truncate", k.str, k.len)) { + if ((workp->truncate = v.val) != 1) + goto err; + /* There can only be one Truncate thread. */ + if (F_ISSET(wtperf, CFG_TRUNCATE)) + goto err; + F_SET(wtperf, CFG_TRUNCATE); + continue; + } + if (STRING_MATCH("truncate_pct", k.str, k.len)) { + if (v.val <= 0) + goto err; + workp->truncate_pct = (uint64_t)v.val; + continue; + } + if (STRING_MATCH("truncate_count", k.str, k.len)) { + if (v.val <= 0) + goto err; + workp->truncate_count = (uint64_t)v.val; + continue; + } + if (STRING_MATCH("update", k.str, k.len) || STRING_MATCH("updates", k.str, k.len)) { + if ((workp->update = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("update_delta", k.str, k.len)) { + if (v.type == WT_CONFIG_ITEM_STRING || v.type == WT_CONFIG_ITEM_ID) { + if (strncmp(v.str, "rand", 4) != 0) + goto err; + /* Special random value */ + workp->update_delta = INT64_MAX; + F_SET(wtperf, CFG_GROW); + } else { + workp->update_delta = v.val; + if (v.val > 0) + F_SET(wtperf, CFG_GROW); + if (v.val < 0) + F_SET(wtperf, CFG_SHRINK); + } + continue; + } + goto err; + } + if (ret == WT_NOTFOUND) + ret = 0; + if (ret != 0) + goto err; + ret = scan->close(scan); + scan = NULL; + if (ret != 0) + goto err; + if (workp->insert == 0 && workp->read == 0 && workp->update == 0 && workp->truncate == 0) + goto err; + /* Why run with truncate if we don't want any truncation. */ + if (workp->truncate != 0 && workp->truncate_pct == 0 && workp->truncate_count == 0) + goto err; + if (workp->truncate != 0 && (workp->truncate_pct < 1 || workp->truncate_pct > 99)) + goto err; + /* Truncate should have its own exclusive thread. */ + if (workp->truncate != 0 && workp->threads > 1) + goto err; + if (workp->truncate != 0 && (workp->insert > 0 || workp->read > 0 || workp->update > 0)) + goto err; + wtperf->workers_cnt += (u_int)workp->threads; + } + + ret = group->close(group); + group = NULL; + if (ret != 0) + goto err; + + return (0); + +err: + if (group != NULL) + testutil_check(group->close(group)); + if (scan != NULL) + testutil_check(scan->close(scan)); + + fprintf(stderr, "invalid thread configuration or scan error: %.*s\n", (int)len, config); + return (EINVAL); } /* * config_opt -- - * Check a single key=value returned by the config parser against our table - * of valid keys, along with the expected type. If everything is okay, set the - * value. + * Check a single key=value returned by the config parser against our table of valid keys, along + * with the expected type. If everything is okay, set the value. */ static int config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) { - CONFIG_OPTS *opts; - CONFIG_OPT *desc; - char *begin, *newstr, **strp; - int ret; - size_t i, newlen; - void *valueloc; - - opts = wtperf->opts; - - desc = NULL; - for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) - if (strlen(config_opts_desc[i].name) == k->len && - strncmp(config_opts_desc[i].name, k->str, k->len) == 0) { - desc = &config_opts_desc[i]; - break; - } - if (desc == NULL) { - fprintf(stderr, "wtperf: Error: " - "unknown option \'%.*s\'\n", (int)k->len, k->str); - fprintf(stderr, "Options:\n"); - for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) - fprintf(stderr, "\t%s\n", config_opts_desc[i].name); - return (EINVAL); - } - valueloc = ((uint8_t *)opts + desc->offset); - switch (desc->type) { - case BOOL_TYPE: - if (v->type != WT_CONFIG_ITEM_BOOL) { - fprintf(stderr, "wtperf: Error: " - "bad bool value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); - return (EINVAL); - } - *(int *)valueloc = (int)v->val; - break; - case INT_TYPE: - if (v->type != WT_CONFIG_ITEM_NUM) { - fprintf(stderr, "wtperf: Error: " - "bad int value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); - return (EINVAL); - } - if (v->val > INT_MAX) { - fprintf(stderr, "wtperf: Error: " - "int value out of range for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); - return (EINVAL); - } - *(int *)valueloc = (int)v->val; - break; - case UINT32_TYPE: - if (v->type != WT_CONFIG_ITEM_NUM) { - fprintf(stderr, "wtperf: Error: " - "bad uint32 value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); - return (EINVAL); - } - if (v->val < 0 || v->val > UINT_MAX) { - fprintf(stderr, "wtperf: Error: " - "uint32 value out of range for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); - return (EINVAL); - } - *(uint32_t *)valueloc = (uint32_t)v->val; - break; - case CONFIG_STRING_TYPE: - /* - * Configuration parsing uses string/ID to distinguish - * between quoted and unquoted values. - */ - if (v->type != WT_CONFIG_ITEM_STRING && - v->type != WT_CONFIG_ITEM_ID) { - fprintf(stderr, "wtperf: Error: " - "bad string value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); - return (EINVAL); - } - strp = (char **)valueloc; - if (*strp == NULL) - begin = newstr = dstrdup(v->str); - else { - newlen = strlen(*strp) + v->len + strlen(",") + 1; - newstr = dmalloc(newlen); - testutil_check(__wt_snprintf(newstr, newlen, - "%s,%.*s", *strp, (int)v->len, v->str)); - /* Free the old value now we've copied it. */ - free(*strp); - begin = &newstr[(newlen - 1) - v->len]; - } - if ((ret = config_unescape(begin)) != 0) { - free(newstr); - return (ret); - } - *strp = newstr; - break; - case STRING_TYPE: - /* - * Thread configuration is the one case where the type isn't a - * "string", it's a "struct". - */ - if (v->type == WT_CONFIG_ITEM_STRUCT && - STRING_MATCH("threads", k->str, k->len)) - return (config_threads(wtperf, v->str, v->len)); - - if (v->type != WT_CONFIG_ITEM_STRING && - v->type != WT_CONFIG_ITEM_ID) { - fprintf(stderr, "wtperf: Error: " - "bad string value for \'%.*s=%.*s\'\n", - (int)k->len, k->str, (int)v->len, v->str); - return (EINVAL); - } - strp = (char **)valueloc; - free(*strp); - /* - * We duplicate the string to len rather than len+1 as we want - * to truncate the trailing quotation mark. - */ - newstr = dstrndup(v->str, v->len); - *strp = newstr; - break; - } - return (0); + CONFIG_OPTS *opts; + CONFIG_OPT *desc; + char *begin, *newstr, **strp; + int ret; + size_t i, newlen; + void *valueloc; + + opts = wtperf->opts; + + desc = NULL; + for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) + if (strlen(config_opts_desc[i].name) == k->len && + strncmp(config_opts_desc[i].name, k->str, k->len) == 0) { + desc = &config_opts_desc[i]; + break; + } + if (desc == NULL) { + fprintf(stderr, + "wtperf: Error: " + "unknown option \'%.*s\'\n", + (int)k->len, k->str); + fprintf(stderr, "Options:\n"); + for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) + fprintf(stderr, "\t%s\n", config_opts_desc[i].name); + return (EINVAL); + } + valueloc = ((uint8_t *)opts + desc->offset); + switch (desc->type) { + case BOOL_TYPE: + if (v->type != WT_CONFIG_ITEM_BOOL) { + fprintf(stderr, + "wtperf: Error: " + "bad bool value for \'%.*s=%.*s\'\n", + (int)k->len, k->str, (int)v->len, v->str); + return (EINVAL); + } + *(int *)valueloc = (int)v->val; + break; + case INT_TYPE: + if (v->type != WT_CONFIG_ITEM_NUM) { + fprintf(stderr, + "wtperf: Error: " + "bad int value for \'%.*s=%.*s\'\n", + (int)k->len, k->str, (int)v->len, v->str); + return (EINVAL); + } + if (v->val > INT_MAX) { + fprintf(stderr, + "wtperf: Error: " + "int value out of range for \'%.*s=%.*s\'\n", + (int)k->len, k->str, (int)v->len, v->str); + return (EINVAL); + } + *(int *)valueloc = (int)v->val; + break; + case UINT32_TYPE: + if (v->type != WT_CONFIG_ITEM_NUM) { + fprintf(stderr, + "wtperf: Error: " + "bad uint32 value for \'%.*s=%.*s\'\n", + (int)k->len, k->str, (int)v->len, v->str); + return (EINVAL); + } + if (v->val < 0 || v->val > UINT_MAX) { + fprintf(stderr, + "wtperf: Error: " + "uint32 value out of range for \'%.*s=%.*s\'\n", + (int)k->len, k->str, (int)v->len, v->str); + return (EINVAL); + } + *(uint32_t *)valueloc = (uint32_t)v->val; + break; + case CONFIG_STRING_TYPE: + /* + * Configuration parsing uses string/ID to distinguish between quoted and unquoted values. + */ + if (v->type != WT_CONFIG_ITEM_STRING && v->type != WT_CONFIG_ITEM_ID) { + fprintf(stderr, + "wtperf: Error: " + "bad string value for \'%.*s=%.*s\'\n", + (int)k->len, k->str, (int)v->len, v->str); + return (EINVAL); + } + strp = (char **)valueloc; + if (*strp == NULL) + begin = newstr = dstrdup(v->str); + else { + newlen = strlen(*strp) + v->len + strlen(",") + 1; + newstr = dmalloc(newlen); + testutil_check(__wt_snprintf(newstr, newlen, "%s,%.*s", *strp, (int)v->len, v->str)); + /* Free the old value now we've copied it. */ + free(*strp); + begin = &newstr[(newlen - 1) - v->len]; + } + if ((ret = config_unescape(begin)) != 0) { + free(newstr); + return (ret); + } + *strp = newstr; + break; + case STRING_TYPE: + /* + * Thread configuration is the one case where the type isn't a + * "string", it's a "struct". + */ + if (v->type == WT_CONFIG_ITEM_STRUCT && STRING_MATCH("threads", k->str, k->len)) + return (config_threads(wtperf, v->str, v->len)); + + if (v->type != WT_CONFIG_ITEM_STRING && v->type != WT_CONFIG_ITEM_ID) { + fprintf(stderr, + "wtperf: Error: " + "bad string value for \'%.*s=%.*s\'\n", + (int)k->len, k->str, (int)v->len, v->str); + return (EINVAL); + } + strp = (char **)valueloc; + free(*strp); + /* + * We duplicate the string to len rather than len+1 as we want to truncate the trailing + * quotation mark. + */ + newstr = dstrndup(v->str, v->len); + *strp = newstr; + break; + } + return (0); } /* * config_opt_file -- - * Parse a configuration file. We recognize comments '#' and continuation - * via lines ending in '\'. + * Parse a configuration file. We recognize comments '#' and continuation via lines ending in + * '\'. */ int config_opt_file(WTPERF *wtperf, const char *filename) { - FILE *fp; - size_t linelen, optionpos; - int linenum, ret; - bool contline; - char line[4 * 1024], option[4 * 1024]; - char *comment, *ltrim, *rtrim; - - ret = 0; - - if ((fp = fopen(filename, "r")) == NULL) { - fprintf(stderr, "wtperf: %s: %s\n", filename, strerror(errno)); - return (errno); - } - - optionpos = 0; - linenum = 0; - while (fgets(line, sizeof(line), fp) != NULL) { - linenum++; - - /* Skip leading space. */ - for (ltrim = line; *ltrim && isspace((u_char)*ltrim); - ltrim++) - ; - - /* - * Find the end of the line; if there's no trailing newline, the - * the line is too long for the buffer or the file was corrupted - * (there's no terminating newline in the file). - */ - for (rtrim = line; *rtrim && *rtrim != '\n'; rtrim++) - ; - if (*rtrim != '\n') { - fprintf(stderr, - "wtperf: %s: %d: configuration line too long\n", - filename, linenum); - ret = EINVAL; - break; - } - - /* Skip trailing space. */ - while (rtrim > ltrim && isspace((u_char)rtrim[-1])) - rtrim--; - - /* - * If the last non-space character in the line is an escape, the - * line will be continued. Checked early because the line might - * otherwise be empty. - */ - contline = rtrim > ltrim && rtrim[-1] == '\\'; - if (contline) - rtrim--; - - /* - * Discard anything after the first hash character. Check after - * the escape character, the escape can appear after a comment. - */ - if ((comment = strchr(ltrim, '#')) != NULL) - rtrim = comment; - - /* Skip trailing space again. */ - while (rtrim > ltrim && isspace((u_char)rtrim[-1])) - rtrim--; - - /* - * Check for empty lines: note that the right-hand boundary can - * cross over the left-hand boundary, less-than or equal to is - * the correct test. - */ - if (rtrim <= ltrim) { - /* - * If we're continuing from this line, or we haven't - * started building an option, ignore this line. - */ - if (contline || optionpos == 0) - continue; - - /* - * An empty line terminating an option we're building; - * clean things up so we can proceed. - */ - linelen = 0; - } else - linelen = (size_t)(rtrim - ltrim); - ltrim[linelen] = '\0'; - - if (linelen + optionpos + 1 > sizeof(option)) { - fprintf(stderr, - "wtperf: %s: %d: option value overflow\n", - filename, linenum); - ret = EINVAL; - break; - } - - memcpy(&option[optionpos], ltrim, linelen); - option[optionpos + linelen] = '\0'; - if (contline) - optionpos += linelen; - else { - if ((ret = config_opt_str(wtperf, option)) != 0) { - fprintf(stderr, "wtperf: %s: %d: parse error\n", - filename, linenum); - break; - } - optionpos = 0; - } - } - if (ret == 0) { - if (ferror(fp)) { - fprintf(stderr, "wtperf: %s: read error\n", filename); - ret = errno; - } - if (optionpos > 0) { - fprintf(stderr, "wtperf: %s: %d: last line continues\n", - filename, linenum); - ret = EINVAL; - } - } - - (void)fclose(fp); - return (ret); + FILE *fp; + size_t linelen, optionpos; + int linenum, ret; + bool contline; + char line[4 * 1024], option[4 * 1024]; + char *comment, *ltrim, *rtrim; + + ret = 0; + + if ((fp = fopen(filename, "r")) == NULL) { + fprintf(stderr, "wtperf: %s: %s\n", filename, strerror(errno)); + return (errno); + } + + optionpos = 0; + linenum = 0; + while (fgets(line, sizeof(line), fp) != NULL) { + linenum++; + + /* Skip leading space. */ + for (ltrim = line; *ltrim && isspace((u_char)*ltrim); ltrim++) + ; + + /* + * Find the end of the line; if there's no trailing newline, the + * the line is too long for the buffer or the file was corrupted + * (there's no terminating newline in the file). + */ + for (rtrim = line; *rtrim && *rtrim != '\n'; rtrim++) + ; + if (*rtrim != '\n') { + fprintf(stderr, "wtperf: %s: %d: configuration line too long\n", filename, linenum); + ret = EINVAL; + break; + } + + /* Skip trailing space. */ + while (rtrim > ltrim && isspace((u_char)rtrim[-1])) + rtrim--; + + /* + * If the last non-space character in the line is an escape, the line will be continued. + * Checked early because the line might otherwise be empty. + */ + contline = rtrim > ltrim && rtrim[-1] == '\\'; + if (contline) + rtrim--; + + /* + * Discard anything after the first hash character. Check after the escape character, the + * escape can appear after a comment. + */ + if ((comment = strchr(ltrim, '#')) != NULL) + rtrim = comment; + + /* Skip trailing space again. */ + while (rtrim > ltrim && isspace((u_char)rtrim[-1])) + rtrim--; + + /* + * Check for empty lines: note that the right-hand boundary can cross over the left-hand + * boundary, less-than or equal to is the correct test. + */ + if (rtrim <= ltrim) { + /* + * If we're continuing from this line, or we haven't started building an option, ignore + * this line. + */ + if (contline || optionpos == 0) + continue; + + /* + * An empty line terminating an option we're building; clean things up so we can + * proceed. + */ + linelen = 0; + } else + linelen = (size_t)(rtrim - ltrim); + ltrim[linelen] = '\0'; + + if (linelen + optionpos + 1 > sizeof(option)) { + fprintf(stderr, "wtperf: %s: %d: option value overflow\n", filename, linenum); + ret = EINVAL; + break; + } + + memcpy(&option[optionpos], ltrim, linelen); + option[optionpos + linelen] = '\0'; + if (contline) + optionpos += linelen; + else { + if ((ret = config_opt_str(wtperf, option)) != 0) { + fprintf(stderr, "wtperf: %s: %d: parse error\n", filename, linenum); + break; + } + optionpos = 0; + } + } + if (ret == 0) { + if (ferror(fp)) { + fprintf(stderr, "wtperf: %s: read error\n", filename); + ret = errno; + } + if (optionpos > 0) { + fprintf(stderr, "wtperf: %s: %d: last line continues\n", filename, linenum); + ret = EINVAL; + } + } + + (void)fclose(fp); + return (ret); } /* * config_opt_str -- - * Parse a single line of config options. Continued lines have already - * been joined. + * Parse a single line of config options. Continued lines have already been joined. */ int config_opt_str(WTPERF *wtperf, const char *optstr) { - CONFIG_OPTS *opts; - CONFIG_QUEUE_ENTRY *config_line; - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *scan; - size_t len; - int ret, t_ret; - - opts = wtperf->opts; - - len = strlen(optstr); - if ((ret = wiredtiger_config_parser_open( - NULL, optstr, len, &scan)) != 0) { - lprintf(wtperf, ret, 0, "Error in config_scan_begin"); - return (ret); - } - - while (ret == 0) { - size_t pos; - - if ((ret = scan->next(scan, &k, &v)) != 0) { - /* Any parse error has already been reported. */ - if (ret == WT_NOTFOUND) - ret = 0; - break; - } - ret = config_opt(wtperf, &k, &v); - - /* - * Append the key-value pair to our copy of the config. - * The config is stored in the order it is processed, so added - * options will be after any parsed from the original config. - */ - config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); - /* - * If key or value is a string, consider extra space for the - * quotes. Add 2 to the required space for '=' and the ending - * null character in "key=value". - */ - config_line->string = dcalloc( - k.len + (k.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + - v.len + (v.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + 2, 1); - pos = 0; - if (k.type == WT_CONFIG_ITEM_STRING) { - config_line->string[pos] = '"'; - pos++; - } - strncpy(config_line->string + pos, k.str, k.len); - pos += k.len; - if (k.type == WT_CONFIG_ITEM_STRING) { - config_line->string[pos] = '"'; - pos++; - } - config_line->string[pos] = '='; - pos++; - if (v.type == WT_CONFIG_ITEM_STRING) { - config_line->string[pos] = '"'; - pos++; - } - strncpy(config_line->string + pos, v.str, v.len); - pos += v.len; - if (v.type == WT_CONFIG_ITEM_STRING) { - config_line->string[pos] = '"'; - pos++; - } - config_line->string[pos] = '\0'; - TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); - } - if ((t_ret = scan->close(scan)) != 0) { - lprintf(wtperf, ret, 0, "Error in config_scan_end"); - if (ret == 0) - ret = t_ret; - } - - return (ret); + CONFIG_OPTS *opts; + CONFIG_QUEUE_ENTRY *config_line; + WT_CONFIG_ITEM k, v; + WT_CONFIG_PARSER *scan; + size_t len; + int ret, t_ret; + + opts = wtperf->opts; + + len = strlen(optstr); + if ((ret = wiredtiger_config_parser_open(NULL, optstr, len, &scan)) != 0) { + lprintf(wtperf, ret, 0, "Error in config_scan_begin"); + return (ret); + } + + while (ret == 0) { + size_t pos; + + if ((ret = scan->next(scan, &k, &v)) != 0) { + /* Any parse error has already been reported. */ + if (ret == WT_NOTFOUND) + ret = 0; + break; + } + ret = config_opt(wtperf, &k, &v); + + /* + * Append the key-value pair to our copy of the config. The config is stored in the order it + * is processed, so added options will be after any parsed from the original config. + */ + config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); + /* + * If key or value is a string, consider extra space for the quotes. Add 2 to the required + * space for '=' and the ending null character in "key=value". + */ + config_line->string = dcalloc(k.len + (k.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + v.len + + (v.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + 2, + 1); + pos = 0; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, k.str, k.len); + pos += k.len; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '='; + pos++; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, v.str, v.len); + pos += v.len; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '\0'; + TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); + } + if ((t_ret = scan->close(scan)) != 0) { + lprintf(wtperf, ret, 0, "Error in config_scan_end"); + if (ret == 0) + ret = t_ret; + } + + return (ret); } /* * config_opt_name_value -- - * Set a name/value configuration pair. + * Set a name/value configuration pair. */ int config_opt_name_value(WTPERF *wtperf, const char *name, const char *value) { - size_t len; - int ret; - char *optstr; - /* name="value" */ - len = strlen(name) + strlen(value) + 4; - optstr = dmalloc(len); - testutil_check(__wt_snprintf(optstr, len, "%s=\"%s\"", name, value)); - ret = config_opt_str(wtperf, optstr); - free(optstr); - return (ret); + size_t len; + int ret; + char *optstr; + /* name="value" */ + len = strlen(name) + strlen(value) + 4; + optstr = dmalloc(len); + testutil_check(__wt_snprintf(optstr, len, "%s=\"%s\"", name, value)); + ret = config_opt_str(wtperf, optstr); + free(optstr); + return (ret); } /* * config_sanity -- - * Configuration sanity checks. + * Configuration sanity checks. */ int config_sanity(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WORKLOAD *workp; - u_int i; - - opts = wtperf->opts; - - /* Various intervals should be less than the run-time. */ - if (opts->run_time > 0 && - ((opts->checkpoint_threads != 0 && - opts->checkpoint_interval > opts->run_time) || - opts->report_interval > opts->run_time || - opts->sample_interval > opts->run_time || - opts->scan_interval > opts->run_time)) { - fprintf(stderr, "interval value longer than the run-time\n"); - return (EINVAL); - } - /* The maximum is here to keep file name construction simple. */ - if (opts->table_count < 1 || opts->table_count > 99999) { - fprintf(stderr, - "invalid table count, less than 1 or greater than 99999\n"); - return (EINVAL); - } - if (opts->database_count < 1 || opts->database_count > 99) { - fprintf(stderr, - "invalid database count, less than 1 or greater than 99\n"); - return (EINVAL); - } - - if (opts->pareto > 100) { - fprintf(stderr, - "Invalid pareto distribution - should be a percentage\n"); - return (EINVAL); - } - - if (opts->scan_pct > 100) { - fprintf(stderr, - "Invalid scan_pct - should be a percentage\n"); - return (EINVAL); - } - - /* If we have separate tables for scanning, we need a separate count. */ - if ((opts->scan_icount > 0 && opts->scan_table_count == 0) || - (opts->scan_icount == 0 && opts->scan_table_count > 0)) { - fprintf(stderr, - "scan_icount %" PRIu32 - " and scan_table_count %" PRIu32 - " must both be zero or nonzero.\n", - opts->scan_icount, opts->scan_table_count); - return (EINVAL); - } - if (opts->scan_interval > 0 && opts->icount == 0 && - opts->scan_icount == 0) { - fprintf(stderr, - "Invalid scan_interval - requires icount to be non-zero\n"); - return (EINVAL); - } - - if (opts->value_sz_max < opts->value_sz) { - if (F_ISSET(wtperf, CFG_GROW)) { - fprintf(stderr, "value_sz_max %" PRIu32 - " must be greater than or equal to value_sz %" - PRIu32 "\n", opts->value_sz_max, opts->value_sz); - return (EINVAL); - } else - opts->value_sz_max = opts->value_sz; - } - if (opts->value_sz_min > opts->value_sz) { - if (F_ISSET(wtperf, CFG_SHRINK)) { - fprintf(stderr, "value_sz_min %" PRIu32 - " must be less than or equal to value_sz %" - PRIu32 "\n", opts->value_sz_min, opts->value_sz); - return (EINVAL); - } else - opts->value_sz_min = opts->value_sz; - } - - if (wtperf->workload != NULL) - for (i = 0, workp = wtperf->workload; - i < wtperf->workload_cnt; ++i, ++workp) { - if (opts->readonly && - (workp->insert != 0 || workp->update != 0 || - workp->truncate != 0)) { - fprintf(stderr, - "Invalid workload: insert, update or " - "truncate specified with readonly\n"); - return (EINVAL); - } - if (workp->insert != 0 && - workp->table_index != INT32_MAX) { - fprintf(stderr, - "Invalid workload: Cannot insert into " - "specific table only\n"); - return (EINVAL); - } - if (workp->table_index != INT32_MAX && - workp->table_index >= (int32_t)opts->table_count) { - fprintf(stderr, - "Workload table index %" PRId32 - " is larger than table count %" PRIu32, - workp->table_index, opts->table_count); - return (EINVAL); - } - } - return (0); + CONFIG_OPTS *opts; + WORKLOAD *workp; + u_int i; + + opts = wtperf->opts; + + /* Various intervals should be less than the run-time. */ + if (opts->run_time > 0 && + ((opts->checkpoint_threads != 0 && opts->checkpoint_interval > opts->run_time) || + opts->report_interval > opts->run_time || opts->sample_interval > opts->run_time || + opts->scan_interval > opts->run_time)) { + fprintf(stderr, "interval value longer than the run-time\n"); + return (EINVAL); + } + /* The maximum is here to keep file name construction simple. */ + if (opts->table_count < 1 || opts->table_count > 99999) { + fprintf(stderr, "invalid table count, less than 1 or greater than 99999\n"); + return (EINVAL); + } + if (opts->database_count < 1 || opts->database_count > 99) { + fprintf(stderr, "invalid database count, less than 1 or greater than 99\n"); + return (EINVAL); + } + + if (opts->pareto > 100) { + fprintf(stderr, "Invalid pareto distribution - should be a percentage\n"); + return (EINVAL); + } + + if (opts->scan_pct > 100) { + fprintf(stderr, "Invalid scan_pct - should be a percentage\n"); + return (EINVAL); + } + + /* If we have separate tables for scanning, we need a separate count. */ + if ((opts->scan_icount > 0 && opts->scan_table_count == 0) || + (opts->scan_icount == 0 && opts->scan_table_count > 0)) { + fprintf(stderr, "scan_icount %" PRIu32 " and scan_table_count %" PRIu32 + " must both be zero or nonzero.\n", + opts->scan_icount, opts->scan_table_count); + return (EINVAL); + } + if (opts->scan_interval > 0 && opts->icount == 0 && opts->scan_icount == 0) { + fprintf(stderr, "Invalid scan_interval - requires icount to be non-zero\n"); + return (EINVAL); + } + + if (opts->value_sz_max < opts->value_sz) { + if (F_ISSET(wtperf, CFG_GROW)) { + fprintf(stderr, + "value_sz_max %" PRIu32 " must be greater than or equal to value_sz %" PRIu32 "\n", + opts->value_sz_max, opts->value_sz); + return (EINVAL); + } else + opts->value_sz_max = opts->value_sz; + } + if (opts->value_sz_min > opts->value_sz) { + if (F_ISSET(wtperf, CFG_SHRINK)) { + fprintf(stderr, + "value_sz_min %" PRIu32 " must be less than or equal to value_sz %" PRIu32 "\n", + opts->value_sz_min, opts->value_sz); + return (EINVAL); + } else + opts->value_sz_min = opts->value_sz; + } + + if (wtperf->workload != NULL) + for (i = 0, workp = wtperf->workload; i < wtperf->workload_cnt; ++i, ++workp) { + if (opts->readonly && + (workp->insert != 0 || workp->update != 0 || workp->truncate != 0)) { + fprintf(stderr, + "Invalid workload: insert, update or " + "truncate specified with readonly\n"); + return (EINVAL); + } + if (workp->insert != 0 && workp->table_index != INT32_MAX) { + fprintf(stderr, + "Invalid workload: Cannot insert into " + "specific table only\n"); + return (EINVAL); + } + if (workp->table_index != INT32_MAX && + workp->table_index >= (int32_t)opts->table_count) { + fprintf(stderr, + "Workload table index %" PRId32 " is larger than table count %" PRIu32, + workp->table_index, opts->table_count); + return (EINVAL); + } + } + return (0); } /* * config_consolidate -- - * Consolidate repeated configuration settings so that it only appears - * once in the configuration output file. + * Consolidate repeated configuration settings so that it only appears once in the configuration + * output file. */ static void config_consolidate(CONFIG_OPTS *opts) { - CONFIG_QUEUE_ENTRY *conf_line, *test_line, *tmp; - char *string_key; - - /* - * This loop iterates over the config queue and for each entry checks if - * a later queue entry has the same key. If there's a match, and key is - * "conn_config" or "table_config", the later queue entry is replaced - * with a concatenated entry of the two queue entries, the current queue - * entry is removed. For any other key, if there is a match, the current - * queue entry is removed. - */ - conf_line = TAILQ_FIRST(&opts->config_head); - while (conf_line != NULL) { - string_key = strchr(conf_line->string, '='); - tmp = test_line = TAILQ_NEXT(conf_line, q); - while (test_line != NULL) { - /* - * The + 1 here forces the '=' sign to be matched - * ensuring we don't match keys that have a common - * prefix such as "table_count" and "table_count_idle" - * as being the same key. - */ - if (strncmp(conf_line->string, test_line->string, - (size_t)((string_key - conf_line->string) + 1)) - == 0) { - if ((strncmp("conn_config=", conf_line->string, - (size_t)((string_key - conf_line->string) + - 1)) == 0) || - (strncmp("table_config=", conf_line->string, - (size_t)((string_key - conf_line->string) + - 1)) == 0)) { - char *concat_str, *val_pointer; - - /* - * To concatenate the two config - * strings, copy the first string to a - * new one, replace the ending '"' with - * a ',' and then concatenate the second - * string's value after its starting '"' - */ - val_pointer = - strchr(test_line->string, '=') + 2; - concat_str = - dmalloc(strlen(conf_line->string) + - strlen(val_pointer) + 1); - strcpy(concat_str, conf_line->string); - concat_str[strlen(concat_str) - 1] = - ','; - strcat(concat_str, val_pointer); - free(test_line->string); - test_line->string = concat_str; - } - - TAILQ_REMOVE(&opts->config_head, conf_line, q); - free(conf_line->string); - free(conf_line); - break; - } - test_line = TAILQ_NEXT(test_line, q); - } - conf_line = tmp; - } + CONFIG_QUEUE_ENTRY *conf_line, *test_line, *tmp; + char *string_key; + + /* + * This loop iterates over the config queue and for each entry checks if + * a later queue entry has the same key. If there's a match, and key is + * "conn_config" or "table_config", the later queue entry is replaced + * with a concatenated entry of the two queue entries, the current queue + * entry is removed. For any other key, if there is a match, the current + * queue entry is removed. + */ + conf_line = TAILQ_FIRST(&opts->config_head); + while (conf_line != NULL) { + string_key = strchr(conf_line->string, '='); + tmp = test_line = TAILQ_NEXT(conf_line, q); + while (test_line != NULL) { + /* + * The + 1 here forces the '=' sign to be matched ensuring we don't match keys that have + * a common prefix such as "table_count" and "table_count_idle" as being the same key. + */ + if (strncmp(conf_line->string, test_line->string, + (size_t)((string_key - conf_line->string) + 1)) == 0) { + if ((strncmp("conn_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + 1)) == 0) || + (strncmp("table_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + 1)) == 0)) { + char *concat_str, *val_pointer; + + /* + * To concatenate the two config strings, copy the first string to a new one, + * replace the ending '"' with a ',' and then concatenate the second string's + * value after its starting '"' + */ + val_pointer = strchr(test_line->string, '=') + 2; + concat_str = dmalloc(strlen(conf_line->string) + strlen(val_pointer) + 1); + strcpy(concat_str, conf_line->string); + concat_str[strlen(concat_str) - 1] = ','; + strcat(concat_str, val_pointer); + free(test_line->string); + test_line->string = concat_str; + } + + TAILQ_REMOVE(&opts->config_head, conf_line, q); + free(conf_line->string); + free(conf_line); + break; + } + test_line = TAILQ_NEXT(test_line, q); + } + conf_line = tmp; + } } /* * config_opt_log -- - * Write the final config used in this execution to a file. + * Write the final config used in this execution to a file. */ void config_opt_log(CONFIG_OPTS *opts, const char *path) { - CONFIG_QUEUE_ENTRY *config_line; - FILE *fp; + CONFIG_QUEUE_ENTRY *config_line; + FILE *fp; - testutil_checkfmt(((fp = fopen(path, "w")) == NULL), "%s", path); + testutil_checkfmt(((fp = fopen(path, "w")) == NULL), "%s", path); - config_consolidate(opts); + config_consolidate(opts); - fprintf(fp,"# Warning: This config includes " - "unwritten, implicit configuration defaults.\n" - "# Changes to those values may cause differences in behavior.\n"); - TAILQ_FOREACH(config_line, &opts->config_head, q) - fprintf(fp, "%s\n", config_line->string); - testutil_check(fclose(fp)); + fprintf(fp, + "# Warning: This config includes " + "unwritten, implicit configuration defaults.\n" + "# Changes to those values may cause differences in behavior.\n"); + TAILQ_FOREACH (config_line, &opts->config_head, q) + fprintf(fp, "%s\n", config_line->string); + testutil_check(fclose(fp)); } /* * config_opt_print -- - * Print out the configuration in verbose mode. + * Print out the configuration in verbose mode. */ void config_opt_print(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WORKLOAD *workp; - u_int i; - - opts = wtperf->opts; - - printf("Workload configuration:\n"); - printf("\t" "Home: %s\n", wtperf->home); - printf("\t" "Table name: %s\n", opts->table_name); - printf("\t" "Connection configuration: %s\n", opts->conn_config); - if (opts->sess_config != NULL) - printf("\t" "Session configuration: %s\n", opts->sess_config); - - printf("\t%s table: %s\n", - opts->create ? "Creating new" : "Using existing", - opts->table_config); - printf("\t" "Key size: %" PRIu32 ", value size: %" PRIu32 "\n", - opts->key_sz, opts->value_sz); - if (opts->create) - printf("\t" "Populate threads: %" PRIu32 ", inserting %" PRIu32 - " rows\n", - opts->populate_threads, opts->icount); - - printf("\t" "Workload seconds, operations: %" PRIu32 ", %" PRIu32 "\n", - opts->run_time, opts->run_ops); - if (wtperf->workload != NULL) { - printf("\t" "Workload configuration(s):\n"); - for (i = 0, workp = wtperf->workload; - i < wtperf->workload_cnt; ++i, ++workp) - printf("\t\t%" PRId64 " threads (inserts=%" PRId64 - ", reads=%" PRId64 ", updates=%" PRId64 - ", truncates=% " PRId64 ")\n", - workp->threads, - workp->insert, workp->read, - workp->update, workp->truncate); - } - - printf("\t" "Checkpoint threads, interval: %" PRIu32 ", %" PRIu32 "\n", - opts->checkpoint_threads, opts->checkpoint_interval); - printf("\t" "Reporting interval: %" PRIu32 "\n", opts->report_interval); - printf("\t" "Sampling interval: %" PRIu32 "\n", opts->sample_interval); - printf("\t" "Scan interval: %" PRIu32 "\n", opts->scan_interval); - - printf("\t" "Verbosity: %" PRIu32 "\n", opts->verbose); + CONFIG_OPTS *opts; + WORKLOAD *workp; + u_int i; + + opts = wtperf->opts; + + printf("Workload configuration:\n"); + printf( + "\t" + "Home: %s\n", + wtperf->home); + printf( + "\t" + "Table name: %s\n", + opts->table_name); + printf( + "\t" + "Connection configuration: %s\n", + opts->conn_config); + if (opts->sess_config != NULL) + printf( + "\t" + "Session configuration: %s\n", + opts->sess_config); + + printf( + "\t%s table: %s\n", opts->create ? "Creating new" : "Using existing", opts->table_config); + printf( + "\t" + "Key size: %" PRIu32 ", value size: %" PRIu32 "\n", + opts->key_sz, opts->value_sz); + if (opts->create) + printf( + "\t" + "Populate threads: %" PRIu32 ", inserting %" PRIu32 " rows\n", + opts->populate_threads, opts->icount); + + printf( + "\t" + "Workload seconds, operations: %" PRIu32 ", %" PRIu32 "\n", + opts->run_time, opts->run_ops); + if (wtperf->workload != NULL) { + printf( + "\t" + "Workload configuration(s):\n"); + for (i = 0, workp = wtperf->workload; i < wtperf->workload_cnt; ++i, ++workp) + printf("\t\t%" PRId64 " threads (inserts=%" PRId64 ", reads=%" PRId64 + ", updates=%" PRId64 ", truncates=% " PRId64 ")\n", + workp->threads, workp->insert, workp->read, workp->update, workp->truncate); + } + + printf( + "\t" + "Checkpoint threads, interval: %" PRIu32 ", %" PRIu32 "\n", + opts->checkpoint_threads, opts->checkpoint_interval); + printf( + "\t" + "Reporting interval: %" PRIu32 "\n", + opts->report_interval); + printf( + "\t" + "Sampling interval: %" PRIu32 "\n", + opts->sample_interval); + printf( + "\t" + "Scan interval: %" PRIu32 "\n", + opts->scan_interval); + + printf( + "\t" + "Verbosity: %" PRIu32 "\n", + opts->verbose); } /* * pretty_print -- - * Print out lines of text for a 80 character window. + * Print out lines of text for a 80 character window. */ static void pretty_print(const char *p, const char *indent) { - const char *t; - - for (;; p = t + 1) { - if (strlen(p) <= 70) - break; - for (t = p + 70; t > p && *t != ' '; --t) - ; - if (t == p) /* No spaces? */ - break; - printf("%s%.*s\n", - indent == NULL ? "" : indent, (int)(t - p), p); - } - if (*p != '\0') - printf("%s%s\n", indent == NULL ? "" : indent, p); + const char *t; + + for (;; p = t + 1) { + if (strlen(p) <= 70) + break; + for (t = p + 70; t > p && *t != ' '; --t) + ; + if (t == p) /* No spaces? */ + break; + printf("%s%.*s\n", indent == NULL ? "" : indent, (int)(t - p), p); + } + if (*p != '\0') + printf("%s%s\n", indent == NULL ? "" : indent, p); } /* * config_opt_usage -- - * Configuration usage error message. + * Configuration usage error message. */ void config_opt_usage(void) { - size_t i; - const char *defaultval, *typestr; - - pretty_print( - "The following are options settable using -o or -O, showing the " - "type and default value.\n", NULL); - pretty_print( - "String values must be enclosed in \" quotes, boolean values must " - "be either true or false.\n", NULL); - - for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) { - defaultval = config_opts_desc[i].defaultval; - typestr = "string"; - switch (config_opts_desc[i].type) { - case BOOL_TYPE: - typestr = "boolean"; - if (strcmp(defaultval, "0") == 0) - defaultval = "false"; - else - defaultval = "true"; - break; - case CONFIG_STRING_TYPE: - case STRING_TYPE: - break; - case INT_TYPE: - typestr = "int"; - break; - case UINT32_TYPE: - typestr = "unsigned int"; - break; - } - printf("%s (%s, default=%s)\n", - config_opts_desc[i].name, typestr, defaultval); - pretty_print(config_opts_desc[i].description, "\t"); - } + size_t i; + const char *defaultval, *typestr; + + pretty_print( + "The following are options settable using -o or -O, showing the " + "type and default value.\n", + NULL); + pretty_print( + "String values must be enclosed in \" quotes, boolean values must " + "be either true or false.\n", + NULL); + + for (i = 0; i < WT_ELEMENTS(config_opts_desc); i++) { + defaultval = config_opts_desc[i].defaultval; + typestr = "string"; + switch (config_opts_desc[i].type) { + case BOOL_TYPE: + typestr = "boolean"; + if (strcmp(defaultval, "0") == 0) + defaultval = "false"; + else + defaultval = "true"; + break; + case CONFIG_STRING_TYPE: + case STRING_TYPE: + break; + case INT_TYPE: + typestr = "int"; + break; + case UINT32_TYPE: + typestr = "unsigned int"; + break; + } + printf("%s (%s, default=%s)\n", config_opts_desc[i].name, typestr, defaultval); + pretty_print(config_opts_desc[i].description, "\t"); + } } diff --git a/src/third_party/wiredtiger/bench/wtperf/config_opt.h b/src/third_party/wiredtiger/bench/wtperf/config_opt.h index ec1cf7a8e67..c89b00b4991 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config_opt.h +++ b/src/third_party/wiredtiger/bench/wtperf/config_opt.h @@ -26,28 +26,26 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -typedef enum { - BOOL_TYPE, CONFIG_STRING_TYPE, INT_TYPE, STRING_TYPE, UINT32_TYPE -} CONFIG_OPT_TYPE; +typedef enum { BOOL_TYPE, CONFIG_STRING_TYPE, INT_TYPE, STRING_TYPE, UINT32_TYPE } CONFIG_OPT_TYPE; typedef struct { - const char *name; - const char *description; - const char *defaultval; - CONFIG_OPT_TYPE type; - size_t offset; + const char *name; + const char *description; + const char *defaultval; + CONFIG_OPT_TYPE type; + size_t offset; } CONFIG_OPT; typedef struct __config_queue_entry { - char *string; - TAILQ_ENTRY(__config_queue_entry) q; + char *string; + TAILQ_ENTRY(__config_queue_entry) q; } CONFIG_QUEUE_ENTRY; -typedef struct { /* Option structure */ -#define OPT_DECLARE_STRUCT +typedef struct { /* Option structure */ +#define OPT_DECLARE_STRUCT #include "wtperf_opt.i" -#undef OPT_DECLARE_STRUCT +#undef OPT_DECLARE_STRUCT - /* Queue head to save a copy of the config to be output */ - TAILQ_HEAD(__config_qh, __config_queue_entry) config_head; + /* Queue head to save a copy of the config to be output */ + TAILQ_HEAD(__config_qh, __config_queue_entry) config_head; } CONFIG_OPTS; diff --git a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c index 822c4661ea3..a8703249c82 100644 --- a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c +++ b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c @@ -29,154 +29,140 @@ #include "wtperf.h" static int -check_timing(WTPERF *wtperf, - const char *name, struct timespec start, struct timespec *stop) +check_timing(WTPERF *wtperf, const char *name, struct timespec start, struct timespec *stop) { - CONFIG_OPTS *opts; - uint64_t last_interval; + CONFIG_OPTS *opts; + uint64_t last_interval; - opts = wtperf->opts; + opts = wtperf->opts; - __wt_epoch(NULL, stop); + __wt_epoch(NULL, stop); - last_interval = (uint64_t)(WT_TIMEDIFF_SEC(*stop, start)); + last_interval = (uint64_t)(WT_TIMEDIFF_SEC(*stop, start)); - if (last_interval > opts->idle_table_cycle) { - lprintf(wtperf, ETIMEDOUT, 0, - "Cycling idle table failed because %s took %" PRIu64 - " seconds which is longer than configured acceptable" - " maximum of %" PRIu32 ".", - name, last_interval, opts->idle_table_cycle); - wtperf->error = true; - return (ETIMEDOUT); - } - return (0); + if (last_interval > opts->idle_table_cycle) { + lprintf(wtperf, ETIMEDOUT, 0, "Cycling idle table failed because %s took %" PRIu64 + " seconds which is longer than configured acceptable" + " maximum of %" PRIu32 ".", + name, last_interval, opts->idle_table_cycle); + wtperf->error = true; + return (ETIMEDOUT); + } + return (0); } /* - * Regularly create, open a cursor and drop a table. - * Measure how long each step takes, and flag an error if it exceeds the - * configured maximum. + * Regularly create, open a cursor and drop a table. Measure how long each step takes, and flag an + * error if it exceeds the configured maximum. */ static WT_THREAD_RET cycle_idle_tables(void *arg) { - struct timespec start, stop; - CONFIG_OPTS *opts; - WTPERF *wtperf; - WT_CURSOR *cursor; - WT_SESSION *session; - int cycle_count, ret; - char uri[512]; - - wtperf = (WTPERF *)arg; - opts = wtperf->opts; - cycle_count = 0; - - if ((ret = wtperf->conn->open_session( - wtperf->conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, - "Error opening a session on %s", wtperf->home); - return (WT_THREAD_RET_VALUE); - } - - for (cycle_count = 0; wtperf->idle_cycle_run; ++cycle_count) { - testutil_check(__wt_snprintf(uri, sizeof(uri), - "%s_cycle%07d", wtperf->uris[0], cycle_count)); - /* Don't busy cycle in this loop. */ - __wt_sleep(1, 0); - - /* Setup a start timer. */ - __wt_epoch(NULL, &start); - - /* Create a table. */ - if ((ret = session->create( - session, uri, opts->table_config)) != 0) { - if (ret == EBUSY) - continue; - lprintf(wtperf, ret, 0, - "Table create failed in cycle_idle_tables."); - wtperf->error = true; - return (WT_THREAD_RET_VALUE); - } - if (check_timing(wtperf, "create", start, &stop) != 0) - return (WT_THREAD_RET_VALUE); - start = stop; - - /* Open and close cursor. */ - if ((ret = session->open_cursor( - session, uri, NULL, NULL, &cursor)) != 0) { - lprintf(wtperf, ret, 0, - "Cursor open failed in cycle_idle_tables."); - wtperf->error = true; - return (WT_THREAD_RET_VALUE); - } - if ((ret = cursor->close(cursor)) != 0) { - lprintf(wtperf, ret, 0, - "Cursor close failed in cycle_idle_tables."); - wtperf->error = true; - return (WT_THREAD_RET_VALUE); - } - if (check_timing(wtperf, "cursor", start, &stop) != 0) - return (WT_THREAD_RET_VALUE); - start = stop; + struct timespec start, stop; + CONFIG_OPTS *opts; + WTPERF *wtperf; + WT_CURSOR *cursor; + WT_SESSION *session; + int cycle_count, ret; + char uri[512]; + + wtperf = (WTPERF *)arg; + opts = wtperf->opts; + cycle_count = 0; + + if ((ret = wtperf->conn->open_session(wtperf->conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "Error opening a session on %s", wtperf->home); + return (WT_THREAD_RET_VALUE); + } + + for (cycle_count = 0; wtperf->idle_cycle_run; ++cycle_count) { + testutil_check( + __wt_snprintf(uri, sizeof(uri), "%s_cycle%07d", wtperf->uris[0], cycle_count)); + /* Don't busy cycle in this loop. */ + __wt_sleep(1, 0); + + /* Setup a start timer. */ + __wt_epoch(NULL, &start); + + /* Create a table. */ + if ((ret = session->create(session, uri, opts->table_config)) != 0) { + if (ret == EBUSY) + continue; + lprintf(wtperf, ret, 0, "Table create failed in cycle_idle_tables."); + wtperf->error = true; + return (WT_THREAD_RET_VALUE); + } + if (check_timing(wtperf, "create", start, &stop) != 0) + return (WT_THREAD_RET_VALUE); + start = stop; + + /* Open and close cursor. */ + if ((ret = session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) { + lprintf(wtperf, ret, 0, "Cursor open failed in cycle_idle_tables."); + wtperf->error = true; + return (WT_THREAD_RET_VALUE); + } + if ((ret = cursor->close(cursor)) != 0) { + lprintf(wtperf, ret, 0, "Cursor close failed in cycle_idle_tables."); + wtperf->error = true; + return (WT_THREAD_RET_VALUE); + } + if (check_timing(wtperf, "cursor", start, &stop) != 0) + return (WT_THREAD_RET_VALUE); + start = stop; #if 1 - /* - * Drop the table. Keep retrying on EBUSY failure - it is an - * expected return when checkpoints are happening. - */ - while ((ret = session->drop( - session, uri, "force,checkpoint_wait=false")) == EBUSY) - __wt_sleep(1, 0); - - if (ret != 0) { - lprintf(wtperf, ret, 0, - "Table drop failed in cycle_idle_tables."); - wtperf->error = true; - return (WT_THREAD_RET_VALUE); - } - if (check_timing(wtperf, "drop", start, &stop) != 0) - return (WT_THREAD_RET_VALUE); + /* + * Drop the table. Keep retrying on EBUSY failure - it is an expected return when + * checkpoints are happening. + */ + while ((ret = session->drop(session, uri, "force,checkpoint_wait=false")) == EBUSY) + __wt_sleep(1, 0); + + if (ret != 0) { + lprintf(wtperf, ret, 0, "Table drop failed in cycle_idle_tables."); + wtperf->error = true; + return (WT_THREAD_RET_VALUE); + } + if (check_timing(wtperf, "drop", start, &stop) != 0) + return (WT_THREAD_RET_VALUE); #endif - } + } - return (WT_THREAD_RET_VALUE); + return (WT_THREAD_RET_VALUE); } /* - * Start a thread the creates and drops tables regularly. - * TODO: Currently accepts a pthread_t as a parameter, since it is not - * possible to portably statically initialize it in the global configuration - * structure. Should reshuffle the configuration structure so explicit static + * Start a thread the creates and drops tables regularly. TODO: Currently accepts a pthread_t as a + * parameter, since it is not possible to portably statically initialize it in the global + * configuration structure. Should reshuffle the configuration structure so explicit static * initialization isn't necessary. */ void start_idle_table_cycle(WTPERF *wtperf, wt_thread_t *idle_table_cycle_thread) { - CONFIG_OPTS *opts; - wt_thread_t thread_id; + CONFIG_OPTS *opts; + wt_thread_t thread_id; - opts = wtperf->opts; + opts = wtperf->opts; - if (opts->idle_table_cycle == 0) - return; + if (opts->idle_table_cycle == 0) + return; - wtperf->idle_cycle_run = true; - testutil_check(__wt_thread_create( - NULL, &thread_id, cycle_idle_tables, wtperf)); - *idle_table_cycle_thread = thread_id; + wtperf->idle_cycle_run = true; + testutil_check(__wt_thread_create(NULL, &thread_id, cycle_idle_tables, wtperf)); + *idle_table_cycle_thread = thread_id; } void stop_idle_table_cycle(WTPERF *wtperf, wt_thread_t idle_table_cycle_thread) { - CONFIG_OPTS *opts; + CONFIG_OPTS *opts; - opts = wtperf->opts; + opts = wtperf->opts; - if (opts->idle_table_cycle == 0 || !wtperf->idle_cycle_run) - return; + if (opts->idle_table_cycle == 0 || !wtperf->idle_cycle_run) + return; - wtperf->idle_cycle_run = false; - testutil_check(__wt_thread_join(NULL, &idle_table_cycle_thread)); + wtperf->idle_cycle_run = false; + testutil_check(__wt_thread_join(NULL, &idle_table_cycle_thread)); } diff --git a/src/third_party/wiredtiger/bench/wtperf/misc.c b/src/third_party/wiredtiger/bench/wtperf/misc.c index 9f68aeddb6f..0528a2fe552 100644 --- a/src/third_party/wiredtiger/bench/wtperf/misc.c +++ b/src/third_party/wiredtiger/bench/wtperf/misc.c @@ -32,33 +32,31 @@ int setup_log_file(WTPERF *wtperf) { - CONFIG_OPTS *opts; - size_t len; - int ret; - char *fname; + CONFIG_OPTS *opts; + size_t len; + int ret; + char *fname; - opts = wtperf->opts; - ret = 0; + opts = wtperf->opts; + ret = 0; - if (opts->verbose < 1) - return (0); + if (opts->verbose < 1) + return (0); - len = strlen(wtperf->monitor_dir) + - strlen(opts->table_name) + strlen(".stat") + 2; - fname = dmalloc(len); - testutil_check(__wt_snprintf(fname, len, - "%s/%s.stat", wtperf->monitor_dir, opts->table_name)); - if ((wtperf->logf = fopen(fname, "w")) == NULL) { - ret = errno; - fprintf(stderr, "%s: %s\n", fname, strerror(ret)); - } - free(fname); - if (wtperf->logf == NULL) - return (ret); + len = strlen(wtperf->monitor_dir) + strlen(opts->table_name) + strlen(".stat") + 2; + fname = dmalloc(len); + testutil_check(__wt_snprintf(fname, len, "%s/%s.stat", wtperf->monitor_dir, opts->table_name)); + if ((wtperf->logf = fopen(fname, "w")) == NULL) { + ret = errno; + fprintf(stderr, "%s: %s\n", fname, strerror(ret)); + } + free(fname); + if (wtperf->logf == NULL) + return (ret); - /* Use line buffering for the log file. */ - __wt_stream_set_line_buffer(wtperf->logf); - return (0); + /* Use line buffering for the log file. */ + __wt_stream_set_line_buffer(wtperf->logf); + return (0); } /* @@ -67,40 +65,40 @@ setup_log_file(WTPERF *wtperf) void lprintf(const WTPERF *wtperf, int err, uint32_t level, const char *fmt, ...) { - CONFIG_OPTS *opts; - va_list ap; + CONFIG_OPTS *opts; + va_list ap; - opts = wtperf->opts; + opts = wtperf->opts; - if (err == 0 && level <= opts->verbose) { - va_start(ap, fmt); - vfprintf(wtperf->logf, fmt, ap); - va_end(ap); - fprintf(wtperf->logf, "\n"); + if (err == 0 && level <= opts->verbose) { + va_start(ap, fmt); + vfprintf(wtperf->logf, fmt, ap); + va_end(ap); + fprintf(wtperf->logf, "\n"); - if (level < opts->verbose) { - va_start(ap, fmt); - vprintf(fmt, ap); - va_end(ap); - printf("\n"); - } - } - if (err == 0) - return; + if (level < opts->verbose) { + va_start(ap, fmt); + vprintf(fmt, ap); + va_end(ap); + printf("\n"); + } + } + if (err == 0) + return; - /* We are dealing with an error. */ - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - fprintf(stderr, " Error: %s\n", wiredtiger_strerror(err)); - if (wtperf->logf != NULL) { - va_start(ap, fmt); - vfprintf(wtperf->logf, fmt, ap); - va_end(ap); - fprintf(wtperf->logf, " Error: %s\n", wiredtiger_strerror(err)); - } + /* We are dealing with an error. */ + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " Error: %s\n", wiredtiger_strerror(err)); + if (wtperf->logf != NULL) { + va_start(ap, fmt); + vfprintf(wtperf->logf, fmt, ap); + va_end(ap); + fprintf(wtperf->logf, " Error: %s\n", wiredtiger_strerror(err)); + } - /* Never attempt to continue if we got a panic from WiredTiger. */ - if (err == WT_PANIC) - abort(); + /* Never attempt to continue if we got a panic from WiredTiger. */ + if (err == WT_PANIC) + abort(); } diff --git a/src/third_party/wiredtiger/bench/wtperf/track.c b/src/third_party/wiredtiger/bench/wtperf/track.c index 3b8832dc6bf..cf0e98061ff 100644 --- a/src/third_party/wiredtiger/bench/wtperf/track.c +++ b/src/third_party/wiredtiger/bench/wtperf/track.c @@ -34,18 +34,18 @@ uint64_t sum_pop_ops(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WTPERF_THREAD *thread; - uint64_t total; - u_int i; - - opts = wtperf->opts; - total = 0; - - for (i = 0, thread = wtperf->popthreads; - thread != NULL && i < opts->populate_threads; ++i, ++thread) - total += thread->insert.ops; - return (total); + CONFIG_OPTS *opts; + WTPERF_THREAD *thread; + uint64_t total; + u_int i; + + opts = wtperf->opts; + total = 0; + + for (i = 0, thread = wtperf->popthreads; thread != NULL && i < opts->populate_threads; + ++i, ++thread) + total += thread->insert.ops; + return (total); } /* @@ -54,18 +54,18 @@ sum_pop_ops(WTPERF *wtperf) uint64_t sum_ckpt_ops(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WTPERF_THREAD *thread; - uint64_t total; - u_int i; - - opts = wtperf->opts; - total = 0; - - for (i = 0, thread = wtperf->ckptthreads; - thread != NULL && i < opts->checkpoint_threads; ++i, ++thread) - total += thread->ckpt.ops; - return (total); + CONFIG_OPTS *opts; + WTPERF_THREAD *thread; + uint64_t total; + u_int i; + + opts = wtperf->opts; + total = 0; + + for (i = 0, thread = wtperf->ckptthreads; thread != NULL && i < opts->checkpoint_threads; + ++i, ++thread) + total += thread->ckpt.ops; + return (total); } /* @@ -74,16 +74,16 @@ sum_ckpt_ops(WTPERF *wtperf) uint64_t sum_scan_ops(WTPERF *wtperf) { - CONFIG_OPTS *opts; - uint64_t total; + CONFIG_OPTS *opts; + uint64_t total; - opts = wtperf->opts; + opts = wtperf->opts; - if (opts->scan_interval > 0) - total = wtperf->scanthreads->scan.ops; - else - total = 0; - return (total); + if (opts->scan_interval > 0) + total = wtperf->scanthreads->scan.ops; + else + total = 0; + return (total); } /* @@ -92,267 +92,260 @@ sum_scan_ops(WTPERF *wtperf) static uint64_t sum_ops(WTPERF *wtperf, size_t field_offset) { - CONFIG_OPTS *opts; - WTPERF_THREAD *thread; - uint64_t total; - int64_t i, th_cnt; - - opts = wtperf->opts; - total = 0; - - if (wtperf->popthreads == NULL) { - thread = wtperf->workers; - th_cnt = wtperf->workers_cnt; - } else { - thread = wtperf->popthreads; - th_cnt = opts->populate_threads; - } - for (i = 0; thread != NULL && i < th_cnt; ++i, ++thread) - total += ((TRACK *)((uint8_t *)thread + field_offset))->ops; - - return (total); + CONFIG_OPTS *opts; + WTPERF_THREAD *thread; + uint64_t total; + int64_t i, th_cnt; + + opts = wtperf->opts; + total = 0; + + if (wtperf->popthreads == NULL) { + thread = wtperf->workers; + th_cnt = wtperf->workers_cnt; + } else { + thread = wtperf->popthreads; + th_cnt = opts->populate_threads; + } + for (i = 0; thread != NULL && i < th_cnt; ++i, ++thread) + total += ((TRACK *)((uint8_t *)thread + field_offset))->ops; + + return (total); } uint64_t sum_insert_ops(WTPERF *wtperf) { - return (sum_ops(wtperf, offsetof(WTPERF_THREAD, insert))); + return (sum_ops(wtperf, offsetof(WTPERF_THREAD, insert))); } uint64_t sum_read_ops(WTPERF *wtperf) { - return (sum_ops(wtperf, offsetof(WTPERF_THREAD, read))); + return (sum_ops(wtperf, offsetof(WTPERF_THREAD, read))); } uint64_t sum_truncate_ops(WTPERF *wtperf) { - return (sum_ops(wtperf, offsetof(WTPERF_THREAD, truncate))); + return (sum_ops(wtperf, offsetof(WTPERF_THREAD, truncate))); } uint64_t sum_update_ops(WTPERF *wtperf) { - return (sum_ops(wtperf, offsetof(WTPERF_THREAD, update))); + return (sum_ops(wtperf, offsetof(WTPERF_THREAD, update))); } /* * latency_op -- - * Get average, minimum and maximum latency for this period for a - * particular operation. + * Get average, minimum and maximum latency for this period for a particular operation. */ static void -latency_op(WTPERF *wtperf, - size_t field_offset, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) +latency_op(WTPERF *wtperf, size_t field_offset, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) { - CONFIG_OPTS *opts; - TRACK *track; - WTPERF_THREAD *thread; - uint64_t ops, latency, tmp; - int64_t i, th_cnt; - uint32_t max, min; - - opts = wtperf->opts; - ops = latency = 0; - max = 0; - min = UINT32_MAX; - - if (wtperf->popthreads == NULL) { - thread = wtperf->workers; - th_cnt = wtperf->workers_cnt; - } else { - thread = wtperf->popthreads; - th_cnt = opts->populate_threads; - } - for (i = 0; thread != NULL && i < th_cnt; ++i, ++thread) { - track = (TRACK *)((uint8_t *)thread + field_offset); - tmp = track->latency_ops; - ops += tmp - track->last_latency_ops; - track->last_latency_ops = tmp; - tmp = track->latency; - latency += tmp - track->last_latency; - track->last_latency = tmp; - - if (min > track->min_latency) - min = track->min_latency; - track->min_latency = UINT32_MAX; - if (max < track->max_latency) - max = track->max_latency; - track->max_latency = 0; - } - - if (ops == 0) - *avgp = *minp = *maxp = 0; - else { - *minp = min; - *maxp = max; - *avgp = (uint32_t)(latency / ops); - } + CONFIG_OPTS *opts; + TRACK *track; + WTPERF_THREAD *thread; + uint64_t ops, latency, tmp; + int64_t i, th_cnt; + uint32_t max, min; + + opts = wtperf->opts; + ops = latency = 0; + max = 0; + min = UINT32_MAX; + + if (wtperf->popthreads == NULL) { + thread = wtperf->workers; + th_cnt = wtperf->workers_cnt; + } else { + thread = wtperf->popthreads; + th_cnt = opts->populate_threads; + } + for (i = 0; thread != NULL && i < th_cnt; ++i, ++thread) { + track = (TRACK *)((uint8_t *)thread + field_offset); + tmp = track->latency_ops; + ops += tmp - track->last_latency_ops; + track->last_latency_ops = tmp; + tmp = track->latency; + latency += tmp - track->last_latency; + track->last_latency = tmp; + + if (min > track->min_latency) + min = track->min_latency; + track->min_latency = UINT32_MAX; + if (max < track->max_latency) + max = track->max_latency; + track->max_latency = 0; + } + + if (ops == 0) + *avgp = *minp = *maxp = 0; + else { + *minp = min; + *maxp = max; + *avgp = (uint32_t)(latency / ops); + } } void latency_read(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) { - static uint32_t last_avg = 0, last_max = 0, last_min = 0; - - latency_op(wtperf, offsetof(WTPERF_THREAD, read), avgp, minp, maxp); - - /* - * If nothing happened, graph the average, minimum and maximum as they - * were the last time, it keeps the graphs from having discontinuities. - */ - if (*minp == 0) { - *avgp = last_avg; - *minp = last_min; - *maxp = last_max; - } else { - last_avg = *avgp; - last_min = *minp; - last_max = *maxp; - } + static uint32_t last_avg = 0, last_max = 0, last_min = 0; + + latency_op(wtperf, offsetof(WTPERF_THREAD, read), avgp, minp, maxp); + + /* + * If nothing happened, graph the average, minimum and maximum as they were the last time, it + * keeps the graphs from having discontinuities. + */ + if (*minp == 0) { + *avgp = last_avg; + *minp = last_min; + *maxp = last_max; + } else { + last_avg = *avgp; + last_min = *minp; + last_max = *maxp; + } } void latency_insert(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) { - static uint32_t last_avg = 0, last_max = 0, last_min = 0; - - latency_op(wtperf, offsetof(WTPERF_THREAD, insert), avgp, minp, maxp); - - /* - * If nothing happened, graph the average, minimum and maximum as they - * were the last time, it keeps the graphs from having discontinuities. - */ - if (*minp == 0) { - *avgp = last_avg; - *minp = last_min; - *maxp = last_max; - } else { - last_avg = *avgp; - last_min = *minp; - last_max = *maxp; - } + static uint32_t last_avg = 0, last_max = 0, last_min = 0; + + latency_op(wtperf, offsetof(WTPERF_THREAD, insert), avgp, minp, maxp); + + /* + * If nothing happened, graph the average, minimum and maximum as they were the last time, it + * keeps the graphs from having discontinuities. + */ + if (*minp == 0) { + *avgp = last_avg; + *minp = last_min; + *maxp = last_max; + } else { + last_avg = *avgp; + last_min = *minp; + last_max = *maxp; + } } void latency_update(WTPERF *wtperf, uint32_t *avgp, uint32_t *minp, uint32_t *maxp) { - static uint32_t last_avg = 0, last_max = 0, last_min = 0; - - latency_op(wtperf, offsetof(WTPERF_THREAD, update), avgp, minp, maxp); - - /* - * If nothing happened, graph the average, minimum and maximum as they - * were the last time, it keeps the graphs from having discontinuities. - */ - if (*minp == 0) { - *avgp = last_avg; - *minp = last_min; - *maxp = last_max; - } else { - last_avg = *avgp; - last_min = *minp; - last_max = *maxp; - } + static uint32_t last_avg = 0, last_max = 0, last_min = 0; + + latency_op(wtperf, offsetof(WTPERF_THREAD, update), avgp, minp, maxp); + + /* + * If nothing happened, graph the average, minimum and maximum as they were the last time, it + * keeps the graphs from having discontinuities. + */ + if (*minp == 0) { + *avgp = last_avg; + *minp = last_min; + *maxp = last_max; + } else { + last_avg = *avgp; + last_min = *minp; + last_max = *maxp; + } } /* * sum_latency -- - * Sum latency for a set of threads. + * Sum latency for a set of threads. */ static void sum_latency(WTPERF *wtperf, size_t field_offset, TRACK *total) { - WTPERF_THREAD *thread; - TRACK *trk; - int64_t i; - u_int j; - - memset(total, 0, sizeof(*total)); - - for (i = 0, thread = wtperf->workers; - thread != NULL && i < wtperf->workers_cnt; ++i, ++thread) { - trk = (TRACK *)((uint8_t *)thread + field_offset); - - for (j = 0; j < ELEMENTS(trk->us); ++j) { - total->ops += trk->us[j]; - total->us[j] += trk->us[j]; - } - for (j = 0; j < ELEMENTS(trk->ms); ++j) { - total->ops += trk->ms[j]; - total->ms[j] += trk->ms[j]; - } - for (j = 0; j < ELEMENTS(trk->sec); ++j) { - total->ops += trk->sec[j]; - total->sec[j] += trk->sec[j]; - } - } + WTPERF_THREAD *thread; + TRACK *trk; + int64_t i; + u_int j; + + memset(total, 0, sizeof(*total)); + + for (i = 0, thread = wtperf->workers; thread != NULL && i < wtperf->workers_cnt; + ++i, ++thread) { + trk = (TRACK *)((uint8_t *)thread + field_offset); + + for (j = 0; j < ELEMENTS(trk->us); ++j) { + total->ops += trk->us[j]; + total->us[j] += trk->us[j]; + } + for (j = 0; j < ELEMENTS(trk->ms); ++j) { + total->ops += trk->ms[j]; + total->ms[j] += trk->ms[j]; + } + for (j = 0; j < ELEMENTS(trk->sec); ++j) { + total->ops += trk->sec[j]; + total->sec[j] += trk->sec[j]; + } + } } static void sum_insert_latency(WTPERF *wtperf, TRACK *total) { - sum_latency(wtperf, offsetof(WTPERF_THREAD, insert), total); + sum_latency(wtperf, offsetof(WTPERF_THREAD, insert), total); } static void sum_read_latency(WTPERF *wtperf, TRACK *total) { - sum_latency(wtperf, offsetof(WTPERF_THREAD, read), total); + sum_latency(wtperf, offsetof(WTPERF_THREAD, read), total); } static void sum_update_latency(WTPERF *wtperf, TRACK *total) { - sum_latency(wtperf, offsetof(WTPERF_THREAD, update), total); + sum_latency(wtperf, offsetof(WTPERF_THREAD, update), total); } static void latency_print_single(WTPERF *wtperf, TRACK *total, const char *name) { - FILE *fp; - u_int i; - uint64_t cumops; - char path[1024]; - - testutil_check(__wt_snprintf(path, sizeof(path), - "%s/latency.%s", wtperf->monitor_dir, name)); - if ((fp = fopen(path, "w")) == NULL) { - lprintf(wtperf, errno, 0, "%s", path); - return; - } - - fprintf(fp, - "#usecs,operations,cumulative-operations,total-operations\n"); - cumops = 0; - for (i = 0; i < ELEMENTS(total->us); ++i) { - if (total->us[i] == 0) - continue; - cumops += total->us[i]; - fprintf(fp, - "%u,%" PRIu32 ",%" PRIu64 ",%" PRIu64 "\n", - (i + 1), total->us[i], cumops, total->ops); - } - for (i = 1; i < ELEMENTS(total->ms); ++i) { - if (total->ms[i] == 0) - continue; - cumops += total->ms[i]; - fprintf(fp, - "%llu,%" PRIu32 ",%" PRIu64 ",%" PRIu64 "\n", - ms_to_us(i + 1), total->ms[i], cumops, total->ops); - } - for (i = 1; i < ELEMENTS(total->sec); ++i) { - if (total->sec[i] == 0) - continue; - cumops += total->sec[i]; - fprintf(fp, - "%llu,%" PRIu32 ",%" PRIu64 ",%" PRIu64 "\n", - sec_to_us(i + 1), total->sec[i], cumops, total->ops); - } - - (void)fclose(fp); + FILE *fp; + u_int i; + uint64_t cumops; + char path[1024]; + + testutil_check(__wt_snprintf(path, sizeof(path), "%s/latency.%s", wtperf->monitor_dir, name)); + if ((fp = fopen(path, "w")) == NULL) { + lprintf(wtperf, errno, 0, "%s", path); + return; + } + + fprintf(fp, "#usecs,operations,cumulative-operations,total-operations\n"); + cumops = 0; + for (i = 0; i < ELEMENTS(total->us); ++i) { + if (total->us[i] == 0) + continue; + cumops += total->us[i]; + fprintf(fp, "%u,%" PRIu32 ",%" PRIu64 ",%" PRIu64 "\n", (i + 1), total->us[i], cumops, + total->ops); + } + for (i = 1; i < ELEMENTS(total->ms); ++i) { + if (total->ms[i] == 0) + continue; + cumops += total->ms[i]; + fprintf(fp, "%llu,%" PRIu32 ",%" PRIu64 ",%" PRIu64 "\n", ms_to_us(i + 1), total->ms[i], + cumops, total->ops); + } + for (i = 1; i < ELEMENTS(total->sec); ++i) { + if (total->sec[i] == 0) + continue; + cumops += total->sec[i]; + fprintf(fp, "%llu,%" PRIu32 ",%" PRIu64 ",%" PRIu64 "\n", sec_to_us(i + 1), total->sec[i], + cumops, total->ops); + } + + (void)fclose(fp); } void latency_print(WTPERF *wtperf) { - TRACK total; - - sum_insert_latency(wtperf, &total); - latency_print_single(wtperf, &total, "insert"); - sum_read_latency(wtperf, &total); - latency_print_single(wtperf, &total, "read"); - sum_update_latency(wtperf, &total); - latency_print_single(wtperf, &total, "update"); + TRACK total; + + sum_insert_latency(wtperf, &total); + latency_print_single(wtperf, &total, "insert"); + sum_read_latency(wtperf, &total); + latency_print_single(wtperf, &total, "read"); + sum_update_latency(wtperf, &total); + latency_print_single(wtperf, &total, "update"); } diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 5f0f402c6f3..3c13304f1c1 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -29,89 +29,85 @@ #include "wtperf.h" /* Default values. */ -#define DEFAULT_HOME "WT_TEST" -#define DEFAULT_MONITOR_DIR "WT_TEST" +#define DEFAULT_HOME "WT_TEST" +#define DEFAULT_MONITOR_DIR "WT_TEST" static WT_THREAD_RET checkpoint_worker(void *); -static int drop_all_tables(WTPERF *); -static int execute_populate(WTPERF *); -static int execute_workload(WTPERF *); -static int find_table_count(WTPERF *); +static int drop_all_tables(WTPERF *); +static int execute_populate(WTPERF *); +static int execute_workload(WTPERF *); +static int find_table_count(WTPERF *); static WT_THREAD_RET monitor(void *); static WT_THREAD_RET populate_thread(void *); -static void randomize_value(WTPERF_THREAD *, char *); -static void recreate_dir(const char *); +static void randomize_value(WTPERF_THREAD *, char *); +static void recreate_dir(const char *); static WT_THREAD_RET scan_worker(void *); -static int start_all_runs(WTPERF *); -static int start_run(WTPERF *); -static void start_threads(WTPERF *, WORKLOAD *, - WTPERF_THREAD *, u_int, WT_THREAD_CALLBACK(*)(void *)); -static void stop_threads(u_int, WTPERF_THREAD *); +static int start_all_runs(WTPERF *); +static int start_run(WTPERF *); +static void start_threads( + WTPERF *, WORKLOAD *, WTPERF_THREAD *, u_int, WT_THREAD_CALLBACK (*)(void *)); +static void stop_threads(u_int, WTPERF_THREAD *); static WT_THREAD_RET thread_run_wtperf(void *); -static void update_value_delta(WTPERF_THREAD *); +static void update_value_delta(WTPERF_THREAD *); static WT_THREAD_RET worker(void *); -static uint64_t wtperf_rand(WTPERF_THREAD *); -static uint64_t wtperf_value_range(WTPERF *); +static uint64_t wtperf_rand(WTPERF_THREAD *); +static uint64_t wtperf_value_range(WTPERF *); -#define INDEX_COL_NAMES "columns=(key,val)" +#define INDEX_COL_NAMES "columns=(key,val)" /* Retrieve an ID for the next insert operation. */ static inline uint64_t get_next_incr(WTPERF *wtperf) { - return (__wt_atomic_add64(&wtperf->insert_key, 1)); + return (__wt_atomic_add64(&wtperf->insert_key, 1)); } /* - * Each time this function is called we will overwrite the first and one - * other element in the value buffer. + * Each time this function is called we will overwrite the first and one other element in the value + * buffer. */ static void randomize_value(WTPERF_THREAD *thread, char *value_buf) { - CONFIG_OPTS *opts; - uint8_t *vb; - uint32_t i, max_range, rand_val; - - opts = thread->wtperf->opts; - - /* - * Limit how much of the buffer we validate for length, this means - * that only threads that do growing updates will ever make changes to - * values outside of the initial value size, but that's a fair trade - * off for avoiding figuring out how long the value is more accurately - * in this performance sensitive function. - */ - if (thread->workload == NULL || thread->workload->update_delta == 0) - max_range = opts->value_sz; - else if (thread->workload->update_delta > 0) - max_range = opts->value_sz_max; - else - max_range = opts->value_sz_min; - - /* - * Generate a single random value and re-use it. We generally only - * have small ranges in this function, so avoiding a bunch of calls - * is worthwhile. - */ - rand_val = __wt_random(&thread->rnd); - i = rand_val % (max_range - 1); - - /* - * Ensure we don't write past the end of a value when configured for - * randomly sized values. - */ - while (value_buf[i] == '\0' && i > 0) - --i; - - vb = (uint8_t *)value_buf; - vb[0] = ((rand_val >> 8) % 255) + 1; - /* - * If i happened to be 0, we'll be re-writing the same value - * twice, but that doesn't matter. - */ - vb[i] = ((rand_val >> 16) % 255) + 1; + CONFIG_OPTS *opts; + uint8_t *vb; + uint32_t i, max_range, rand_val; + + opts = thread->wtperf->opts; + + /* + * Limit how much of the buffer we validate for length, this means that only threads that do + * growing updates will ever make changes to values outside of the initial value size, but + * that's a fair trade off for avoiding figuring out how long the value is more accurately in + * this performance sensitive function. + */ + if (thread->workload == NULL || thread->workload->update_delta == 0) + max_range = opts->value_sz; + else if (thread->workload->update_delta > 0) + max_range = opts->value_sz_max; + else + max_range = opts->value_sz_min; + + /* + * Generate a single random value and re-use it. We generally only have small ranges in this + * function, so avoiding a bunch of calls is worthwhile. + */ + rand_val = __wt_random(&thread->rnd); + i = rand_val % (max_range - 1); + + /* + * Ensure we don't write past the end of a value when configured for randomly sized values. + */ + while (value_buf[i] == '\0' && i > 0) + --i; + + vb = (uint8_t *)value_buf; + vb[0] = ((rand_val >> 8) % 255) + 1; + /* + * If i happened to be 0, we'll be re-writing the same value twice, but that doesn't matter. + */ + vb[i] = ((rand_val >> 16) % 255) + 1; } /* @@ -120,367 +116,353 @@ randomize_value(WTPERF_THREAD *thread, char *value_buf) static uint32_t map_key_to_table(CONFIG_OPTS *opts, uint64_t k) { - /* - * The first part of the key range is reserved for dedicated - * scan tables, if any. The scan tables do not grow, but the - * rest of the key space may. - */ - if (k < opts->scan_icount) - return ((uint32_t) - (opts->table_count + k % opts->scan_table_count)); - k -= opts->scan_icount; - if (opts->range_partition) { - /* Take care to return a result in [0..table_count-1]. */ - if (k > opts->icount + opts->random_range) - return (0); - return ((uint32_t)((k - 1) / - ((opts->icount + opts->random_range + - opts->table_count - 1) / opts->table_count))); - } else - return ((uint32_t)(k % opts->table_count)); + /* + * The first part of the key range is reserved for dedicated scan tables, if any. The scan + * tables do not grow, but the rest of the key space may. + */ + if (k < opts->scan_icount) + return ((uint32_t)(opts->table_count + k % opts->scan_table_count)); + k -= opts->scan_icount; + if (opts->range_partition) { + /* Take care to return a result in [0..table_count-1]. */ + if (k > opts->icount + opts->random_range) + return (0); + return ((uint32_t)((k - 1) / + ((opts->icount + opts->random_range + opts->table_count - 1) / opts->table_count))); + } else + return ((uint32_t)(k % opts->table_count)); } /* - * Figure out and extend the size of the value string, used for growing - * updates. We know that the value to be updated is in the threads value - * scratch buffer. + * Figure out and extend the size of the value string, used for growing updates. We know that the + * value to be updated is in the threads value scratch buffer. */ static inline void update_value_delta(WTPERF_THREAD *thread) { - CONFIG_OPTS *opts; - WTPERF *wtperf; - char * value; - int64_t delta, len, new_len; - - wtperf = thread->wtperf; - opts = wtperf->opts; - value = thread->value_buf; - delta = thread->workload->update_delta; - len = (int64_t)strlen(value); - - if (delta == INT64_MAX) - delta = __wt_random(&thread->rnd) % - (opts->value_sz_max - opts->value_sz); - - /* Ensure we aren't changing across boundaries */ - if (delta > 0 && len + delta > opts->value_sz_max) - delta = opts->value_sz_max - len; - else if (delta < 0 && len + delta < opts->value_sz_min) - delta = opts->value_sz_min - len; - - /* Bail if there isn't anything to do */ - if (delta == 0) - return; - - if (delta < 0) - value[len + delta] = '\0'; - else { - /* Extend the value by the configured amount. */ - for (new_len = len; - new_len < opts->value_sz_max && new_len - len < delta; - new_len++) - value[new_len] = 'a'; - } + CONFIG_OPTS *opts; + WTPERF *wtperf; + char *value; + int64_t delta, len, new_len; + + wtperf = thread->wtperf; + opts = wtperf->opts; + value = thread->value_buf; + delta = thread->workload->update_delta; + len = (int64_t)strlen(value); + + if (delta == INT64_MAX) + delta = __wt_random(&thread->rnd) % (opts->value_sz_max - opts->value_sz); + + /* Ensure we aren't changing across boundaries */ + if (delta > 0 && len + delta > opts->value_sz_max) + delta = opts->value_sz_max - len; + else if (delta < 0 && len + delta < opts->value_sz_min) + delta = opts->value_sz_min - len; + + /* Bail if there isn't anything to do */ + if (delta == 0) + return; + + if (delta < 0) + value[len + delta] = '\0'; + else { + /* Extend the value by the configured amount. */ + for (new_len = len; new_len < opts->value_sz_max && new_len - len < delta; new_len++) + value[new_len] = 'a'; + } } static int cb_asyncop(WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP *op, int ret, uint32_t flags) { - TRACK *trk; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_ASYNC_OPTYPE type; - uint32_t *tables; - int t_ret; - char *value; - - (void)cb; - (void)flags; - - wtperf = NULL; /* -Wconditional-uninitialized */ - thread = NULL; /* -Wconditional-uninitialized */ - - type = op->get_type(op); - if (type != WT_AOP_COMPACT) { - thread = (WTPERF_THREAD *)op->app_private; - wtperf = thread->wtperf; - } - - trk = NULL; - switch (type) { - case WT_AOP_COMPACT: - tables = (uint32_t *)op->app_private; - (void)__wt_atomic_add32(tables, (uint32_t)-1); - break; - case WT_AOP_INSERT: - trk = &thread->insert; - break; - case WT_AOP_SEARCH: - trk = &thread->read; - if (ret == 0 && - (t_ret = op->get_value(op, &value)) != 0) { - ret = t_ret; - lprintf(wtperf, ret, 0, "get_value in read."); - goto err; - } - break; - case WT_AOP_UPDATE: - trk = &thread->update; - break; - case WT_AOP_NONE: - case WT_AOP_REMOVE: - /* We never expect this type. */ - lprintf(wtperf, - ret, 0, "No type in op %" PRIu64, op->get_id(op)); - goto err; - } - - /* - * Either we have success and we track it, or failure and panic. - * - * Reads and updates can fail with WT_NOTFOUND: we may be searching - * in a random range, or an insert op might have updated the - * last record in the table but not yet finished the actual insert. - */ - if (type == WT_AOP_COMPACT) - return (0); - if (ret == 0 || (ret == WT_NOTFOUND && type != WT_AOP_INSERT)) { - if (!wtperf->in_warmup) - (void)__wt_atomic_add64(&trk->ops, 1); - return (0); - } + TRACK *trk; + WTPERF *wtperf; + WTPERF_THREAD *thread; + WT_ASYNC_OPTYPE type; + uint32_t *tables; + int t_ret; + char *value; + + (void)cb; + (void)flags; + + wtperf = NULL; /* -Wconditional-uninitialized */ + thread = NULL; /* -Wconditional-uninitialized */ + + type = op->get_type(op); + if (type != WT_AOP_COMPACT) { + thread = (WTPERF_THREAD *)op->app_private; + wtperf = thread->wtperf; + } + + trk = NULL; + switch (type) { + case WT_AOP_COMPACT: + tables = (uint32_t *)op->app_private; + (void)__wt_atomic_add32(tables, (uint32_t)-1); + break; + case WT_AOP_INSERT: + trk = &thread->insert; + break; + case WT_AOP_SEARCH: + trk = &thread->read; + if (ret == 0 && (t_ret = op->get_value(op, &value)) != 0) { + ret = t_ret; + lprintf(wtperf, ret, 0, "get_value in read."); + goto err; + } + break; + case WT_AOP_UPDATE: + trk = &thread->update; + break; + case WT_AOP_NONE: + case WT_AOP_REMOVE: + /* We never expect this type. */ + lprintf(wtperf, ret, 0, "No type in op %" PRIu64, op->get_id(op)); + goto err; + } + + /* + * Either we have success and we track it, or failure and panic. + * + * Reads and updates can fail with WT_NOTFOUND: we may be searching + * in a random range, or an insert op might have updated the + * last record in the table but not yet finished the actual insert. + */ + if (type == WT_AOP_COMPACT) + return (0); + if (ret == 0 || (ret == WT_NOTFOUND && type != WT_AOP_INSERT)) { + if (!wtperf->in_warmup) + (void)__wt_atomic_add64(&trk->ops, 1); + return (0); + } err: - /* Panic if error */ - lprintf(wtperf, ret, 0, "Error in op %" PRIu64, op->get_id(op)); - wtperf->error = wtperf->stop = true; - return (1); + /* Panic if error */ + lprintf(wtperf, ret, 0, "Error in op %" PRIu64, op->get_id(op)); + wtperf->error = wtperf->stop = true; + return (1); } -static WT_ASYNC_CALLBACK cb = { cb_asyncop }; +static WT_ASYNC_CALLBACK cb = {cb_asyncop}; /* * track_operation -- - * Update an operation's tracking structure with new latency information. + * Update an operation's tracking structure with new latency information. */ static inline void track_operation(TRACK *trk, uint64_t usecs) { - uint64_t v; - - /* average microseconds per call */ - v = (uint64_t)usecs; - - trk->latency += usecs; /* track total latency */ - - if (v > trk->max_latency) /* track max/min latency */ - trk->max_latency = (uint32_t)v; - if (v < trk->min_latency) - trk->min_latency = (uint32_t)v; - - /* - * Update a latency bucket. - * First buckets: usecs from 100us to 1000us at 100us each. - */ - if (v < 1000) - ++trk->us[v]; - - /* - * Second buckets: milliseconds from 1ms to 1000ms, at 1ms each. - */ - else if (v < ms_to_us(1000)) - ++trk->ms[us_to_ms(v)]; - - /* - * Third buckets are seconds from 1s to 100s, at 1s each. - */ - else if (v < sec_to_us(100)) - ++trk->sec[us_to_sec(v)]; - - /* >100 seconds, accumulate in the biggest bucket. */ - else - ++trk->sec[ELEMENTS(trk->sec) - 1]; + uint64_t v; + + /* average microseconds per call */ + v = (uint64_t)usecs; + + trk->latency += usecs; /* track total latency */ + + if (v > trk->max_latency) /* track max/min latency */ + trk->max_latency = (uint32_t)v; + if (v < trk->min_latency) + trk->min_latency = (uint32_t)v; + + /* + * Update a latency bucket. First buckets: usecs from 100us to 1000us at 100us each. + */ + if (v < 1000) + ++trk->us[v]; + + /* + * Second buckets: milliseconds from 1ms to 1000ms, at 1ms each. + */ + else if (v < ms_to_us(1000)) + ++trk->ms[us_to_ms(v)]; + + /* + * Third buckets are seconds from 1s to 100s, at 1s each. + */ + else if (v < sec_to_us(100)) + ++trk->sec[us_to_sec(v)]; + + /* >100 seconds, accumulate in the biggest bucket. */ + else + ++trk->sec[ELEMENTS(trk->sec) - 1]; } static const char * op_name(uint8_t *op) { - switch (*op) { - case WORKER_INSERT: - return ("insert"); - case WORKER_INSERT_RMW: - return ("insert_rmw"); - case WORKER_READ: - return ("read"); - case WORKER_TRUNCATE: - return ("truncate"); - case WORKER_UPDATE: - return ("update"); - default: - return ("unknown"); - } - /* NOTREACHED */ + switch (*op) { + case WORKER_INSERT: + return ("insert"); + case WORKER_INSERT_RMW: + return ("insert_rmw"); + case WORKER_READ: + return ("read"); + case WORKER_TRUNCATE: + return ("truncate"); + case WORKER_UPDATE: + return ("update"); + default: + return ("unknown"); + } + /* NOTREACHED */ } static WT_THREAD_RET worker_async(void *arg) { - CONFIG_OPTS *opts; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_ASYNC_OP *asyncop; - WT_CONNECTION *conn; - uint64_t next_val; - uint8_t *op, *op_end; - int ret; - char *key_buf, *value_buf; - - thread = (WTPERF_THREAD *)arg; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - - key_buf = thread->key_buf; - value_buf = thread->value_buf; - - op = thread->workload->ops; - op_end = op + sizeof(thread->workload->ops); - - while (!wtperf->stop) { - /* - * Generate the next key and setup operation specific - * statistics tracking objects. - */ - switch (*op) { - case WORKER_INSERT: - case WORKER_INSERT_RMW: - if (opts->random_range) - next_val = wtperf_rand(thread); - else - next_val = opts->icount + get_next_incr(wtperf); - break; - case WORKER_READ: - case WORKER_UPDATE: - next_val = wtperf_rand(thread); - - /* - * If the workload is started without a populate phase - * we rely on at least one insert to get a valid item - * id. - */ - if (wtperf_value_range(wtperf) < next_val) - continue; - break; - default: - lprintf(wtperf, 0, 0, "invalid op!"); - goto err; /* can't happen */ - } - - generate_key(opts, key_buf, next_val); - - /* - * Spread the data out around the multiple databases. - * Sleep to allow workers a chance to run and process async ops. - * Then retry to get an async op. - */ - while ((ret = conn->async_new_op(conn, - wtperf->uris[map_key_to_table(wtperf->opts, next_val)], - NULL, &cb, &asyncop)) == EBUSY) - (void)usleep(10000); - if (ret != 0) { - lprintf(wtperf, ret, 0, "failed async_new_op"); - goto err; - } - - asyncop->app_private = thread; - asyncop->set_key(asyncop, key_buf); - switch (*op) { - case WORKER_READ: - ret = asyncop->search(asyncop); - if (ret == 0) - break; - goto op_err; - case WORKER_INSERT: - if (opts->random_value) - randomize_value(thread, value_buf); - asyncop->set_value(asyncop, value_buf); - if ((ret = asyncop->insert(asyncop)) == 0) - break; - goto op_err; - case WORKER_UPDATE: - if (opts->random_value) - randomize_value(thread, value_buf); - asyncop->set_value(asyncop, value_buf); - if ((ret = asyncop->update(asyncop)) == 0) - break; - goto op_err; - default: -op_err: lprintf(wtperf, ret, 0, - "%s failed for: %s, range: %"PRIu64, - op_name(op), key_buf, wtperf_value_range(wtperf)); - goto err; /* can't happen */ - } - - /* Schedule the next operation */ - if (++op == op_end) - op = thread->workload->ops; - } - - if (conn->async_flush(conn) != 0) - goto err; - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: wtperf->error = wtperf->stop = true; - } - return (WT_THREAD_RET_VALUE); + CONFIG_OPTS *opts; + WTPERF *wtperf; + WTPERF_THREAD *thread; + WT_ASYNC_OP *asyncop; + WT_CONNECTION *conn; + uint64_t next_val; + uint8_t *op, *op_end; + int ret; + char *key_buf, *value_buf; + + thread = (WTPERF_THREAD *)arg; + wtperf = thread->wtperf; + opts = wtperf->opts; + conn = wtperf->conn; + + key_buf = thread->key_buf; + value_buf = thread->value_buf; + + op = thread->workload->ops; + op_end = op + sizeof(thread->workload->ops); + + while (!wtperf->stop) { + /* + * Generate the next key and setup operation specific statistics tracking objects. + */ + switch (*op) { + case WORKER_INSERT: + case WORKER_INSERT_RMW: + if (opts->random_range) + next_val = wtperf_rand(thread); + else + next_val = opts->icount + get_next_incr(wtperf); + break; + case WORKER_READ: + case WORKER_UPDATE: + next_val = wtperf_rand(thread); + + /* + * If the workload is started without a populate phase we rely on at least one insert to + * get a valid item id. + */ + if (wtperf_value_range(wtperf) < next_val) + continue; + break; + default: + lprintf(wtperf, 0, 0, "invalid op!"); + goto err; /* can't happen */ + } + + generate_key(opts, key_buf, next_val); + + /* + * Spread the data out around the multiple databases. Sleep to allow workers a chance to run + * and process async ops. Then retry to get an async op. + */ + while ( + (ret = conn->async_new_op(conn, wtperf->uris[map_key_to_table(wtperf->opts, next_val)], + NULL, &cb, &asyncop)) == EBUSY) + (void)usleep(10000); + if (ret != 0) { + lprintf(wtperf, ret, 0, "failed async_new_op"); + goto err; + } + + asyncop->app_private = thread; + asyncop->set_key(asyncop, key_buf); + switch (*op) { + case WORKER_READ: + ret = asyncop->search(asyncop); + if (ret == 0) + break; + goto op_err; + case WORKER_INSERT: + if (opts->random_value) + randomize_value(thread, value_buf); + asyncop->set_value(asyncop, value_buf); + if ((ret = asyncop->insert(asyncop)) == 0) + break; + goto op_err; + case WORKER_UPDATE: + if (opts->random_value) + randomize_value(thread, value_buf); + asyncop->set_value(asyncop, value_buf); + if ((ret = asyncop->update(asyncop)) == 0) + break; + goto op_err; + default: + op_err: + lprintf(wtperf, ret, 0, "%s failed for: %s, range: %" PRIu64, op_name(op), key_buf, + wtperf_value_range(wtperf)); + goto err; /* can't happen */ + } + + /* Schedule the next operation */ + if (++op == op_end) + op = thread->workload->ops; + } + + if (conn->async_flush(conn) != 0) + goto err; + + /* Notify our caller we failed and shut the system down. */ + if (0) { +err: + wtperf->error = wtperf->stop = true; + } + return (WT_THREAD_RET_VALUE); } /* * do_range_reads -- - * If configured to execute a sequence of next operations after each - * search do them. Ensuring the keys we see are always in order. + * If configured to execute a sequence of next operations after each search do them. Ensuring + * the keys we see are always in order. */ static int do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor, int64_t read_range) { - uint64_t next_val, prev_val; - int64_t range; - char *range_key_buf; - char buf[512]; - int ret; - - ret = 0; - - if (read_range == 0) - return (0); - - memset(&buf[0], 0, 512 * sizeof(char)); - range_key_buf = &buf[0]; - - /* Save where the first key is for comparisons. */ - testutil_check(cursor->get_key(cursor, &range_key_buf)); - extract_key(range_key_buf, &next_val); - - for (range = 0; range < read_range; ++range) { - prev_val = next_val; - ret = cursor->next(cursor); - /* We are done if we reach the end. */ - if (ret != 0) - break; - - /* Retrieve and decode the key */ - testutil_check(cursor->get_key(cursor, &range_key_buf)); - extract_key(range_key_buf, &next_val); - if (next_val < prev_val) { - lprintf(wtperf, EINVAL, 0, - "Out of order keys %" PRIu64 - " came before %" PRIu64, - prev_val, next_val); - return (EINVAL); - } - } - return (0); + uint64_t next_val, prev_val; + int64_t range; + char *range_key_buf; + char buf[512]; + int ret; + + ret = 0; + + if (read_range == 0) + return (0); + + memset(&buf[0], 0, 512 * sizeof(char)); + range_key_buf = &buf[0]; + + /* Save where the first key is for comparisons. */ + testutil_check(cursor->get_key(cursor, &range_key_buf)); + extract_key(range_key_buf, &next_val); + + for (range = 0; range < read_range; ++range) { + prev_val = next_val; + ret = cursor->next(cursor); + /* We are done if we reach the end. */ + if (ret != 0) + break; + + /* Retrieve and decode the key */ + testutil_check(cursor->get_key(cursor, &range_key_buf)); + extract_key(range_key_buf, &next_val); + if (next_val < prev_val) { + lprintf(wtperf, EINVAL, 0, "Out of order keys %" PRIu64 " came before %" PRIu64, + prev_val, next_val); + return (EINVAL); + } + } + return (0); } /* pre_load_data -- @@ -489,1631 +471,1465 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor, int64_t read_range) static void pre_load_data(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - size_t i; - int ret; - char *key; - - opts = wtperf->opts; - conn = wtperf->conn; - - testutil_check(conn->open_session( - conn, NULL, opts->sess_config, &session)); - for (i = 0; i < opts->table_count; i++) { - testutil_check(session->open_cursor( - session, wtperf->uris[i], NULL, NULL, &cursor)); - while ((ret = cursor->next(cursor)) == 0) - testutil_check(cursor->get_key(cursor, &key)); - testutil_assert(ret == WT_NOTFOUND); - testutil_check(cursor->close(cursor)); - } - testutil_check(session->close(session, NULL)); + CONFIG_OPTS *opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + size_t i; + int ret; + char *key; + + opts = wtperf->opts; + conn = wtperf->conn; + + testutil_check(conn->open_session(conn, NULL, opts->sess_config, &session)); + for (i = 0; i < opts->table_count; i++) { + testutil_check(session->open_cursor(session, wtperf->uris[i], NULL, NULL, &cursor)); + while ((ret = cursor->next(cursor)) == 0) + testutil_check(cursor->get_key(cursor, &key)); + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + } + testutil_check(session->close(session, NULL)); } static WT_THREAD_RET worker(void *arg) { - struct timespec start, stop; - CONFIG_OPTS *opts; - TRACK *trk; - WORKLOAD *workload; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_CONNECTION *conn; - WT_CURSOR **cursors, *cursor, *log_table_cursor, *tmp_cursor; - WT_SESSION *session; - size_t i; - uint32_t total_table_count; - int64_t ops, ops_per_txn; - uint64_t log_id, next_val, usecs; - uint8_t *op, *op_end; - int measure_latency, ret, truncated; - char *value_buf, *key_buf, *value; - char buf[512]; - - thread = (WTPERF_THREAD *)arg; - workload = thread->workload; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - cursors = NULL; - cursor = log_table_cursor = NULL; /* -Wconditional-initialized */ - ops = 0; - ops_per_txn = workload->ops_per_txn; - session = NULL; - trk = NULL; - - if ((ret = conn->open_session( - conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); - goto err; - } - for (i = 0; i < opts->table_count_idle; i++) { - testutil_check(__wt_snprintf( - buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); - if ((ret = session->open_cursor( - session, buf, NULL, NULL, &tmp_cursor)) != 0) { - lprintf(wtperf, ret, 0, - "Error opening idle table %s", buf); - goto err; - } - if ((ret = tmp_cursor->close(tmp_cursor)) != 0) { - lprintf(wtperf, ret, 0, - "Error closing idle table %s", buf); - goto err; - } - } - if (workload->table_index != INT32_MAX) { - if ((ret = session->open_cursor(session, - wtperf->uris[workload->table_index], - NULL, NULL, &cursor)) != 0) { - lprintf(wtperf, ret, 0, - "worker: WT_SESSION.open_cursor: %s", - wtperf->uris[workload->table_index]); - goto err; - } - if ((ret = session->open_cursor(session, - wtperf->uris[workload->table_index], - NULL, "next_random=true", &thread->rand_cursor)) != 0) { - lprintf(wtperf, ret, 0, - "worker: WT_SESSION.open_cursor: random %s", - wtperf->uris[workload->table_index]); - goto err; - } - } else { - total_table_count = opts->table_count + opts->scan_table_count; - cursors = dcalloc(total_table_count, sizeof(WT_CURSOR *)); - for (i = 0; i < total_table_count; i++) { - if ((ret = session->open_cursor(session, - wtperf->uris[i], NULL, NULL, &cursors[i])) != 0) { - lprintf(wtperf, ret, 0, - "worker: WT_SESSION.open_cursor: %s", - wtperf->uris[i]); - goto err; - } - } - } - if (opts->log_like_table && (ret = session->open_cursor(session, - wtperf->log_table_uri, NULL, NULL, &log_table_cursor)) != 0) { - lprintf(wtperf, ret, 0, - "worker: WT_SESSION.open_cursor: %s", - wtperf->log_table_uri); - goto err; - } - - /* Setup the timer for throttling. */ - if (workload->throttle != 0) - setup_throttle(thread); - - /* Setup for truncate */ - if (workload->truncate != 0) - setup_truncate(wtperf, thread, session); - - key_buf = thread->key_buf; - value_buf = thread->value_buf; - - op = workload->ops; - op_end = op + sizeof(workload->ops); - - if ((ops_per_txn != 0 || opts->log_like_table) && - (ret = session->begin_transaction(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "First transaction begin failed"); - goto err; - } - - while (!wtperf->stop) { - if (workload->pause != 0) - (void)sleep((unsigned int)workload->pause); - /* - * Generate the next key and setup operation specific - * statistics tracking objects. - */ - switch (*op) { - case WORKER_INSERT: - case WORKER_INSERT_RMW: - trk = &thread->insert; - if (opts->random_range) - next_val = wtperf_rand(thread); - else - next_val = opts->icount + get_next_incr(wtperf); - break; - case WORKER_READ: - trk = &thread->read; - /* FALLTHROUGH */ - case WORKER_UPDATE: - if (*op == WORKER_UPDATE) - trk = &thread->update; - next_val = wtperf_rand(thread); - - /* - * If the workload is started without a populate phase - * we rely on at least one insert to get a valid item - * id. - */ - if (wtperf_value_range(wtperf) < next_val) - continue; - break; - case WORKER_TRUNCATE: - /* Required but not used. */ - next_val = wtperf_rand(thread); - break; - default: - goto err; /* can't happen */ - } - - generate_key(opts, key_buf, next_val); - - if (workload->table_index == INT32_MAX) - /* - * Spread the data out around the multiple databases. - */ - cursor = cursors[ - map_key_to_table(wtperf->opts, next_val)]; - - /* - * Skip the first time we do an operation, when trk->ops - * is 0, to avoid first time latency spikes. - */ - measure_latency = - opts->sample_interval != 0 && trk != NULL && - trk->ops != 0 && (trk->ops % opts->sample_rate == 0); - if (measure_latency) - __wt_epoch(NULL, &start); - - cursor->set_key(cursor, key_buf); - switch (*op) { - case WORKER_READ: - /* - * Reads can fail with WT_NOTFOUND: we may be searching - * in a random range, or an insert thread might have - * updated the last record in the table but not yet - * finished the actual insert. Count failed search in - * a random range as a "read". - */ - ret = cursor->search(cursor); - if (ret == 0) { - if ((ret = cursor->get_value( - cursor, &value)) != 0) { - lprintf(wtperf, ret, 0, - "get_value in read."); - goto err; - } - /* - * If we want to read a range, then call next - * for several operations, confirming that the - * next key is in the correct order. - */ - ret = do_range_reads(wtperf, - cursor, workload->read_range); - } - - if (ret == 0 || ret == WT_NOTFOUND) - break; - goto op_err; - case WORKER_INSERT_RMW: - if ((ret = cursor->search(cursor)) != WT_NOTFOUND) - goto op_err; - - /* The error return reset the cursor's key. */ - cursor->set_key(cursor, key_buf); - - /* FALLTHROUGH */ - case WORKER_INSERT: - if (opts->random_value) - randomize_value(thread, value_buf); - cursor->set_value(cursor, value_buf); - if ((ret = cursor->insert(cursor)) == 0) - break; - goto op_err; - case WORKER_TRUNCATE: - if ((ret = run_truncate(wtperf, - thread, cursor, session, &truncated)) == 0) { - if (truncated) - trk = &thread->truncate; - else - trk = &thread->truncate_sleep; - /* Pause between truncate attempts */ - (void)usleep(1000); - break; - } - goto op_err; - case WORKER_UPDATE: - if ((ret = cursor->search(cursor)) == 0) { - if ((ret = cursor->get_value( - cursor, &value)) != 0) { - lprintf(wtperf, ret, 0, - "get_value in update."); - goto err; - } - /* - * Copy as much of the previous value as is - * safe, and be sure to NUL-terminate. - */ - strncpy(value_buf, - value, opts->value_sz_max - 1); - if (workload->update_delta != 0) - update_value_delta(thread); - if (value_buf[0] == 'a') - value_buf[0] = 'b'; - else - value_buf[0] = 'a'; - if (opts->random_value) - randomize_value(thread, value_buf); - cursor->set_value(cursor, value_buf); - if ((ret = cursor->update(cursor)) == 0) - break; - goto op_err; - } - - /* - * Reads can fail with WT_NOTFOUND: we may be searching - * in a random range, or an insert thread might have - * updated the last record in the table but not yet - * finished the actual insert. Count failed search in - * a random range as a "read". - */ - if (ret == WT_NOTFOUND) - break; - -op_err: if (ret == WT_ROLLBACK && - (ops_per_txn != 0 || opts->log_like_table)) { - /* - * If we are running with explicit transactions - * configured and we hit a WT_ROLLBACK, then we - * should rollback the current transaction and - * attempt to continue. - * This does break the guarantee of insertion - * order in cases of ordered inserts, as we - * aren't retrying here. - */ - lprintf(wtperf, ret, 1, - "%s for: %s, range: %"PRIu64, op_name(op), - key_buf, wtperf_value_range(wtperf)); - if ((ret = session->rollback_transaction( - session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Failed rollback_transaction"); - goto err; - } - if ((ret = session->begin_transaction( - session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Worker begin transaction failed"); - goto err; - } - break; - } - lprintf(wtperf, ret, 0, - "%s failed for: %s, range: %"PRIu64, - op_name(op), key_buf, wtperf_value_range(wtperf)); - goto err; - default: - goto err; /* can't happen */ - } - - /* Update the log-like table. */ - if (opts->log_like_table && - (*op != WORKER_READ && *op != WORKER_TRUNCATE)) { - log_id = - __wt_atomic_add64(&wtperf->log_like_table_key, 1); - log_table_cursor->set_key(log_table_cursor, log_id); - log_table_cursor->set_value( - log_table_cursor, value_buf); - if ((ret = - log_table_cursor->insert(log_table_cursor)) != 0) { - lprintf(wtperf, ret, 1, "Cursor insert failed"); - if (ret == WT_ROLLBACK && ops_per_txn == 0) { - lprintf(wtperf, ret, 1, - "log-table: ROLLBACK"); - if ((ret = - session->rollback_transaction( - session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Failed" - " rollback_transaction"); - goto err; - } - if ((ret = session->begin_transaction( - session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Worker begin " - "transaction failed"); - goto err; - } - } else - goto err; - } - } - - /* Release the cursor, if we have multiple tables. */ - if (opts->table_count > 1 && ret == 0 && - *op != WORKER_INSERT && *op != WORKER_INSERT_RMW) { - if ((ret = cursor->reset(cursor)) != 0) { - lprintf(wtperf, ret, 0, "Cursor reset failed"); - goto err; - } - } - - /* Gather statistics */ - if (!wtperf->in_warmup) { - if (measure_latency) { - __wt_epoch(NULL, &stop); - ++trk->latency_ops; - usecs = WT_TIMEDIFF_US(stop, start); - track_operation(trk, usecs); - } - /* Increment operation count */ - ++trk->ops; - } - - /* - * Commit the transaction if grouping operations together - * or tracking changes in our log table. - */ - if ((opts->log_like_table && ops_per_txn == 0) || - (ops_per_txn != 0 && ops++ % ops_per_txn == 0)) { - if ((ret = session->commit_transaction( - session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Worker transaction commit failed"); - goto err; - } - if ((ret = session->begin_transaction( - session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Worker begin transaction failed"); - goto err; - } - } - - /* Schedule the next operation */ - if (++op == op_end) - op = workload->ops; - - /* - * Decrement throttle ops and check if we should sleep - * and then get more work to perform. - */ - if (--thread->throttle_cfg.ops_count == 0) - worker_throttle(thread); - - } - - if ((ret = session->close(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Session close in worker failed"); - goto err; - } - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: wtperf->error = wtperf->stop = true; - } - free(cursors); - - return (WT_THREAD_RET_VALUE); + struct timespec start, stop; + CONFIG_OPTS *opts; + TRACK *trk; + WORKLOAD *workload; + WTPERF *wtperf; + WTPERF_THREAD *thread; + WT_CONNECTION *conn; + WT_CURSOR **cursors, *cursor, *log_table_cursor, *tmp_cursor; + WT_SESSION *session; + size_t i; + uint32_t total_table_count; + int64_t ops, ops_per_txn; + uint64_t log_id, next_val, usecs; + uint8_t *op, *op_end; + int measure_latency, ret, truncated; + char *value_buf, *key_buf, *value; + char buf[512]; + + thread = (WTPERF_THREAD *)arg; + workload = thread->workload; + wtperf = thread->wtperf; + opts = wtperf->opts; + conn = wtperf->conn; + cursors = NULL; + cursor = log_table_cursor = NULL; /* -Wconditional-initialized */ + ops = 0; + ops_per_txn = workload->ops_per_txn; + session = NULL; + trk = NULL; + + if ((ret = conn->open_session(conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); + goto err; + } + for (i = 0; i < opts->table_count_idle; i++) { + testutil_check(__wt_snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); + if ((ret = session->open_cursor(session, buf, NULL, NULL, &tmp_cursor)) != 0) { + lprintf(wtperf, ret, 0, "Error opening idle table %s", buf); + goto err; + } + if ((ret = tmp_cursor->close(tmp_cursor)) != 0) { + lprintf(wtperf, ret, 0, "Error closing idle table %s", buf); + goto err; + } + } + if (workload->table_index != INT32_MAX) { + if ((ret = session->open_cursor( + session, wtperf->uris[workload->table_index], NULL, NULL, &cursor)) != 0) { + lprintf(wtperf, ret, 0, "worker: WT_SESSION.open_cursor: %s", + wtperf->uris[workload->table_index]); + goto err; + } + if ((ret = session->open_cursor(session, wtperf->uris[workload->table_index], NULL, + "next_random=true", &thread->rand_cursor)) != 0) { + lprintf(wtperf, ret, 0, "worker: WT_SESSION.open_cursor: random %s", + wtperf->uris[workload->table_index]); + goto err; + } + } else { + total_table_count = opts->table_count + opts->scan_table_count; + cursors = dcalloc(total_table_count, sizeof(WT_CURSOR *)); + for (i = 0; i < total_table_count; i++) { + if ((ret = session->open_cursor(session, wtperf->uris[i], NULL, NULL, &cursors[i])) != + 0) { + lprintf(wtperf, ret, 0, "worker: WT_SESSION.open_cursor: %s", wtperf->uris[i]); + goto err; + } + } + } + if (opts->log_like_table && + (ret = session->open_cursor(session, wtperf->log_table_uri, NULL, NULL, &log_table_cursor)) != + 0) { + lprintf(wtperf, ret, 0, "worker: WT_SESSION.open_cursor: %s", wtperf->log_table_uri); + goto err; + } + + /* Setup the timer for throttling. */ + if (workload->throttle != 0) + setup_throttle(thread); + + /* Setup for truncate */ + if (workload->truncate != 0) + setup_truncate(wtperf, thread, session); + + key_buf = thread->key_buf; + value_buf = thread->value_buf; + + op = workload->ops; + op_end = op + sizeof(workload->ops); + + if ((ops_per_txn != 0 || opts->log_like_table) && + (ret = session->begin_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "First transaction begin failed"); + goto err; + } + + while (!wtperf->stop) { + if (workload->pause != 0) + (void)sleep((unsigned int)workload->pause); + /* + * Generate the next key and setup operation specific statistics tracking objects. + */ + switch (*op) { + case WORKER_INSERT: + case WORKER_INSERT_RMW: + trk = &thread->insert; + if (opts->random_range) + next_val = wtperf_rand(thread); + else + next_val = opts->icount + get_next_incr(wtperf); + break; + case WORKER_READ: + trk = &thread->read; + /* FALLTHROUGH */ + case WORKER_UPDATE: + if (*op == WORKER_UPDATE) + trk = &thread->update; + next_val = wtperf_rand(thread); + + /* + * If the workload is started without a populate phase we rely on at least one insert to + * get a valid item id. + */ + if (wtperf_value_range(wtperf) < next_val) + continue; + break; + case WORKER_TRUNCATE: + /* Required but not used. */ + next_val = wtperf_rand(thread); + break; + default: + goto err; /* can't happen */ + } + + generate_key(opts, key_buf, next_val); + + if (workload->table_index == INT32_MAX) + /* + * Spread the data out around the multiple databases. + */ + cursor = cursors[map_key_to_table(wtperf->opts, next_val)]; + + /* + * Skip the first time we do an operation, when trk->ops is 0, to avoid first time latency + * spikes. + */ + measure_latency = opts->sample_interval != 0 && trk != NULL && trk->ops != 0 && + (trk->ops % opts->sample_rate == 0); + if (measure_latency) + __wt_epoch(NULL, &start); + + cursor->set_key(cursor, key_buf); + switch (*op) { + case WORKER_READ: + /* + * Reads can fail with WT_NOTFOUND: we may be searching in a random range, or an insert + * thread might have updated the last record in the table but not yet finished the + * actual insert. Count failed search in a random range as a "read". + */ + ret = cursor->search(cursor); + if (ret == 0) { + if ((ret = cursor->get_value(cursor, &value)) != 0) { + lprintf(wtperf, ret, 0, "get_value in read."); + goto err; + } + /* + * If we want to read a range, then call next for several operations, confirming + * that the next key is in the correct order. + */ + ret = do_range_reads(wtperf, cursor, workload->read_range); + } + + if (ret == 0 || ret == WT_NOTFOUND) + break; + goto op_err; + case WORKER_INSERT_RMW: + if ((ret = cursor->search(cursor)) != WT_NOTFOUND) + goto op_err; + + /* The error return reset the cursor's key. */ + cursor->set_key(cursor, key_buf); + + /* FALLTHROUGH */ + case WORKER_INSERT: + if (opts->random_value) + randomize_value(thread, value_buf); + cursor->set_value(cursor, value_buf); + if ((ret = cursor->insert(cursor)) == 0) + break; + goto op_err; + case WORKER_TRUNCATE: + if ((ret = run_truncate(wtperf, thread, cursor, session, &truncated)) == 0) { + if (truncated) + trk = &thread->truncate; + else + trk = &thread->truncate_sleep; + /* Pause between truncate attempts */ + (void)usleep(1000); + break; + } + goto op_err; + case WORKER_UPDATE: + if ((ret = cursor->search(cursor)) == 0) { + if ((ret = cursor->get_value(cursor, &value)) != 0) { + lprintf(wtperf, ret, 0, "get_value in update."); + goto err; + } + /* + * Copy as much of the previous value as is safe, and be sure to NUL-terminate. + */ + strncpy(value_buf, value, opts->value_sz_max - 1); + if (workload->update_delta != 0) + update_value_delta(thread); + if (value_buf[0] == 'a') + value_buf[0] = 'b'; + else + value_buf[0] = 'a'; + if (opts->random_value) + randomize_value(thread, value_buf); + cursor->set_value(cursor, value_buf); + if ((ret = cursor->update(cursor)) == 0) + break; + goto op_err; + } + + /* + * Reads can fail with WT_NOTFOUND: we may be searching in a random range, or an insert + * thread might have updated the last record in the table but not yet finished the + * actual insert. Count failed search in a random range as a "read". + */ + if (ret == WT_NOTFOUND) + break; + + op_err: + if (ret == WT_ROLLBACK && (ops_per_txn != 0 || opts->log_like_table)) { + /* + * If we are running with explicit transactions configured and we hit a WT_ROLLBACK, + * then we should rollback the current transaction and attempt to continue. This + * does break the guarantee of insertion order in cases of ordered inserts, as we + * aren't retrying here. + */ + lprintf(wtperf, ret, 1, "%s for: %s, range: %" PRIu64, op_name(op), key_buf, + wtperf_value_range(wtperf)); + if ((ret = session->rollback_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Failed rollback_transaction"); + goto err; + } + if ((ret = session->begin_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Worker begin transaction failed"); + goto err; + } + break; + } + lprintf(wtperf, ret, 0, "%s failed for: %s, range: %" PRIu64, op_name(op), key_buf, + wtperf_value_range(wtperf)); + goto err; + default: + goto err; /* can't happen */ + } + + /* Update the log-like table. */ + if (opts->log_like_table && (*op != WORKER_READ && *op != WORKER_TRUNCATE)) { + log_id = __wt_atomic_add64(&wtperf->log_like_table_key, 1); + log_table_cursor->set_key(log_table_cursor, log_id); + log_table_cursor->set_value(log_table_cursor, value_buf); + if ((ret = log_table_cursor->insert(log_table_cursor)) != 0) { + lprintf(wtperf, ret, 1, "Cursor insert failed"); + if (ret == WT_ROLLBACK && ops_per_txn == 0) { + lprintf(wtperf, ret, 1, "log-table: ROLLBACK"); + if ((ret = session->rollback_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, + "Failed" + " rollback_transaction"); + goto err; + } + if ((ret = session->begin_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, + "Worker begin " + "transaction failed"); + goto err; + } + } else + goto err; + } + } + + /* Release the cursor, if we have multiple tables. */ + if (opts->table_count > 1 && ret == 0 && *op != WORKER_INSERT && *op != WORKER_INSERT_RMW) { + if ((ret = cursor->reset(cursor)) != 0) { + lprintf(wtperf, ret, 0, "Cursor reset failed"); + goto err; + } + } + + /* Gather statistics */ + if (!wtperf->in_warmup) { + if (measure_latency) { + __wt_epoch(NULL, &stop); + ++trk->latency_ops; + usecs = WT_TIMEDIFF_US(stop, start); + track_operation(trk, usecs); + } + /* Increment operation count */ + ++trk->ops; + } + + /* + * Commit the transaction if grouping operations together or tracking changes in our log + * table. + */ + if ((opts->log_like_table && ops_per_txn == 0) || + (ops_per_txn != 0 && ops++ % ops_per_txn == 0)) { + if ((ret = session->commit_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Worker transaction commit failed"); + goto err; + } + if ((ret = session->begin_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Worker begin transaction failed"); + goto err; + } + } + + /* Schedule the next operation */ + if (++op == op_end) + op = workload->ops; + + /* + * Decrement throttle ops and check if we should sleep and then get more work to perform. + */ + if (--thread->throttle_cfg.ops_count == 0) + worker_throttle(thread); + } + + if ((ret = session->close(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Session close in worker failed"); + goto err; + } + + /* Notify our caller we failed and shut the system down. */ + if (0) { +err: + wtperf->error = wtperf->stop = true; + } + free(cursors); + + return (WT_THREAD_RET_VALUE); } /* * run_mix_schedule_op -- - * Replace read operations with another operation, in the configured - * percentage. + * Replace read operations with another operation, in the configured percentage. */ static void run_mix_schedule_op(WORKLOAD *workp, int op, int64_t op_cnt) { - int jump, pass; - uint8_t *p, *end; - - /* Jump around the array to roughly spread out the operations. */ - jump = (int)(100 / op_cnt); - - /* - * Find a read operation and replace it with another operation. This - * is roughly n-squared, but it's an N of 100, leave it. - */ - p = workp->ops; - end = workp->ops + sizeof(workp->ops); - while (op_cnt-- > 0) { - for (pass = 0; *p != WORKER_READ; ++p) - if (p == end) { - /* - * Passed a percentage of total operations and - * should always be a read operation to replace, - * but don't allow infinite loops. - */ - if (++pass > 1) - return; - p = workp->ops; - } - *p = (uint8_t)op; - - if (end - jump < p) - p = workp->ops; - else - p += jump; - } + int jump, pass; + uint8_t *p, *end; + + /* Jump around the array to roughly spread out the operations. */ + jump = (int)(100 / op_cnt); + + /* + * Find a read operation and replace it with another operation. This is roughly n-squared, but + * it's an N of 100, leave it. + */ + p = workp->ops; + end = workp->ops + sizeof(workp->ops); + while (op_cnt-- > 0) { + for (pass = 0; *p != WORKER_READ; ++p) + if (p == end) { + /* + * Passed a percentage of total operations and should always be a read operation to + * replace, but don't allow infinite loops. + */ + if (++pass > 1) + return; + p = workp->ops; + } + *p = (uint8_t)op; + + if (end - jump < p) + p = workp->ops; + else + p += jump; + } } /* * run_mix_schedule -- - * Schedule the mixed-run operations. + * Schedule the mixed-run operations. */ static int run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) { - CONFIG_OPTS *opts; - int64_t pct; - - opts = wtperf->opts; - - if (workp->truncate != 0) { - if (workp->insert != 0 || - workp->read != 0 || workp->update != 0) { - lprintf(wtperf, EINVAL, 0, - "Can't configure truncate in a mixed workload"); - return (EINVAL); - } - memset(workp->ops, WORKER_TRUNCATE, sizeof(workp->ops)); - return (0); - } - - /* Confirm reads, inserts and updates cannot all be zero. */ - if (workp->insert == 0 && workp->read == 0 && workp->update == 0) { - lprintf(wtperf, EINVAL, 0, "no operations scheduled"); - return (EINVAL); - } - - /* - * Check for a simple case where the thread is only doing insert or - * update operations (because the default operation for a - * job-mix is read, the subsequent code works fine if only reads are - * specified). - */ - if (workp->insert != 0 && workp->read == 0 && workp->update == 0) { - memset(workp->ops, - opts->insert_rmw ? WORKER_INSERT_RMW : WORKER_INSERT, - sizeof(workp->ops)); - return (0); - } - if (workp->insert == 0 && workp->read == 0 && workp->update != 0) { - memset(workp->ops, WORKER_UPDATE, sizeof(workp->ops)); - return (0); - } - - /* - * The worker thread configuration is done as ratios of operations. If - * the caller gives us something insane like "reads=77,updates=23" (do - * 77 reads for every 23 updates), we don't want to do 77 reads followed - * by 23 updates, we want to uniformly distribute the read and update - * operations across the space. Convert to percentages and then lay out - * the operations across an array. - * - * Percentage conversion is lossy, the application can do stupid stuff - * here, for example, imagine a configured ratio of "reads=1,inserts=2, - * updates=999999". First, if the percentages are skewed enough, some - * operations might never be done. Second, we set the base operation to - * read, which means any fractional results from percentage conversion - * will be reads, implying read operations in some cases where reads - * weren't configured. We should be fine if the application configures - * something approaching a rational set of ratios. - */ - memset(workp->ops, WORKER_READ, sizeof(workp->ops)); - - pct = (workp->insert * 100) / - (workp->insert + workp->read + workp->update); - if (pct != 0) - run_mix_schedule_op(workp, - opts->insert_rmw ? WORKER_INSERT_RMW : WORKER_INSERT, pct); - pct = (workp->update * 100) / - (workp->insert + workp->read + workp->update); - if (pct != 0) - run_mix_schedule_op(workp, WORKER_UPDATE, pct); - return (0); + CONFIG_OPTS *opts; + int64_t pct; + + opts = wtperf->opts; + + if (workp->truncate != 0) { + if (workp->insert != 0 || workp->read != 0 || workp->update != 0) { + lprintf(wtperf, EINVAL, 0, "Can't configure truncate in a mixed workload"); + return (EINVAL); + } + memset(workp->ops, WORKER_TRUNCATE, sizeof(workp->ops)); + return (0); + } + + /* Confirm reads, inserts and updates cannot all be zero. */ + if (workp->insert == 0 && workp->read == 0 && workp->update == 0) { + lprintf(wtperf, EINVAL, 0, "no operations scheduled"); + return (EINVAL); + } + + /* + * Check for a simple case where the thread is only doing insert or update operations (because + * the default operation for a job-mix is read, the subsequent code works fine if only reads are + * specified). + */ + if (workp->insert != 0 && workp->read == 0 && workp->update == 0) { + memset( + workp->ops, opts->insert_rmw ? WORKER_INSERT_RMW : WORKER_INSERT, sizeof(workp->ops)); + return (0); + } + if (workp->insert == 0 && workp->read == 0 && workp->update != 0) { + memset(workp->ops, WORKER_UPDATE, sizeof(workp->ops)); + return (0); + } + + /* + * The worker thread configuration is done as ratios of operations. If + * the caller gives us something insane like "reads=77,updates=23" (do + * 77 reads for every 23 updates), we don't want to do 77 reads followed + * by 23 updates, we want to uniformly distribute the read and update + * operations across the space. Convert to percentages and then lay out + * the operations across an array. + * + * Percentage conversion is lossy, the application can do stupid stuff + * here, for example, imagine a configured ratio of "reads=1,inserts=2, + * updates=999999". First, if the percentages are skewed enough, some + * operations might never be done. Second, we set the base operation to + * read, which means any fractional results from percentage conversion + * will be reads, implying read operations in some cases where reads + * weren't configured. We should be fine if the application configures + * something approaching a rational set of ratios. + */ + memset(workp->ops, WORKER_READ, sizeof(workp->ops)); + + pct = (workp->insert * 100) / (workp->insert + workp->read + workp->update); + if (pct != 0) + run_mix_schedule_op(workp, opts->insert_rmw ? WORKER_INSERT_RMW : WORKER_INSERT, pct); + pct = (workp->update * 100) / (workp->insert + workp->read + workp->update); + if (pct != 0) + run_mix_schedule_op(workp, WORKER_UPDATE, pct); + return (0); } static WT_THREAD_RET populate_thread(void *arg) { - struct timespec start, stop; - CONFIG_OPTS *opts; - TRACK *trk; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_CONNECTION *conn; - WT_CURSOR **cursors, *cursor; - WT_SESSION *session; - size_t i; - uint64_t op, usecs; - uint32_t opcount, total_table_count; - int intxn, measure_latency, ret, stress_checkpoint_due; - char *value_buf, *key_buf; - const char *cursor_config; - - thread = (WTPERF_THREAD *)arg; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - session = NULL; - cursors = NULL; - ret = stress_checkpoint_due = 0; - trk = &thread->insert; - total_table_count = opts->table_count + opts->scan_table_count; - - key_buf = thread->key_buf; - value_buf = thread->value_buf; - - if ((ret = conn->open_session( - conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, "populate: WT_CONNECTION.open_session"); - goto err; - } - - /* Do bulk loads if populate is single-threaded. */ - cursor_config = - (opts->populate_threads == 1 && !opts->index) ? "bulk" : NULL; - /* Create the cursors. */ - cursors = dcalloc(total_table_count, sizeof(WT_CURSOR *)); - for (i = 0; i < total_table_count; i++) { - if ((ret = session->open_cursor( - session, wtperf->uris[i], NULL, - cursor_config, &cursors[i])) != 0) { - lprintf(wtperf, ret, 0, - "populate: WT_SESSION.open_cursor: %s", - wtperf->uris[i]); - goto err; - } - } - - /* Populate the databases. */ - for (intxn = 0, opcount = 0;;) { - op = get_next_incr(wtperf); - if (op > (uint64_t)opts->icount + (uint64_t)opts->scan_icount) - break; - - if (opts->populate_ops_per_txn != 0 && !intxn) { - if ((ret = session->begin_transaction( - session, opts->transaction_config)) != 0) { - lprintf(wtperf, ret, 0, - "Failed starting transaction."); - goto err; - } - intxn = 1; - } - /* - * Figure out which table this op belongs to. - */ - cursor = cursors[map_key_to_table(wtperf->opts, op)]; - generate_key(opts, key_buf, op); - measure_latency = - opts->sample_interval != 0 && - trk->ops != 0 && (trk->ops % opts->sample_rate == 0); - if (measure_latency) - __wt_epoch(NULL, &start); - cursor->set_key(cursor, key_buf); - if (opts->random_value) - randomize_value(thread, value_buf); - cursor->set_value(cursor, value_buf); - if ((ret = cursor->insert(cursor)) == WT_ROLLBACK) { - lprintf(wtperf, ret, 0, "insert retrying"); - if ((ret = session->rollback_transaction( - session, NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Failed rollback_transaction"); - goto err; - } - intxn = 0; - continue; - } else if (ret != 0) { - lprintf(wtperf, ret, 0, "Failed inserting"); - goto err; - } - /* - * Gather statistics. - * We measure the latency of inserting a single key. If there - * are multiple tables, it is the time for insertion into all - * of them. - */ - if (measure_latency) { - __wt_epoch(NULL, &stop); - ++trk->latency_ops; - usecs = WT_TIMEDIFF_US(stop, start); - track_operation(trk, usecs); - } - ++thread->insert.ops; /* Same as trk->ops */ - - if (opts->checkpoint_stress_rate != 0 && - (op % opts->checkpoint_stress_rate) == 0) - stress_checkpoint_due = 1; - - if (opts->populate_ops_per_txn != 0) { - if (++opcount < opts->populate_ops_per_txn) - continue; - opcount = 0; - - if ((ret = session->commit_transaction( - session, NULL)) != 0) - lprintf(wtperf, ret, 0, - "Fail committing, transaction was aborted"); - intxn = 0; - } - - if (stress_checkpoint_due && intxn == 0) { - stress_checkpoint_due = 0; - if ((ret = session->checkpoint(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Checkpoint failed"); - goto err; - } - } - } - if (intxn && - (ret = session->commit_transaction(session, NULL)) != 0) - lprintf(wtperf, ret, 0, - "Fail committing, transaction was aborted"); - - if ((ret = session->close(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error closing session in populate"); - goto err; - } - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: wtperf->error = wtperf->stop = true; - } - free(cursors); - return (WT_THREAD_RET_VALUE); + struct timespec start, stop; + CONFIG_OPTS *opts; + TRACK *trk; + WTPERF *wtperf; + WTPERF_THREAD *thread; + WT_CONNECTION *conn; + WT_CURSOR **cursors, *cursor; + WT_SESSION *session; + size_t i; + uint64_t op, usecs; + uint32_t opcount, total_table_count; + int intxn, measure_latency, ret, stress_checkpoint_due; + char *value_buf, *key_buf; + const char *cursor_config; + + thread = (WTPERF_THREAD *)arg; + wtperf = thread->wtperf; + opts = wtperf->opts; + conn = wtperf->conn; + session = NULL; + cursors = NULL; + ret = stress_checkpoint_due = 0; + trk = &thread->insert; + total_table_count = opts->table_count + opts->scan_table_count; + + key_buf = thread->key_buf; + value_buf = thread->value_buf; + + if ((ret = conn->open_session(conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "populate: WT_CONNECTION.open_session"); + goto err; + } + + /* Do bulk loads if populate is single-threaded. */ + cursor_config = (opts->populate_threads == 1 && !opts->index) ? "bulk" : NULL; + /* Create the cursors. */ + cursors = dcalloc(total_table_count, sizeof(WT_CURSOR *)); + for (i = 0; i < total_table_count; i++) { + if ((ret = session->open_cursor( + session, wtperf->uris[i], NULL, cursor_config, &cursors[i])) != 0) { + lprintf(wtperf, ret, 0, "populate: WT_SESSION.open_cursor: %s", wtperf->uris[i]); + goto err; + } + } + + /* Populate the databases. */ + for (intxn = 0, opcount = 0;;) { + op = get_next_incr(wtperf); + if (op > (uint64_t)opts->icount + (uint64_t)opts->scan_icount) + break; + + if (opts->populate_ops_per_txn != 0 && !intxn) { + if ((ret = session->begin_transaction(session, opts->transaction_config)) != 0) { + lprintf(wtperf, ret, 0, "Failed starting transaction."); + goto err; + } + intxn = 1; + } + /* + * Figure out which table this op belongs to. + */ + cursor = cursors[map_key_to_table(wtperf->opts, op)]; + generate_key(opts, key_buf, op); + measure_latency = + opts->sample_interval != 0 && trk->ops != 0 && (trk->ops % opts->sample_rate == 0); + if (measure_latency) + __wt_epoch(NULL, &start); + cursor->set_key(cursor, key_buf); + if (opts->random_value) + randomize_value(thread, value_buf); + cursor->set_value(cursor, value_buf); + if ((ret = cursor->insert(cursor)) == WT_ROLLBACK) { + lprintf(wtperf, ret, 0, "insert retrying"); + if ((ret = session->rollback_transaction(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Failed rollback_transaction"); + goto err; + } + intxn = 0; + continue; + } else if (ret != 0) { + lprintf(wtperf, ret, 0, "Failed inserting"); + goto err; + } + /* + * Gather statistics. We measure the latency of inserting a single key. If there are + * multiple tables, it is the time for insertion into all of them. + */ + if (measure_latency) { + __wt_epoch(NULL, &stop); + ++trk->latency_ops; + usecs = WT_TIMEDIFF_US(stop, start); + track_operation(trk, usecs); + } + ++thread->insert.ops; /* Same as trk->ops */ + + if (opts->checkpoint_stress_rate != 0 && (op % opts->checkpoint_stress_rate) == 0) + stress_checkpoint_due = 1; + + if (opts->populate_ops_per_txn != 0) { + if (++opcount < opts->populate_ops_per_txn) + continue; + opcount = 0; + + if ((ret = session->commit_transaction(session, NULL)) != 0) + lprintf(wtperf, ret, 0, "Fail committing, transaction was aborted"); + intxn = 0; + } + + if (stress_checkpoint_due && intxn == 0) { + stress_checkpoint_due = 0; + if ((ret = session->checkpoint(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Checkpoint failed"); + goto err; + } + } + } + if (intxn && (ret = session->commit_transaction(session, NULL)) != 0) + lprintf(wtperf, ret, 0, "Fail committing, transaction was aborted"); + + if ((ret = session->close(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Error closing session in populate"); + goto err; + } + + /* Notify our caller we failed and shut the system down. */ + if (0) { +err: + wtperf->error = wtperf->stop = true; + } + free(cursors); + return (WT_THREAD_RET_VALUE); } static WT_THREAD_RET populate_async(void *arg) { - struct timespec start, stop; - CONFIG_OPTS *opts; - TRACK *trk; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_ASYNC_OP *asyncop; - WT_CONNECTION *conn; - WT_SESSION *session; - uint64_t op, usecs; - int measure_latency, ret; - char *value_buf, *key_buf; - - thread = (WTPERF_THREAD *)arg; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - session = NULL; - ret = 0; - trk = &thread->insert; - - key_buf = thread->key_buf; - value_buf = thread->value_buf; - - if ((ret = conn->open_session( - conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, "populate: WT_CONNECTION.open_session"); - goto err; - } - - /* - * Measuring latency of one async op is not meaningful. We - * will measure the time it takes to do all of them, including - * the time to process by workers. - */ - measure_latency = - opts->sample_interval != 0 && - trk->ops != 0 && (trk->ops % opts->sample_rate == 0); - if (measure_latency) - __wt_epoch(NULL, &start); - - /* Populate the databases. */ - for (;;) { - op = get_next_incr(wtperf); - if (op > (uint64_t)opts->icount + (uint64_t)opts->scan_icount) - break; - /* - * Allocate an async op for whichever table. - */ - while ((ret = conn->async_new_op( - conn, wtperf->uris[map_key_to_table(wtperf->opts, op)], - NULL, &cb, &asyncop)) == EBUSY) - (void)usleep(10000); - if (ret != 0) { - lprintf(wtperf, ret, 0, "Failed async_new_op"); - goto err; - } - - asyncop->app_private = thread; - generate_key(opts, key_buf, op); - asyncop->set_key(asyncop, key_buf); - if (opts->random_value) - randomize_value(thread, value_buf); - asyncop->set_value(asyncop, value_buf); - if ((ret = asyncop->insert(asyncop)) != 0) { - lprintf(wtperf, ret, 0, "Failed inserting"); - goto err; - } - } - - /* - * Gather statistics. - * We measure the latency of inserting a single key. If there - * are multiple tables, it is the time for insertion into all - * of them. Note that currently every populate thread will call - * async_flush and those calls will convoy. That is not the - * most efficient way, but we want to flush before measuring latency. - */ - if (conn->async_flush(conn) != 0) { - lprintf(wtperf, ret, 0, "Failed async flush"); - goto err; - } - if (measure_latency) { - __wt_epoch(NULL, &stop); - ++trk->latency_ops; - usecs = WT_TIMEDIFF_US(stop, start); - track_operation(trk, usecs); - } - if ((ret = session->close(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error closing session in populate"); - goto err; - } - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: wtperf->error = wtperf->stop = true; - } - return (WT_THREAD_RET_VALUE); + struct timespec start, stop; + CONFIG_OPTS *opts; + TRACK *trk; + WTPERF *wtperf; + WTPERF_THREAD *thread; + WT_ASYNC_OP *asyncop; + WT_CONNECTION *conn; + WT_SESSION *session; + uint64_t op, usecs; + int measure_latency, ret; + char *value_buf, *key_buf; + + thread = (WTPERF_THREAD *)arg; + wtperf = thread->wtperf; + opts = wtperf->opts; + conn = wtperf->conn; + session = NULL; + ret = 0; + trk = &thread->insert; + + key_buf = thread->key_buf; + value_buf = thread->value_buf; + + if ((ret = conn->open_session(conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "populate: WT_CONNECTION.open_session"); + goto err; + } + + /* + * Measuring latency of one async op is not meaningful. We will measure the time it takes to do + * all of them, including the time to process by workers. + */ + measure_latency = + opts->sample_interval != 0 && trk->ops != 0 && (trk->ops % opts->sample_rate == 0); + if (measure_latency) + __wt_epoch(NULL, &start); + + /* Populate the databases. */ + for (;;) { + op = get_next_incr(wtperf); + if (op > (uint64_t)opts->icount + (uint64_t)opts->scan_icount) + break; + /* + * Allocate an async op for whichever table. + */ + while ((ret = conn->async_new_op(conn, wtperf->uris[map_key_to_table(wtperf->opts, op)], + NULL, &cb, &asyncop)) == EBUSY) + (void)usleep(10000); + if (ret != 0) { + lprintf(wtperf, ret, 0, "Failed async_new_op"); + goto err; + } + + asyncop->app_private = thread; + generate_key(opts, key_buf, op); + asyncop->set_key(asyncop, key_buf); + if (opts->random_value) + randomize_value(thread, value_buf); + asyncop->set_value(asyncop, value_buf); + if ((ret = asyncop->insert(asyncop)) != 0) { + lprintf(wtperf, ret, 0, "Failed inserting"); + goto err; + } + } + + /* + * Gather statistics. We measure the latency of inserting a single key. If there are multiple + * tables, it is the time for insertion into all of them. Note that currently every populate + * thread will call async_flush and those calls will convoy. That is not the most efficient way, + * but we want to flush before measuring latency. + */ + if (conn->async_flush(conn) != 0) { + lprintf(wtperf, ret, 0, "Failed async flush"); + goto err; + } + if (measure_latency) { + __wt_epoch(NULL, &stop); + ++trk->latency_ops; + usecs = WT_TIMEDIFF_US(stop, start); + track_operation(trk, usecs); + } + if ((ret = session->close(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Error closing session in populate"); + goto err; + } + + /* Notify our caller we failed and shut the system down. */ + if (0) { +err: + wtperf->error = wtperf->stop = true; + } + return (WT_THREAD_RET_VALUE); } static WT_THREAD_RET monitor(void *arg) { - struct timespec t; - struct tm localt; - CONFIG_OPTS *opts; - FILE *fp, *jfp; - WTPERF *wtperf; - size_t len; - uint64_t min_thr, reads, inserts, updates; - uint64_t cur_reads, cur_inserts, cur_updates; - uint64_t last_reads, last_inserts, last_updates; - uint32_t read_avg, read_min, read_max; - uint32_t insert_avg, insert_min, insert_max; - uint32_t update_avg, update_min, update_max; - uint32_t latency_max, level; - u_int i; - size_t buf_size; - int msg_err; - const char *str; - char buf[64], *path; - bool first; - - wtperf = (WTPERF *)arg; - opts = wtperf->opts; - assert(opts->sample_interval != 0); - - fp = jfp = NULL; - first = true; - path = NULL; - - min_thr = (uint64_t)opts->min_throughput; - latency_max = (uint32_t)ms_to_us(opts->max_latency); - - /* Open the logging file. */ - len = strlen(wtperf->monitor_dir) + 100; - path = dmalloc(len); - testutil_check(__wt_snprintf( - path, len, "%s/monitor", wtperf->monitor_dir)); - if ((fp = fopen(path, "w")) == NULL) { - lprintf(wtperf, errno, 0, "%s", path); - goto err; - } - testutil_check(__wt_snprintf( - path, len, "%s/monitor.json", wtperf->monitor_dir)); - if ((jfp = fopen(path, "w")) == NULL) { - lprintf(wtperf, errno, 0, "%s", path); - goto err; - } - /* Set line buffering for monitor file. */ - __wt_stream_set_line_buffer(fp); - __wt_stream_set_line_buffer(jfp); - fprintf(fp, - "#time," - "totalsec," - "read ops per second," - "insert ops per second," - "update ops per second," - "checkpoints," - "scans," - "read average latency(uS)," - "read minimum latency(uS)," - "read maximum latency(uS)," - "insert average latency(uS)," - "insert min latency(uS)," - "insert maximum latency(uS)," - "update average latency(uS)," - "update min latency(uS)," - "update maximum latency(uS)" - "\n"); - last_reads = last_inserts = last_updates = 0; - while (!wtperf->stop) { - for (i = 0; i < opts->sample_interval; i++) { - sleep(1); - if (wtperf->stop) - break; - } - /* If the workers are done, don't bother with a final call. */ - if (wtperf->stop) - break; - if (wtperf->in_warmup) - continue; - - __wt_epoch(NULL, &t); - testutil_check(__wt_localtime(NULL, &t.tv_sec, &localt)); - testutil_assert( - strftime(buf, sizeof(buf), "%b %d %H:%M:%S", &localt) != 0); - - reads = sum_read_ops(wtperf); - inserts = sum_insert_ops(wtperf); - updates = sum_update_ops(wtperf); - latency_read(wtperf, &read_avg, &read_min, &read_max); - latency_insert(wtperf, &insert_avg, &insert_min, &insert_max); - latency_update(wtperf, &update_avg, &update_min, &update_max); - - cur_reads = (reads - last_reads) / opts->sample_interval; - cur_updates = (updates - last_updates) / opts->sample_interval; - /* - * For now the only item we need to worry about changing is - * inserts when we transition from the populate phase to - * workload phase. - */ - if (inserts < last_inserts) - cur_inserts = 0; - else - cur_inserts = - (inserts - last_inserts) / opts->sample_interval; - - (void)fprintf(fp, - "%s,%" PRIu32 - ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 - ",%c,%c" - ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 - ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 - ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 - "\n", - buf, wtperf->totalsec, - cur_reads, cur_inserts, cur_updates, - wtperf->ckpt ? 'Y' : 'N', - wtperf->scan ? 'Y' : 'N', - read_avg, read_min, read_max, - insert_avg, insert_min, insert_max, - update_avg, update_min, update_max); - if (jfp != NULL) { - buf_size = strftime(buf, - sizeof(buf), "%Y-%m-%dT%H:%M:%S", &localt); - testutil_assert(buf_size != 0); - testutil_check(__wt_snprintf(&buf[buf_size], - sizeof(buf) - buf_size, - ".%3.3" PRIu64 "Z", - (uint64_t)ns_to_ms((uint64_t)t.tv_nsec))); - (void)fprintf(jfp, "{"); - if (first) { - (void)fprintf(jfp, "\"version\":\"%s\",", - WIREDTIGER_VERSION_STRING); - first = false; - } - (void)fprintf(jfp, - "\"localTime\":\"%s\",\"wtperf\":{", buf); - /* Note does not have initial comma before "read" */ - (void)fprintf(jfp, - "\"read\":{\"ops per sec\":%" PRIu64 - ",\"average latency\":%" PRIu32 - ",\"min latency\":%" PRIu32 - ",\"max latency\":%" PRIu32 "}", - cur_reads, read_avg, read_min, read_max); - (void)fprintf(jfp, - ",\"insert\":{\"ops per sec\":%" PRIu64 - ",\"average latency\":%" PRIu32 - ",\"min latency\":%" PRIu32 - ",\"max latency\":%" PRIu32 "}", - cur_inserts, insert_avg, insert_min, insert_max); - (void)fprintf(jfp, - ",\"update\":{\"ops per sec\":%" PRIu64 - ",\"average latency\":%" PRIu32 - ",\"min latency\":%" PRIu32 - ",\"max latency\":%" PRIu32 "}", - cur_updates, update_avg, update_min, update_max); - fprintf(jfp, "}}\n"); - } - - if (latency_max != 0 && - (read_max > latency_max || insert_max > latency_max || - update_max > latency_max)) { - if (opts->max_latency_fatal) { - level = 1; - msg_err = WT_PANIC; - str = "ERROR"; - } else { - level = 0; - msg_err = 0; - str = "WARNING"; - } - lprintf(wtperf, msg_err, level, - "%s: max latency exceeded: threshold %" PRIu32 - " read max %" PRIu32 " insert max %" PRIu32 - " update max %" PRIu32, str, latency_max, - read_max, insert_max, update_max); - } - if (min_thr != 0 && - ((cur_reads != 0 && cur_reads < min_thr) || - (cur_inserts != 0 && cur_inserts < min_thr) || - (cur_updates != 0 && cur_updates < min_thr))) { - if (opts->min_throughput_fatal) { - level = 1; - msg_err = WT_PANIC; - str = "ERROR"; - } else { - level = 0; - msg_err = 0; - str = "WARNING"; - } - lprintf(wtperf, msg_err, level, - "%s: minimum throughput not met: threshold %" PRIu64 - " reads %" PRIu64 " inserts %" PRIu64 - " updates %" PRIu64, str, min_thr, cur_reads, - cur_inserts, cur_updates); - } - last_reads = reads; - last_inserts = inserts; - last_updates = updates; - } - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: wtperf->error = wtperf->stop = true; - } - - if (fp != NULL) - (void)fclose(fp); - if (jfp != NULL) - (void)fclose(jfp); - free(path); - - return (WT_THREAD_RET_VALUE); + struct timespec t; + struct tm localt; + CONFIG_OPTS *opts; + FILE *fp, *jfp; + WTPERF *wtperf; + size_t len; + uint64_t min_thr, reads, inserts, updates; + uint64_t cur_reads, cur_inserts, cur_updates; + uint64_t last_reads, last_inserts, last_updates; + uint32_t read_avg, read_min, read_max; + uint32_t insert_avg, insert_min, insert_max; + uint32_t update_avg, update_min, update_max; + uint32_t latency_max, level; + u_int i; + size_t buf_size; + int msg_err; + const char *str; + char buf[64], *path; + bool first; + + wtperf = (WTPERF *)arg; + opts = wtperf->opts; + assert(opts->sample_interval != 0); + + fp = jfp = NULL; + first = true; + path = NULL; + + min_thr = (uint64_t)opts->min_throughput; + latency_max = (uint32_t)ms_to_us(opts->max_latency); + + /* Open the logging file. */ + len = strlen(wtperf->monitor_dir) + 100; + path = dmalloc(len); + testutil_check(__wt_snprintf(path, len, "%s/monitor", wtperf->monitor_dir)); + if ((fp = fopen(path, "w")) == NULL) { + lprintf(wtperf, errno, 0, "%s", path); + goto err; + } + testutil_check(__wt_snprintf(path, len, "%s/monitor.json", wtperf->monitor_dir)); + if ((jfp = fopen(path, "w")) == NULL) { + lprintf(wtperf, errno, 0, "%s", path); + goto err; + } + /* Set line buffering for monitor file. */ + __wt_stream_set_line_buffer(fp); + __wt_stream_set_line_buffer(jfp); + fprintf(fp, + "#time," + "totalsec," + "read ops per second," + "insert ops per second," + "update ops per second," + "checkpoints," + "scans," + "read average latency(uS)," + "read minimum latency(uS)," + "read maximum latency(uS)," + "insert average latency(uS)," + "insert min latency(uS)," + "insert maximum latency(uS)," + "update average latency(uS)," + "update min latency(uS)," + "update maximum latency(uS)" + "\n"); + last_reads = last_inserts = last_updates = 0; + while (!wtperf->stop) { + for (i = 0; i < opts->sample_interval; i++) { + sleep(1); + if (wtperf->stop) + break; + } + /* If the workers are done, don't bother with a final call. */ + if (wtperf->stop) + break; + if (wtperf->in_warmup) + continue; + + __wt_epoch(NULL, &t); + testutil_check(__wt_localtime(NULL, &t.tv_sec, &localt)); + testutil_assert(strftime(buf, sizeof(buf), "%b %d %H:%M:%S", &localt) != 0); + + reads = sum_read_ops(wtperf); + inserts = sum_insert_ops(wtperf); + updates = sum_update_ops(wtperf); + latency_read(wtperf, &read_avg, &read_min, &read_max); + latency_insert(wtperf, &insert_avg, &insert_min, &insert_max); + latency_update(wtperf, &update_avg, &update_min, &update_max); + + cur_reads = (reads - last_reads) / opts->sample_interval; + cur_updates = (updates - last_updates) / opts->sample_interval; + /* + * For now the only item we need to worry about changing is inserts when we transition from + * the populate phase to workload phase. + */ + if (inserts < last_inserts) + cur_inserts = 0; + else + cur_inserts = (inserts - last_inserts) / opts->sample_interval; + + (void)fprintf(fp, "%s,%" PRIu32 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 + ",%c,%c" + ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 + ",%" PRIu32 ",%" PRIu32 ",%" PRIu32 "\n", + buf, wtperf->totalsec, cur_reads, cur_inserts, cur_updates, wtperf->ckpt ? 'Y' : 'N', + wtperf->scan ? 'Y' : 'N', read_avg, read_min, read_max, insert_avg, insert_min, + insert_max, update_avg, update_min, update_max); + if (jfp != NULL) { + buf_size = strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &localt); + testutil_assert(buf_size != 0); + testutil_check(__wt_snprintf(&buf[buf_size], sizeof(buf) - buf_size, ".%3.3" PRIu64 "Z", + (uint64_t)ns_to_ms((uint64_t)t.tv_nsec))); + (void)fprintf(jfp, "{"); + if (first) { + (void)fprintf(jfp, "\"version\":\"%s\",", WIREDTIGER_VERSION_STRING); + first = false; + } + (void)fprintf(jfp, "\"localTime\":\"%s\",\"wtperf\":{", buf); + /* Note does not have initial comma before "read" */ + (void)fprintf(jfp, "\"read\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 + ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", + cur_reads, read_avg, read_min, read_max); + (void)fprintf(jfp, + ",\"insert\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 + ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", + cur_inserts, insert_avg, insert_min, insert_max); + (void)fprintf(jfp, + ",\"update\":{\"ops per sec\":%" PRIu64 ",\"average latency\":%" PRIu32 + ",\"min latency\":%" PRIu32 ",\"max latency\":%" PRIu32 "}", + cur_updates, update_avg, update_min, update_max); + fprintf(jfp, "}}\n"); + } + + if (latency_max != 0 && + (read_max > latency_max || insert_max > latency_max || update_max > latency_max)) { + if (opts->max_latency_fatal) { + level = 1; + msg_err = WT_PANIC; + str = "ERROR"; + } else { + level = 0; + msg_err = 0; + str = "WARNING"; + } + lprintf(wtperf, msg_err, level, + "%s: max latency exceeded: threshold %" PRIu32 " read max %" PRIu32 + " insert max %" PRIu32 " update max %" PRIu32, + str, latency_max, read_max, insert_max, update_max); + } + if (min_thr != 0 && + ((cur_reads != 0 && cur_reads < min_thr) || (cur_inserts != 0 && cur_inserts < min_thr) || + (cur_updates != 0 && cur_updates < min_thr))) { + if (opts->min_throughput_fatal) { + level = 1; + msg_err = WT_PANIC; + str = "ERROR"; + } else { + level = 0; + msg_err = 0; + str = "WARNING"; + } + lprintf(wtperf, msg_err, level, + "%s: minimum throughput not met: threshold %" PRIu64 " reads %" PRIu64 + " inserts %" PRIu64 " updates %" PRIu64, + str, min_thr, cur_reads, cur_inserts, cur_updates); + } + last_reads = reads; + last_inserts = inserts; + last_updates = updates; + } + + /* Notify our caller we failed and shut the system down. */ + if (0) { +err: + wtperf->error = wtperf->stop = true; + } + + if (fp != NULL) + (void)fclose(fp); + if (jfp != NULL) + (void)fclose(jfp); + free(path); + + return (WT_THREAD_RET_VALUE); } static WT_THREAD_RET checkpoint_worker(void *arg) { - CONFIG_OPTS *opts; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_CONNECTION *conn; - WT_SESSION *session; - struct timespec e, s; - uint32_t i; - int ret; - - thread = (WTPERF_THREAD *)arg; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - session = NULL; - - if ((ret = conn->open_session( - conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, - "open_session failed in checkpoint thread."); - goto err; - } - - while (!wtperf->stop) { - /* Break the sleep up, so we notice interrupts faster. */ - for (i = 0; i < opts->checkpoint_interval; i++) { - sleep(1); - if (wtperf->stop) - break; - } - /* If the workers are done, don't bother with a final call. */ - if (wtperf->stop) - break; - - __wt_epoch(NULL, &s); - - wtperf->ckpt = true; - if ((ret = session->checkpoint(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Checkpoint failed."); - goto err; - } - wtperf->ckpt = false; - ++thread->ckpt.ops; - - __wt_epoch(NULL, &e); - } - - if (session != NULL && - ((ret = session->close(session, NULL)) != 0)) { - lprintf(wtperf, ret, 0, - "Error closing session in checkpoint worker."); - goto err; - } - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: wtperf->error = wtperf->stop = true; - } - - return (WT_THREAD_RET_VALUE); + CONFIG_OPTS *opts; + WTPERF *wtperf; + WTPERF_THREAD *thread; + WT_CONNECTION *conn; + WT_SESSION *session; + struct timespec e, s; + uint32_t i; + int ret; + + thread = (WTPERF_THREAD *)arg; + wtperf = thread->wtperf; + opts = wtperf->opts; + conn = wtperf->conn; + session = NULL; + + if ((ret = conn->open_session(conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "open_session failed in checkpoint thread."); + goto err; + } + + while (!wtperf->stop) { + /* Break the sleep up, so we notice interrupts faster. */ + for (i = 0; i < opts->checkpoint_interval; i++) { + sleep(1); + if (wtperf->stop) + break; + } + /* If the workers are done, don't bother with a final call. */ + if (wtperf->stop) + break; + + __wt_epoch(NULL, &s); + + wtperf->ckpt = true; + if ((ret = session->checkpoint(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Checkpoint failed."); + goto err; + } + wtperf->ckpt = false; + ++thread->ckpt.ops; + + __wt_epoch(NULL, &e); + } + + if (session != NULL && ((ret = session->close(session, NULL)) != 0)) { + lprintf(wtperf, ret, 0, "Error closing session in checkpoint worker."); + goto err; + } + + /* Notify our caller we failed and shut the system down. */ + if (0) { +err: + wtperf->error = wtperf->stop = true; + } + + return (WT_THREAD_RET_VALUE); } static WT_THREAD_RET scan_worker(void *arg) { - CONFIG_OPTS *opts; - WTPERF *wtperf; - WTPERF_THREAD *thread; - WT_CONNECTION *conn; - WT_CURSOR *cursor, **cursors; - WT_SESSION *session; - char *key_buf; - struct timespec e, s; - uint32_t i, ntables, pct, table_start; - uint64_t cur_id, end_id, incr, items, start_id, tot_items; - int ret; - - thread = (WTPERF_THREAD *)arg; - key_buf = thread->key_buf; - wtperf = thread->wtperf; - opts = wtperf->opts; - conn = wtperf->conn; - session = NULL; - cursors = NULL; - items = 0; - - /* - * Figure out how many items we should scan. - * We base the percentage on the icount. - */ - pct = opts->scan_pct == 0 ? 100 : opts->scan_pct; - start_id = cur_id = 1; - - /* - * When we scan the tables, we will increment the key by an amount - * that causes us to visit each table in order, and jump ahead in - * the key space when returning to a table. By doing this, we don't - * repeat keys until we visit them all, but we don't visit keys in - * sequential order. This might better emulate the access pattern - * to a main table when an index is scanned, or a more complex query - * is performed. - */ - if (opts->scan_icount != 0) { - end_id = opts->scan_icount; - tot_items = ((uint64_t)opts->scan_icount * pct) / 100; - incr = (uint64_t)opts->scan_table_count * 1000 + 1; - table_start = opts->table_count; - ntables = opts->scan_table_count; - } else { - end_id = opts->icount; - tot_items = ((uint64_t)opts->icount * pct) / 100; - incr = (uint64_t)opts->table_count * 1000 + 1; - table_start = 0; - ntables = opts->table_count; - } - if ((ret = conn->open_session( - conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, - "open_session failed in scan thread."); - goto err; - } - cursors = dmalloc(ntables * sizeof(WT_CURSOR *)); - for (i = 0; i < ntables; i++) - if ((ret = session->open_cursor( - session, wtperf->uris[i + table_start], NULL, NULL, - &cursors[i])) != 0) { - lprintf(wtperf, ret, 0, - "open_cursor failed in scan thread."); - goto err; - } - - while (!wtperf->stop) { - /* Break the sleep up, so we notice interrupts faster. */ - for (i = 0; i < opts->scan_interval; i++) { - sleep(1); - if (wtperf->stop) - break; - } - /* If the workers are done, don't bother with a final call. */ - if (wtperf->stop) - break; - - __wt_epoch(NULL, &s); - - wtperf->scan = true; - items = 0; - while (items < tot_items && !wtperf->stop) { - cursor = cursors[map_key_to_table(opts, cur_id) - - table_start]; - generate_key(opts, key_buf, cur_id); - cursor->set_key(cursor, key_buf); - if ((ret = cursor->search(cursor)) != 0) { - lprintf(wtperf, ret, 0, "Failed scan search " - "key %s, items %d", key_buf, (int)items); - goto err; - } - - items++; - cur_id += incr; - if (cur_id >= end_id) { - /* - * Continue with the next slice of the key - * space. - */ - cur_id = ++start_id; - if (cur_id >= end_id) - cur_id = start_id = 1; - } - } - wtperf->scan = false; - ++thread->scan.ops; - __wt_epoch(NULL, &e); - } - - if (session != NULL && - ((ret = session->close(session, NULL)) != 0)) { - lprintf(wtperf, ret, 0, - "Error closing session in scan worker."); - goto err; - } - - /* Notify our caller we failed and shut the system down. */ - if (0) { -err: wtperf->error = wtperf->stop = true; - } - free(cursors); - return (WT_THREAD_RET_VALUE); + CONFIG_OPTS *opts; + WTPERF *wtperf; + WTPERF_THREAD *thread; + WT_CONNECTION *conn; + WT_CURSOR *cursor, **cursors; + WT_SESSION *session; + char *key_buf; + struct timespec e, s; + uint32_t i, ntables, pct, table_start; + uint64_t cur_id, end_id, incr, items, start_id, tot_items; + int ret; + + thread = (WTPERF_THREAD *)arg; + key_buf = thread->key_buf; + wtperf = thread->wtperf; + opts = wtperf->opts; + conn = wtperf->conn; + session = NULL; + cursors = NULL; + items = 0; + + /* + * Figure out how many items we should scan. We base the percentage on the icount. + */ + pct = opts->scan_pct == 0 ? 100 : opts->scan_pct; + start_id = cur_id = 1; + + /* + * When we scan the tables, we will increment the key by an amount that causes us to visit each + * table in order, and jump ahead in the key space when returning to a table. By doing this, we + * don't repeat keys until we visit them all, but we don't visit keys in sequential order. This + * might better emulate the access pattern to a main table when an index is scanned, or a more + * complex query is performed. + */ + if (opts->scan_icount != 0) { + end_id = opts->scan_icount; + tot_items = ((uint64_t)opts->scan_icount * pct) / 100; + incr = (uint64_t)opts->scan_table_count * 1000 + 1; + table_start = opts->table_count; + ntables = opts->scan_table_count; + } else { + end_id = opts->icount; + tot_items = ((uint64_t)opts->icount * pct) / 100; + incr = (uint64_t)opts->table_count * 1000 + 1; + table_start = 0; + ntables = opts->table_count; + } + if ((ret = conn->open_session(conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "open_session failed in scan thread."); + goto err; + } + cursors = dmalloc(ntables * sizeof(WT_CURSOR *)); + for (i = 0; i < ntables; i++) + if ((ret = session->open_cursor( + session, wtperf->uris[i + table_start], NULL, NULL, &cursors[i])) != 0) { + lprintf(wtperf, ret, 0, "open_cursor failed in scan thread."); + goto err; + } + + while (!wtperf->stop) { + /* Break the sleep up, so we notice interrupts faster. */ + for (i = 0; i < opts->scan_interval; i++) { + sleep(1); + if (wtperf->stop) + break; + } + /* If the workers are done, don't bother with a final call. */ + if (wtperf->stop) + break; + + __wt_epoch(NULL, &s); + + wtperf->scan = true; + items = 0; + while (items < tot_items && !wtperf->stop) { + cursor = cursors[map_key_to_table(opts, cur_id) - table_start]; + generate_key(opts, key_buf, cur_id); + cursor->set_key(cursor, key_buf); + if ((ret = cursor->search(cursor)) != 0) { + lprintf(wtperf, ret, 0, + "Failed scan search " + "key %s, items %d", + key_buf, (int)items); + goto err; + } + + items++; + cur_id += incr; + if (cur_id >= end_id) { + /* + * Continue with the next slice of the key space. + */ + cur_id = ++start_id; + if (cur_id >= end_id) + cur_id = start_id = 1; + } + } + wtperf->scan = false; + ++thread->scan.ops; + __wt_epoch(NULL, &e); + } + + if (session != NULL && ((ret = session->close(session, NULL)) != 0)) { + lprintf(wtperf, ret, 0, "Error closing session in scan worker."); + goto err; + } + + /* Notify our caller we failed and shut the system down. */ + if (0) { +err: + wtperf->error = wtperf->stop = true; + } + free(cursors); + return (WT_THREAD_RET_VALUE); } static int execute_populate(WTPERF *wtperf) { - struct timespec start, stop; - CONFIG_OPTS *opts; - WT_ASYNC_OP *asyncop; - WTPERF_THREAD *popth; - WT_THREAD_CALLBACK(*pfunc)(void *); - size_t i; - uint64_t last_ops, msecs, print_ops_sec, max_key; - uint32_t interval, tables; - wt_thread_t idle_table_cycle_thread; - double print_secs; - int elapsed, ret; - - opts = wtperf->opts; - max_key = (uint64_t)opts->icount + (uint64_t)opts->scan_icount; - - lprintf(wtperf, 0, 1, - "Starting %" PRIu32 - " populate thread(s) for %" PRIu64 " items", - opts->populate_threads, max_key); - - /* Start cycling idle tables if configured. */ - start_idle_table_cycle(wtperf, &idle_table_cycle_thread); - - wtperf->insert_key = 0; - - wtperf->popthreads = - dcalloc(opts->populate_threads, sizeof(WTPERF_THREAD)); - if (wtperf->use_asyncops) { - lprintf(wtperf, 0, 1, "Starting %" PRIu32 " async thread(s)", - opts->async_threads); - pfunc = populate_async; - } else - pfunc = populate_thread; - start_threads(wtperf, NULL, - wtperf->popthreads, opts->populate_threads, pfunc); - - __wt_epoch(NULL, &start); - for (elapsed = 0, interval = 0, last_ops = 0; - wtperf->insert_key < max_key && !wtperf->error;) { - /* - * Sleep for 100th of a second, report_interval is in second - * granularity, each 100th increment of elapsed is a single - * increment of interval. - */ - (void)usleep(10000); - if (opts->report_interval == 0 || ++elapsed < 100) - continue; - elapsed = 0; - if (++interval < opts->report_interval) - continue; - interval = 0; - wtperf->totalsec += opts->report_interval; - wtperf->insert_ops = sum_pop_ops(wtperf); - lprintf(wtperf, 0, 1, - "%" PRIu64 " populate inserts (%" PRIu64 " of %" - PRIu32 ") in %" PRIu32 " secs (%" PRIu32 " total secs)", - wtperf->insert_ops - last_ops, wtperf->insert_ops, - opts->icount, opts->report_interval, wtperf->totalsec); - last_ops = wtperf->insert_ops; - } - __wt_epoch(NULL, &stop); - - /* - * Move popthreads aside to narrow possible race with the monitor - * thread. The latency tracking code also requires that popthreads be - * NULL when the populate phase is finished, to know that the workload - * phase has started. - */ - popth = wtperf->popthreads; - wtperf->popthreads = NULL; - stop_threads(opts->populate_threads, popth); - free(popth); - - /* Report if any worker threads didn't finish. */ - if (wtperf->error) { - lprintf(wtperf, WT_ERROR, 0, - "Populate thread(s) exited without finishing."); - return (WT_ERROR); - } - - lprintf(wtperf, - 0, 1, "Finished load of %" PRIu32 " items", opts->icount); - msecs = WT_TIMEDIFF_MS(stop, start); - - /* - * This is needed as the divisions will fail if the insert takes no time - * which will only be the case when there is no data to insert. - */ - if (msecs == 0) { - print_secs = 0; - print_ops_sec = 0; - } else { - print_secs = (double)msecs / (double)MSEC_PER_SEC; - print_ops_sec = (uint64_t)(opts->icount / print_secs); - } - lprintf(wtperf, 0, 1, - "Load time: %.2f\n" "load ops/sec: %" PRIu64, - print_secs, print_ops_sec); - - /* - * If configured, compact to allow LSM merging to complete. We - * set an unlimited timeout because if we close the connection - * then any in-progress compact/merge is aborted. - */ - if (opts->compact) { - assert(opts->async_threads > 0); - lprintf(wtperf, 0, 1, "Compact after populate"); - __wt_epoch(NULL, &start); - tables = opts->table_count; - for (i = 0; i < opts->table_count; i++) { - /* - * If no ops are available, retry. Any other error, - * return. - */ - while ((ret = wtperf->conn->async_new_op( - wtperf->conn, wtperf->uris[i], - "timeout=0", &cb, &asyncop)) == EBUSY) - (void)usleep(10000); - if (ret != 0) - return (ret); - - asyncop->app_private = &tables; - if ((ret = asyncop->compact(asyncop)) != 0) { - lprintf(wtperf, - ret, 0, "Async compact failed."); - return (ret); - } - } - if ((ret = wtperf->conn->async_flush(wtperf->conn)) != 0) { - lprintf(wtperf, ret, 0, "Populate async flush failed."); - return (ret); - } - __wt_epoch(NULL, &stop); - lprintf(wtperf, 0, 1, - "Compact completed in %" PRIu64 " seconds", - (uint64_t)(WT_TIMEDIFF_SEC(stop, start))); - assert(tables == 0); - } - - /* Stop cycling idle tables. */ - stop_idle_table_cycle(wtperf, idle_table_cycle_thread); - - return (0); + struct timespec start, stop; + CONFIG_OPTS *opts; + WT_ASYNC_OP *asyncop; + WTPERF_THREAD *popth; + WT_THREAD_CALLBACK (*pfunc)(void *); + size_t i; + uint64_t last_ops, msecs, print_ops_sec, max_key; + uint32_t interval, tables; + wt_thread_t idle_table_cycle_thread; + double print_secs; + int elapsed, ret; + + opts = wtperf->opts; + max_key = (uint64_t)opts->icount + (uint64_t)opts->scan_icount; + + lprintf(wtperf, 0, 1, "Starting %" PRIu32 " populate thread(s) for %" PRIu64 " items", + opts->populate_threads, max_key); + + /* Start cycling idle tables if configured. */ + start_idle_table_cycle(wtperf, &idle_table_cycle_thread); + + wtperf->insert_key = 0; + + wtperf->popthreads = dcalloc(opts->populate_threads, sizeof(WTPERF_THREAD)); + if (wtperf->use_asyncops) { + lprintf(wtperf, 0, 1, "Starting %" PRIu32 " async thread(s)", opts->async_threads); + pfunc = populate_async; + } else + pfunc = populate_thread; + start_threads(wtperf, NULL, wtperf->popthreads, opts->populate_threads, pfunc); + + __wt_epoch(NULL, &start); + for (elapsed = 0, interval = 0, last_ops = 0; wtperf->insert_key < max_key && !wtperf->error;) { + /* + * Sleep for 100th of a second, report_interval is in second granularity, each 100th + * increment of elapsed is a single increment of interval. + */ + (void)usleep(10000); + if (opts->report_interval == 0 || ++elapsed < 100) + continue; + elapsed = 0; + if (++interval < opts->report_interval) + continue; + interval = 0; + wtperf->totalsec += opts->report_interval; + wtperf->insert_ops = sum_pop_ops(wtperf); + lprintf(wtperf, 0, 1, "%" PRIu64 " populate inserts (%" PRIu64 " of %" PRIu32 + ") in %" PRIu32 " secs (%" PRIu32 " total secs)", + wtperf->insert_ops - last_ops, wtperf->insert_ops, opts->icount, opts->report_interval, + wtperf->totalsec); + last_ops = wtperf->insert_ops; + } + __wt_epoch(NULL, &stop); + + /* + * Move popthreads aside to narrow possible race with the monitor thread. The latency tracking + * code also requires that popthreads be NULL when the populate phase is finished, to know that + * the workload phase has started. + */ + popth = wtperf->popthreads; + wtperf->popthreads = NULL; + stop_threads(opts->populate_threads, popth); + free(popth); + + /* Report if any worker threads didn't finish. */ + if (wtperf->error) { + lprintf(wtperf, WT_ERROR, 0, "Populate thread(s) exited without finishing."); + return (WT_ERROR); + } + + lprintf(wtperf, 0, 1, "Finished load of %" PRIu32 " items", opts->icount); + msecs = WT_TIMEDIFF_MS(stop, start); + + /* + * This is needed as the divisions will fail if the insert takes no time which will only be the + * case when there is no data to insert. + */ + if (msecs == 0) { + print_secs = 0; + print_ops_sec = 0; + } else { + print_secs = (double)msecs / (double)MSEC_PER_SEC; + print_ops_sec = (uint64_t)(opts->icount / print_secs); + } + lprintf(wtperf, 0, 1, + "Load time: %.2f\n" + "load ops/sec: %" PRIu64, + print_secs, print_ops_sec); + + /* + * If configured, compact to allow LSM merging to complete. We set an unlimited timeout because + * if we close the connection then any in-progress compact/merge is aborted. + */ + if (opts->compact) { + assert(opts->async_threads > 0); + lprintf(wtperf, 0, 1, "Compact after populate"); + __wt_epoch(NULL, &start); + tables = opts->table_count; + for (i = 0; i < opts->table_count; i++) { + /* + * If no ops are available, retry. Any other error, return. + */ + while ((ret = wtperf->conn->async_new_op( + wtperf->conn, wtperf->uris[i], "timeout=0", &cb, &asyncop)) == EBUSY) + (void)usleep(10000); + if (ret != 0) + return (ret); + + asyncop->app_private = &tables; + if ((ret = asyncop->compact(asyncop)) != 0) { + lprintf(wtperf, ret, 0, "Async compact failed."); + return (ret); + } + } + if ((ret = wtperf->conn->async_flush(wtperf->conn)) != 0) { + lprintf(wtperf, ret, 0, "Populate async flush failed."); + return (ret); + } + __wt_epoch(NULL, &stop); + lprintf(wtperf, 0, 1, "Compact completed in %" PRIu64 " seconds", + (uint64_t)(WT_TIMEDIFF_SEC(stop, start))); + assert(tables == 0); + } + + /* Stop cycling idle tables. */ + stop_idle_table_cycle(wtperf, idle_table_cycle_thread); + + return (0); } static int close_reopen(WTPERF *wtperf) { - CONFIG_OPTS *opts; - int ret; - - opts = wtperf->opts; - - if (opts->in_memory) - return (0); - - if (!opts->readonly && !opts->reopen_connection) - return (0); - /* - * Reopen the connection. We do this so that the workload phase always - * starts with the on-disk files, and so that read-only workloads can - * be identified. This is particularly important for LSM, where the - * merge algorithm is more aggressive for read-only trees. - */ - /* wtperf->conn is released no matter the return value from close(). */ - ret = wtperf->conn->close(wtperf->conn, NULL); - wtperf->conn = NULL; - if (ret != 0) { - lprintf(wtperf, ret, 0, "Closing the connection failed"); - return (ret); - } - if ((ret = wiredtiger_open( - wtperf->home, NULL, wtperf->reopen_config, &wtperf->conn)) != 0) { - lprintf(wtperf, ret, 0, "Re-opening the connection failed"); - return (ret); - } - /* - * If we started async threads only for the purposes of compact, - * then turn it off before starting the workload so that those extra - * threads looking for work that will never arrive don't affect - * performance. - */ - if (opts->compact && !wtperf->use_asyncops) { - if ((ret = wtperf->conn->reconfigure( - wtperf->conn, "async=(enabled=false)")) != 0) { - lprintf(wtperf, ret, 0, "Reconfigure async off failed"); - return (ret); - } - } - return (0); + CONFIG_OPTS *opts; + int ret; + + opts = wtperf->opts; + + if (opts->in_memory) + return (0); + + if (!opts->readonly && !opts->reopen_connection) + return (0); + /* + * Reopen the connection. We do this so that the workload phase always starts with the on-disk + * files, and so that read-only workloads can be identified. This is particularly important for + * LSM, where the merge algorithm is more aggressive for read-only trees. + */ + /* wtperf->conn is released no matter the return value from close(). */ + ret = wtperf->conn->close(wtperf->conn, NULL); + wtperf->conn = NULL; + if (ret != 0) { + lprintf(wtperf, ret, 0, "Closing the connection failed"); + return (ret); + } + if ((ret = wiredtiger_open(wtperf->home, NULL, wtperf->reopen_config, &wtperf->conn)) != 0) { + lprintf(wtperf, ret, 0, "Re-opening the connection failed"); + return (ret); + } + /* + * If we started async threads only for the purposes of compact, then turn it off before + * starting the workload so that those extra threads looking for work that will never arrive + * don't affect performance. + */ + if (opts->compact && !wtperf->use_asyncops) { + if ((ret = wtperf->conn->reconfigure(wtperf->conn, "async=(enabled=false)")) != 0) { + lprintf(wtperf, ret, 0, "Reconfigure async off failed"); + return (ret); + } + } + return (0); } static int execute_workload(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WORKLOAD *workp; - WTPERF_THREAD *threads; - WT_CONNECTION *conn; - WT_SESSION **sessions; - WT_THREAD_CALLBACK(*pfunc)(void *); - wt_thread_t idle_table_cycle_thread; - uint64_t last_ckpts, last_scans; - uint64_t last_inserts, last_reads, last_truncates, last_updates; - uint32_t interval, run_ops, run_time; - u_int i; - int ret; - - opts = wtperf->opts; - - wtperf->insert_key = 0; - wtperf->insert_ops = wtperf->read_ops = wtperf->truncate_ops = 0; - wtperf->update_ops = 0; - - last_ckpts = last_scans = 0; - last_inserts = last_reads = last_truncates = last_updates = 0; - ret = 0; - - sessions = NULL; - - /* Start cycling idle tables. */ - start_idle_table_cycle(wtperf, &idle_table_cycle_thread); - - if (opts->warmup != 0) - wtperf->in_warmup = true; - - /* Allocate memory for the worker threads. */ - wtperf->workers = - dcalloc((size_t)wtperf->workers_cnt, sizeof(WTPERF_THREAD)); - - if (wtperf->use_asyncops) { - lprintf(wtperf, 0, 1, "Starting %" PRIu32 " async thread(s)", - opts->async_threads); - pfunc = worker_async; - } else - pfunc = worker; - - if (opts->session_count_idle != 0) { - sessions = dcalloc((size_t)opts->session_count_idle, - sizeof(WT_SESSION *)); - conn = wtperf->conn; - for (i = 0; i < opts->session_count_idle; ++i) - if ((ret = conn->open_session(conn, - NULL, opts->sess_config, &sessions[i])) != 0) { - lprintf(wtperf, ret, 0, - "execute_workload: idle open_session"); - goto err; - } - } - /* Start each workload. */ - for (threads = wtperf->workers, i = 0, - workp = wtperf->workload; i < wtperf->workload_cnt; ++i, ++workp) { - lprintf(wtperf, 0, 1, - "Starting workload #%u: %" PRId64 " threads, inserts=%" - PRId64 ", reads=%" PRId64 ", updates=%" PRId64 - ", truncate=%" PRId64 ", throttle=%" PRIu64, - i + 1, workp->threads, workp->insert, - workp->read, workp->update, workp->truncate, - workp->throttle); - - /* Figure out the workload's schedule. */ - if ((ret = run_mix_schedule(wtperf, workp)) != 0) - goto err; - - /* Start the workload's threads. */ - start_threads( - wtperf, workp, threads, (u_int)workp->threads, pfunc); - threads += workp->threads; - } - - if (opts->warmup != 0) { - lprintf(wtperf, 0, 1, - "Waiting for warmup duration of %" PRIu32, opts->warmup); - sleep(opts->warmup); - wtperf->in_warmup = false; - } - - for (interval = opts->report_interval, - run_time = opts->run_time, run_ops = opts->run_ops; - !wtperf->error;) { - /* - * Sleep for one second at a time. - * If we are tracking run time, check to see if we're done, and - * if we're only tracking run time, go back to sleep. - */ - sleep(1); - if (run_time != 0) { - if (--run_time == 0) - break; - if (!interval && !run_ops) - continue; - } - - /* Sum the operations we've done. */ - wtperf->ckpt_ops = sum_ckpt_ops(wtperf); - wtperf->scan_ops = sum_scan_ops(wtperf); - wtperf->insert_ops = sum_insert_ops(wtperf); - wtperf->read_ops = sum_read_ops(wtperf); - wtperf->update_ops = sum_update_ops(wtperf); - wtperf->truncate_ops = sum_truncate_ops(wtperf); - - /* If we're checking total operations, see if we're done. */ - if (run_ops != 0 && run_ops <= - wtperf->insert_ops + wtperf->read_ops + wtperf->update_ops) - break; - - /* If writing out throughput information, see if it's time. */ - if (interval == 0 || --interval > 0) - continue; - interval = opts->report_interval; - wtperf->totalsec += opts->report_interval; - - lprintf(wtperf, 0, 1, - "%" PRIu64 " reads, %" PRIu64 " inserts, %" PRIu64 - " updates, %" PRIu64 " truncates, %" PRIu64 - " checkpoints, %" PRIu64 " scans in %" PRIu32 - " secs (%" PRIu32 " total secs)", - wtperf->read_ops - last_reads, - wtperf->insert_ops - last_inserts, - wtperf->update_ops - last_updates, - wtperf->truncate_ops - last_truncates, - wtperf->ckpt_ops - last_ckpts, - wtperf->scan_ops - last_scans, - opts->report_interval, wtperf->totalsec); - last_reads = wtperf->read_ops; - last_inserts = wtperf->insert_ops; - last_updates = wtperf->update_ops; - last_truncates = wtperf->truncate_ops; - last_ckpts = wtperf->ckpt_ops; - last_scans = wtperf->scan_ops; - } - - /* Notify the worker threads they are done. */ -err: wtperf->stop = true; - - /* Stop cycling idle tables. */ - stop_idle_table_cycle(wtperf, idle_table_cycle_thread); - - stop_threads((u_int)wtperf->workers_cnt, wtperf->workers); - - /* Drop tables if configured to and this isn't an error path */ - if (ret == 0 && - opts->drop_tables && (ret = drop_all_tables(wtperf)) != 0) - lprintf(wtperf, ret, 0, "Drop tables failed."); - - free(sessions); - /* Report if any worker threads didn't finish. */ - if (wtperf->error) { - lprintf(wtperf, WT_ERROR, 0, - "Worker thread(s) exited without finishing."); - if (ret == 0) - ret = WT_ERROR; - } - return (ret); + CONFIG_OPTS *opts; + WORKLOAD *workp; + WTPERF_THREAD *threads; + WT_CONNECTION *conn; + WT_SESSION **sessions; + WT_THREAD_CALLBACK (*pfunc)(void *); + wt_thread_t idle_table_cycle_thread; + uint64_t last_ckpts, last_scans; + uint64_t last_inserts, last_reads, last_truncates, last_updates; + uint32_t interval, run_ops, run_time; + u_int i; + int ret; + + opts = wtperf->opts; + + wtperf->insert_key = 0; + wtperf->insert_ops = wtperf->read_ops = wtperf->truncate_ops = 0; + wtperf->update_ops = 0; + + last_ckpts = last_scans = 0; + last_inserts = last_reads = last_truncates = last_updates = 0; + ret = 0; + + sessions = NULL; + + /* Start cycling idle tables. */ + start_idle_table_cycle(wtperf, &idle_table_cycle_thread); + + if (opts->warmup != 0) + wtperf->in_warmup = true; + + /* Allocate memory for the worker threads. */ + wtperf->workers = dcalloc((size_t)wtperf->workers_cnt, sizeof(WTPERF_THREAD)); + + if (wtperf->use_asyncops) { + lprintf(wtperf, 0, 1, "Starting %" PRIu32 " async thread(s)", opts->async_threads); + pfunc = worker_async; + } else + pfunc = worker; + + if (opts->session_count_idle != 0) { + sessions = dcalloc((size_t)opts->session_count_idle, sizeof(WT_SESSION *)); + conn = wtperf->conn; + for (i = 0; i < opts->session_count_idle; ++i) + if ((ret = conn->open_session(conn, NULL, opts->sess_config, &sessions[i])) != 0) { + lprintf(wtperf, ret, 0, "execute_workload: idle open_session"); + goto err; + } + } + /* Start each workload. */ + for (threads = wtperf->workers, i = 0, workp = wtperf->workload; i < wtperf->workload_cnt; + ++i, ++workp) { + lprintf(wtperf, 0, 1, + "Starting workload #%u: %" PRId64 " threads, inserts=%" PRId64 ", reads=%" PRId64 + ", updates=%" PRId64 ", truncate=%" PRId64 ", throttle=%" PRIu64, + i + 1, workp->threads, workp->insert, workp->read, workp->update, workp->truncate, + workp->throttle); + + /* Figure out the workload's schedule. */ + if ((ret = run_mix_schedule(wtperf, workp)) != 0) + goto err; + + /* Start the workload's threads. */ + start_threads(wtperf, workp, threads, (u_int)workp->threads, pfunc); + threads += workp->threads; + } + + if (opts->warmup != 0) { + lprintf(wtperf, 0, 1, "Waiting for warmup duration of %" PRIu32, opts->warmup); + sleep(opts->warmup); + wtperf->in_warmup = false; + } + + for (interval = opts->report_interval, run_time = opts->run_time, run_ops = opts->run_ops; + !wtperf->error;) { + /* + * Sleep for one second at a time. If we are tracking run time, check to see if we're done, + * and if we're only tracking run time, go back to sleep. + */ + sleep(1); + if (run_time != 0) { + if (--run_time == 0) + break; + if (!interval && !run_ops) + continue; + } + + /* Sum the operations we've done. */ + wtperf->ckpt_ops = sum_ckpt_ops(wtperf); + wtperf->scan_ops = sum_scan_ops(wtperf); + wtperf->insert_ops = sum_insert_ops(wtperf); + wtperf->read_ops = sum_read_ops(wtperf); + wtperf->update_ops = sum_update_ops(wtperf); + wtperf->truncate_ops = sum_truncate_ops(wtperf); + + /* If we're checking total operations, see if we're done. */ + if (run_ops != 0 && run_ops <= wtperf->insert_ops + wtperf->read_ops + wtperf->update_ops) + break; + + /* If writing out throughput information, see if it's time. */ + if (interval == 0 || --interval > 0) + continue; + interval = opts->report_interval; + wtperf->totalsec += opts->report_interval; + + lprintf(wtperf, 0, 1, "%" PRIu64 " reads, %" PRIu64 " inserts, %" PRIu64 + " updates, %" PRIu64 " truncates, %" PRIu64 " checkpoints, %" PRIu64 + " scans in %" PRIu32 " secs (%" PRIu32 " total secs)", + wtperf->read_ops - last_reads, wtperf->insert_ops - last_inserts, + wtperf->update_ops - last_updates, wtperf->truncate_ops - last_truncates, + wtperf->ckpt_ops - last_ckpts, wtperf->scan_ops - last_scans, opts->report_interval, + wtperf->totalsec); + last_reads = wtperf->read_ops; + last_inserts = wtperf->insert_ops; + last_updates = wtperf->update_ops; + last_truncates = wtperf->truncate_ops; + last_ckpts = wtperf->ckpt_ops; + last_scans = wtperf->scan_ops; + } + +/* Notify the worker threads they are done. */ +err: + wtperf->stop = true; + + /* Stop cycling idle tables. */ + stop_idle_table_cycle(wtperf, idle_table_cycle_thread); + + stop_threads((u_int)wtperf->workers_cnt, wtperf->workers); + + /* Drop tables if configured to and this isn't an error path */ + if (ret == 0 && opts->drop_tables && (ret = drop_all_tables(wtperf)) != 0) + lprintf(wtperf, ret, 0, "Drop tables failed."); + + free(sessions); + /* Report if any worker threads didn't finish. */ + if (wtperf->error) { + lprintf(wtperf, WT_ERROR, 0, "Worker thread(s) exited without finishing."); + if (ret == 0) + ret = WT_ERROR; + } + return (ret); } /* - * Ensure that icount matches the number of records in the - * existing table. + * Ensure that icount matches the number of records in the existing table. */ static int find_table_count(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - uint32_t i, max_icount, table_icount; - int ret, t_ret; - char *key; - - opts = wtperf->opts; - conn = wtperf->conn; - - max_icount = 0; - if ((ret = conn->open_session( - conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, - "find_table_count: open_session failed"); - goto out; - } - for (i = 0; i < opts->table_count; i++) { - if ((ret = session->open_cursor(session, wtperf->uris[i], - NULL, NULL, &cursor)) != 0) { - lprintf(wtperf, ret, 0, - "find_table_count: open_cursor failed"); - goto err; - } - if ((ret = cursor->prev(cursor)) != 0) { - lprintf(wtperf, ret, 0, - "find_table_count: cursor prev failed"); - goto err; - } - if ((ret = cursor->get_key(cursor, &key)) != 0) { - lprintf(wtperf, ret, 0, - "find_table_count: cursor get_key failed"); - goto err; - } - table_icount = (uint32_t)atoi(key); - if (table_icount > max_icount) - max_icount = table_icount; - - if ((ret = cursor->close(cursor)) != 0) { - lprintf(wtperf, ret, 0, - "find_table_count: cursor close failed"); - goto err; - } - } -err: if ((t_ret = session->close(session, NULL)) != 0) { - if (ret == 0) - ret = t_ret; - lprintf(wtperf, ret, 0, - "find_table_count: session close failed"); - } - opts->icount = max_icount; -out: return (ret); + CONFIG_OPTS *opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + uint32_t i, max_icount, table_icount; + int ret, t_ret; + char *key; + + opts = wtperf->opts; + conn = wtperf->conn; + + max_icount = 0; + if ((ret = conn->open_session(conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "find_table_count: open_session failed"); + goto out; + } + for (i = 0; i < opts->table_count; i++) { + if ((ret = session->open_cursor(session, wtperf->uris[i], NULL, NULL, &cursor)) != 0) { + lprintf(wtperf, ret, 0, "find_table_count: open_cursor failed"); + goto err; + } + if ((ret = cursor->prev(cursor)) != 0) { + lprintf(wtperf, ret, 0, "find_table_count: cursor prev failed"); + goto err; + } + if ((ret = cursor->get_key(cursor, &key)) != 0) { + lprintf(wtperf, ret, 0, "find_table_count: cursor get_key failed"); + goto err; + } + table_icount = (uint32_t)atoi(key); + if (table_icount > max_icount) + max_icount = table_icount; + + if ((ret = cursor->close(cursor)) != 0) { + lprintf(wtperf, ret, 0, "find_table_count: cursor close failed"); + goto err; + } + } +err: + if ((t_ret = session->close(session, NULL)) != 0) { + if (ret == 0) + ret = t_ret; + lprintf(wtperf, ret, 0, "find_table_count: session close failed"); + } + opts->icount = max_icount; +out: + return (ret); } /* @@ -2122,481 +1938,440 @@ out: return (ret); static void create_uris(WTPERF *wtperf) { - CONFIG_OPTS *opts; - size_t len; - uint32_t i, total_table_count; - - opts = wtperf->opts; - - total_table_count = opts->table_count + opts->scan_table_count; - wtperf->uris = dcalloc(total_table_count, sizeof(char *)); - len = strlen("table:") + strlen(opts->table_name) + 20; - for (i = 0; i < total_table_count; i++) { - /* If there is only one table, just use the base name. */ - wtperf->uris[i] = dmalloc(len); - if (total_table_count == 1) - testutil_check(__wt_snprintf(wtperf->uris[i], - len, "table:%s", opts->table_name)); - else - testutil_check(__wt_snprintf(wtperf->uris[i], - len, "table:%s%05" PRIu32, opts->table_name, i)); - } - - /* Create the log-like-table URI. */ - len = strlen("table:") + - strlen(opts->table_name) + strlen("_log_table") + 1; - wtperf->log_table_uri = dmalloc(len); - testutil_check(__wt_snprintf(wtperf->log_table_uri, - len, "table:%s_log_table", opts->table_name)); + CONFIG_OPTS *opts; + size_t len; + uint32_t i, total_table_count; + + opts = wtperf->opts; + + total_table_count = opts->table_count + opts->scan_table_count; + wtperf->uris = dcalloc(total_table_count, sizeof(char *)); + len = strlen("table:") + strlen(opts->table_name) + 20; + for (i = 0; i < total_table_count; i++) { + /* If there is only one table, just use the base name. */ + wtperf->uris[i] = dmalloc(len); + if (total_table_count == 1) + testutil_check(__wt_snprintf(wtperf->uris[i], len, "table:%s", opts->table_name)); + else + testutil_check( + __wt_snprintf(wtperf->uris[i], len, "table:%s%05" PRIu32, opts->table_name, i)); + } + + /* Create the log-like-table URI. */ + len = strlen("table:") + strlen(opts->table_name) + strlen("_log_table") + 1; + wtperf->log_table_uri = dmalloc(len); + testutil_check( + __wt_snprintf(wtperf->log_table_uri, len, "table:%s_log_table", opts->table_name)); } static int create_tables(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WT_SESSION *session; - size_t i; - int ret; - uint32_t total_table_count; - char buf[512]; - - opts = wtperf->opts; - - if ((ret = wtperf->conn->open_session( - wtperf->conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, - "Error opening a session on %s", wtperf->home); - return (ret); - } - - for (i = 0; i < opts->table_count_idle; i++) { - testutil_check(__wt_snprintf( - buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); - if ((ret = session->create( - session, buf, opts->table_config)) != 0) { - lprintf(wtperf, ret, 0, - "Error creating idle table %s", buf); - return (ret); - } - } - if (opts->log_like_table && (ret = session->create(session, - wtperf->log_table_uri, "key_format=Q,value_format=S")) != 0) { - lprintf(wtperf, ret, 0, "Error creating log table %s", buf); - return (ret); - } - - total_table_count = opts->table_count + opts->scan_table_count; - for (i = 0; i < total_table_count; i++) { - if (opts->log_partial && i > 0) { - if (((ret = session->create(session, - wtperf->uris[i], wtperf->partial_config)) != 0)) { - lprintf(wtperf, ret, 0, - "Error creating table %s", wtperf->uris[i]); - return (ret); - } - } else if ((ret = session->create( - session, wtperf->uris[i], opts->table_config)) != 0) { - lprintf(wtperf, ret, 0, - "Error creating table %s", wtperf->uris[i]); - return (ret); - } - if (opts->index) { - testutil_check(__wt_snprintf(buf, 512, - "index:%s:val_idx", - wtperf->uris[i] + strlen("table:"))); - if ((ret = session->create( - session, buf, "columns=(val)")) != 0) { - lprintf(wtperf, ret, 0, - "Error creating index %s", buf); - return (ret); - } - } - } - - if ((ret = session->close(session, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error closing session"); - return (ret); - } - - return (0); + CONFIG_OPTS *opts; + WT_SESSION *session; + size_t i; + int ret; + uint32_t total_table_count; + char buf[512]; + + opts = wtperf->opts; + + if ((ret = wtperf->conn->open_session(wtperf->conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "Error opening a session on %s", wtperf->home); + return (ret); + } + + for (i = 0; i < opts->table_count_idle; i++) { + testutil_check(__wt_snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); + if ((ret = session->create(session, buf, opts->table_config)) != 0) { + lprintf(wtperf, ret, 0, "Error creating idle table %s", buf); + return (ret); + } + } + if (opts->log_like_table && + (ret = session->create(session, wtperf->log_table_uri, "key_format=Q,value_format=S")) != 0) { + lprintf(wtperf, ret, 0, "Error creating log table %s", buf); + return (ret); + } + + total_table_count = opts->table_count + opts->scan_table_count; + for (i = 0; i < total_table_count; i++) { + if (opts->log_partial && i > 0) { + if (((ret = session->create(session, wtperf->uris[i], wtperf->partial_config)) != 0)) { + lprintf(wtperf, ret, 0, "Error creating table %s", wtperf->uris[i]); + return (ret); + } + } else if ((ret = session->create(session, wtperf->uris[i], opts->table_config)) != 0) { + lprintf(wtperf, ret, 0, "Error creating table %s", wtperf->uris[i]); + return (ret); + } + if (opts->index) { + testutil_check( + __wt_snprintf(buf, 512, "index:%s:val_idx", wtperf->uris[i] + strlen("table:"))); + if ((ret = session->create(session, buf, "columns=(val)")) != 0) { + lprintf(wtperf, ret, 0, "Error creating index %s", buf); + return (ret); + } + } + } + + if ((ret = session->close(session, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Error closing session"); + return (ret); + } + + return (0); } /* * wtperf_copy -- - * Create a new WTPERF structure as a duplicate of a previous one. + * Create a new WTPERF structure as a duplicate of a previous one. */ static void wtperf_copy(const WTPERF *src, WTPERF **retp) { - CONFIG_OPTS *opts; - WTPERF *dest; - size_t i; - uint32_t total_table_count; + CONFIG_OPTS *opts; + WTPERF *dest; + size_t i; + uint32_t total_table_count; - opts = src->opts; - total_table_count = opts->table_count + opts->scan_table_count; + opts = src->opts; + total_table_count = opts->table_count + opts->scan_table_count; - dest = dcalloc(1, sizeof(WTPERF)); + dest = dcalloc(1, sizeof(WTPERF)); - /* - * Don't copy the home and monitor directories, they are filled in by - * our caller, explicitly. - */ + /* + * Don't copy the home and monitor directories, they are filled in by our caller, explicitly. + */ - if (src->partial_config != NULL) - dest->partial_config = dstrdup(src->partial_config); - if (src->reopen_config != NULL) - dest->reopen_config = dstrdup(src->reopen_config); + if (src->partial_config != NULL) + dest->partial_config = dstrdup(src->partial_config); + if (src->reopen_config != NULL) + dest->reopen_config = dstrdup(src->reopen_config); - if (src->uris != NULL) { - dest->uris = dcalloc(total_table_count, sizeof(char *)); - for (i = 0; i < total_table_count; i++) - dest->uris[i] = dstrdup(src->uris[i]); - } + if (src->uris != NULL) { + dest->uris = dcalloc(total_table_count, sizeof(char *)); + for (i = 0; i < total_table_count; i++) + dest->uris[i] = dstrdup(src->uris[i]); + } - if (src->async_config != NULL) - dest->async_config = dstrdup(src->async_config); + if (src->async_config != NULL) + dest->async_config = dstrdup(src->async_config); - dest->ckptthreads = NULL; - dest->scanthreads = NULL; - dest->popthreads = NULL; + dest->ckptthreads = NULL; + dest->scanthreads = NULL; + dest->popthreads = NULL; - dest->workers = NULL; - dest->workers_cnt = src->workers_cnt; - if (src->workload_cnt != 0) { - dest->workload_cnt = src->workload_cnt; - dest->workload = dcalloc(src->workload_cnt, sizeof(WORKLOAD)); - memcpy(dest->workload, - src->workload, src->workload_cnt * sizeof(WORKLOAD)); - } + dest->workers = NULL; + dest->workers_cnt = src->workers_cnt; + if (src->workload_cnt != 0) { + dest->workload_cnt = src->workload_cnt; + dest->workload = dcalloc(src->workload_cnt, sizeof(WORKLOAD)); + memcpy(dest->workload, src->workload, src->workload_cnt * sizeof(WORKLOAD)); + } - TAILQ_INIT(&dest->stone_head); + TAILQ_INIT(&dest->stone_head); - dest->opts = src->opts; + dest->opts = src->opts; - *retp = dest; + *retp = dest; } /* * wtperf_free -- - * Free any storage allocated in the WTPERF structure. + * Free any storage allocated in the WTPERF structure. */ static void wtperf_free(WTPERF *wtperf) { - CONFIG_OPTS *opts; - size_t i; + CONFIG_OPTS *opts; + size_t i; - opts = wtperf->opts; + opts = wtperf->opts; - free(wtperf->home); - free(wtperf->monitor_dir); - free(wtperf->partial_config); - free(wtperf->reopen_config); - free(wtperf->log_table_uri); + free(wtperf->home); + free(wtperf->monitor_dir); + free(wtperf->partial_config); + free(wtperf->reopen_config); + free(wtperf->log_table_uri); - if (wtperf->uris != NULL) { - for (i = 0; i < opts->table_count + opts->scan_table_count; i++) - free(wtperf->uris[i]); - free(wtperf->uris); - } + if (wtperf->uris != NULL) { + for (i = 0; i < opts->table_count + opts->scan_table_count; i++) + free(wtperf->uris[i]); + free(wtperf->uris); + } - free(wtperf->async_config); + free(wtperf->async_config); - free(wtperf->ckptthreads); - free(wtperf->scanthreads); - free(wtperf->popthreads); + free(wtperf->ckptthreads); + free(wtperf->scanthreads); + free(wtperf->popthreads); - free(wtperf->workers); - free(wtperf->workload); + free(wtperf->workers); + free(wtperf->workload); - cleanup_truncate_config(wtperf); + cleanup_truncate_config(wtperf); } /* * config_compress -- - * Parse the compression configuration. + * Parse the compression configuration. */ static int config_compress(WTPERF *wtperf) { - CONFIG_OPTS *opts; - int ret; - const char *s; - - opts = wtperf->opts; - ret = 0; - - s = opts->compression; - if (strcmp(s, "none") == 0) { - wtperf->compress_ext = NULL; - wtperf->compress_table = NULL; - } else if (strcmp(s, "lz4") == 0) { + CONFIG_OPTS *opts; + int ret; + const char *s; + + opts = wtperf->opts; + ret = 0; + + s = opts->compression; + if (strcmp(s, "none") == 0) { + wtperf->compress_ext = NULL; + wtperf->compress_table = NULL; + } else if (strcmp(s, "lz4") == 0) { #ifndef HAVE_BUILTIN_EXTENSION_LZ4 - wtperf->compress_ext = LZ4_EXT; + wtperf->compress_ext = LZ4_EXT; #endif - wtperf->compress_table = LZ4_BLK; - } else if (strcmp(s, "snappy") == 0) { + wtperf->compress_table = LZ4_BLK; + } else if (strcmp(s, "snappy") == 0) { #ifndef HAVE_BUILTIN_EXTENSION_SNAPPY - wtperf->compress_ext = SNAPPY_EXT; + wtperf->compress_ext = SNAPPY_EXT; #endif - wtperf->compress_table = SNAPPY_BLK; - } else if (strcmp(s, "zlib") == 0) { + wtperf->compress_table = SNAPPY_BLK; + } else if (strcmp(s, "zlib") == 0) { #ifndef HAVE_BUILTIN_EXTENSION_ZLIB - wtperf->compress_ext = ZLIB_EXT; + wtperf->compress_ext = ZLIB_EXT; #endif - wtperf->compress_table = ZLIB_BLK; - } else if (strcmp(s, "zstd") == 0) { + wtperf->compress_table = ZLIB_BLK; + } else if (strcmp(s, "zstd") == 0) { #ifndef HAVE_BUILTIN_EXTENSION_ZSTD - wtperf->compress_ext = ZSTD_EXT; + wtperf->compress_ext = ZSTD_EXT; #endif - wtperf->compress_table = ZSTD_BLK; - } else { - fprintf(stderr, - "invalid compression configuration: %s\n", s); - ret = EINVAL; - } - return (ret); - + wtperf->compress_table = ZSTD_BLK; + } else { + fprintf(stderr, "invalid compression configuration: %s\n", s); + ret = EINVAL; + } + return (ret); } static int start_all_runs(WTPERF *wtperf) { - CONFIG_OPTS *opts; - WTPERF *next_wtperf, **wtperfs; - size_t i, len; - wt_thread_t *threads; - int ret; - - opts = wtperf->opts; - wtperfs = NULL; - ret = 0; - - if (opts->database_count == 1) - return (start_run(wtperf)); - - /* Allocate an array to hold our WTPERF copies. */ - wtperfs = dcalloc(opts->database_count, sizeof(WTPERF *)); - - /* Allocate an array to hold our thread IDs. */ - threads = dcalloc(opts->database_count, sizeof(*threads)); - - for (i = 0; i < opts->database_count; i++) { - wtperf_copy(wtperf, &next_wtperf); - wtperfs[i] = next_wtperf; - - /* - * Set up unique home/monitor directories for each database. - * Re-create the directories if creating the databases. - */ - len = strlen(wtperf->home) + 5; - next_wtperf->home = dmalloc(len); - testutil_check(__wt_snprintf( - next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i)); - if (opts->create != 0) - recreate_dir(next_wtperf->home); - - len = strlen(wtperf->monitor_dir) + 5; - next_wtperf->monitor_dir = dmalloc(len); - testutil_check(__wt_snprintf(next_wtperf->monitor_dir, - len, "%s/D%02d", wtperf->monitor_dir, (int)i)); - if (opts->create != 0 && - strcmp(next_wtperf->home, next_wtperf->monitor_dir) != 0) - recreate_dir(next_wtperf->monitor_dir); - - testutil_check(__wt_thread_create(NULL, - &threads[i], thread_run_wtperf, next_wtperf)); - } - - /* Wait for threads to finish. */ - for (i = 0; i < opts->database_count; i++) - testutil_check(__wt_thread_join(NULL, &threads[i])); - - for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { - wtperf_free(wtperfs[i]); - free(wtperfs[i]); - } - free(wtperfs); - free(threads); - - return (ret); + CONFIG_OPTS *opts; + WTPERF *next_wtperf, **wtperfs; + size_t i, len; + wt_thread_t *threads; + int ret; + + opts = wtperf->opts; + wtperfs = NULL; + ret = 0; + + if (opts->database_count == 1) + return (start_run(wtperf)); + + /* Allocate an array to hold our WTPERF copies. */ + wtperfs = dcalloc(opts->database_count, sizeof(WTPERF *)); + + /* Allocate an array to hold our thread IDs. */ + threads = dcalloc(opts->database_count, sizeof(*threads)); + + for (i = 0; i < opts->database_count; i++) { + wtperf_copy(wtperf, &next_wtperf); + wtperfs[i] = next_wtperf; + + /* + * Set up unique home/monitor directories for each database. Re-create the directories if + * creating the databases. + */ + len = strlen(wtperf->home) + 5; + next_wtperf->home = dmalloc(len); + testutil_check(__wt_snprintf(next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i)); + if (opts->create != 0) + recreate_dir(next_wtperf->home); + + len = strlen(wtperf->monitor_dir) + 5; + next_wtperf->monitor_dir = dmalloc(len); + testutil_check( + __wt_snprintf(next_wtperf->monitor_dir, len, "%s/D%02d", wtperf->monitor_dir, (int)i)); + if (opts->create != 0 && strcmp(next_wtperf->home, next_wtperf->monitor_dir) != 0) + recreate_dir(next_wtperf->monitor_dir); + + testutil_check(__wt_thread_create(NULL, &threads[i], thread_run_wtperf, next_wtperf)); + } + + /* Wait for threads to finish. */ + for (i = 0; i < opts->database_count; i++) + testutil_check(__wt_thread_join(NULL, &threads[i])); + + for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { + wtperf_free(wtperfs[i]); + free(wtperfs[i]); + } + free(wtperfs); + free(threads); + + return (ret); } /* Run an instance of wtperf for a given configuration. */ static WT_THREAD_RET thread_run_wtperf(void *arg) { - WTPERF *wtperf; - int ret; + WTPERF *wtperf; + int ret; - wtperf = (WTPERF *)arg; - if ((ret = start_run(wtperf)) != 0) - lprintf(wtperf, ret, 0, "Run failed for: %s.", wtperf->home); - return (WT_THREAD_RET_VALUE); + wtperf = (WTPERF *)arg; + if ((ret = start_run(wtperf)) != 0) + lprintf(wtperf, ret, 0, "Run failed for: %s.", wtperf->home); + return (WT_THREAD_RET_VALUE); } static int start_run(WTPERF *wtperf) { - CONFIG_OPTS *opts; - wt_thread_t monitor_thread; - uint64_t total_ops; - uint32_t run_time; - int monitor_created, ret, t_ret; - - opts = wtperf->opts; - monitor_created = ret = 0; - /* [-Wconditional-uninitialized] */ - memset(&monitor_thread, 0, sizeof(monitor_thread)); - - if ((ret = setup_log_file(wtperf)) != 0) - goto err; - - if ((ret = wiredtiger_open( /* Open the real connection. */ - wtperf->home, NULL, opts->conn_config, &wtperf->conn)) != 0) { - lprintf(wtperf, ret, 0, "Error connecting to %s", wtperf->home); - goto err; - } - - create_uris(wtperf); - - /* If creating, create the tables. */ - if (opts->create != 0 && (ret = create_tables(wtperf)) != 0) - goto err; - - /* Start the monitor thread. */ - if (opts->sample_interval != 0) { - testutil_check(__wt_thread_create( - NULL, &monitor_thread, monitor, wtperf)); - monitor_created = 1; - } - - /* If creating, populate the table. */ - if (opts->create != 0 && execute_populate(wtperf) != 0) - goto err; - - /* Optional workload. */ - if (wtperf->workers_cnt != 0 && - (opts->run_time != 0 || opts->run_ops != 0)) { - /* - * If we have a workload, close and reopen the connection so - * that LSM can detect read-only workloads. - */ - if (close_reopen(wtperf) != 0) - goto err; - - /* Didn't create, set insert count. */ - if (opts->create == 0 && - opts->random_range == 0 && find_table_count(wtperf) != 0) - goto err; - /* Start the checkpoint thread. */ - if (opts->checkpoint_threads != 0) { - lprintf(wtperf, 0, 1, - "Starting %" PRIu32 " checkpoint thread(s)", - opts->checkpoint_threads); - wtperf->ckptthreads = dcalloc( - opts->checkpoint_threads, sizeof(WTPERF_THREAD)); - start_threads(wtperf, NULL, wtperf->ckptthreads, - opts->checkpoint_threads, checkpoint_worker); - } - /* Start the scan thread. */ - if (opts->scan_interval != 0) { - lprintf(wtperf, 0, 1, - "Starting 1 scan thread"); - wtperf->scanthreads = dcalloc( - 1, sizeof(WTPERF_THREAD)); - start_threads(wtperf, NULL, wtperf->scanthreads, - 1, scan_worker); - } - if (opts->pre_load_data) - pre_load_data(wtperf); - - /* Execute the workload. */ - if ((ret = execute_workload(wtperf)) != 0) - goto err; - - /* One final summation of the operations we've completed. */ - wtperf->read_ops = sum_read_ops(wtperf); - wtperf->insert_ops = sum_insert_ops(wtperf); - wtperf->truncate_ops = sum_truncate_ops(wtperf); - wtperf->update_ops = sum_update_ops(wtperf); - wtperf->ckpt_ops = sum_ckpt_ops(wtperf); - wtperf->scan_ops = sum_scan_ops(wtperf); - total_ops = - wtperf->read_ops + wtperf->insert_ops + wtperf->update_ops; - - run_time = opts->run_time == 0 ? 1 : opts->run_time; - lprintf(wtperf, 0, 1, - "Executed %" PRIu64 " read operations (%" PRIu64 - "%%) %" PRIu64 " ops/sec", - wtperf->read_ops, (wtperf->read_ops * 100) / total_ops, - wtperf->read_ops / run_time); - lprintf(wtperf, 0, 1, - "Executed %" PRIu64 " insert operations (%" PRIu64 - "%%) %" PRIu64 " ops/sec", - wtperf->insert_ops, (wtperf->insert_ops * 100) / total_ops, - wtperf->insert_ops / run_time); - lprintf(wtperf, 0, 1, - "Executed %" PRIu64 " truncate operations (%" PRIu64 - "%%) %" PRIu64 " ops/sec", - wtperf->truncate_ops, - (wtperf->truncate_ops * 100) / total_ops, - wtperf->truncate_ops / run_time); - lprintf(wtperf, 0, 1, - "Executed %" PRIu64 " update operations (%" PRIu64 - "%%) %" PRIu64 " ops/sec", - wtperf->update_ops, (wtperf->update_ops * 100) / total_ops, - wtperf->update_ops / run_time); - lprintf(wtperf, 0, 1, - "Executed %" PRIu64 " checkpoint operations", - wtperf->ckpt_ops); - lprintf(wtperf, 0, 1, - "Executed %" PRIu64 " scan operations", - wtperf->scan_ops); - - latency_print(wtperf); - } - - if (0) { -err: if (ret == 0) - ret = EXIT_FAILURE; - } - - /* Notify the worker threads they are done. */ - wtperf->stop = true; - - stop_threads(1, wtperf->ckptthreads); - stop_threads(1, wtperf->scanthreads); - - if (monitor_created != 0) - testutil_check(__wt_thread_join(NULL, &monitor_thread)); - - if (wtperf->conn != NULL && opts->close_conn && - (t_ret = wtperf->conn->close(wtperf->conn, NULL)) != 0) { - lprintf(wtperf, t_ret, 0, - "Error closing connection to %s", wtperf->home); - if (ret == 0) - ret = t_ret; - } - - if (ret == 0) { - if (opts->run_time == 0 && opts->run_ops == 0) - lprintf(wtperf, 0, 1, "Run completed"); - else - lprintf(wtperf, 0, 1, "Run completed: %" PRIu32 " %s", - opts->run_time == 0 ? - opts->run_ops : opts->run_time, - opts->run_time == 0 ? "operations" : "seconds"); - } - - if (wtperf->logf != NULL) { - if ((t_ret = fflush(wtperf->logf)) != 0 && ret == 0) - ret = t_ret; - if ((t_ret = fclose(wtperf->logf)) != 0 && ret == 0) - ret = t_ret; - } - return (ret); + CONFIG_OPTS *opts; + wt_thread_t monitor_thread; + uint64_t total_ops; + uint32_t run_time; + int monitor_created, ret, t_ret; + + opts = wtperf->opts; + monitor_created = ret = 0; + /* [-Wconditional-uninitialized] */ + memset(&monitor_thread, 0, sizeof(monitor_thread)); + + if ((ret = setup_log_file(wtperf)) != 0) + goto err; + + if ((ret = wiredtiger_open(/* Open the real connection. */ + wtperf->home, NULL, opts->conn_config, &wtperf->conn)) != 0) { + lprintf(wtperf, ret, 0, "Error connecting to %s", wtperf->home); + goto err; + } + + create_uris(wtperf); + + /* If creating, create the tables. */ + if (opts->create != 0 && (ret = create_tables(wtperf)) != 0) + goto err; + + /* Start the monitor thread. */ + if (opts->sample_interval != 0) { + testutil_check(__wt_thread_create(NULL, &monitor_thread, monitor, wtperf)); + monitor_created = 1; + } + + /* If creating, populate the table. */ + if (opts->create != 0 && execute_populate(wtperf) != 0) + goto err; + + /* Optional workload. */ + if (wtperf->workers_cnt != 0 && (opts->run_time != 0 || opts->run_ops != 0)) { + /* + * If we have a workload, close and reopen the connection so that LSM can detect read-only + * workloads. + */ + if (close_reopen(wtperf) != 0) + goto err; + + /* Didn't create, set insert count. */ + if (opts->create == 0 && opts->random_range == 0 && find_table_count(wtperf) != 0) + goto err; + /* Start the checkpoint thread. */ + if (opts->checkpoint_threads != 0) { + lprintf( + wtperf, 0, 1, "Starting %" PRIu32 " checkpoint thread(s)", opts->checkpoint_threads); + wtperf->ckptthreads = dcalloc(opts->checkpoint_threads, sizeof(WTPERF_THREAD)); + start_threads( + wtperf, NULL, wtperf->ckptthreads, opts->checkpoint_threads, checkpoint_worker); + } + /* Start the scan thread. */ + if (opts->scan_interval != 0) { + lprintf(wtperf, 0, 1, "Starting 1 scan thread"); + wtperf->scanthreads = dcalloc(1, sizeof(WTPERF_THREAD)); + start_threads(wtperf, NULL, wtperf->scanthreads, 1, scan_worker); + } + if (opts->pre_load_data) + pre_load_data(wtperf); + + /* Execute the workload. */ + if ((ret = execute_workload(wtperf)) != 0) + goto err; + + /* One final summation of the operations we've completed. */ + wtperf->read_ops = sum_read_ops(wtperf); + wtperf->insert_ops = sum_insert_ops(wtperf); + wtperf->truncate_ops = sum_truncate_ops(wtperf); + wtperf->update_ops = sum_update_ops(wtperf); + wtperf->ckpt_ops = sum_ckpt_ops(wtperf); + wtperf->scan_ops = sum_scan_ops(wtperf); + total_ops = wtperf->read_ops + wtperf->insert_ops + wtperf->update_ops; + + run_time = opts->run_time == 0 ? 1 : opts->run_time; + lprintf(wtperf, 0, 1, + "Executed %" PRIu64 " read operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", + wtperf->read_ops, (wtperf->read_ops * 100) / total_ops, wtperf->read_ops / run_time); + lprintf(wtperf, 0, 1, + "Executed %" PRIu64 " insert operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", + wtperf->insert_ops, (wtperf->insert_ops * 100) / total_ops, + wtperf->insert_ops / run_time); + lprintf(wtperf, 0, 1, + "Executed %" PRIu64 " truncate operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", + wtperf->truncate_ops, (wtperf->truncate_ops * 100) / total_ops, + wtperf->truncate_ops / run_time); + lprintf(wtperf, 0, 1, + "Executed %" PRIu64 " update operations (%" PRIu64 "%%) %" PRIu64 " ops/sec", + wtperf->update_ops, (wtperf->update_ops * 100) / total_ops, + wtperf->update_ops / run_time); + lprintf(wtperf, 0, 1, "Executed %" PRIu64 " checkpoint operations", wtperf->ckpt_ops); + lprintf(wtperf, 0, 1, "Executed %" PRIu64 " scan operations", wtperf->scan_ops); + + latency_print(wtperf); + } + + if (0) { +err: + if (ret == 0) + ret = EXIT_FAILURE; + } + + /* Notify the worker threads they are done. */ + wtperf->stop = true; + + stop_threads(1, wtperf->ckptthreads); + stop_threads(1, wtperf->scanthreads); + + if (monitor_created != 0) + testutil_check(__wt_thread_join(NULL, &monitor_thread)); + + if (wtperf->conn != NULL && opts->close_conn && + (t_ret = wtperf->conn->close(wtperf->conn, NULL)) != 0) { + lprintf(wtperf, t_ret, 0, "Error closing connection to %s", wtperf->home); + if (ret == 0) + ret = t_ret; + } + + if (ret == 0) { + if (opts->run_time == 0 && opts->run_ops == 0) + lprintf(wtperf, 0, 1, "Run completed"); + else + lprintf(wtperf, 0, 1, "Run completed: %" PRIu32 " %s", + opts->run_time == 0 ? opts->run_ops : opts->run_time, + opts->run_time == 0 ? "operations" : "seconds"); + } + + if (wtperf->logf != NULL) { + if ((t_ret = fflush(wtperf->logf)) != 0 && ret == 0) + ret = t_ret; + if ((t_ret = fclose(wtperf->logf)) != 0 && ret == 0) + ret = t_ret; + } + return (ret); } extern int __wt_optind, __wt_optreset; @@ -2604,554 +2379,507 @@ extern char *__wt_optarg; /* * usage -- - * wtperf usage print, no error. + * wtperf usage print, no error. */ static void usage(void) { - printf("wtperf [-C config] " - "[-h home] [-O file] [-o option] [-T config]\n"); - printf("\t-C <string> additional connection configuration\n"); - printf("\t (added to option conn_config)\n"); - printf("\t-h <string> Wired Tiger home must exist, default WT_TEST\n"); - printf("\t-O <file> file contains options as listed below\n"); - printf("\t-o option=val[,option=val,...] set options listed below\n"); - printf("\t-T <string> additional table configuration\n"); - printf("\t (added to option table_config)\n"); - printf("\n"); - config_opt_usage(); + printf( + "wtperf [-C config] " + "[-h home] [-O file] [-o option] [-T config]\n"); + printf("\t-C <string> additional connection configuration\n"); + printf("\t (added to option conn_config)\n"); + printf("\t-h <string> Wired Tiger home must exist, default WT_TEST\n"); + printf("\t-O <file> file contains options as listed below\n"); + printf("\t-o option=val[,option=val,...] set options listed below\n"); + printf("\t-T <string> additional table configuration\n"); + printf("\t (added to option table_config)\n"); + printf("\n"); + config_opt_usage(); } int main(int argc, char *argv[]) { - CONFIG_OPTS *opts; - WTPERF *wtperf, _wtperf; - size_t pos, req_len, sreq_len; - bool monitor_set; - int ch, ret; - const char *cmdflags = "C:h:m:O:o:T:"; - const char *append_comma, *config_opts; - char *cc_buf, *path, *sess_cfg, *tc_buf, *user_cconfig, *user_tconfig; - - /* The first WTPERF structure (from which all others are derived). */ - wtperf = &_wtperf; - memset(wtperf, 0, sizeof(*wtperf)); - wtperf->home = dstrdup(DEFAULT_HOME); - wtperf->monitor_dir = dstrdup(DEFAULT_MONITOR_DIR); - TAILQ_INIT(&wtperf->stone_head); - config_opt_init(&wtperf->opts); - - opts = wtperf->opts; - monitor_set = false; - ret = 0; - config_opts = NULL; - cc_buf = sess_cfg = tc_buf = user_cconfig = user_tconfig = NULL; - - /* Do a basic validation of options, and home is needed before open. */ - while ((ch = __wt_getopt("wtperf", argc, argv, cmdflags)) != EOF) - switch (ch) { - case 'C': - if (user_cconfig == NULL) - user_cconfig = dstrdup(__wt_optarg); - else { - user_cconfig = drealloc(user_cconfig, - strlen(user_cconfig) + - strlen(__wt_optarg) + 2); - strcat(user_cconfig, ","); - strcat(user_cconfig, __wt_optarg); - } - break; - case 'h': - free(wtperf->home); - wtperf->home = dstrdup(__wt_optarg); - break; - case 'm': - free(wtperf->monitor_dir); - wtperf->monitor_dir = dstrdup(__wt_optarg); - monitor_set = true; - break; - case 'O': - config_opts = __wt_optarg; - break; - case 'T': - if (user_tconfig == NULL) - user_tconfig = dstrdup(__wt_optarg); - else { - user_tconfig = drealloc(user_tconfig, - strlen(user_tconfig) + - strlen(__wt_optarg) + 2); - strcat(user_tconfig, ","); - strcat(user_tconfig, __wt_optarg); - } - break; - case '?': - usage(); - goto einval; - } - - /* - * If the user did not specify a monitor directory then set the - * monitor directory to the home dir. - */ - if (!monitor_set) { - free(wtperf->monitor_dir); - wtperf->monitor_dir = dstrdup(wtperf->home); - } - - /* Parse configuration settings from configuration file. */ - if (config_opts != NULL && config_opt_file(wtperf, config_opts) != 0) - goto einval; - - /* Parse options that override values set via a configuration file. */ - __wt_optreset = __wt_optind = 1; - while ((ch = __wt_getopt("wtperf", argc, argv, cmdflags)) != EOF) - switch (ch) { - case 'o': - /* Allow -o key=value */ - if (config_opt_str(wtperf, __wt_optarg) != 0) - goto einval; - break; - } - - if (opts->populate_threads == 0 && opts->icount != 0) { - lprintf(wtperf, 1, 0, - "Cannot have 0 populate threads when icount is set\n"); - goto err; - } - - wtperf->async_config = NULL; - /* - * If the user specified async_threads we use async for all ops. - * If the user wants compaction, then we also enable async for - * the compact operation, but not for the workloads. - */ - if (opts->async_threads > 0) { - if (F_ISSET(wtperf, CFG_TRUNCATE)) { - lprintf(wtperf, - 1, 0, "Cannot run truncate and async\n"); - goto err; - } - wtperf->use_asyncops = true; - } - if (opts->compact && opts->async_threads == 0) - opts->async_threads = 2; - if (opts->async_threads > 0) { - /* - * The maximum number of async threads is two digits, so just - * use that to compute the space we need. Assume the default - * of 1024 for the max ops. Although we could bump that up - * to 4096 if needed. - */ - req_len = strlen(",async=(enabled=true,threads=)") + 4; - wtperf->async_config = dmalloc(req_len); - testutil_check(__wt_snprintf(wtperf->async_config, req_len, - ",async=(enabled=true,threads=%" PRIu32 ")", - opts->async_threads)); - } - if ((ret = config_compress(wtperf)) != 0) - goto err; - - /* You can't have truncate on a random collection. */ - if (F_ISSET(wtperf, CFG_TRUNCATE) && opts->random_range) { - lprintf(wtperf, 1, 0, "Cannot run truncate and random_range\n"); - goto err; - } - - /* We can't run truncate with more than one table. */ - if (F_ISSET(wtperf, CFG_TRUNCATE) && opts->table_count > 1) { - lprintf(wtperf, 1, 0, "Cannot truncate more than 1 table\n"); - goto err; - } - - /* Make stdout line buffered, so verbose output appears quickly. */ - __wt_stream_set_line_buffer(stdout); - - /* Concatenate non-default configuration strings. */ - if (user_cconfig != NULL || opts->session_count_idle > 0 || - wtperf->compress_ext != NULL || wtperf->async_config != NULL || - opts->in_memory) { - req_len = 20; - req_len += wtperf->async_config != NULL ? - strlen(wtperf->async_config) : 0; - req_len += wtperf->compress_ext != NULL ? - strlen(wtperf->compress_ext) : 0; - if (opts->session_count_idle > 0) { - sreq_len = strlen("session_max=") + 6; - req_len += sreq_len; - sess_cfg = dmalloc(sreq_len); - testutil_check(__wt_snprintf(sess_cfg, sreq_len, - "session_max=%" PRIu32, - opts->session_count_idle + - wtperf->workers_cnt + opts->populate_threads + 10)); - } - req_len += opts->in_memory ? strlen("in_memory=true") : 0; - req_len += user_cconfig != NULL ? strlen(user_cconfig) : 0; - cc_buf = dmalloc(req_len); - - pos = 0; - append_comma = ""; - if (wtperf->async_config != NULL && - strlen(wtperf->async_config) != 0) { - testutil_check(__wt_snprintf_len_incr( - cc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, wtperf->async_config)); - append_comma = ","; - } - if (wtperf->compress_ext != NULL && - strlen(wtperf->compress_ext) != 0) { - testutil_check(__wt_snprintf_len_incr( - cc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, wtperf->compress_ext)); - append_comma = ","; - } - if (opts->in_memory) { - testutil_check(__wt_snprintf_len_incr( - cc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, "in_memory=true")); - append_comma = ","; - } - if (sess_cfg != NULL && strlen(sess_cfg) != 0) { - testutil_check(__wt_snprintf_len_incr( - cc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, sess_cfg)); - append_comma = ","; - } - if (user_cconfig != NULL && strlen(user_cconfig) != 0) { - testutil_check(__wt_snprintf_len_incr( - cc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, user_cconfig)); - } - - if (strlen(cc_buf) != 0 && (ret = - config_opt_name_value(wtperf, "conn_config", cc_buf)) != 0) - goto err; - } - if (opts->index || - user_tconfig != NULL || wtperf->compress_table != NULL) { - req_len = 20; - req_len += wtperf->compress_table != NULL ? - strlen(wtperf->compress_table) : 0; - req_len += opts->index ? strlen(INDEX_COL_NAMES) : 0; - req_len += user_tconfig != NULL ? strlen(user_tconfig) : 0; - tc_buf = dmalloc(req_len); - - pos = 0; - append_comma = ""; - if (wtperf->compress_table != NULL && - strlen(wtperf->compress_table) != 0) { - testutil_check(__wt_snprintf_len_incr( - tc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, wtperf->compress_table)); - append_comma = ","; - } - if (opts->index) { - testutil_check(__wt_snprintf_len_incr( - tc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, INDEX_COL_NAMES)); - append_comma = ","; - } - if (user_tconfig != NULL && strlen(user_tconfig) != 0) { - testutil_check(__wt_snprintf_len_incr( - tc_buf + pos, req_len - pos, &pos, "%s%s", - append_comma, user_tconfig)); - } - - if (strlen(tc_buf) != 0 && (ret = - config_opt_name_value(wtperf, "table_config", tc_buf)) != 0) - goto err; - } - if (opts->log_partial && opts->table_count > 1) { - req_len = strlen(opts->table_config) + - strlen(LOG_PARTIAL_CONFIG) + 1; - wtperf->partial_config = dmalloc(req_len); - testutil_check(__wt_snprintf( - wtperf->partial_config, req_len, "%s%s", - opts->table_config, LOG_PARTIAL_CONFIG)); - } - /* - * Set the config for reopen. If readonly add in that string. - * If not readonly then just copy the original conn_config. - */ - if (opts->readonly) - req_len = strlen(opts->conn_config) + - strlen(READONLY_CONFIG) + 1; - else - req_len = strlen(opts->conn_config) + 1; - wtperf->reopen_config = dmalloc(req_len); - if (opts->readonly) - testutil_check(__wt_snprintf( - wtperf->reopen_config, req_len, "%s%s", - opts->conn_config, READONLY_CONFIG)); - else - testutil_check(__wt_snprintf( - wtperf->reopen_config, req_len, "%s", opts->conn_config)); - - /* Sanity-check the configuration. */ - if ((ret = config_sanity(wtperf)) != 0) - goto err; - - /* If creating, remove and re-create the home directory. */ - if (opts->create != 0) - recreate_dir(wtperf->home); - - /* Write a copy of the config. */ - req_len = strlen(wtperf->home) + strlen("/CONFIG.wtperf") + 1; - path = dmalloc(req_len); - testutil_check(__wt_snprintf( - path, req_len, "%s/CONFIG.wtperf", wtperf->home)); - config_opt_log(opts, path); - free(path); - - /* Display the configuration. */ - if (opts->verbose > 1) - config_opt_print(wtperf); - - if ((ret = start_all_runs(wtperf)) != 0) - goto err; - - if (0) { -einval: ret = EINVAL; - } - -err: wtperf_free(wtperf); - config_opt_cleanup(opts); - - free(cc_buf); - free(sess_cfg); - free(tc_buf); - free(user_cconfig); - free(user_tconfig); - - return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); + CONFIG_OPTS *opts; + WTPERF *wtperf, _wtperf; + size_t pos, req_len, sreq_len; + bool monitor_set; + int ch, ret; + const char *cmdflags = "C:h:m:O:o:T:"; + const char *append_comma, *config_opts; + char *cc_buf, *path, *sess_cfg, *tc_buf, *user_cconfig, *user_tconfig; + + /* The first WTPERF structure (from which all others are derived). */ + wtperf = &_wtperf; + memset(wtperf, 0, sizeof(*wtperf)); + wtperf->home = dstrdup(DEFAULT_HOME); + wtperf->monitor_dir = dstrdup(DEFAULT_MONITOR_DIR); + TAILQ_INIT(&wtperf->stone_head); + config_opt_init(&wtperf->opts); + + opts = wtperf->opts; + monitor_set = false; + ret = 0; + config_opts = NULL; + cc_buf = sess_cfg = tc_buf = user_cconfig = user_tconfig = NULL; + + /* Do a basic validation of options, and home is needed before open. */ + while ((ch = __wt_getopt("wtperf", argc, argv, cmdflags)) != EOF) + switch (ch) { + case 'C': + if (user_cconfig == NULL) + user_cconfig = dstrdup(__wt_optarg); + else { + user_cconfig = + drealloc(user_cconfig, strlen(user_cconfig) + strlen(__wt_optarg) + 2); + strcat(user_cconfig, ","); + strcat(user_cconfig, __wt_optarg); + } + break; + case 'h': + free(wtperf->home); + wtperf->home = dstrdup(__wt_optarg); + break; + case 'm': + free(wtperf->monitor_dir); + wtperf->monitor_dir = dstrdup(__wt_optarg); + monitor_set = true; + break; + case 'O': + config_opts = __wt_optarg; + break; + case 'T': + if (user_tconfig == NULL) + user_tconfig = dstrdup(__wt_optarg); + else { + user_tconfig = + drealloc(user_tconfig, strlen(user_tconfig) + strlen(__wt_optarg) + 2); + strcat(user_tconfig, ","); + strcat(user_tconfig, __wt_optarg); + } + break; + case '?': + usage(); + goto einval; + } + + /* + * If the user did not specify a monitor directory then set the monitor directory to the home + * dir. + */ + if (!monitor_set) { + free(wtperf->monitor_dir); + wtperf->monitor_dir = dstrdup(wtperf->home); + } + + /* Parse configuration settings from configuration file. */ + if (config_opts != NULL && config_opt_file(wtperf, config_opts) != 0) + goto einval; + + /* Parse options that override values set via a configuration file. */ + __wt_optreset = __wt_optind = 1; + while ((ch = __wt_getopt("wtperf", argc, argv, cmdflags)) != EOF) + switch (ch) { + case 'o': + /* Allow -o key=value */ + if (config_opt_str(wtperf, __wt_optarg) != 0) + goto einval; + break; + } + + if (opts->populate_threads == 0 && opts->icount != 0) { + lprintf(wtperf, 1, 0, "Cannot have 0 populate threads when icount is set\n"); + goto err; + } + + wtperf->async_config = NULL; + /* + * If the user specified async_threads we use async for all ops. If the user wants compaction, + * then we also enable async for the compact operation, but not for the workloads. + */ + if (opts->async_threads > 0) { + if (F_ISSET(wtperf, CFG_TRUNCATE)) { + lprintf(wtperf, 1, 0, "Cannot run truncate and async\n"); + goto err; + } + wtperf->use_asyncops = true; + } + if (opts->compact && opts->async_threads == 0) + opts->async_threads = 2; + if (opts->async_threads > 0) { + /* + * The maximum number of async threads is two digits, so just use that to compute the space + * we need. Assume the default of 1024 for the max ops. Although we could bump that up to + * 4096 if needed. + */ + req_len = strlen(",async=(enabled=true,threads=)") + 4; + wtperf->async_config = dmalloc(req_len); + testutil_check(__wt_snprintf(wtperf->async_config, req_len, + ",async=(enabled=true,threads=%" PRIu32 ")", opts->async_threads)); + } + if ((ret = config_compress(wtperf)) != 0) + goto err; + + /* You can't have truncate on a random collection. */ + if (F_ISSET(wtperf, CFG_TRUNCATE) && opts->random_range) { + lprintf(wtperf, 1, 0, "Cannot run truncate and random_range\n"); + goto err; + } + + /* We can't run truncate with more than one table. */ + if (F_ISSET(wtperf, CFG_TRUNCATE) && opts->table_count > 1) { + lprintf(wtperf, 1, 0, "Cannot truncate more than 1 table\n"); + goto err; + } + + /* Make stdout line buffered, so verbose output appears quickly. */ + __wt_stream_set_line_buffer(stdout); + + /* Concatenate non-default configuration strings. */ + if (user_cconfig != NULL || opts->session_count_idle > 0 || wtperf->compress_ext != NULL || + wtperf->async_config != NULL || opts->in_memory) { + req_len = 20; + req_len += wtperf->async_config != NULL ? strlen(wtperf->async_config) : 0; + req_len += wtperf->compress_ext != NULL ? strlen(wtperf->compress_ext) : 0; + if (opts->session_count_idle > 0) { + sreq_len = strlen("session_max=") + 6; + req_len += sreq_len; + sess_cfg = dmalloc(sreq_len); + testutil_check(__wt_snprintf(sess_cfg, sreq_len, "session_max=%" PRIu32, + opts->session_count_idle + wtperf->workers_cnt + opts->populate_threads + 10)); + } + req_len += opts->in_memory ? strlen("in_memory=true") : 0; + req_len += user_cconfig != NULL ? strlen(user_cconfig) : 0; + cc_buf = dmalloc(req_len); + + pos = 0; + append_comma = ""; + if (wtperf->async_config != NULL && strlen(wtperf->async_config) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, wtperf->async_config)); + append_comma = ","; + } + if (wtperf->compress_ext != NULL && strlen(wtperf->compress_ext) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, wtperf->compress_ext)); + append_comma = ","; + } + if (opts->in_memory) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, "in_memory=true")); + append_comma = ","; + } + if (sess_cfg != NULL && strlen(sess_cfg) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, sess_cfg)); + append_comma = ","; + } + if (user_cconfig != NULL && strlen(user_cconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, user_cconfig)); + } + + if (strlen(cc_buf) != 0 && + (ret = config_opt_name_value(wtperf, "conn_config", cc_buf)) != 0) + goto err; + } + if (opts->index || user_tconfig != NULL || wtperf->compress_table != NULL) { + req_len = 20; + req_len += wtperf->compress_table != NULL ? strlen(wtperf->compress_table) : 0; + req_len += opts->index ? strlen(INDEX_COL_NAMES) : 0; + req_len += user_tconfig != NULL ? strlen(user_tconfig) : 0; + tc_buf = dmalloc(req_len); + + pos = 0; + append_comma = ""; + if (wtperf->compress_table != NULL && strlen(wtperf->compress_table) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, wtperf->compress_table)); + append_comma = ","; + } + if (opts->index) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, INDEX_COL_NAMES)); + append_comma = ","; + } + if (user_tconfig != NULL && strlen(user_tconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", append_comma, user_tconfig)); + } + + if (strlen(tc_buf) != 0 && + (ret = config_opt_name_value(wtperf, "table_config", tc_buf)) != 0) + goto err; + } + if (opts->log_partial && opts->table_count > 1) { + req_len = strlen(opts->table_config) + strlen(LOG_PARTIAL_CONFIG) + 1; + wtperf->partial_config = dmalloc(req_len); + testutil_check(__wt_snprintf( + wtperf->partial_config, req_len, "%s%s", opts->table_config, LOG_PARTIAL_CONFIG)); + } + /* + * Set the config for reopen. If readonly add in that string. If not readonly then just copy the + * original conn_config. + */ + if (opts->readonly) + req_len = strlen(opts->conn_config) + strlen(READONLY_CONFIG) + 1; + else + req_len = strlen(opts->conn_config) + 1; + wtperf->reopen_config = dmalloc(req_len); + if (opts->readonly) + testutil_check(__wt_snprintf( + wtperf->reopen_config, req_len, "%s%s", opts->conn_config, READONLY_CONFIG)); + else + testutil_check(__wt_snprintf(wtperf->reopen_config, req_len, "%s", opts->conn_config)); + + /* Sanity-check the configuration. */ + if ((ret = config_sanity(wtperf)) != 0) + goto err; + + /* If creating, remove and re-create the home directory. */ + if (opts->create != 0) + recreate_dir(wtperf->home); + + /* Write a copy of the config. */ + req_len = strlen(wtperf->home) + strlen("/CONFIG.wtperf") + 1; + path = dmalloc(req_len); + testutil_check(__wt_snprintf(path, req_len, "%s/CONFIG.wtperf", wtperf->home)); + config_opt_log(opts, path); + free(path); + + /* Display the configuration. */ + if (opts->verbose > 1) + config_opt_print(wtperf); + + if ((ret = start_all_runs(wtperf)) != 0) + goto err; + + if (0) { +einval: + ret = EINVAL; + } + +err: + wtperf_free(wtperf); + config_opt_cleanup(opts); + + free(cc_buf); + free(sess_cfg); + free(tc_buf); + free(user_cconfig); + free(user_tconfig); + + return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); } static void -start_threads(WTPERF *wtperf, WORKLOAD *workp, - WTPERF_THREAD *base, u_int num, WT_THREAD_CALLBACK(*func)(void *)) +start_threads(WTPERF *wtperf, WORKLOAD *workp, WTPERF_THREAD *base, u_int num, + WT_THREAD_CALLBACK (*func)(void *)) { - CONFIG_OPTS *opts; - WTPERF_THREAD *thread; - u_int i; - - opts = wtperf->opts; - - /* Initialize the threads. */ - for (i = 0, thread = base; i < num; ++i, ++thread) { - thread->wtperf = wtperf; - thread->workload = workp; - - /* - * We don't want the threads executing in lock-step, seed each - * one differently. - */ - __wt_random_init_seed(NULL, &thread->rnd); - - /* - * Every thread gets a key/data buffer because we don't bother - * to distinguish between threads needing them and threads that - * don't, it's not enough memory to bother. These buffers hold - * strings: trailing NUL is included in the size. - */ - thread->key_buf = dcalloc(opts->key_sz, 1); - thread->value_buf = dcalloc(opts->value_sz_max, 1); - - /* - * Initialize and then toss in a bit of random values if needed. - */ - memset(thread->value_buf, 'a', opts->value_sz - 1); - if (opts->random_value) - randomize_value(thread, thread->value_buf); - - /* - * Every thread gets tracking information and is initialized - * for latency measurements, for the same reason. - */ - thread->ckpt.min_latency = - thread->scan.min_latency = - thread->insert.min_latency = thread->read.min_latency = - thread->update.min_latency = UINT32_MAX; - thread->ckpt.max_latency = thread->scan.max_latency = - thread->insert.max_latency = - thread->read.max_latency = thread->update.max_latency = 0; - } - - /* Start the threads. */ - for (i = 0, thread = base; i < num; ++i, ++thread) - testutil_check(__wt_thread_create( - NULL, &thread->handle, func, thread)); + CONFIG_OPTS *opts; + WTPERF_THREAD *thread; + u_int i; + + opts = wtperf->opts; + + /* Initialize the threads. */ + for (i = 0, thread = base; i < num; ++i, ++thread) { + thread->wtperf = wtperf; + thread->workload = workp; + + /* + * We don't want the threads executing in lock-step, seed each one differently. + */ + __wt_random_init_seed(NULL, &thread->rnd); + + /* + * Every thread gets a key/data buffer because we don't bother to distinguish between + * threads needing them and threads that don't, it's not enough memory to bother. These + * buffers hold strings: trailing NUL is included in the size. + */ + thread->key_buf = dcalloc(opts->key_sz, 1); + thread->value_buf = dcalloc(opts->value_sz_max, 1); + + /* + * Initialize and then toss in a bit of random values if needed. + */ + memset(thread->value_buf, 'a', opts->value_sz - 1); + if (opts->random_value) + randomize_value(thread, thread->value_buf); + + /* + * Every thread gets tracking information and is initialized for latency measurements, for + * the same reason. + */ + thread->ckpt.min_latency = thread->scan.min_latency = thread->insert.min_latency = + thread->read.min_latency = thread->update.min_latency = UINT32_MAX; + thread->ckpt.max_latency = thread->scan.max_latency = thread->insert.max_latency = + thread->read.max_latency = thread->update.max_latency = 0; + } + + /* Start the threads. */ + for (i = 0, thread = base; i < num; ++i, ++thread) + testutil_check(__wt_thread_create(NULL, &thread->handle, func, thread)); } static void stop_threads(u_int num, WTPERF_THREAD *threads) { - u_int i; - - if (num == 0 || threads == NULL) - return; - - for (i = 0; i < num; ++i, ++threads) { - testutil_check(__wt_thread_join(NULL, &threads->handle)); - - free(threads->key_buf); - threads->key_buf = NULL; - free(threads->value_buf); - threads->value_buf = NULL; - } - - /* - * We don't free the thread structures or any memory referenced, or NULL - * the reference when we stop the threads; the thread structure is still - * being read by the monitor thread (among others). As a standalone - * program, leaking memory isn't a concern, and it's simpler that way. - */ + u_int i; + + if (num == 0 || threads == NULL) + return; + + for (i = 0; i < num; ++i, ++threads) { + testutil_check(__wt_thread_join(NULL, &threads->handle)); + + free(threads->key_buf); + threads->key_buf = NULL; + free(threads->value_buf); + threads->value_buf = NULL; + } + + /* + * We don't free the thread structures or any memory referenced, or NULL the reference when we + * stop the threads; the thread structure is still being read by the monitor thread (among + * others). As a standalone program, leaking memory isn't a concern, and it's simpler that way. + */ } static void recreate_dir(const char *name) { - char *buf; - size_t len; - - len = strlen(name) * 2 + 100; - buf = dmalloc(len); - testutil_check(__wt_snprintf( - buf, len, "rm -rf %s && mkdir %s", name, name)); - testutil_checkfmt(system(buf), "system: %s", buf); - free(buf); + char *buf; + size_t len; + + len = strlen(name) * 2 + 100; + buf = dmalloc(len); + testutil_check(__wt_snprintf(buf, len, "rm -rf %s && mkdir %s", name, name)); + testutil_checkfmt(system(buf), "system: %s", buf); + free(buf); } static int drop_all_tables(WTPERF *wtperf) { - struct timespec start, stop; - CONFIG_OPTS *opts; - WT_SESSION *session; - size_t i; - uint32_t total_table_count; - uint64_t msecs; - int ret, t_ret; - - opts = wtperf->opts; - total_table_count = opts->table_count + opts->scan_table_count; - - /* Drop any tables. */ - if ((ret = wtperf->conn->open_session( - wtperf->conn, NULL, opts->sess_config, &session)) != 0) { - lprintf(wtperf, ret, 0, - "Error opening a session on %s", wtperf->home); - return (ret); - } - __wt_epoch(NULL, &start); - for (i = 0; i < total_table_count; i++) { - if ((ret = - session->drop(session, wtperf->uris[i], NULL)) != 0) { - lprintf(wtperf, ret, 0, - "Error dropping table %s", wtperf->uris[i]); - goto err; - } - } - __wt_epoch(NULL, &stop); - msecs = WT_TIMEDIFF_MS(stop, start); - lprintf(wtperf, 0, 1, - "Executed %" PRIu32 " drop operations average time %" PRIu64 "ms", - total_table_count, msecs / total_table_count); - -err: if ((t_ret = session->close(session, NULL)) != 0 && ret == 0) - ret = t_ret; - return (ret); + struct timespec start, stop; + CONFIG_OPTS *opts; + WT_SESSION *session; + size_t i; + uint32_t total_table_count; + uint64_t msecs; + int ret, t_ret; + + opts = wtperf->opts; + total_table_count = opts->table_count + opts->scan_table_count; + + /* Drop any tables. */ + if ((ret = wtperf->conn->open_session(wtperf->conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "Error opening a session on %s", wtperf->home); + return (ret); + } + __wt_epoch(NULL, &start); + for (i = 0; i < total_table_count; i++) { + if ((ret = session->drop(session, wtperf->uris[i], NULL)) != 0) { + lprintf(wtperf, ret, 0, "Error dropping table %s", wtperf->uris[i]); + goto err; + } + } + __wt_epoch(NULL, &stop); + msecs = WT_TIMEDIFF_MS(stop, start); + lprintf(wtperf, 0, 1, "Executed %" PRIu32 " drop operations average time %" PRIu64 "ms", + total_table_count, msecs / total_table_count); + +err: + if ((t_ret = session->close(session, NULL)) != 0 && ret == 0) + ret = t_ret; + return (ret); } static uint64_t wtperf_value_range(WTPERF *wtperf) { - CONFIG_OPTS *opts; - uint64_t total_icount; - - opts = wtperf->opts; - total_icount = (uint64_t)opts->scan_icount + (uint64_t)opts->icount; - - if (opts->random_range) - return (total_icount + opts->random_range); - /* - * It is legal to configure a zero size populate phase, hide that - * from other code by pretending the range is 1 in that case. - */ - if (total_icount + wtperf->insert_key == 0) - return (1); - return (total_icount + - wtperf->insert_key - (u_int)(wtperf->workers_cnt + 1)); + CONFIG_OPTS *opts; + uint64_t total_icount; + + opts = wtperf->opts; + total_icount = (uint64_t)opts->scan_icount + (uint64_t)opts->icount; + + if (opts->random_range) + return (total_icount + opts->random_range); + /* + * It is legal to configure a zero size populate phase, hide that from other code by pretending + * the range is 1 in that case. + */ + if (total_icount + wtperf->insert_key == 0) + return (1); + return (total_icount + wtperf->insert_key - (u_int)(wtperf->workers_cnt + 1)); } static uint64_t wtperf_rand(WTPERF_THREAD *thread) { - CONFIG_OPTS *opts; - WT_CURSOR *rnd_cursor; - WTPERF *wtperf; - double S1, S2, U; - uint64_t end_range, range, rval, start_range; - int ret; - char *key_buf; - - wtperf = thread->wtperf; - opts = wtperf->opts; - end_range = wtperf_value_range(wtperf); - start_range = opts->scan_icount; - range = end_range - start_range; - - /* - * If we have a random cursor set up then use it. - */ - if ((rnd_cursor = thread->rand_cursor) != NULL) { - if ((ret = rnd_cursor->next(rnd_cursor)) != 0) { - lprintf(wtperf, ret, 0, "worker: rand next failed"); - /* 0 is outside the expected range. */ - return (0); - } - if ((ret = rnd_cursor->get_key(rnd_cursor, &key_buf)) != 0) { - lprintf(wtperf, ret, 0, - "worker: rand next key retrieval"); - return (0); - } - /* - * Resetting the cursor is not fatal. We still return the - * value we retrieved above. We do it so that we don't - * leave a cursor positioned. - */ - if ((ret = rnd_cursor->reset(rnd_cursor)) != 0) - lprintf(wtperf, ret, 0, - "worker: rand cursor reset failed"); - extract_key(key_buf, &rval); - return (rval); - } - - /* - * Use WiredTiger's random number routine: it's lock-free and fairly - * good. - */ - rval = __wt_random(&thread->rnd); - - /* Use Pareto distribution to give 80/20 hot/cold values. */ - if (opts->pareto != 0) { -#define PARETO_SHAPE 1.5 - S1 = (-1 / PARETO_SHAPE); - S2 = range * - (opts->pareto / 100.0) * (PARETO_SHAPE - 1); - U = 1 - (double)rval / (double)UINT32_MAX; - rval = (uint64_t)((pow(U, S1) - 1) * S2); - /* - * This Pareto calculation chooses out of range values about - * 2% of the time, from my testing. That will lead to the - * first item in the table being "hot". - */ - if (rval > end_range) - rval = 0; - } - /* - * Wrap the key to within the expected range and avoid zero: we never - * insert that key. - */ - rval = (rval % range) + 1; - return (start_range + rval); + CONFIG_OPTS *opts; + WT_CURSOR *rnd_cursor; + WTPERF *wtperf; + double S1, S2, U; + uint64_t end_range, range, rval, start_range; + int ret; + char *key_buf; + + wtperf = thread->wtperf; + opts = wtperf->opts; + end_range = wtperf_value_range(wtperf); + start_range = opts->scan_icount; + range = end_range - start_range; + + /* + * If we have a random cursor set up then use it. + */ + if ((rnd_cursor = thread->rand_cursor) != NULL) { + if ((ret = rnd_cursor->next(rnd_cursor)) != 0) { + lprintf(wtperf, ret, 0, "worker: rand next failed"); + /* 0 is outside the expected range. */ + return (0); + } + if ((ret = rnd_cursor->get_key(rnd_cursor, &key_buf)) != 0) { + lprintf(wtperf, ret, 0, "worker: rand next key retrieval"); + return (0); + } + /* + * Resetting the cursor is not fatal. We still return the value we retrieved above. We do it + * so that we don't leave a cursor positioned. + */ + if ((ret = rnd_cursor->reset(rnd_cursor)) != 0) + lprintf(wtperf, ret, 0, "worker: rand cursor reset failed"); + extract_key(key_buf, &rval); + return (rval); + } + + /* + * Use WiredTiger's random number routine: it's lock-free and fairly good. + */ + rval = __wt_random(&thread->rnd); + + /* Use Pareto distribution to give 80/20 hot/cold values. */ + if (opts->pareto != 0) { +#define PARETO_SHAPE 1.5 + S1 = (-1 / PARETO_SHAPE); + S2 = range * (opts->pareto / 100.0) * (PARETO_SHAPE - 1); + U = 1 - (double)rval / (double)UINT32_MAX; + rval = (uint64_t)((pow(U, S1) - 1) * S2); + /* + * This Pareto calculation chooses out of range values about + * 2% of the time, from my testing. That will lead to the + * first item in the table being "hot". + */ + if (rval > end_range) + rval = 0; + } + /* + * Wrap the key to within the expected range and avoid zero: we never insert that key. + */ + rval = (rval % range) + 1; + return (start_range + rval); } diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index e5163409b4e..de36109309d 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -26,8 +26,8 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#ifndef HAVE_WTPERF_H -#define HAVE_WTPERF_H +#ifndef HAVE_WTPERF_H +#define HAVE_WTPERF_H #include "test_util.h" @@ -40,242 +40,236 @@ typedef struct __wtperf WTPERF; typedef struct __wtperf_thread WTPERF_THREAD; typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY; -#define EXT_PFX ",extensions=(" -#define EXT_SFX ")" -#define EXTPATH "../../ext/compressors/" /* Extensions path */ -#define BLKCMP_PFX "block_compressor=" - -#define LZ4_BLK BLKCMP_PFX "lz4" -#define LZ4_EXT \ - EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX -#define SNAPPY_BLK BLKCMP_PFX "snappy" -#define SNAPPY_EXT \ - EXT_PFX EXTPATH "snappy/.libs/libwiredtiger_snappy.so" EXT_SFX -#define ZLIB_BLK BLKCMP_PFX "zlib" -#define ZLIB_EXT \ - EXT_PFX EXTPATH "zlib/.libs/libwiredtiger_zlib.so" EXT_SFX -#define ZSTD_BLK BLKCMP_PFX "zstd" -#define ZSTD_EXT \ - EXT_PFX EXTPATH "zstd/.libs/libwiredtiger_zstd.so" EXT_SFX +#define EXT_PFX ",extensions=(" +#define EXT_SFX ")" +#define EXTPATH "../../ext/compressors/" /* Extensions path */ +#define BLKCMP_PFX "block_compressor=" + +#define LZ4_BLK BLKCMP_PFX "lz4" +#define LZ4_EXT EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX +#define SNAPPY_BLK BLKCMP_PFX "snappy" +#define SNAPPY_EXT EXT_PFX EXTPATH "snappy/.libs/libwiredtiger_snappy.so" EXT_SFX +#define ZLIB_BLK BLKCMP_PFX "zlib" +#define ZLIB_EXT EXT_PFX EXTPATH "zlib/.libs/libwiredtiger_zlib.so" EXT_SFX +#define ZSTD_BLK BLKCMP_PFX "zstd" +#define ZSTD_EXT EXT_PFX EXTPATH "zstd/.libs/libwiredtiger_zstd.so" EXT_SFX typedef struct { - int64_t threads; /* Thread count */ - int64_t insert; /* Insert ratio */ - int64_t read; /* Read ratio */ - int64_t update; /* Update ratio */ - uint64_t throttle; /* Maximum operations/second */ - /* Number of operations per transaction. Zero for autocommit */ - int64_t ops_per_txn; - int64_t pause; /* Time between scans */ - int64_t read_range; /* Range of reads */ - int32_t table_index; /* Table to focus ops on */ - int64_t truncate; /* Truncate ratio */ - uint64_t truncate_pct; /* Truncate Percent */ - uint64_t truncate_count; /* Truncate Count */ - int64_t update_delta; /* Value size change on update */ - -#define WORKER_INSERT 1 /* Insert */ -#define WORKER_INSERT_RMW 2 /* Insert with read-modify-write */ -#define WORKER_READ 3 /* Read */ -#define WORKER_TRUNCATE 4 /* Truncate */ -#define WORKER_UPDATE 5 /* Update */ - uint8_t ops[100]; /* Operation schedule */ + int64_t threads; /* Thread count */ + int64_t insert; /* Insert ratio */ + int64_t read; /* Read ratio */ + int64_t update; /* Update ratio */ + uint64_t throttle; /* Maximum operations/second */ + /* Number of operations per transaction. Zero for autocommit */ + int64_t ops_per_txn; + int64_t pause; /* Time between scans */ + int64_t read_range; /* Range of reads */ + int32_t table_index; /* Table to focus ops on */ + int64_t truncate; /* Truncate ratio */ + uint64_t truncate_pct; /* Truncate Percent */ + uint64_t truncate_count; /* Truncate Count */ + int64_t update_delta; /* Value size change on update */ + +#define WORKER_INSERT 1 /* Insert */ +#define WORKER_INSERT_RMW 2 /* Insert with read-modify-write */ +#define WORKER_READ 3 /* Read */ +#define WORKER_TRUNCATE 4 /* Truncate */ +#define WORKER_UPDATE 5 /* Update */ + uint8_t ops[100]; /* Operation schedule */ } WORKLOAD; /* Steering items for the truncate workload */ typedef struct { - uint64_t stone_gap; - uint64_t needed_stones; - uint64_t expected_total; - uint64_t total_inserts; - uint64_t last_total_inserts; - uint64_t num_stones; - uint64_t last_key; - uint64_t catchup_multiplier; + uint64_t stone_gap; + uint64_t needed_stones; + uint64_t expected_total; + uint64_t total_inserts; + uint64_t last_total_inserts; + uint64_t num_stones; + uint64_t last_key; + uint64_t catchup_multiplier; } TRUNCATE_CONFIG; /* Queue entry for use with the Truncate Logic */ struct __truncate_queue_entry { - char *key; /* Truncation point */ - uint64_t diff; /* Number of items to be truncated*/ - TAILQ_ENTRY(__truncate_queue_entry) q; + char *key; /* Truncation point */ + uint64_t diff; /* Number of items to be truncated*/ + TAILQ_ENTRY(__truncate_queue_entry) q; }; /* Steering for the throttle configuration */ typedef struct { - struct timespec last_increment; /* Time that we last added more ops */ - uint64_t ops_count; /* The number of ops this increment */ - uint64_t ops_per_increment; /* Ops to add per increment */ - uint64_t usecs_increment; /* Time interval of each increment */ + struct timespec last_increment; /* Time that we last added more ops */ + uint64_t ops_count; /* The number of ops this increment */ + uint64_t ops_per_increment; /* Ops to add per increment */ + uint64_t usecs_increment; /* Time interval of each increment */ } THROTTLE_CONFIG; -#define LOG_PARTIAL_CONFIG ",log=(enabled=false)" -#define READONLY_CONFIG ",readonly=true" -struct __wtperf { /* Per-database structure */ - char *home; /* WiredTiger home */ - char *monitor_dir; /* Monitor output dir */ - char *partial_config; /* Config string for partial logging */ - char *reopen_config; /* Config string for conn reopen */ - char *log_table_uri; /* URI for log table */ - char **uris; /* URIs */ +#define LOG_PARTIAL_CONFIG ",log=(enabled=false)" +#define READONLY_CONFIG ",readonly=true" +struct __wtperf { /* Per-database structure */ + char *home; /* WiredTiger home */ + char *monitor_dir; /* Monitor output dir */ + char *partial_config; /* Config string for partial logging */ + char *reopen_config; /* Config string for conn reopen */ + char *log_table_uri; /* URI for log table */ + char **uris; /* URIs */ - WT_CONNECTION *conn; /* Database connection */ + WT_CONNECTION *conn; /* Database connection */ - FILE *logf; /* Logging handle */ + FILE *logf; /* Logging handle */ - char *async_config; /* Config string for async */ - bool use_asyncops; /* Use async operations */ + char *async_config; /* Config string for async */ + bool use_asyncops; /* Use async operations */ - const char *compress_ext; /* Compression extension for conn */ - const char *compress_table; /* Compression arg to table create */ + const char *compress_ext; /* Compression extension for conn */ + const char *compress_table; /* Compression arg to table create */ - WTPERF_THREAD *ckptthreads; /* Checkpoint threads */ - WTPERF_THREAD *popthreads; /* Populate threads */ - WTPERF_THREAD *scanthreads; /* Scan threads */ + WTPERF_THREAD *ckptthreads; /* Checkpoint threads */ + WTPERF_THREAD *popthreads; /* Populate threads */ + WTPERF_THREAD *scanthreads; /* Scan threads */ -#define WORKLOAD_MAX 50 - WTPERF_THREAD *workers; /* Worker threads */ - u_int workers_cnt; +#define WORKLOAD_MAX 50 + WTPERF_THREAD *workers; /* Worker threads */ + u_int workers_cnt; - WORKLOAD *workload; /* Workloads */ - u_int workload_cnt; + WORKLOAD *workload; /* Workloads */ + u_int workload_cnt; - /* State tracking variables. */ - uint64_t ckpt_ops; /* checkpoint operations */ - uint64_t scan_ops; /* scan operations */ - uint64_t insert_ops; /* insert operations */ - uint64_t read_ops; /* read operations */ - uint64_t truncate_ops; /* truncate operations */ - uint64_t update_ops; /* update operations */ + /* State tracking variables. */ + uint64_t ckpt_ops; /* checkpoint operations */ + uint64_t scan_ops; /* scan operations */ + uint64_t insert_ops; /* insert operations */ + uint64_t read_ops; /* read operations */ + uint64_t truncate_ops; /* truncate operations */ + uint64_t update_ops; /* update operations */ - uint64_t insert_key; /* insert key */ - uint64_t log_like_table_key; /* used to allocate IDs for log table */ + uint64_t insert_key; /* insert key */ + uint64_t log_like_table_key; /* used to allocate IDs for log table */ - volatile bool ckpt; /* checkpoint in progress */ - volatile bool scan; /* scan in progress */ - volatile bool error; /* thread error */ - volatile bool stop; /* notify threads to stop */ - volatile bool in_warmup; /* running warmup phase */ + volatile bool ckpt; /* checkpoint in progress */ + volatile bool scan; /* scan in progress */ + volatile bool error; /* thread error */ + volatile bool stop; /* notify threads to stop */ + volatile bool in_warmup; /* running warmup phase */ - volatile bool idle_cycle_run; /* Signal for idle cycle thread */ + volatile bool idle_cycle_run; /* Signal for idle cycle thread */ - volatile uint32_t totalsec; /* total seconds running */ + volatile uint32_t totalsec; /* total seconds running */ -#define CFG_GROW 0x0001 /* There is a grow workload */ -#define CFG_SHRINK 0x0002 /* There is a shrink workload */ -#define CFG_TRUNCATE 0x0004 /* There is a truncate workload */ - uint32_t flags; /* flags */ +#define CFG_GROW 0x0001 /* There is a grow workload */ +#define CFG_SHRINK 0x0002 /* There is a shrink workload */ +#define CFG_TRUNCATE 0x0004 /* There is a truncate workload */ + uint32_t flags; /* flags */ - /* Queue head for use with the Truncate Logic */ - TAILQ_HEAD(__truncate_qh, __truncate_queue_entry) stone_head; + /* Queue head for use with the Truncate Logic */ + TAILQ_HEAD(__truncate_qh, __truncate_queue_entry) stone_head; - CONFIG_OPTS *opts; /* Global configuration */ + CONFIG_OPTS *opts; /* Global configuration */ }; -#define ELEMENTS(a) (sizeof(a) / sizeof(a[0])) +#define ELEMENTS(a) (sizeof(a) / sizeof(a[0])) -#define READ_RANGE_OPS 10 -#define THROTTLE_OPS 100 +#define READ_RANGE_OPS 10 +#define THROTTLE_OPS 100 -#define THOUSAND (1000ULL) -#define MILLION (1000000ULL) -#define BILLION (1000000000ULL) +#define THOUSAND (1000ULL) +#define MILLION (1000000ULL) +#define BILLION (1000000000ULL) -#define NSEC_PER_SEC BILLION -#define USEC_PER_SEC MILLION -#define MSEC_PER_SEC THOUSAND +#define NSEC_PER_SEC BILLION +#define USEC_PER_SEC MILLION +#define MSEC_PER_SEC THOUSAND -#define ns_to_ms(v) ((v) / MILLION) -#define ns_to_sec(v) ((v) / BILLION) -#define ns_to_us(v) ((v) / THOUSAND) +#define ns_to_ms(v) ((v) / MILLION) +#define ns_to_sec(v) ((v) / BILLION) +#define ns_to_us(v) ((v) / THOUSAND) -#define us_to_ms(v) ((v) / THOUSAND) -#define us_to_ns(v) ((v) * THOUSAND) -#define us_to_sec(v) ((v) / MILLION) +#define us_to_ms(v) ((v) / THOUSAND) +#define us_to_ns(v) ((v)*THOUSAND) +#define us_to_sec(v) ((v) / MILLION) -#define ms_to_ns(v) ((v) * MILLION) -#define ms_to_us(v) ((v) * THOUSAND) -#define ms_to_sec(v) ((v) / THOUSAND) +#define ms_to_ns(v) ((v)*MILLION) +#define ms_to_us(v) ((v)*THOUSAND) +#define ms_to_sec(v) ((v) / THOUSAND) -#define sec_to_ns(v) ((v) * BILLION) -#define sec_to_us(v) ((v) * MILLION) -#define sec_to_ms(v) ((v) * THOUSAND) +#define sec_to_ns(v) ((v)*BILLION) +#define sec_to_us(v) ((v)*MILLION) +#define sec_to_ms(v) ((v)*THOUSAND) typedef struct { - /* - * Threads maintain the total thread operation and total latency they've - * experienced; the monitor thread periodically copies these values into - * the last_XXX fields. - */ - uint64_t ops; /* Total operations */ - uint64_t latency_ops; /* Total ops sampled for latency */ - uint64_t latency; /* Total latency */ - - uint64_t last_latency_ops; /* Last read by monitor thread */ - uint64_t last_latency; - - /* - * Minimum/maximum latency, shared with the monitor thread, that is, the - * monitor thread clears it so it's recalculated again for each period. - */ - uint32_t min_latency; /* Minimum latency (uS) */ - uint32_t max_latency; /* Maximum latency (uS) */ - - /* - * Latency buckets. - */ - uint32_t us[1000]; /* < 1us ... 1000us */ - uint32_t ms[1000]; /* < 1ms ... 1000ms */ - uint32_t sec[100]; /* < 1s 2s ... 100s */ + /* + * Threads maintain the total thread operation and total latency they've experienced; the + * monitor thread periodically copies these values into the last_XXX fields. + */ + uint64_t ops; /* Total operations */ + uint64_t latency_ops; /* Total ops sampled for latency */ + uint64_t latency; /* Total latency */ + + uint64_t last_latency_ops; /* Last read by monitor thread */ + uint64_t last_latency; + + /* + * Minimum/maximum latency, shared with the monitor thread, that is, the monitor thread clears + * it so it's recalculated again for each period. + */ + uint32_t min_latency; /* Minimum latency (uS) */ + uint32_t max_latency; /* Maximum latency (uS) */ + + /* + * Latency buckets. + */ + uint32_t us[1000]; /* < 1us ... 1000us */ + uint32_t ms[1000]; /* < 1ms ... 1000ms */ + uint32_t sec[100]; /* < 1s 2s ... 100s */ } TRACK; -struct __wtperf_thread { /* Per-thread structure */ - WTPERF *wtperf; /* Enclosing configuration */ - WT_CURSOR *rand_cursor; /* Random key cursor */ +struct __wtperf_thread { /* Per-thread structure */ + WTPERF *wtperf; /* Enclosing configuration */ + WT_CURSOR *rand_cursor; /* Random key cursor */ - WT_RAND_STATE rnd; /* Random number generation state */ + WT_RAND_STATE rnd; /* Random number generation state */ - wt_thread_t handle; /* Handle */ + wt_thread_t handle; /* Handle */ - char *key_buf, *value_buf; /* Key/value memory */ + char *key_buf, *value_buf; /* Key/value memory */ - WORKLOAD *workload; /* Workload */ + WORKLOAD *workload; /* Workload */ - THROTTLE_CONFIG throttle_cfg; /* Throttle configuration */ + THROTTLE_CONFIG throttle_cfg; /* Throttle configuration */ - TRUNCATE_CONFIG trunc_cfg; /* Truncate configuration */ + TRUNCATE_CONFIG trunc_cfg; /* Truncate configuration */ - TRACK ckpt; /* Checkpoint operations */ - TRACK insert; /* Insert operations */ - TRACK read; /* Read operations */ - TRACK scan; /* Scan operations */ - TRACK update; /* Update operations */ - TRACK truncate; /* Truncate operations */ - TRACK truncate_sleep; /* Truncate sleep operations */ + TRACK ckpt; /* Checkpoint operations */ + TRACK insert; /* Insert operations */ + TRACK read; /* Read operations */ + TRACK scan; /* Scan operations */ + TRACK update; /* Update operations */ + TRACK truncate; /* Truncate operations */ + TRACK truncate_sleep; /* Truncate sleep operations */ }; -void cleanup_truncate_config(WTPERF *); -int config_opt_file(WTPERF *, const char *); -void config_opt_cleanup(CONFIG_OPTS *); -void config_opt_init(CONFIG_OPTS **); -void config_opt_log(CONFIG_OPTS *, const char *); -int config_opt_name_value(WTPERF *, const char *, const char *); -void config_opt_print(WTPERF *); -int config_opt_str(WTPERF *, const char *); -void config_opt_usage(void); -int config_sanity(WTPERF *); -void latency_insert(WTPERF *, uint32_t *, uint32_t *, uint32_t *); -void latency_print(WTPERF *); -void latency_read(WTPERF *, uint32_t *, uint32_t *, uint32_t *); -void latency_update(WTPERF *, uint32_t *, uint32_t *, uint32_t *); -int run_truncate( - WTPERF *, WTPERF_THREAD *, WT_CURSOR *, WT_SESSION *, int *); -int setup_log_file(WTPERF *); -void setup_throttle(WTPERF_THREAD *); -void setup_truncate(WTPERF *, WTPERF_THREAD *, WT_SESSION *); -void start_idle_table_cycle(WTPERF *, wt_thread_t *); -void stop_idle_table_cycle(WTPERF *, wt_thread_t); -void worker_throttle(WTPERF_THREAD *); +void cleanup_truncate_config(WTPERF *); +int config_opt_file(WTPERF *, const char *); +void config_opt_cleanup(CONFIG_OPTS *); +void config_opt_init(CONFIG_OPTS **); +void config_opt_log(CONFIG_OPTS *, const char *); +int config_opt_name_value(WTPERF *, const char *, const char *); +void config_opt_print(WTPERF *); +int config_opt_str(WTPERF *, const char *); +void config_opt_usage(void); +int config_sanity(WTPERF *); +void latency_insert(WTPERF *, uint32_t *, uint32_t *, uint32_t *); +void latency_print(WTPERF *); +void latency_read(WTPERF *, uint32_t *, uint32_t *, uint32_t *); +void latency_update(WTPERF *, uint32_t *, uint32_t *, uint32_t *); +int run_truncate(WTPERF *, WTPERF_THREAD *, WT_CURSOR *, WT_SESSION *, int *); +int setup_log_file(WTPERF *); +void setup_throttle(WTPERF_THREAD *); +void setup_truncate(WTPERF *, WTPERF_THREAD *, WT_SESSION *); +void start_idle_table_cycle(WTPERF *, wt_thread_t *); +void stop_idle_table_cycle(WTPERF *, wt_thread_t); +void worker_throttle(WTPERF_THREAD *); uint64_t sum_ckpt_ops(WTPERF *); uint64_t sum_scan_ops(WTPERF *); uint64_t sum_insert_ops(WTPERF *); @@ -284,35 +278,33 @@ uint64_t sum_read_ops(WTPERF *); uint64_t sum_truncate_ops(WTPERF *); uint64_t sum_update_ops(WTPERF *); -void lprintf(const WTPERF *, int err, uint32_t, const char *, ...) +void lprintf(const WTPERF *, int err, uint32_t, const char *, ...) #if defined(__GNUC__) -__attribute__((format (printf, 4, 5))) + __attribute__((format(printf, 4, 5))) #endif -; + ; static inline void generate_key(CONFIG_OPTS *opts, char *key_buf, uint64_t keyno) { - u64_to_string_zf(keyno, key_buf, opts->key_sz); + u64_to_string_zf(keyno, key_buf, opts->key_sz); } static inline void extract_key(char *key_buf, uint64_t *keynop) { - (void)sscanf(key_buf, "%" SCNu64, keynop); + (void)sscanf(key_buf, "%" SCNu64, keynop); } /* * die -- - * Print message and exit on failure. + * Print message and exit on failure. */ -static inline void -die(int, const char *) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static inline void die(int, const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static inline void die(int e, const char *str) { - fprintf(stderr, "Call to %s failed: %s", str, wiredtiger_strerror(e)); - exit(EXIT_FAILURE); + fprintf(stderr, "Call to %s failed: %s", str, wiredtiger_strerror(e)); + exit(EXIT_FAILURE); } #endif diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index 079c419908f..6c0eb481de0 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -30,40 +30,35 @@ */ #ifdef OPT_DECLARE_STRUCT -#define DEF_OPT_AS_BOOL(name, initval, desc) int name; -#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) const char *name; -#define DEF_OPT_AS_STRING(name, initval, desc) const char *name; -#define DEF_OPT_AS_UINT32(name, initval, desc) uint32_t name; +#define DEF_OPT_AS_BOOL(name, initval, desc) int name; +#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) const char *name; +#define DEF_OPT_AS_STRING(name, initval, desc) const char *name; +#define DEF_OPT_AS_UINT32(name, initval, desc) uint32_t name; #endif #ifdef OPT_DEFINE_DESC -#define DEF_OPT_AS_BOOL(name, initval, desc) \ - { #name, desc, #initval, BOOL_TYPE, offsetof(CONFIG_OPTS, name) }, -#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) \ - { #name, desc, initval, CONFIG_STRING_TYPE, \ - offsetof(CONFIG_OPTS, name) }, -#define DEF_OPT_AS_STRING(name, initval, desc) \ - { #name, desc, initval, STRING_TYPE, offsetof(CONFIG_OPTS, name) }, -#define DEF_OPT_AS_UINT32(name, initval, desc) \ - { #name, desc, #initval, UINT32_TYPE, offsetof(CONFIG_OPTS, name) }, +#define DEF_OPT_AS_BOOL(name, initval, desc) \ + {#name, desc, #initval, BOOL_TYPE, offsetof(CONFIG_OPTS, name)}, +#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) \ + {#name, desc, initval, CONFIG_STRING_TYPE, offsetof(CONFIG_OPTS, name)}, +#define DEF_OPT_AS_STRING(name, initval, desc) \ + {#name, desc, initval, STRING_TYPE, offsetof(CONFIG_OPTS, name)}, +#define DEF_OPT_AS_UINT32(name, initval, desc) \ + {#name, desc, #initval, UINT32_TYPE, offsetof(CONFIG_OPTS, name)}, #endif #ifdef OPT_DEFINE_DEFAULT -#define DEF_OPT_AS_BOOL(name, initval, desc) initval, -#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) initval, -#define DEF_OPT_AS_STRING(name, initval, desc) initval, -#define DEF_OPT_AS_UINT32(name, initval, desc) initval, +#define DEF_OPT_AS_BOOL(name, initval, desc) initval, +#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) initval, +#define DEF_OPT_AS_STRING(name, initval, desc) initval, +#define DEF_OPT_AS_UINT32(name, initval, desc) initval, #endif #ifdef OPT_DEFINE_DOXYGEN -#define DEF_OPT_AS_BOOL(name, initval, desc) \ - OPTION #name, desc, #initval, boolean -#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) \ - OPTION #name, desc, initval, string -#define DEF_OPT_AS_STRING(name, initval, desc) \ - OPTION #name, desc, initval, string -#define DEF_OPT_AS_UINT32(name, initval, desc) \ - OPTION #name, desc, #initval, unsigned int +#define DEF_OPT_AS_BOOL(name, initval, desc) OPTION #name, desc, #initval, boolean +#define DEF_OPT_AS_CONFIG_STRING(name, initval, desc) OPTION #name, desc, initval, string +#define DEF_OPT_AS_STRING(name, initval, desc) OPTION #name, desc, initval, string +#define DEF_OPT_AS_UINT32(name, initval, desc) OPTION #name, desc, #initval, unsigned int #endif /* @@ -86,144 +81,134 @@ * options are appended to existing content, whereas STRING options overwrite. */ DEF_OPT_AS_UINT32(async_threads, 0, "number of async worker threads") -DEF_OPT_AS_UINT32(checkpoint_interval, 120, - "checkpoint every interval seconds during the workload phase.") +DEF_OPT_AS_UINT32( + checkpoint_interval, 120, "checkpoint every interval seconds during the workload phase.") DEF_OPT_AS_UINT32(checkpoint_stress_rate, 0, - "checkpoint every rate operations during the populate phase in the " - "populate thread(s), 0 to disable") + "checkpoint every rate operations during the populate phase in the " + "populate thread(s), 0 to disable") DEF_OPT_AS_UINT32(checkpoint_threads, 0, "number of checkpoint threads") -DEF_OPT_AS_CONFIG_STRING(conn_config, - "create,statistics=(fast),statistics_log=(json,wait=1)", - "connection configuration string") -DEF_OPT_AS_BOOL(close_conn, 1, "properly close connection at end of test. " - "Setting to false does not sync data to disk and can result in lost " - "data after test exits.") +DEF_OPT_AS_CONFIG_STRING(conn_config, "create,statistics=(fast),statistics_log=(json,wait=1)", + "connection configuration string") +DEF_OPT_AS_BOOL(close_conn, 1, + "properly close connection at end of test. " + "Setting to false does not sync data to disk and can result in lost " + "data after test exits.") DEF_OPT_AS_BOOL(compact, 0, "post-populate compact for LSM merging activity") DEF_OPT_AS_STRING(compression, "none", - "compression extension. Allowed configuration values are: " - "'none', 'lz4', 'snappy', 'zlib', 'zstd'") -DEF_OPT_AS_BOOL(create, 1, - "do population phase; false to use existing database") + "compression extension. Allowed configuration values are: " + "'none', 'lz4', 'snappy', 'zlib', 'zstd'") +DEF_OPT_AS_BOOL(create, 1, "do population phase; false to use existing database") DEF_OPT_AS_UINT32(database_count, 1, - "number of WiredTiger databases to use. Each database will execute the" - " workload using a separate home directory and complete set of worker" - " threads") + "number of WiredTiger databases to use. Each database will execute the" + " workload using a separate home directory and complete set of worker" + " threads") DEF_OPT_AS_BOOL(drop_tables, 0, - "Whether to drop all tables at the end of the run, and report time taken" - " to do the drop.") -DEF_OPT_AS_BOOL(in_memory, 0, - "Whether to create the database in-memory.") + "Whether to drop all tables at the end of the run, and report time taken" + " to do the drop.") +DEF_OPT_AS_BOOL(in_memory, 0, "Whether to create the database in-memory.") DEF_OPT_AS_UINT32(icount, 5000, - "number of records to initially populate. If multiple tables are " - "configured the count is spread evenly across all tables.") + "number of records to initially populate. If multiple tables are " + "configured the count is spread evenly across all tables.") DEF_OPT_AS_UINT32(idle_table_cycle, 0, - "Enable regular create and drop of idle tables, value is the maximum " - "number of seconds a create or drop is allowed before flagging an error. " - "Default 0 which means disabled.") -DEF_OPT_AS_BOOL(index, 0, - "Whether to create an index on the value field.") -DEF_OPT_AS_BOOL(insert_rmw, 0, - "execute a read prior to each insert in workload phase") + "Enable regular create and drop of idle tables, value is the maximum " + "number of seconds a create or drop is allowed before flagging an error. " + "Default 0 which means disabled.") +DEF_OPT_AS_BOOL(index, 0, "Whether to create an index on the value field.") +DEF_OPT_AS_BOOL(insert_rmw, 0, "execute a read prior to each insert in workload phase") DEF_OPT_AS_UINT32(key_sz, 20, "key size") DEF_OPT_AS_BOOL(log_partial, 0, "perform partial logging on first table only.") -DEF_OPT_AS_BOOL(log_like_table, 0, - "Append all modification operations to another shared table.") +DEF_OPT_AS_BOOL(log_like_table, 0, "Append all modification operations to another shared table.") DEF_OPT_AS_UINT32(min_throughput, 0, - "notify if any throughput measured is less than this amount. " - "Aborts or prints warning based on min_throughput_fatal setting. " - "Requires sample_interval to be configured") -DEF_OPT_AS_BOOL(min_throughput_fatal, 0, - "print warning (false) or abort (true) of min_throughput failure.") + "notify if any throughput measured is less than this amount. " + "Aborts or prints warning based on min_throughput_fatal setting. " + "Requires sample_interval to be configured") +DEF_OPT_AS_BOOL( + min_throughput_fatal, 0, "print warning (false) or abort (true) of min_throughput failure.") DEF_OPT_AS_UINT32(max_latency, 0, - "notify if any latency measured exceeds this number of milliseconds." - "Aborts or prints warning based on min_throughput_fatal setting. " - "Requires sample_interval to be configured") -DEF_OPT_AS_BOOL(max_latency_fatal, 0, - "print warning (false) or abort (true) of max_latency failure.") -DEF_OPT_AS_UINT32(pareto, 0, "use pareto distribution for random numbers. Zero " - "to disable, otherwise a percentage indicating how aggressive the " - "distribution should be.") + "notify if any latency measured exceeds this number of milliseconds." + "Aborts or prints warning based on min_throughput_fatal setting. " + "Requires sample_interval to be configured") +DEF_OPT_AS_BOOL( + max_latency_fatal, 0, "print warning (false) or abort (true) of max_latency failure.") +DEF_OPT_AS_UINT32(pareto, 0, + "use pareto distribution for random numbers. Zero " + "to disable, otherwise a percentage indicating how aggressive the " + "distribution should be.") DEF_OPT_AS_UINT32(populate_ops_per_txn, 0, - "number of operations to group into each transaction in the populate " - "phase, zero for auto-commit") -DEF_OPT_AS_UINT32(populate_threads, 1, - "number of populate threads, 1 for bulk load") -DEF_OPT_AS_BOOL(pre_load_data, 0, - "Scan all data prior to starting the workload phase to warm the cache") + "number of operations to group into each transaction in the populate " + "phase, zero for auto-commit") +DEF_OPT_AS_UINT32(populate_threads, 1, "number of populate threads, 1 for bulk load") +DEF_OPT_AS_BOOL( + pre_load_data, 0, "Scan all data prior to starting the workload phase to warm the cache") DEF_OPT_AS_UINT32(random_range, 0, - "if non zero choose a value from within this range as the key for " - "insert operations") + "if non zero choose a value from within this range as the key for " + "insert operations") DEF_OPT_AS_BOOL(random_value, 0, "generate random content for the value") DEF_OPT_AS_BOOL(range_partition, 0, "partition data by range (vs hash)") DEF_OPT_AS_BOOL(readonly, 0, - "reopen the connection between populate and workload phases in readonly " - "mode. Requires reopen_connection turned on (default). Requires that " - "read be the only workload specified") -DEF_OPT_AS_BOOL(reopen_connection, 1, - "close and reopen the connection between populate and workload phases") -DEF_OPT_AS_UINT32(report_interval, 2, - "output throughput information every interval seconds, 0 to disable") -DEF_OPT_AS_UINT32(run_ops, 0, - "total read, insert and update workload operations") -DEF_OPT_AS_UINT32(run_time, 0, - "total workload seconds") -DEF_OPT_AS_UINT32(sample_interval, 0, - "performance logging every interval seconds, 0 to disable") + "reopen the connection between populate and workload phases in readonly " + "mode. Requires reopen_connection turned on (default). Requires that " + "read be the only workload specified") +DEF_OPT_AS_BOOL( + reopen_connection, 1, "close and reopen the connection between populate and workload phases") +DEF_OPT_AS_UINT32( + report_interval, 2, "output throughput information every interval seconds, 0 to disable") +DEF_OPT_AS_UINT32(run_ops, 0, "total read, insert and update workload operations") +DEF_OPT_AS_UINT32(run_time, 0, "total workload seconds") +DEF_OPT_AS_UINT32(sample_interval, 0, "performance logging every interval seconds, 0 to disable") DEF_OPT_AS_UINT32(sample_rate, 50, - "how often the latency of operations is measured. One for every operation," - "two for every second operation, three for every third operation etc.") -DEF_OPT_AS_UINT32(scan_icount, 0, - "number of records in scan tables to populate") + "how often the latency of operations is measured. One for every operation," + "two for every second operation, three for every third operation etc.") +DEF_OPT_AS_UINT32(scan_icount, 0, "number of records in scan tables to populate") DEF_OPT_AS_UINT32(scan_interval, 0, - "scan tables every interval seconds during the workload phase," - " 0 to disable") -DEF_OPT_AS_UINT32(scan_pct, 10, - "percentage of entire data set scanned, if scan_interval is enabled") + "scan tables every interval seconds during the workload phase," + " 0 to disable") +DEF_OPT_AS_UINT32( + scan_pct, 10, "percentage of entire data set scanned, if scan_interval is enabled") DEF_OPT_AS_UINT32(scan_table_count, 0, - "number of separate tables to be used for scanning. Zero indicates " - "that tables are shared with other operations") + "number of separate tables to be used for scanning. Zero indicates " + "that tables are shared with other operations") DEF_OPT_AS_CONFIG_STRING(sess_config, "", "session configuration string") -DEF_OPT_AS_UINT32(session_count_idle, 0, - "number of idle sessions to create. Default 0.") +DEF_OPT_AS_UINT32(session_count_idle, 0, "number of idle sessions to create. Default 0.") DEF_OPT_AS_CONFIG_STRING(table_config, - "key_format=S,value_format=S,type=lsm,exclusive=true," - "allocation_size=4kb,internal_page_max=64kb,leaf_page_max=4kb," - "split_pct=100", - "table configuration string") + "key_format=S,value_format=S,type=lsm,exclusive=true," + "allocation_size=4kb,internal_page_max=64kb,leaf_page_max=4kb," + "split_pct=100", + "table configuration string") DEF_OPT_AS_UINT32(table_count, 1, - "number of tables to run operations over. Keys are divided evenly " - "over the tables. Cursors are held open on all tables. Default 1, maximum " - "99999.") -DEF_OPT_AS_UINT32(table_count_idle, 0, - "number of tables to create, that won't be populated. Default 0.") -DEF_OPT_AS_STRING(threads, "", "workload configuration: each 'count' " - "entry is the total number of threads, and the 'insert', 'read' and " - "'update' entries are the ratios of insert, read and update operations " - "done by each worker thread; If a throttle value is provided each thread " - "will do a maximum of that number of operations per second; multiple " - "workload configurations may be specified per threads configuration; " - "for example, a more complex threads configuration might be " - "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' " - "which would create 2 threads doing nothing but reads and 8 threads " - "each doing 50% inserts and 25% reads and updates. Allowed configuration " - "values are 'count', 'throttle', 'update_delta', 'reads', 'read_range', " - "'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. " - "There are also behavior modifiers, supported modifiers are " - "'ops_per_txn'") + "number of tables to run operations over. Keys are divided evenly " + "over the tables. Cursors are held open on all tables. Default 1, maximum " + "99999.") +DEF_OPT_AS_UINT32( + table_count_idle, 0, "number of tables to create, that won't be populated. Default 0.") +DEF_OPT_AS_STRING(threads, "", + "workload configuration: each 'count' " + "entry is the total number of threads, and the 'insert', 'read' and " + "'update' entries are the ratios of insert, read and update operations " + "done by each worker thread; If a throttle value is provided each thread " + "will do a maximum of that number of operations per second; multiple " + "workload configurations may be specified per threads configuration; " + "for example, a more complex threads configuration might be " + "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' " + "which would create 2 threads doing nothing but reads and 8 threads " + "each doing 50% inserts and 25% reads and updates. Allowed configuration " + "values are 'count', 'throttle', 'update_delta', 'reads', 'read_range', " + "'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. " + "There are also behavior modifiers, supported modifiers are " + "'ops_per_txn'") DEF_OPT_AS_CONFIG_STRING(transaction_config, "", - "WT_SESSION.begin_transaction configuration string, applied during the " - "populate phase when populate_ops_per_txn is nonzero") + "WT_SESSION.begin_transaction configuration string, applied during the " + "populate phase when populate_ops_per_txn is nonzero") DEF_OPT_AS_STRING(table_name, "test", "table name") -DEF_OPT_AS_BOOL(truncate_single_ops, 0, - "Implement truncate via cursor remove instead of session API") -DEF_OPT_AS_UINT32(value_sz_max, 1000, - "maximum value size when delta updates are present. Default disabled") -DEF_OPT_AS_UINT32(value_sz_min, 1, - "minimum value size when delta updates are present. Default disabled") +DEF_OPT_AS_BOOL( + truncate_single_ops, 0, "Implement truncate via cursor remove instead of session API") +DEF_OPT_AS_UINT32( + value_sz_max, 1000, "maximum value size when delta updates are present. Default disabled") +DEF_OPT_AS_UINT32( + value_sz_min, 1, "minimum value size when delta updates are present. Default disabled") DEF_OPT_AS_UINT32(value_sz, 100, "value size") DEF_OPT_AS_UINT32(verbose, 1, "verbosity") -DEF_OPT_AS_UINT32(warmup, 0, - "How long to run the workload phase before starting measurements") +DEF_OPT_AS_UINT32(warmup, 0, "How long to run the workload phase before starting measurements") #undef DEF_OPT_AS_BOOL #undef DEF_OPT_AS_CONFIG_STRING diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_throttle.c b/src/third_party/wiredtiger/bench/wtperf/wtperf_throttle.c index 883acdbe355..16d178eed54 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_throttle.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_throttle.c @@ -34,84 +34,77 @@ void setup_throttle(WTPERF_THREAD *thread) { - THROTTLE_CONFIG *throttle_cfg; + THROTTLE_CONFIG *throttle_cfg; - throttle_cfg = &thread->throttle_cfg; + throttle_cfg = &thread->throttle_cfg; - /* - * Setup how the number of operations to run each interval in order to - * meet our desired max throughput. - * - If we have a very small number of them we can do one op - * on a larger increment. Given there is overhead in throttle logic - * we want to avoid running the throttle check regularly. - * - For most workloads, we aim to do 100 ops per interval and adjust - * the sleep period accordingly. - * - For high throughput workloads, we aim to do many ops in 100us - * increments. - */ + /* + * Setup how the number of operations to run each interval in order to + * meet our desired max throughput. + * - If we have a very small number of them we can do one op + * on a larger increment. Given there is overhead in throttle logic + * we want to avoid running the throttle check regularly. + * - For most workloads, we aim to do 100 ops per interval and adjust + * the sleep period accordingly. + * - For high throughput workloads, we aim to do many ops in 100us + * increments. + */ - if (thread->workload->throttle < THROTTLE_OPS) { - /* If the interval is very small, we do one operation */ - throttle_cfg->usecs_increment = - USEC_PER_SEC / thread->workload->throttle; - throttle_cfg->ops_per_increment = 1; - } else if (thread->workload->throttle < USEC_PER_SEC / THROTTLE_OPS) { - throttle_cfg->usecs_increment = - USEC_PER_SEC / thread->workload->throttle * THROTTLE_OPS; - throttle_cfg->ops_per_increment = THROTTLE_OPS; - } else { - /* If the interval is large, we do more ops per interval */ - throttle_cfg->usecs_increment = USEC_PER_SEC / THROTTLE_OPS; - throttle_cfg->ops_per_increment = - thread->workload->throttle / THROTTLE_OPS; - } + if (thread->workload->throttle < THROTTLE_OPS) { + /* If the interval is very small, we do one operation */ + throttle_cfg->usecs_increment = USEC_PER_SEC / thread->workload->throttle; + throttle_cfg->ops_per_increment = 1; + } else if (thread->workload->throttle < USEC_PER_SEC / THROTTLE_OPS) { + throttle_cfg->usecs_increment = USEC_PER_SEC / thread->workload->throttle * THROTTLE_OPS; + throttle_cfg->ops_per_increment = THROTTLE_OPS; + } else { + /* If the interval is large, we do more ops per interval */ + throttle_cfg->usecs_increment = USEC_PER_SEC / THROTTLE_OPS; + throttle_cfg->ops_per_increment = thread->workload->throttle / THROTTLE_OPS; + } - /* Give the queue some initial operations to work with */ - throttle_cfg->ops_count = throttle_cfg->ops_per_increment; + /* Give the queue some initial operations to work with */ + throttle_cfg->ops_count = throttle_cfg->ops_per_increment; - /* Set the first timestamp of when we incremented */ - __wt_epoch(NULL, &throttle_cfg->last_increment); + /* Set the first timestamp of when we incremented */ + __wt_epoch(NULL, &throttle_cfg->last_increment); } /* - * Run the throttle function. We will sleep if needed and then reload the - * counter to perform more operations. + * Run the throttle function. We will sleep if needed and then reload the counter to perform more + * operations. */ void worker_throttle(WTPERF_THREAD *thread) { - THROTTLE_CONFIG *throttle_cfg; - struct timespec now; - uint64_t usecs_delta; + THROTTLE_CONFIG *throttle_cfg; + struct timespec now; + uint64_t usecs_delta; - throttle_cfg = &thread->throttle_cfg; + throttle_cfg = &thread->throttle_cfg; - __wt_epoch(NULL, &now); + __wt_epoch(NULL, &now); - /* - * If we did enough operations in the current interval, sleep for - * the rest of the interval. Then add more operations to the queue. - */ - usecs_delta = WT_TIMEDIFF_US(now, throttle_cfg->last_increment); - if (usecs_delta < throttle_cfg->usecs_increment) { - (void)usleep( - (useconds_t)(throttle_cfg->usecs_increment - usecs_delta)); - throttle_cfg->ops_count = - throttle_cfg->ops_per_increment; - /* - * After sleeping, set the interval to the current time. - */ - __wt_epoch(NULL, &throttle_cfg->last_increment); - } else { - throttle_cfg->ops_count = (usecs_delta * - throttle_cfg->ops_per_increment) / - throttle_cfg->usecs_increment; - throttle_cfg->last_increment = now; - } + /* + * If we did enough operations in the current interval, sleep for the rest of the interval. Then + * add more operations to the queue. + */ + usecs_delta = WT_TIMEDIFF_US(now, throttle_cfg->last_increment); + if (usecs_delta < throttle_cfg->usecs_increment) { + (void)usleep((useconds_t)(throttle_cfg->usecs_increment - usecs_delta)); + throttle_cfg->ops_count = throttle_cfg->ops_per_increment; + /* + * After sleeping, set the interval to the current time. + */ + __wt_epoch(NULL, &throttle_cfg->last_increment); + } else { + throttle_cfg->ops_count = + (usecs_delta * throttle_cfg->ops_per_increment) / throttle_cfg->usecs_increment; + throttle_cfg->last_increment = now; + } - /* - * Take the minimum so we don't overfill the queue. - */ - throttle_cfg->ops_count = - WT_MIN(throttle_cfg->ops_count, thread->workload->throttle); + /* + * Take the minimum so we don't overfill the queue. + */ + throttle_cfg->ops_count = WT_MIN(throttle_cfg->ops_count, thread->workload->throttle); } diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c b/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c index 93e6e3ca3a1..1e76514fd1f 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c @@ -31,207 +31,190 @@ static inline uint64_t decode_key(char *key_buf) { - return (strtoull(key_buf, NULL, 10)); + return (strtoull(key_buf, NULL, 10)); } void setup_truncate(WTPERF *wtperf, WTPERF_THREAD *thread, WT_SESSION *session) { - CONFIG_OPTS *opts; - TRUNCATE_CONFIG *trunc_cfg; - TRUNCATE_QUEUE_ENTRY *truncate_item; - WORKLOAD *workload; - WT_CURSOR *cursor; - uint64_t end_point, final_stone_gap, i, start_point; - char *key; - - opts = wtperf->opts; - end_point = final_stone_gap = start_point = 0; - trunc_cfg = &thread->trunc_cfg; - workload = thread->workload; - - /* We are limited to only one table when running truncate. */ - testutil_check(session->open_cursor( - session, wtperf->uris[0], NULL, NULL, &cursor)); - - /* - * If we find the workload getting behind we multiply the number of - * records to be truncated. - */ - trunc_cfg->catchup_multiplier = 1; - - /* How many entries between each stone. */ - trunc_cfg->stone_gap = - (workload->truncate_count * workload->truncate_pct) / 100; - /* How many stones we need. */ - trunc_cfg->needed_stones = - workload->truncate_count / trunc_cfg->stone_gap; - - final_stone_gap = trunc_cfg->stone_gap; - - /* Reset this value for use again. */ - trunc_cfg->stone_gap = 0; - - /* - * Here we check if there is data in the collection. If there is - * data available, then we need to setup some initial truncation - * stones. - */ - testutil_check(cursor->next(cursor)); - testutil_check(cursor->get_key(cursor, &key)); - - start_point = decode_key(key); - testutil_check(cursor->reset(cursor)); - testutil_check(cursor->prev(cursor)); - testutil_check(cursor->get_key(cursor, &key)); - end_point = decode_key(key); - - /* Assign stones if there are enough documents. */ - if (start_point + trunc_cfg->needed_stones > end_point) - trunc_cfg->stone_gap = 0; - else - trunc_cfg->stone_gap = - (end_point - start_point) / trunc_cfg->needed_stones; - - /* If we have enough data allocate some stones. */ - if (trunc_cfg->stone_gap != 0) { - trunc_cfg->expected_total = (end_point - start_point); - for (i = 1; i <= trunc_cfg->needed_stones; i++) { - truncate_item = - dcalloc(sizeof(TRUNCATE_QUEUE_ENTRY), 1); - truncate_item->key = dcalloc(opts->key_sz, 1); - generate_key( - opts, truncate_item->key, trunc_cfg->stone_gap * i); - truncate_item->diff = - (trunc_cfg->stone_gap * i) - trunc_cfg->last_key; - TAILQ_INSERT_TAIL( - &wtperf->stone_head, truncate_item, q); - trunc_cfg->last_key = trunc_cfg->stone_gap * i; - trunc_cfg->num_stones++; - } - } - trunc_cfg->stone_gap = final_stone_gap; - - testutil_check(cursor->close(cursor)); + CONFIG_OPTS *opts; + TRUNCATE_CONFIG *trunc_cfg; + TRUNCATE_QUEUE_ENTRY *truncate_item; + WORKLOAD *workload; + WT_CURSOR *cursor; + uint64_t end_point, final_stone_gap, i, start_point; + char *key; + + opts = wtperf->opts; + end_point = final_stone_gap = start_point = 0; + trunc_cfg = &thread->trunc_cfg; + workload = thread->workload; + + /* We are limited to only one table when running truncate. */ + testutil_check(session->open_cursor(session, wtperf->uris[0], NULL, NULL, &cursor)); + + /* + * If we find the workload getting behind we multiply the number of records to be truncated. + */ + trunc_cfg->catchup_multiplier = 1; + + /* How many entries between each stone. */ + trunc_cfg->stone_gap = (workload->truncate_count * workload->truncate_pct) / 100; + /* How many stones we need. */ + trunc_cfg->needed_stones = workload->truncate_count / trunc_cfg->stone_gap; + + final_stone_gap = trunc_cfg->stone_gap; + + /* Reset this value for use again. */ + trunc_cfg->stone_gap = 0; + + /* + * Here we check if there is data in the collection. If there is data available, then we need to + * setup some initial truncation stones. + */ + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &key)); + + start_point = decode_key(key); + testutil_check(cursor->reset(cursor)); + testutil_check(cursor->prev(cursor)); + testutil_check(cursor->get_key(cursor, &key)); + end_point = decode_key(key); + + /* Assign stones if there are enough documents. */ + if (start_point + trunc_cfg->needed_stones > end_point) + trunc_cfg->stone_gap = 0; + else + trunc_cfg->stone_gap = (end_point - start_point) / trunc_cfg->needed_stones; + + /* If we have enough data allocate some stones. */ + if (trunc_cfg->stone_gap != 0) { + trunc_cfg->expected_total = (end_point - start_point); + for (i = 1; i <= trunc_cfg->needed_stones; i++) { + truncate_item = dcalloc(sizeof(TRUNCATE_QUEUE_ENTRY), 1); + truncate_item->key = dcalloc(opts->key_sz, 1); + generate_key(opts, truncate_item->key, trunc_cfg->stone_gap * i); + truncate_item->diff = (trunc_cfg->stone_gap * i) - trunc_cfg->last_key; + TAILQ_INSERT_TAIL(&wtperf->stone_head, truncate_item, q); + trunc_cfg->last_key = trunc_cfg->stone_gap * i; + trunc_cfg->num_stones++; + } + } + trunc_cfg->stone_gap = final_stone_gap; + + testutil_check(cursor->close(cursor)); } int -run_truncate(WTPERF *wtperf, WTPERF_THREAD *thread, - WT_CURSOR *cursor, WT_SESSION *session, int *truncatedp) +run_truncate( + WTPERF *wtperf, WTPERF_THREAD *thread, WT_CURSOR *cursor, WT_SESSION *session, int *truncatedp) { - CONFIG_OPTS *opts; - TRUNCATE_CONFIG *trunc_cfg; - TRUNCATE_QUEUE_ENTRY *truncate_item; - char *next_key; - int ret, t_ret; - uint64_t used_stone_gap; - - opts = wtperf->opts; - trunc_cfg = &thread->trunc_cfg; - ret = 0; - - *truncatedp = 0; - /* Update the total inserts */ - trunc_cfg->total_inserts = sum_insert_ops(wtperf); - trunc_cfg->expected_total += - (trunc_cfg->total_inserts - trunc_cfg->last_total_inserts); - trunc_cfg->last_total_inserts = trunc_cfg->total_inserts; - - /* We are done if there isn't enough data to trigger a new milestone. */ - if (trunc_cfg->expected_total <= thread->workload->truncate_count) - return (0); - - /* - * If we are falling behind and using more than one stone per lap we - * should widen the stone gap for this lap to try and catch up quicker. - */ - if (trunc_cfg->expected_total > - thread->workload->truncate_count + trunc_cfg->stone_gap) { - /* - * Increase the multiplier until we create stones that are - * almost large enough to truncate the whole expected table size - * in one operation. - */ - trunc_cfg->catchup_multiplier = - WT_MIN(trunc_cfg->catchup_multiplier + 1, - trunc_cfg->needed_stones - 1); - } else { - /* Back off if we start seeing an improvement */ - trunc_cfg->catchup_multiplier = - WT_MAX(trunc_cfg->catchup_multiplier - 1, 1); - } - used_stone_gap = trunc_cfg->stone_gap * trunc_cfg->catchup_multiplier; - - while (trunc_cfg->num_stones < trunc_cfg->needed_stones) { - trunc_cfg->last_key += used_stone_gap; - truncate_item = dcalloc(sizeof(TRUNCATE_QUEUE_ENTRY), 1); - truncate_item->key = dcalloc(opts->key_sz, 1); - generate_key(opts, truncate_item->key, trunc_cfg->last_key); - truncate_item->diff = used_stone_gap; - TAILQ_INSERT_TAIL(&wtperf->stone_head, truncate_item, q); - trunc_cfg->num_stones++; - } - - /* We are done if there isn't enough data to trigger a truncate. */ - if (trunc_cfg->num_stones == 0 || - trunc_cfg->expected_total <= thread->workload->truncate_count) - return (0); - - truncate_item = TAILQ_FIRST(&wtperf->stone_head); - trunc_cfg->num_stones--; - TAILQ_REMOVE(&wtperf->stone_head, truncate_item, q); - - /* - * Truncate the content via a single truncate call or a cursor walk - * depending on the configuration. - */ - if (opts->truncate_single_ops) { - while ((ret = cursor->next(cursor)) == 0) { - testutil_check(cursor->get_key(cursor, &next_key)); - if (strcmp(next_key, truncate_item->key) == 0) - break; - if ((ret = cursor->remove(cursor)) != 0) { - lprintf(wtperf, - ret, 0, "Truncate remove: failed"); - goto err; - } - } - } else { - cursor->set_key(cursor,truncate_item->key); - if ((ret = cursor->search(cursor)) != 0) { - lprintf(wtperf, ret, 0, "Truncate search: failed"); - goto err; - } - - if ((ret = session->truncate( - session, NULL, NULL, cursor, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Truncate: failed"); - goto err; - } - } - - *truncatedp = 1; - trunc_cfg->expected_total -= truncate_item->diff; - -err: free(truncate_item->key); - free(truncate_item); - t_ret = cursor->reset(cursor); - if (t_ret != 0) - lprintf(wtperf, t_ret, 0, "Cursor reset failed"); - if (ret == 0 && t_ret != 0) - ret = t_ret; - return (ret); + CONFIG_OPTS *opts; + TRUNCATE_CONFIG *trunc_cfg; + TRUNCATE_QUEUE_ENTRY *truncate_item; + char *next_key; + int ret, t_ret; + uint64_t used_stone_gap; + + opts = wtperf->opts; + trunc_cfg = &thread->trunc_cfg; + ret = 0; + + *truncatedp = 0; + /* Update the total inserts */ + trunc_cfg->total_inserts = sum_insert_ops(wtperf); + trunc_cfg->expected_total += (trunc_cfg->total_inserts - trunc_cfg->last_total_inserts); + trunc_cfg->last_total_inserts = trunc_cfg->total_inserts; + + /* We are done if there isn't enough data to trigger a new milestone. */ + if (trunc_cfg->expected_total <= thread->workload->truncate_count) + return (0); + + /* + * If we are falling behind and using more than one stone per lap we should widen the stone gap + * for this lap to try and catch up quicker. + */ + if (trunc_cfg->expected_total > thread->workload->truncate_count + trunc_cfg->stone_gap) { + /* + * Increase the multiplier until we create stones that are almost large enough to truncate + * the whole expected table size in one operation. + */ + trunc_cfg->catchup_multiplier = + WT_MIN(trunc_cfg->catchup_multiplier + 1, trunc_cfg->needed_stones - 1); + } else { + /* Back off if we start seeing an improvement */ + trunc_cfg->catchup_multiplier = WT_MAX(trunc_cfg->catchup_multiplier - 1, 1); + } + used_stone_gap = trunc_cfg->stone_gap * trunc_cfg->catchup_multiplier; + + while (trunc_cfg->num_stones < trunc_cfg->needed_stones) { + trunc_cfg->last_key += used_stone_gap; + truncate_item = dcalloc(sizeof(TRUNCATE_QUEUE_ENTRY), 1); + truncate_item->key = dcalloc(opts->key_sz, 1); + generate_key(opts, truncate_item->key, trunc_cfg->last_key); + truncate_item->diff = used_stone_gap; + TAILQ_INSERT_TAIL(&wtperf->stone_head, truncate_item, q); + trunc_cfg->num_stones++; + } + + /* We are done if there isn't enough data to trigger a truncate. */ + if (trunc_cfg->num_stones == 0 || trunc_cfg->expected_total <= thread->workload->truncate_count) + return (0); + + truncate_item = TAILQ_FIRST(&wtperf->stone_head); + trunc_cfg->num_stones--; + TAILQ_REMOVE(&wtperf->stone_head, truncate_item, q); + + /* + * Truncate the content via a single truncate call or a cursor walk depending on the + * configuration. + */ + if (opts->truncate_single_ops) { + while ((ret = cursor->next(cursor)) == 0) { + testutil_check(cursor->get_key(cursor, &next_key)); + if (strcmp(next_key, truncate_item->key) == 0) + break; + if ((ret = cursor->remove(cursor)) != 0) { + lprintf(wtperf, ret, 0, "Truncate remove: failed"); + goto err; + } + } + } else { + cursor->set_key(cursor, truncate_item->key); + if ((ret = cursor->search(cursor)) != 0) { + lprintf(wtperf, ret, 0, "Truncate search: failed"); + goto err; + } + + if ((ret = session->truncate(session, NULL, NULL, cursor, NULL)) != 0) { + lprintf(wtperf, ret, 0, "Truncate: failed"); + goto err; + } + } + + *truncatedp = 1; + trunc_cfg->expected_total -= truncate_item->diff; + +err: + free(truncate_item->key); + free(truncate_item); + t_ret = cursor->reset(cursor); + if (t_ret != 0) + lprintf(wtperf, t_ret, 0, "Cursor reset failed"); + if (ret == 0 && t_ret != 0) + ret = t_ret; + return (ret); } void cleanup_truncate_config(WTPERF *wtperf) { - TRUNCATE_QUEUE_ENTRY *truncate_item; - - while (!TAILQ_EMPTY(&wtperf->stone_head)) { - truncate_item = TAILQ_FIRST(&wtperf->stone_head); - TAILQ_REMOVE(&wtperf->stone_head, truncate_item, q); - free(truncate_item->key); - free(truncate_item); - } + TRUNCATE_QUEUE_ENTRY *truncate_item; + + while (!TAILQ_EMPTY(&wtperf->stone_head)) { + truncate_item = TAILQ_FIRST(&wtperf->stone_head); + TAILQ_REMOVE(&wtperf->stone_head, truncate_item, q); + free(truncate_item->key); + free(truncate_item); + } } |