summaryrefslogtreecommitdiff
path: root/bench/wtperf/wtperf.h
blob: 1bb94db26344af9a92850bd95b7f612473e31afa (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
/*-
 * Public Domain 2014-2016 MongoDB, Inc.
 * Public Domain 2008-2014 WiredTiger, Inc.
 *
 * This is free and unencumbered software released into the public domain.
 *
 * Anyone is free to copy, modify, publish, use, compile, sell, or
 * distribute this software, either in source code form or as a compiled
 * binary, for any purpose, commercial or non-commercial, and by any
 * means.
 *
 * In jurisdictions that recognize copyright laws, the author or authors
 * of this software dedicate any and all copyright interest in the
 * software to the public domain. We make this dedication for the benefit
 * of the public at large and to the detriment of our heirs and
 * successors. We intend this dedication to be an overt act of
 * relinquishment in perpetuity of all present and future rights to this
 * software under copyright law.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef	HAVE_WTPERF_H
#define	HAVE_WTPERF_H

#include "test_util.h"

#include <assert.h>
#include <math.h>

#include "config_opt.h"

typedef struct __config CONFIG;
typedef struct __config_thread CONFIG_THREAD;
typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY;

#define	EXT_PFX	",extensions=("
#define	EXT_SFX	")"
#define	EXTPATH "../../ext/compressors/"		/* Extensions path */
#define	BLKCMP_PFX	"block_compressor="

#define	LZ4_BLK BLKCMP_PFX "lz4"
#define	LZ4_EXT							\
	EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX
#define	SNAPPY_BLK BLKCMP_PFX "snappy"
#define	SNAPPY_EXT							\
	EXT_PFX EXTPATH "snappy/.libs/libwiredtiger_snappy.so" EXT_SFX
#define	ZLIB_BLK BLKCMP_PFX "zlib"
#define	ZLIB_EXT							\
	EXT_PFX EXTPATH "zlib/.libs/libwiredtiger_zlib.so" EXT_SFX

typedef struct {
	int64_t threads;		/* Thread count */
	int64_t insert;			/* Insert ratio */
	int64_t read;			/* Read ratio */
	int64_t update;			/* Update ratio */
	uint64_t throttle;		/* Maximum operations/second */
		/* Number of operations per transaction. Zero for autocommit */
	int64_t ops_per_txn;
	int64_t truncate;		/* Truncate ratio */
	uint64_t truncate_pct;		/* Truncate Percent */
	uint64_t truncate_count;	/* Truncate Count */
	int64_t update_delta;		/* Value size change on update */

#define	WORKER_INSERT		1	/* Insert */
#define	WORKER_INSERT_RMW	2	/* Insert with read-modify-write */
#define	WORKER_READ		3	/* Read */
#define	WORKER_TRUNCATE		4	/* Truncate */
#define	WORKER_UPDATE		5	/* Update */
	uint8_t ops[100];		/* Operation schedule */
} WORKLOAD;

/* Steering items for the truncate workload */
typedef struct {
	uint64_t stone_gap;
	uint64_t needed_stones;
	uint64_t expected_total;
	uint64_t total_inserts;
	uint64_t last_total_inserts;
	uint64_t num_stones;
	uint64_t last_key;
	uint64_t catchup_multiplier;
} TRUNCATE_CONFIG;

/* Queue entry for use with the Truncate Logic */
struct __truncate_queue_entry {
	char *key;			/* Truncation point */
	uint64_t diff;			/* Number of items to be truncated*/
	TAILQ_ENTRY(__truncate_queue_entry) q;
};

struct __config_queue_entry {
	char *string;
	TAILQ_ENTRY(__config_queue_entry) c;
};
typedef struct __config_queue_entry CONFIG_QUEUE_ENTRY;

/* Steering for the throttle configuration */
typedef struct {
	struct timespec last_increment;	/* Time that we last added more ops */
	uint64_t ops_count;		/* The number of ops this increment */
	uint64_t ops_per_increment;	/* Ops to add per increment */
	uint64_t usecs_increment;	/* Time interval of each increment */
} THROTTLE_CONFIG;

#define	LOG_PARTIAL_CONFIG	",log=(enabled=false)"
#define	READONLY_CONFIG		",readonly=true"
/*
 * NOTE:  If you add any fields to this structure here, you must also add
 * an initialization in wtperf.c in the default_cfg.
 */
struct __config {			/* Configuration structure */
	char *home;			/* WiredTiger home */
	char *monitor_dir;		/* Monitor output dir */
	char *partial_config;		/* Config string for partial logging */
	char *reopen_config;		/* Config string for conn reopen */
	char *base_uri;			/* Object URI */
	char *log_table_uri;		/* URI for log table */
	char **uris;			/* URIs if multiple tables */

	WT_CONNECTION *conn;		/* Database connection */

	FILE *logf;			/* Logging handle */

	char *async_config;		/* Config string for async */

	const char *compress_ext;	/* Compression extension for conn */
	const char *compress_table;	/* Compression arg to table create */

	CONFIG_THREAD *ckptthreads, *popthreads;

#define	WORKLOAD_MAX	50
	CONFIG_THREAD	*workers;	/* Worker threads */
	u_int		 workers_cnt;

	WORKLOAD	*workload;	/* Workloads */
	u_int		 workload_cnt;

	uint32_t	 use_asyncops;	/* Use async operations */
	/* State tracking variables. */

	uint64_t ckpt_ops;		/* checkpoint operations */
	uint64_t insert_ops;		/* insert operations */
	uint64_t read_ops;		/* read operations */
	uint64_t truncate_ops;		/* truncate operations */
	uint64_t update_ops;		/* update operations */

	uint64_t insert_key;		/* insert key */
	uint64_t log_like_table_key;	/* used to allocate IDs for log table */

	volatile int ckpt;		/* checkpoint in progress */
	volatile int error;		/* thread error */
	volatile int stop;		/* notify threads to stop */
	volatile int in_warmup;		/* Running warmup phase */

	volatile bool idle_cycle_run;	/* Signal for idle cycle thread */

	volatile uint32_t totalsec;	/* total seconds running */

#define	CFG_GROW	0x0001		/* There is a grow workload */
#define	CFG_SHRINK	0x0002		/* There is a shrink workload */
#define	CFG_TRUNCATE	0x0004		/* There is a truncate workload */
	uint32_t	flags;		/* flags */

	/* Queue head for use with the Truncate Logic */
	TAILQ_HEAD(__truncate_qh, __truncate_queue_entry) stone_head;

	/* Queue head to save a copy of the config to be output */
	TAILQ_HEAD(__config_qh, __config_queue_entry) config_head;

	/* Fields changeable on command line are listed in wtperf_opt.i */
#define	OPT_DECLARE_STRUCT
#include "wtperf_opt.i"
#undef OPT_DECLARE_STRUCT
};

#define	ELEMENTS(a)	(sizeof(a) / sizeof(a[0]))

#define	READ_RANGE_OPS	10
#define	THROTTLE_OPS	100

#define	THOUSAND	(1000ULL)
#define	MILLION		(1000000ULL)
#define	BILLION		(1000000000ULL)

#define	NSEC_PER_SEC	BILLION
#define	USEC_PER_SEC	MILLION
#define	MSEC_PER_SEC	THOUSAND

#define	ns_to_ms(v)	((v) / MILLION)
#define	ns_to_sec(v)	((v) / BILLION)
#define	ns_to_us(v)	((v) / THOUSAND)

#define	us_to_ms(v)	((v) / THOUSAND)
#define	us_to_ns(v)	((v) * THOUSAND)
#define	us_to_sec(v)	((v) / MILLION)

#define	ms_to_ns(v)	((v) * MILLION)
#define	ms_to_us(v)	((v) * THOUSAND)
#define	ms_to_sec(v)	((v) / THOUSAND)

#define	sec_to_ns(v)	((v) * BILLION)
#define	sec_to_us(v)	((v) * MILLION)
#define	sec_to_ms(v)	((v) * THOUSAND)

typedef struct {
	/*
	 * Threads maintain the total thread operation and total latency they've
	 * experienced; the monitor thread periodically copies these values into
	 * the last_XXX fields.
	 */
	uint64_t ops;			/* Total operations */
	uint64_t latency_ops;		/* Total ops sampled for latency */
	uint64_t latency;		/* Total latency */

	uint64_t last_latency_ops;	/* Last read by monitor thread */
	uint64_t last_latency;

	/*
	 * Minimum/maximum latency, shared with the monitor thread, that is, the
	 * monitor thread clears it so it's recalculated again for each period.
	 */
	uint32_t min_latency;		/* Minimum latency (uS) */
	uint32_t max_latency;		/* Maximum latency (uS) */

	/*
	 * Latency buckets.
	 */
	uint32_t us[1000];		/* < 1us ... 1000us */
	uint32_t ms[1000];		/* < 1ms ... 1000ms */
	uint32_t sec[100];		/* < 1s 2s ... 100s */
} TRACK;

struct __config_thread {		/* Per-thread structure */
	CONFIG *cfg;			/* Enclosing configuration */

	WT_RAND_STATE rnd;		/* Random number generation state */

	pthread_t handle;		/* Handle */

	char *key_buf, *value_buf;	/* Key/value memory */

	WORKLOAD *workload;		/* Workload */

	THROTTLE_CONFIG throttle_cfg;   /* Throttle configuration */

	TRUNCATE_CONFIG trunc_cfg;      /* Truncate configuration */

	TRACK ckpt;			/* Checkpoint operations */
	TRACK insert;			/* Insert operations */
	TRACK read;			/* Read operations */
	TRACK update;			/* Update operations */
	TRACK truncate;			/* Truncate operations */
	TRACK truncate_sleep;		/* Truncate sleep operations */
};

void	 cleanup_truncate_config(CONFIG *);
int	 config_compress(CONFIG *);
void	 config_free(CONFIG *);
void	 config_copy(CONFIG *, const CONFIG *);
int	 config_opt_file(CONFIG *, const char *);
int	 config_opt_line(CONFIG *, const char *);
int	 config_opt_str(CONFIG *, const char *, const char *);
void	 config_to_file(CONFIG *);
void	 config_consolidate(CONFIG *);
void	 config_print(CONFIG *);
int	 config_sanity(CONFIG *);
void	 latency_insert(CONFIG *, uint32_t *, uint32_t *, uint32_t *);
void	 latency_read(CONFIG *, uint32_t *, uint32_t *, uint32_t *);
void	 latency_update(CONFIG *, uint32_t *, uint32_t *, uint32_t *);
void	 latency_print(CONFIG *);
int	 run_truncate(
    CONFIG *, CONFIG_THREAD *, WT_CURSOR *, WT_SESSION *, int *);
int	 setup_log_file(CONFIG *);
void	 setup_throttle(CONFIG_THREAD*);
int	 setup_truncate(CONFIG *, CONFIG_THREAD *, WT_SESSION *);
int	 start_idle_table_cycle(CONFIG *, pthread_t *);
int	 stop_idle_table_cycle(CONFIG *, pthread_t);
uint64_t sum_ckpt_ops(CONFIG *);
uint64_t sum_insert_ops(CONFIG *);
uint64_t sum_pop_ops(CONFIG *);
uint64_t sum_read_ops(CONFIG *);
uint64_t sum_truncate_ops(CONFIG *);
uint64_t sum_update_ops(CONFIG *);
void	 usage(void);
void	 worker_throttle(CONFIG_THREAD*);

void	 lprintf(const CONFIG *, int err, uint32_t, const char *, ...)
#if defined(__GNUC__)
__attribute__((format (printf, 4, 5)))
#endif
;

static inline void
generate_key(CONFIG *cfg, char *key_buf, uint64_t keyno)
{
	/*
	 * Don't change to snprintf, sprintf is faster in some tests.
	 */
	sprintf(key_buf, "%0*" PRIu64, cfg->key_sz - 1, keyno);
}

static inline void
extract_key(char *key_buf, uint64_t *keynop)
{
	(void)sscanf(key_buf, "%" SCNu64, keynop);
}

/*
 * die --
 *      Print message and exit on failure.
 */
static inline void
die(int, const char *)
    WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
static inline void
die(int e, const char *str)
{
	fprintf(stderr, "Call to %s failed: %s", str, wiredtiger_strerror(e));
	exit(EXIT_FAILURE);
}
#endif