From 1a7f0516a07ac91c817ccb5d52e3568f0a47482b Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Thu, 13 Jul 2017 21:59:26 -0400 Subject: WT-3425 In workgen, added 'reopen' configuration option for Operations. (#3508) --- bench/workgen/runner/small_btree_reopen.py | 56 +++++++++++++++++++++++++ bench/workgen/workgen.cxx | 65 +++++++++++++++++++++--------- bench/workgen/workgen.h | 8 +++- bench/workgen/workgen.swig | 2 +- bench/workgen/workgen_int.h | 2 +- 5 files changed, 111 insertions(+), 22 deletions(-) create mode 100644 bench/workgen/runner/small_btree_reopen.py diff --git a/bench/workgen/runner/small_btree_reopen.py b/bench/workgen/runner/small_btree_reopen.py new file mode 100644 index 00000000000..252dd02aae9 --- /dev/null +++ b/bench/workgen/runner/small_btree_reopen.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from runner import * +from wiredtiger import * +from workgen import * + +context = Context() +conn = wiredtiger_open("WT_TEST", "create,cache_size=500MB") +s = conn.open_session() +tname = "file:test.wt" +s.create(tname, 'key_format=S,value_format=S') +table = Table(tname) +table.options.key_size = 20 +table.options.value_size = 100 + +op = Operation(Operation.OP_INSERT, table) +thread = Thread(op * 500000) +pop_workload = Workload(context, thread) +print('populate:') +pop_workload.run(conn) + +op = Operation(Operation.OP_SEARCH, table) +op._config = 'reopen' +t = Thread(op) +workload = Workload(context, t * 8) +workload.options.run_time = 120 +workload.options.report_interval = 5 +print('read workload:') +workload.run(conn) diff --git a/bench/workgen/workgen.cxx b/bench/workgen/workgen.cxx index 880b8ca6467..1a0735f9adc 100644 --- a/bench/workgen/workgen.cxx +++ b/bench/workgen/workgen.cxx @@ -534,7 +534,7 @@ void ThreadRunner::op_create_all(Operation *op, size_t &keysize, size_t &valuesize) { tint_t tint; - op->size_check(); + op->create_all(); if (op->_optype != Operation::OP_NONE) { op->kv_compute_max(true); if (OP_HAS_VALUE(op)) @@ -579,6 +579,7 @@ uint64_t ThreadRunner::op_get_key_recno(Operation *op, tint_t tint) { uint64_t recno_count; uint32_t rand; + (void)op; recno_count = _icontext->_recno[tint]; if (recno_count == 0) // The file has no entries, returning 0 forces a WT_NOTFOUND return. @@ -590,13 +591,15 @@ uint64_t ThreadRunner::op_get_key_recno(Operation *op, tint_t tint) { int ThreadRunner::op_run(Operation *op) { Track *track; tint_t tint = op->_table._internal->_tint; - WT_CURSOR *cursor = _cursors[tint]; + WT_CURSOR *cursor; WT_DECL_RET; uint64_t recno; - bool measure_latency; + bool measure_latency, own_cursor; - recno = 0; track = NULL; + cursor = NULL; + recno = 0; + own_cursor = false; if (_throttle != NULL) { if (_throttle_ops >= _throttle_limit && !_in_transaction) { WT_ERR(_throttle->throttle(_throttle_ops, @@ -636,6 +639,13 @@ int ThreadRunner::op_run(Operation *op) { recno = 0; break; } + if ((op->_flags & WORKGEN_OP_REOPEN) != 0 && + op->_optype != Operation::OP_NONE) { + WT_ERR(_session->open_cursor(_session, op->_table._uri.c_str(), NULL, + NULL, &cursor)); + own_cursor = true; + } else + cursor = _cursors[tint]; measure_latency = track != NULL && track->ops != 0 && track->track_latency() && @@ -676,6 +686,7 @@ int ThreadRunner::op_run(Operation *op) { ASSERT(false); } if (ret != 0) { + ASSERT(ret == WT_NOTFOUND); track = &_stats.not_found; ret = 0; // WT_NOTFOUND allowed. } @@ -694,6 +705,8 @@ int ThreadRunner::op_run(Operation *op) { i != op->_group->end(); i++) WT_ERR(op_run(&*i)); err: + if (own_cursor) + WT_TRET(cursor->close(cursor)); if (op->_transaction != NULL) { if (ret != 0 || op->_transaction->_rollback) WT_TRET(_session->rollback_transaction(_session, NULL)); @@ -716,8 +729,8 @@ Throttle::Throttle(ThreadRunner &runner, double throttle, _burst(throttle_burst), _next_div(), _ops_delta(0), _ops_prev(0), _ops_per_div(0), _ms_per_div(0), _started(false) { ts_clear(_next_div); - _ms_per_div = ceill(1000.0 / THROTTLE_PER_SEC); - _ops_per_div = ceill(_throttle / THROTTLE_PER_SEC); + _ms_per_div = (uint64_t)ceill(1000.0 / THROTTLE_PER_SEC); + _ops_per_div = (uint64_t)ceill(_throttle / THROTTLE_PER_SEC); } Throttle::~Throttle() {} @@ -835,36 +848,37 @@ void Thread::describe(std::ostream &os) const { } Operation::Operation() : - _optype(OP_NONE), _table(), _key(), _value(), _transaction(NULL), - _group(NULL), _repeatgroup(0), + _optype(OP_NONE), _table(), _key(), _value(), _config(), _transaction(NULL), + _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { } Operation::Operation(OpType optype, Table table, Key key, Value value) : - _optype(optype), _table(table), _key(key), _value(value), - _transaction(NULL), _group(NULL), _repeatgroup(0), + _optype(optype), _table(table), _key(key), _value(value), _config(), + _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { size_check(); } Operation::Operation(OpType optype, Table table, Key key) : - _optype(optype), _table(table), _key(key), _value(), _transaction(NULL), - _group(NULL), _repeatgroup(0), + _optype(optype), _table(table), _key(key), _value(), _config(), + _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { size_check(); } Operation::Operation(OpType optype, Table table) : - _optype(optype), _table(table), _key(), _value(), _transaction(NULL), - _group(NULL), _repeatgroup(0), + _optype(optype), _table(table), _key(), _value(), _config(), + _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0), _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { size_check(); } Operation::Operation(const Operation &other) : _optype(other._optype), _table(other._table), _key(other._key), - _value(other._value), _transaction(other._transaction), - _group(other._group), _repeatgroup(other._repeatgroup), + _value(other._value), _config(other._config), + _transaction(other._transaction), _group(other._group), + _repeatgroup(other._repeatgroup), _flags(other._flags), _keysize(other._keysize), _valuesize(other._valuesize), _keymax(other._keymax), _valuemax(other._valuemax) { // Creation and destruction of _group and _transaction is managed @@ -890,6 +904,18 @@ Operation& Operation::operator=(const Operation &other) { return (*this); } +void Operation::create_all() { + size_check(); + + _flags = 0; + if (!_config.empty()) { + if (_config == "reopen") + _flags |= WORKGEN_OP_REOPEN; + else + THROW("operation has illegal config: \"" << _config << "\""); + } +} + void Operation::describe(std::ostream &os) const { os << "Operation: " << _optype; if (_optype != OP_NONE) { @@ -897,9 +923,10 @@ void Operation::describe(std::ostream &os) const { os << ", "; _key.describe(os); os << ", "; _value.describe(os); } - if (_transaction != NULL) { + if (!_config.empty()) + os << ", '" << _config; + if (_transaction != NULL) os << ", ["; _transaction->describe(os); os << "]"; - } if (_group != NULL) { os << ", group[" << _repeatgroup << "]: {"; bool first = true; @@ -1648,4 +1675,4 @@ int WorkloadRunner::run_all() { return (ret); } -}; +} diff --git a/bench/workgen/workgen.h b/bench/workgen/workgen.h index c7be8ee0035..a12e4dc4c89 100644 --- a/bench/workgen/workgen.h +++ b/bench/workgen/workgen.h @@ -33,6 +33,7 @@ namespace workgen { struct ContextInternal; +struct OperationInternal; struct TableInternal; struct Thread; struct Transaction; @@ -245,11 +246,15 @@ struct Operation { Table _table; Key _key; Value _value; + std::string _config; Transaction *_transaction; std::vector *_group; int _repeatgroup; #ifndef SWIG +#define WORKGEN_OP_REOPEN 0x0001 // reopen cursor for each op + uint32_t _flags; + int _keysize; // derived from Key._size and Table.options.key_size int _valuesize; uint64_t _keymax; @@ -266,6 +271,7 @@ struct Operation { void describe(std::ostream &os) const; #ifndef SWIG Operation& operator=(const Operation &other); + void create_all(); void get_static_counts(Stats &stats, int multiplier); void kv_compute_max(bool); void kv_gen(bool, uint64_t, char *) const; @@ -408,4 +414,4 @@ struct Workload { int run(WT_CONNECTION *conn); }; -}; +} diff --git a/bench/workgen/workgen.swig b/bench/workgen/workgen.swig index 0f74942169c..a33705f4350 100644 --- a/bench/workgen/workgen.swig +++ b/bench/workgen/workgen.swig @@ -158,7 +158,7 @@ WorkgenFrozenClass(WorkloadOptions) def __add__(self, other): if not isinstance(other, Operation): raise Exception('Operation.__sum__ requires an Operation') - if self._group == None or self._repeatgroup != 1 or self._transaction != None: + if self._group == None or self._repeatgroup != 1 or self._transaction != None or self._config != '': op = Operation() op._group = OpList([self, other]) op._repeatgroup = 1 diff --git a/bench/workgen/workgen_int.h b/bench/workgen/workgen_int.h index 9283aea1d7b..a8d008a3bc5 100644 --- a/bench/workgen/workgen_int.h +++ b/bench/workgen/workgen_int.h @@ -203,4 +203,4 @@ private: WorkloadRunner& operator=(const WorkloadRunner &other); // disallowed }; -}; +} -- cgit v1.2.1