Import wiredtiger: 6173a98979715ed727c432c1a31da64ea8a37048 from branch mongodb-3.6

ref: ff10db8811..6173a98979
for: 3.5.11

WT-3039       Enhance logging so new log files no longer need to wait for the previous log file to be synced
WT-3138       Enhance eviction statistics
WT-3140       Enhance json mode statistics to better support per-dhandle statistics
WT-3310       Add support to WT_SESSION::alter to change table log setting
WT-3329       With a uniform workload and a number of small collections, eviction does a poor job of selecting candidates for eviction.
WT-3381       Improve concurrency in the transaction subsystem
WT-3394       Build WiredTiger with gcc7
WT-3396       test/csuite/wt3363_checkpoint_op_races, test_utility cleanups
WT-3397       Coverity lint
WT-3398       in-memory configurations don't support schema_worker operations
WT-3401       Lint and minor cleanup
WT-3402       Move cached overflow records to the update list.
WT-3403       Automated tests timeout running csuite
WT-3409       WiredTiger generations can silently self-deadlock.
WT-3413       Add more aggressive compile warning flags to Jenkins Windows job
WT-3415       setting timestamp in txn_state requires the global write lock
WT-3421       Compilation error unreachable code in test/csuite/wt3363_checkpoint_op_races
WT-3424       additional gcc 7.1 compile warnings.
WT-3425       workgen: add ability to reopen cursors
WT-3426       Add new wtperf configuration to automated performance testing
WT-3438       Don't tune eviction thread count when the count is fixed

Also explicitly disable v2 log records via a configuration setting in MongoDB -
this will later be plumbed into the feature compatability version mechanism.

8 files changed, 123 insertions, 24 deletions

diff --git a/src/third_party/wiredtiger/bench/workgen/runner/small_btree_reopen.py b/src/third_party/wiredtiger/bench/workgen/runner/small_btree_reopen.py
new file mode 100644
index 00000000000..252dd02aae9
--- /dev/null
+++ b/src/third_party/wiredtiger/bench/workgen/runner/small_btree_reopen.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2017 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+
+from runner import *
+from wiredtiger import *
+from workgen import *
+
+context = Context()
+conn = wiredtiger_open("WT_TEST", "create,cache_size=500MB")
+s = conn.open_session()
+tname = "file:test.wt"
+s.create(tname, 'key_format=S,value_format=S')
+table = Table(tname)
+table.options.key_size = 20
+table.options.value_size = 100
+
+op = Operation(Operation.OP_INSERT, table)
+thread = Thread(op * 500000)
+pop_workload = Workload(context, thread)
+print('populate:')
+pop_workload.run(conn)
+
+op = Operation(Operation.OP_SEARCH, table)
+op._config = 'reopen'
+t = Thread(op)
+workload = Workload(context, t * 8)
+workload.options.run_time = 120
+workload.options.report_interval = 5
+print('read workload:')
+workload.run(conn)
diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx
index 880b8ca6467..1a0735f9adc 100644
--- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx
+++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx
@@ -534,7 +534,7 @@ void ThreadRunner::op_create_all(Operation *op, size_t &keysize,
   size_t &valuesize) {
     tint_t tint;
 
-    op->size_check();
+    op->create_all();
     if (op->_optype != Operation::OP_NONE) {
         op->kv_compute_max(true);
         if (OP_HAS_VALUE(op))
@@ -579,6 +579,7 @@ uint64_t ThreadRunner::op_get_key_recno(Operation *op, tint_t tint) {
     uint64_t recno_count;
     uint32_t rand;
 
+    (void)op;
     recno_count = _icontext->_recno[tint];
     if (recno_count == 0)
         // The file has no entries, returning 0 forces a WT_NOTFOUND return.
@@ -590,13 +591,15 @@ uint64_t ThreadRunner::op_get_key_recno(Operation *op, tint_t tint) {
 int ThreadRunner::op_run(Operation *op) {
     Track *track;
     tint_t tint = op->_table._internal->_tint;
-    WT_CURSOR *cursor = _cursors[tint];
+    WT_CURSOR *cursor;
     WT_DECL_RET;
     uint64_t recno;
-    bool measure_latency;
+    bool measure_latency, own_cursor;
 
-    recno = 0;
     track = NULL;
+    cursor = NULL;
+    recno = 0;
+    own_cursor = false;
     if (_throttle != NULL) {
         if (_throttle_ops >= _throttle_limit && !_in_transaction) {
             WT_ERR(_throttle->throttle(_throttle_ops,
@@ -636,6 +639,13 @@ int ThreadRunner::op_run(Operation *op) {
         recno = 0;
         break;
     }
+    if ((op->_flags & WORKGEN_OP_REOPEN) != 0 &&
+      op->_optype != Operation::OP_NONE) {
+        WT_ERR(_session->open_cursor(_session, op->_table._uri.c_str(), NULL,
+          NULL, &cursor));
+        own_cursor = true;
+    } else
+        cursor = _cursors[tint];
 
     measure_latency = track != NULL && track->ops != 0 &&
       track->track_latency() &&
@@ -676,6 +686,7 @@ int ThreadRunner::op_run(Operation *op) {
             ASSERT(false);
         }
         if (ret != 0) {
+            ASSERT(ret == WT_NOTFOUND);
             track = &_stats.not_found;
             ret = 0;  // WT_NOTFOUND allowed.
         }
@@ -694,6 +705,8 @@ int ThreadRunner::op_run(Operation *op) {
               i != op->_group->end(); i++)
                 WT_ERR(op_run(&*i));
 err:
+    if (own_cursor)
+        WT_TRET(cursor->close(cursor));
     if (op->_transaction != NULL) {
         if (ret != 0 || op->_transaction->_rollback)
             WT_TRET(_session->rollback_transaction(_session, NULL));
@@ -716,8 +729,8 @@ Throttle::Throttle(ThreadRunner &runner, double throttle,
     _burst(throttle_burst), _next_div(), _ops_delta(0), _ops_prev(0),
     _ops_per_div(0), _ms_per_div(0), _started(false) {
     ts_clear(_next_div);
-    _ms_per_div = ceill(1000.0 / THROTTLE_PER_SEC);
-    _ops_per_div = ceill(_throttle / THROTTLE_PER_SEC);
+    _ms_per_div = (uint64_t)ceill(1000.0 / THROTTLE_PER_SEC);
+    _ops_per_div = (uint64_t)ceill(_throttle / THROTTLE_PER_SEC);
 }
 
 Throttle::~Throttle() {}
@@ -835,36 +848,37 @@ void Thread::describe(std::ostream &os) const {
 }
 
 Operation::Operation() :
-    _optype(OP_NONE), _table(), _key(), _value(), _transaction(NULL),
-    _group(NULL), _repeatgroup(0),
+    _optype(OP_NONE), _table(), _key(), _value(), _config(), _transaction(NULL),
+    _group(NULL), _repeatgroup(0), _flags(0),
     _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) {
 }
 
 Operation::Operation(OpType optype, Table table, Key key, Value value) :
-    _optype(optype), _table(table), _key(key), _value(value),
-    _transaction(NULL), _group(NULL), _repeatgroup(0),
+    _optype(optype), _table(table), _key(key), _value(value), _config(),
+    _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0),
     _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) {
     size_check();
 }
 
 Operation::Operation(OpType optype, Table table, Key key) :
-    _optype(optype), _table(table), _key(key), _value(), _transaction(NULL),
-    _group(NULL), _repeatgroup(0),
+    _optype(optype), _table(table), _key(key), _value(), _config(),
+    _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0),
     _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) {
     size_check();
 }
 
 Operation::Operation(OpType optype, Table table) :
-    _optype(optype), _table(table), _key(), _value(), _transaction(NULL),
-    _group(NULL), _repeatgroup(0),
+    _optype(optype), _table(table), _key(), _value(), _config(),
+    _transaction(NULL), _group(NULL), _repeatgroup(0), _flags(0),
     _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) {
     size_check();
 }
 
 Operation::Operation(const Operation &other) :
     _optype(other._optype), _table(other._table), _key(other._key),
-    _value(other._value), _transaction(other._transaction),
-    _group(other._group), _repeatgroup(other._repeatgroup),
+    _value(other._value), _config(other._config),
+    _transaction(other._transaction), _group(other._group),
+    _repeatgroup(other._repeatgroup), _flags(other._flags),
     _keysize(other._keysize), _valuesize(other._valuesize),
     _keymax(other._keymax), _valuemax(other._valuemax) {
     // Creation and destruction of _group and _transaction is managed
@@ -890,6 +904,18 @@ Operation& Operation::operator=(const Operation &other) {
     return (*this);
 }
 
+void Operation::create_all() {
+    size_check();
+
+    _flags = 0;
+    if (!_config.empty()) {
+        if (_config == "reopen")
+            _flags |= WORKGEN_OP_REOPEN;
+        else
+            THROW("operation has illegal config: \"" << _config << "\"");
+    }
+}
+
 void Operation::describe(std::ostream &os) const {
     os << "Operation: " << _optype;
     if (_optype != OP_NONE) {
@@ -897,9 +923,10 @@ void Operation::describe(std::ostream &os) const {
         os << ", "; _key.describe(os);
         os << ", "; _value.describe(os);
     }
-    if (_transaction != NULL) {
+    if (!_config.empty())
+        os << ", '" << _config;
+    if (_transaction != NULL)
         os << ", ["; _transaction->describe(os); os << "]";
-    }
     if (_group != NULL) {
         os << ", group[" << _repeatgroup << "]: {";
         bool first = true;
@@ -1648,4 +1675,4 @@ int WorkloadRunner::run_all() {
     return (ret);
 }
 
-};
+}
diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.h b/src/third_party/wiredtiger/bench/workgen/workgen.h
index c7be8ee0035..a12e4dc4c89 100644
--- a/src/third_party/wiredtiger/bench/workgen/workgen.h
+++ b/src/third_party/wiredtiger/bench/workgen/workgen.h
@@ -33,6 +33,7 @@
 namespace workgen {
 
 struct ContextInternal;
+struct OperationInternal;
 struct TableInternal;
 struct Thread;
 struct Transaction;
@@ -245,11 +246,15 @@ struct Operation {
     Table _table;
     Key _key;
     Value _value;
+    std::string _config;
     Transaction *_transaction;
     std::vector<Operation> *_group;
     int _repeatgroup;
 
 #ifndef SWIG
+#define	WORKGEN_OP_REOPEN		0x0001 // reopen cursor for each op
+    uint32_t _flags;
+
     int _keysize;    // derived from Key._size and Table.options.key_size
     int _valuesize;
     uint64_t _keymax;
@@ -266,6 +271,7 @@ struct Operation {
     void describe(std::ostream &os) const;
 #ifndef SWIG
     Operation& operator=(const Operation &other);
+    void create_all();
     void get_static_counts(Stats &stats, int multiplier);
     void kv_compute_max(bool);
     void kv_gen(bool, uint64_t, char *) const;
@@ -408,4 +414,4 @@ struct Workload {
     int run(WT_CONNECTION *conn);
 };
 
-};
+}
diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.swig b/src/third_party/wiredtiger/bench/workgen/workgen.swig
index 0f74942169c..a33705f4350 100644
--- a/src/third_party/wiredtiger/bench/workgen/workgen.swig
+++ b/src/third_party/wiredtiger/bench/workgen/workgen.swig
@@ -158,7 +158,7 @@ WorkgenFrozenClass(WorkloadOptions)
     def __add__(self, other):
         if not isinstance(other, Operation):
             raise Exception('Operation.__sum__ requires an Operation')
-        if self._group == None or self._repeatgroup != 1 or self._transaction != None:
+        if self._group == None or self._repeatgroup != 1 or self._transaction != None or self._config != '':
             op = Operation()
             op._group = OpList([self, other])
             op._repeatgroup = 1
diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_int.h b/src/third_party/wiredtiger/bench/workgen/workgen_int.h
index 9283aea1d7b..a8d008a3bc5 100644
--- a/src/third_party/wiredtiger/bench/workgen/workgen_int.h
+++ b/src/third_party/wiredtiger/bench/workgen/workgen_int.h
@@ -203,4 +203,4 @@ private:
     WorkloadRunner& operator=(const WorkloadRunner &other); // disallowed
 };
 
-};
+}
diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c
index c5a3dd40032..1a4f46f5c37 100644
--- a/src/third_party/wiredtiger/bench/wtperf/config.c
+++ b/src/third_party/wiredtiger/bench/wtperf/config.c
@@ -798,7 +798,7 @@ config_sanity(WTPERF *wtperf)
 			    workp->table_index >= (int32_t)opts->table_count) {
 				fprintf(stderr,
 				    "Workload table index %" PRId32
-				    " is larger than table count %" PRId32,
+				    " is larger than table count %" PRIu32,
 				    workp->table_index, opts->table_count);
 				return (EINVAL);
 			}
diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/update-only-btree.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/update-only-btree.wtperf
new file mode 100644
index 00000000000..ea37a4d13d1
--- /dev/null
+++ b/src/third_party/wiredtiger/bench/wtperf/runners/update-only-btree.wtperf
@@ -0,0 +1,10 @@
+# wtperf options file: btree with updates, in memory. The goal here is to
+# have a workload that uses a lot of transaction IDs, so stresses the ID
+# allocation/release code performance.
+conn_config="cache_size=5G"
+table_config="type=file"
+icount=5000000
+report_interval=5
+run_time=120
+populate_threads=1
+threads=((count=16,updates=1))
diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c
index a8d3f135280..9e18df9a0e3 100644
--- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c
+++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c
@@ -1750,7 +1750,7 @@ execute_workload(WTPERF *wtperf)
 		lprintf(wtperf, 0, 1,
 		    "Starting workload #%u: %" PRId64 " threads, inserts=%"
 		    PRId64 ", reads=%" PRId64 ", updates=%" PRId64
-		    ", truncate=%" PRId64 ", throttle=%" PRId64,
+		    ", truncate=%" PRId64 ", throttle=%" PRIu64,
 		    i + 1, workp->threads, workp->insert,
 		    workp->read, workp->update, workp->truncate,
 		    workp->throttle);