Import wiredtiger: e99ab771b2aa837199b72c2028569e7f5d3a7815 from branch mongodb-3.4

ref: 0b36171f4a..e99ab771b2
for: 3.4.15

WT-3972       Allow more than 64K cursors to be open on a data source simultaneously
WT-3975       arg format mismatch after rwlock changes

5 files changed, 73 insertions, 9 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index e867147e59e..04b716ec1c7 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
 {
-    "commit": "0b36171f4aa0bea6ab6118b0d3bcf6329a24939e", 
+    "commit": "e99ab771b2aa837199b72c2028569e7f5d3a7815", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-3.4"
diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h
index 36acea810d9..14fe0b2e778 100644
--- a/src/third_party/wiredtiger/src/include/mutex.h
+++ b/src/third_party/wiredtiger/src/include/mutex.h
@@ -44,9 +44,8 @@ struct __wt_rwlock {			/* Read/write lock */
 			uint8_t current;	/* Current ticket */
 			uint8_t next;		/* Next available ticket */
 			uint8_t reader;		/* Read queue ticket */
-			uint8_t __notused;	/* Padding */
-			uint16_t readers_active;/* Count of active readers */
-			uint16_t readers_queued;/* Count of queued readers */
+			uint8_t readers_queued; /* Count of queued readers */
+			uint32_t readers_active;/* Count of active readers */
 		} s;
 	} u;
 
diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c
index 7905241be0e..de0a6d72a1f 100644
--- a/src/third_party/wiredtiger/src/support/mtx_rw.c
+++ b/src/third_party/wiredtiger/src/support/mtx_rw.c
@@ -48,9 +48,8 @@
  *		uint8_t current;		// Current ticket
  *		uint8_t next;			// Next available ticket
  *		uint8_t reader;			// Read queue ticket
- *		uint8_t __notused;		// Padding
- *		uint16_t readers_active;	// Count of active readers
- *		uint16_t readers_queued;	// Count of queued readers
+ *		uint8_t readers_queued;		// Count of queued readers
+ *		uint32_t readers_active;	// Count of active readers
  *	} s;
  * } u;
  *
@@ -75,6 +74,12 @@
  * 'reader' to 'next' (i.e. readers are scheduled after any queued writers,
  * avoiding starvation), then atomically incrementing 'readers_queued'.
  *
+ * We limit how many readers can queue: we don't allow more readers to queue
+ * than there are active writers (calculated as `next - current`): otherwise,
+ * in write-heavy workloads, readers can keep queuing up in front of writers
+ * and throughput is unstable.  The remaining read requests wait without any
+ * ordering.
+ *
  * The 'next' field is a 1-byte value so the available ticket number wraps
  * after 256 requests. If a thread's write lock request would cause the 'next'
  * field to catch up with 'current', instead it waits to avoid the same ticket
diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c
index 04813182478..52e4a12b4fd 100644
--- a/src/third_party/wiredtiger/test/csuite/rwlock/main.c
+++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c
@@ -171,8 +171,8 @@ thread_dump(void *arg) {
 		sleep(1);
 		printf("\n"
 		    "rwlock { current %" PRIu8 ", next %" PRIu8
-		    ", reader %" PRIu8 ", readers_active %" PRIu16
-		    ", readers_queued %" PRIu16 " }\n",
+		    ", reader %" PRIu8 ", readers_active %" PRIu32
+		    ", readers_queued %" PRIu8 " }\n",
 		    rwlock.u.s.current,
 		    rwlock.u.s.next,
 		    rwlock.u.s.reader,
diff --git a/src/third_party/wiredtiger/test/suite/test_cursor14.py b/src/third_party/wiredtiger/test/suite/test_cursor14.py
new file mode 100644
index 00000000000..25bd0cec00a
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_cursor14.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2018 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import wiredtiger, wttest
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
+from wtscenario import make_scenarios
+
+# test_cursor14.py
+#    Test that more than 64K cursors can be opened on a data source
+class test_cursor14(wttest.WiredTigerTestCase):
+    scenarios = make_scenarios([
+        ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)),
+        ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)),
+        ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)),
+        ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)),
+        ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)),
+        ('table-r-complex', dict(type='table:', keyfmt='r',
+            dataset=ComplexDataSet)),
+        ('table-S-complex', dict(type='table:', keyfmt='S',
+            dataset=ComplexDataSet)),
+        ('table-S-complex-lsm', dict(type='table:', keyfmt='S',
+            dataset=ComplexLSMDataSet)),
+    ])
+
+    def test_cursor14(self):
+        uri = self.type + 'cursor14'
+
+        ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+        ds.populate()
+
+        for i in xrange(66000):
+            cursor = self.session.open_cursor(uri, None, None)
+
+if __name__ == '__main__':
+    wttest.run()