Import wiredtiger-wiredtiger-2.8.0-410-g636a7b2.tar.gz from wiredtiger branch mongodb-3.4

ref: eaa7b5f..636a7b2

WT-2103       add incremental backup testing to format
WT-2223       Add stress testing for in-memory
WT-2343       Assert we don't remove or rename when backup cursor is open
WT-2349       Add ability to open databases read-only
WT-2359       WiredTiger with Python will hang if a calloc failure occurs during __wt_connection_close
WT-2360       Allow disjunctions and combinations of operations in join cursors
WT-2446       Estimate WT cache hit ratio
WT-2450       salvage releases pages, then explicitly evicts them.
WT-2453       Throughput drop in wtperf evict Jenkins tests
WT-2479       dump utility discards table config (JSON)
WT-2504       Should READONLY always read basecfg file?
WT-2505       Review clang analyzer warnings
WT-2508       test programs should remove test directories on the "clean" target
WT-2518       LSM checkpoint handle acquisition optimization
WT-2520       WT_SESSION::verify should not alter tables
WT-2526       mixing and matching readonly and read/write handles
WT-2535       Extend test/format to test for transactions reading their writes
WT-2537       cannot open DB written by WT2.6.1 with WT2.8.0 due to WT_NOTFOUND on recovery
WT-2539       Implement file streaming above pluggable filesystems
WT-2540       Separate stream and file handle methods
WT-2542       fixed-length column store reconciliation overwrites original values
WT-2544       Investigate any thread populating eviction queue
WT-2546       Eviction server not help evict pages sometimes
WT-2547       Add 1-eviction-worker jobs to Jenkins
WT-2548       Cap the amount of data handed to raw compression.
WT-2549       joins using recno keys return no values
WT-2550       java ex_schema example fails
WT-2552       Public API for pluggable filesystems
WT-2553       Document in-memory configuration and WT_CACHE_FULL error return
WT-2556       typo in the Java example code
WT-2557       format test program should discard log files after incremental backup
WT-2558       WT_PAGE structure reorganization
WT-2559       Jenkins Windows segfault in logging code
WT-2560       test/format workload stuck trying to update oldest transaction ID
WT-2562       reconfig02 test failing sometimes on PPC
WT-2565       item 3573 on page at [write-check] is a corrupted cell
WT-2566       All lock operations should be barriers
WT-2567       segfault in test/format log truncate
WT-2568       Java PackTest.java compilation error
WT-2569       win_handle_read should always call GetLastError on error
WT-2570       Minor lint cleanups.
WT-2571       join code cleanup
WT-2572       don't select an in-memory format run if incompatible options configured
WT-2573       free of stack-allocated WT_REF
WT-2574       format doesn't free all allocated configure memory
WT-2576       variable-length column-store out-of-order return
WT-2577       core dump discarding non-existent addresses
WT-2579       in-memory configurations break debugging support
WT-2580       potential SWIG naming conflict in Java
WT-2581       assert multi->disk_image == NULL
WT-2582       cache eviction server error: WT_RESTART
WT-2583       incremental backup can prevent future recovery
WT-2584       don't use periods in error messages
WT-2586       Remove ex_config.c until config cursors are supported
WT-2592       Joins using non-recno key types not working
WT-2593       disk full with pre-allocated log files
WT-2595       Fix compiler warning in packing tests
WT-2598       in-memory FS needs fast lookup on file names
WT-2599       split out the checksum code from the support directory
WT-2600       clean up test program #includes
WT-2602       LSM stress hangs with very large uncompressed pages
WT-2609       Incorrect "skips API_END call" error.
WT-2612       The dist/s_prototypes script is creating a debugging file xxx.
WT-2613       WT Compile windows Alt is returning a C4100 error
WT-2615       Enabling checkpoints in test/format leads to reduced concurrency
WT-2616       In-memory deadlock getting size
WT-2621       WiredTiger fails to compile on MSVC 2013
SERVER-23661  $sample takes disproportionately long time on newly created collection
SERVER-23904  WiredTiger changes for MongoDB 3.3.6

5 files changed, 83 insertions, 27 deletions

diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i
index 450cb1d5ab2..2264cb31ef1 100644
--- a/src/third_party/wiredtiger/lang/java/java_doc.i
+++ b/src/third_party/wiredtiger/lang/java/java_doc.i
@@ -63,6 +63,7 @@ COPYDOC(__wt_connection, WT_CONNECTION, add_collator)
 COPYDOC(__wt_connection, WT_CONNECTION, add_compressor)
 COPYDOC(__wt_connection, WT_CONNECTION, add_encryptor)
 COPYDOC(__wt_connection, WT_CONNECTION, add_extractor)
+COPYDOC(__wt_connection, WT_CONNECTION, set_file_system)
 COPYDOC(__wt_config_parser, WT_CONFIG_PARSER, close)
 COPYDOC(__wt_config_parser, WT_CONFIG_PARSER, next)
 COPYDOC(__wt_config_parser, WT_CONFIG_PARSER, get)
diff --git a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java
index 02639bfe77a..4f05e153607 100644
--- a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java
+++ b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java
@@ -40,6 +40,7 @@ import com.wiredtiger.db.WiredTigerPackingException;
 public class PackFormatInputStream {
 
     protected String format;
+    protected boolean isRaw;
     protected int formatOff;
     protected int formatRepeatCount;
 
@@ -48,8 +49,9 @@ public class PackFormatInputStream {
      *
      * \param format the encoded format backing string.
      */
-    protected PackFormatInputStream(String format) {
+    protected PackFormatInputStream(String format, boolean isRaw) {
         this.format = format;
+        this.isRaw = isRaw;
         formatOff = 0;
         formatRepeatCount = 0;
     }
@@ -114,6 +116,9 @@ public class PackFormatInputStream {
     throws WiredTigerPackingException {
 
         char expected = getType();
+        if (isRaw)
+            throw new WiredTigerPackingException(
+                "Format mismatch for raw mode");
         if (Character.toLowerCase(expected) != Character.toLowerCase(asking))
             throw new WiredTigerPackingException(
                 "Format mismatch. Wanted: " + asking + ", got: " + expected);
diff --git a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackInputStream.java b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackInputStream.java
index f265d041d94..732bf450acd 100644
--- a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackInputStream.java
+++ b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackInputStream.java
@@ -43,6 +43,7 @@ public class PackInputStream {
     protected byte[] value;
     protected int valueOff;
     protected int valueLen;
+    protected boolean isRaw;
 
     /**
      * Constructor.
@@ -52,7 +53,7 @@ public class PackInputStream {
      * \param value The raw bytes that back the stream.
      */
     public PackInputStream(String format, byte[] value) {
-        this(format, value, 0, value.length);
+        this(format, value, false, 0, value.length);
     }
 
     /**
@@ -61,14 +62,29 @@ public class PackInputStream {
      * \param format A String that contains the WiredTiger format that
      *               defines the layout of this packed value.
      * \param value The raw bytes that back the stream.
+     * \param isRaw The stream is opened raw.
+     */
+    public PackInputStream(String format, byte[] value, boolean isRaw) {
+        this(format, value, isRaw, 0, value.length);
+    }
+
+    /**
+     * Constructor.
+     *
+     * \param format A String that contains the WiredTiger format that
+     *               defines the layout of this packed value.
+     * \param value The raw bytes that back the stream.
+     * \param isRaw The stream is opened raw.
      * \param off Offset into the value array at which the stream begins.
      * \param len Length of the value array that forms the stream.
      */
-    public PackInputStream(String format, byte[] value, int off, int len) {
-        this.format = new PackFormatInputStream(format);
+    public PackInputStream(
+        String format, byte[] value, boolean isRaw, int off, int len) {
+        this.format = new PackFormatInputStream(format, isRaw);
         this.value = value;
         this.valueOff = off;
         this.valueLen = len;
+        this.isRaw = isRaw;
     }
 
     /**
@@ -117,7 +133,9 @@ public class PackInputStream {
      */
     public void getByteArray(byte[] dest, int off, int len)
     throws WiredTigerPackingException {
-        format.checkType('U', false);
+        if (!isRaw) {
+            format.checkType('U', false);
+        }
         getByteArrayInternal(getByteArrayLength(), dest, off, len);
     }
 
@@ -127,7 +145,9 @@ public class PackInputStream {
      */
     public byte[] getByteArray()
     throws WiredTigerPackingException {
-        format.checkType('U', false);
+        if (!isRaw) {
+            format.checkType('U', false);
+        }
         int itemLen = getByteArrayLength();
         byte[] unpacked = new byte[itemLen];
         getByteArrayInternal(itemLen, unpacked, 0, itemLen);
@@ -142,7 +162,10 @@ public class PackInputStream {
     throws WiredTigerPackingException {
         int itemLen = 0;
 
-        if (format.hasLength()) {
+        if (isRaw) {
+            // The rest of the buffer is a byte array.
+            itemLen = valueLen - valueOff;
+        } else if (format.hasLength()) {
             // If the format has a length, it's always used.
             itemLen = format.getLengthFromFormat(true);
         } else if (format.getType() == 'U') {
diff --git a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java
index 805e34f6ca8..46b3aef0974 100644
--- a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java
+++ b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java
@@ -50,7 +50,7 @@ public class PackOutputStream {
      *               defines the layout of this packed value.
      */
     public PackOutputStream(String format) {
-        this.format = new PackFormatInputStream(format);
+        this.format = new PackFormatInputStream(format, false);
         intBuf = new byte[MAX_INT_BYTES];
         packed = new ByteArrayOutputStream(100);
     }
diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i
index ce013a1939c..c04bae63cbc 100644
--- a/src/third_party/wiredtiger/lang/java/wiredtiger.i
+++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i
@@ -80,6 +80,7 @@ typedef struct {
 	JavaVM *javavm;		/* Used in async threads to craft a jnienv */
 	JNIEnv *jnienv;		/* jni env that created the Session/Cursor */
 	WT_SESSION_IMPL *session; /* session used for alloc/free */
+	bool cursor_raw;	/* is the cursor opened raw? */
 	jobject jobj;		/* the java Session/Cursor/AsyncOp object */
 	jobject jcallback;	/* callback object for async ops */
 	jfieldID cptr_fid;	/* cached Cursor.swigCPtr field id in session */
@@ -576,8 +577,15 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 		return $self->update($self);
 	}
 
-	%javamethodmodifiers java_init "protected";
-	int java_init(jobject jasyncop) {
+	%javamethodmodifiers _java_raw "protected";
+	bool _java_raw(JNIEnv *jenv) {
+		(void)jenv;
+		JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->c.lang_private;
+		return jcb->cursor_raw;
+	}
+
+	%javamethodmodifiers _java_init "protected";
+	int _java_init(jobject jasyncop) {
 		JAVA_CALLBACK *jcb =
 		    (JAVA_CALLBACK *)$self->c.lang_private;
 		jcb->jobj = JCALL1(NewGlobalRef, jcb->jnienv, jasyncop);
@@ -604,7 +612,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
    valueFormat = getValue_format();
    keyPacker = new PackOutputStream(keyFormat);
    valuePacker = new PackOutputStream(valueFormat);
-   wiredtigerJNI.AsyncOp_java_init(swigCPtr, this, this);
+   wiredtigerJNI.AsyncOp__java_init(swigCPtr, this, this);
  }
 
  protected static long getCPtr($javaclassname obj) {
@@ -1090,7 +1098,8 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 	throws WiredTigerPackingException {
 		if (keyUnpacker == null)
 			keyUnpacker =
-			    new PackInputStream(keyFormat, get_key_wrap());
+			    new PackInputStream(keyFormat, get_key_wrap(),
+			    _java_raw());
 		return keyUnpacker;
 	}
 
@@ -1103,7 +1112,8 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 	throws WiredTigerPackingException {
 		if (valueUnpacker == null)
 			valueUnpacker =
-			    new PackInputStream(valueFormat, get_value_wrap());
+			    new PackInputStream(valueFormat, get_value_wrap(),
+			    _java_raw());
 		return valueUnpacker;
 	}
 
@@ -1175,6 +1185,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 		return $self->update($self);
 	}
 
+	%javamethodmodifiers compare_wrap "protected";
 	int compare_wrap(JNIEnv *jenv, WT_CURSOR *other) {
 		int cmp, ret = $self->compare($self, other, &cmp);
 		if (ret != 0)
@@ -1182,6 +1193,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 		return cmp;
 	}
 
+	%javamethodmodifiers equals_wrap "protected";
 	int equals_wrap(JNIEnv *jenv, WT_CURSOR *other) {
 		int cmp, ret = $self->equals($self, other, &cmp);
 		if (ret != 0)
@@ -1189,8 +1201,15 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 		return cmp;
 	}
 
-	%javamethodmodifiers java_init "protected";
-	int java_init(jobject jcursor) {
+	%javamethodmodifiers _java_raw "protected";
+	bool _java_raw(JNIEnv *jenv) {
+		(void)jenv;
+		JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->lang_private;
+		return jcb->cursor_raw;
+	}
+
+	%javamethodmodifiers _java_init "protected";
+	int _java_init(jobject jcursor) {
 		JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->lang_private;
 		jcb->jobj = JCALL1(NewGlobalRef, jcb->jnienv, jcursor);
 		JCALL1(DeleteLocalRef, jcb->jnienv, jcursor);
@@ -1216,7 +1235,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
    valueFormat = getValue_format();
    keyPacker = new PackOutputStream(keyFormat);
    valuePacker = new PackOutputStream(valueFormat);
-   wiredtigerJNI.Cursor_java_init(swigCPtr, this, this);
+   wiredtigerJNI.Cursor__java_init(swigCPtr, this, this);
  }
 
  protected static long getCPtr($javaclassname obj) {
@@ -1773,7 +1792,8 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 		if (!success || keyFormat.equals(""))
 			return null;
 		else
-			return new PackInputStream(keyFormat, get_key_wrap());
+			return new PackInputStream(keyFormat,
+			    get_key_wrap(), _java_raw());
 	}
 
 	/**
@@ -1789,7 +1809,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 			return null;
 		else
 			return new PackInputStream(valueFormat,
-			    get_value_wrap());
+			    get_value_wrap(), _java_raw());
 	}
 %}
 
@@ -1799,20 +1819,22 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
  */
 %javaexception("com.wiredtiger.db.WiredTigerException") { $action; }
 %javaexception("") wiredtiger_strerror { $action; }
+%javaexception("") __wt_async_op::_java_raw { $action; }
 %javaexception("") __wt_async_op::connection { $action; }
 %javaexception("") __wt_async_op::get_type { $action; }
 %javaexception("") __wt_async_op::get_id { $action; }
 %javaexception("") __wt_async_op::key_format { $action; }
 %javaexception("") __wt_async_op::value_format { $action; }
+%javaexception("") __wt_connection::_java_init { $action; }
 %javaexception("") __wt_connection::get_home { $action; }
 %javaexception("") __wt_connection::is_new { $action; }
-%javaexception("") __wt_connection::java_init { $action; }
+%javaexception("") __wt_cursor::_java_raw { $action; }
 %javaexception("") __wt_cursor::key_format { $action; }
 %javaexception("") __wt_cursor::session { $action; }
 %javaexception("") __wt_cursor::uri { $action; }
 %javaexception("") __wt_cursor::value_format { $action; }
+%javaexception("") __wt_session::_java_init { $action; }
 %javaexception("") __wt_session::connection { $action; }
-%javaexception("") __wt_session::java_init { $action; }
 
 /* Remove / rename parts of the C API that we don't want in Java. */
 %immutable __wt_cursor::session;
@@ -1832,6 +1854,9 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
 %ignore __wt_event_handler;
 %ignore __wt_extractor;
 %ignore __wt_connection::add_extractor;
+%ignore __wt_file_system;
+%ignore __wt_file_handle;
+%ignore __wt_connection::set_file_system;
 %ignore __wt_item;
 %ignore __wt_lsn;
 %ignore __wt_session::msg_printf;
@@ -1890,8 +1915,8 @@ REQUIRE_WRAP(WT_ASYNC_OP::get_id, __wt_async_op::get_id,getId)
 %}
 
 %extend ctypename {
-	%javamethodmodifiers java_init "protected";
-	int java_init(jobject jsess) {
+	%javamethodmodifiers _java_init "protected";
+	int _java_init(jobject jsess) {
 		implclass *session = (implclass *)$self;
 		JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)session->lang_private;
 		jcb->jobj = JCALL1(NewGlobalRef, jcb->jnienv, jsess);
@@ -1901,8 +1926,8 @@ REQUIRE_WRAP(WT_ASYNC_OP::get_id, __wt_async_op::get_id,getId)
 }
 %enddef
 
-TRACKED_CLASS(Session, __wt_session, wiredtigerJNI.Session_java_init, WT_SESSION_IMPL)
-TRACKED_CLASS(Connection, __wt_connection, wiredtigerJNI.Connection_java_init, WT_CONNECTION_IMPL)
+TRACKED_CLASS(Session, __wt_session, wiredtigerJNI.Session__java_init, WT_SESSION_IMPL)
+TRACKED_CLASS(Connection, __wt_connection, wiredtigerJNI.Connection__java_init, WT_CONNECTION_IMPL)
 /* Note: Cursor incorporates the elements of TRACKED_CLASS into its
  * custom constructor and %extend clause.
  */
@@ -1996,13 +2021,15 @@ err:		if (ret != 0)
 		if ((ret = $self->open_cursor($self, uri, to_dup, config, &cursor)) != 0)
 			goto err;
 
-		if ((cursor->flags & WT_CURSTD_DUMP_JSON) == 0)
-			cursor->flags |= WT_CURSTD_RAW;
-
 		if ((ret = __wt_calloc_def((WT_SESSION_IMPL *)cursor->session,
 			    1, &jcb)) != 0)
 			goto err;
 
+		if ((cursor->flags & WT_CURSTD_RAW) != 0)
+			jcb->cursor_raw = true;
+		if ((cursor->flags & WT_CURSTD_DUMP_JSON) == 0)
+			cursor->flags |= WT_CURSTD_RAW;
+
 		jcb->jnienv = jenv;
 		jcb->session = (WT_SESSION_IMPL *)cursor->session;
 		cursor->lang_private = jcb;