Import wiredtiger: 1b6c815a3fd34f14c20d5cd627155799d1de535c from branch mongodb-3.6

ref: ca6eee06ff..1b6c815a3f
for: 3.5.1

WT-2336       Add a test validating schema operations via file system call monitoring
WT-2670       Add option to configure read-ahead per table and change default behavior
WT-2960       Inserting multi-megabyte values can cause pathological lookaside usage
WT-2969       Fix a bug that could cause snapshot corruption during compaction
WT-3014       Add GCC/clang support for ELF symbol visibility.
WT-3021       Fixes needed for Java log cursor example, Java raw mode cursors, log cursors in raw mode
WT-3025       fix error path in log_force_sync
WT-3028       Workloads with all dirty pages could trigger diagnostic stuck check
WT-3030       Test failure indicating invalid key order during traversal
WT-3034       Add support for single-writer named snapshots.
WT-3037       Fix some outdated comments in logging
WT-3048       WiredTiger maximum size warning uses the wrong format.
WT-3051       Remove external __wt_hex symbol.
WT-3052       Improve search if an index hint is wrong
WT-3053       Review Python and Java calls to internal WiredTiger functions
WT-3054       Java PackTest, PackTest03 do not compile
WT-3055       Java AsyncTest faults
WT-3057       WiredTiger hazard pointers should use the WT_REF, not the WT_PAGE.
WT-3064       minor tree cleanups: .gitignore, NEWS misspelling

3 files changed, 128 insertions, 86 deletions

diff --git a/src/third_party/wiredtiger/examples/c/ex_file_system.c b/src/third_party/wiredtiger/examples/c/ex_file_system.c
index f51cad328c3..56869171558 100644
--- a/src/third_party/wiredtiger/examples/c/ex_file_system.c
+++ b/src/third_party/wiredtiger/examples/c/ex_file_system.c
@@ -29,35 +29,49 @@
  * 	demonstrates how to use the custom file system interface
  */
 
-/*
- * Include WiredTiger internal functions: we need architecture portable locking
- * in this example, and we use the TAILQ_XXX functions to keep the code simple.
- *
- * Application-writers SHOULD NOT INCLUDE "wt_internal.h", the public WiredTiger
- * include files should be used instead:
- *
- *	#include <wiredtiger.h>
- *	#include <wiredtiger_ext.h>
- */
-#include "wt_internal.h"
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <queue.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef _WIN32
+#include <pthread.h>
+#else
+#include "windows_shim.h"
+#endif
+
+#include <wiredtiger.h>
+#include <wiredtiger_ext.h>
 
 /*
- * This example code uses internal WiredTiger functions for portable locking.
- * We use #defines to clarify the meaning and ignore errors to simplify the
- * code.
- *
- * Application writers SHOULD NOT COPY THIS LOCKING CODE, it's special-case code
- * to make this example portable across platforms.
+ * This example code uses pthread functions for portable locking, we ignore
+ * errors for simplicity.
  */
-#define	ALLOCATE_FILE_SYSTEM_LOCK(demo_fs)				\
-	(void)__wt_spin_init(NULL, &(demo_fs)->lock, "demo file handle lock")
-#define	DESTROY_FILE_SYSTEM_LOCK(wt_session, demo_fs)			\
-	__wt_spin_destroy((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock)
-#define	LOCK_FILE_SYSTEM(wt_session, demo_fs)				\
-	__wt_spin_lock((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock)
-#define	UNLOCK_FILE_SYSTEM(wt_session, demo_fs)				\
-	__wt_spin_unlock(						\
-	    (WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock)
+static void
+allocate_file_system_lock(pthread_rwlock_t *lockp)
+{
+	assert(pthread_rwlock_init(lockp, NULL) == 0);
+}
+
+static void
+destroy_file_system_lock(pthread_rwlock_t *lockp)
+{
+	assert(pthread_rwlock_destroy(lockp) == 0);
+}
+
+static void
+lock_file_system(pthread_rwlock_t *lockp)
+{
+	assert(pthread_rwlock_wrlock(lockp) == 0);
+}
+
+static void
+unlock_file_system(pthread_rwlock_t *lockp)
+{
+	assert(pthread_rwlock_unlock(lockp) == 0);
+}
 
 /*
  * Example file system implementation, using memory buffers to represent files.
@@ -72,7 +86,7 @@ typedef struct {
 	 * might require finer granularity, for example, a single lock for the
 	 * file system handle list and per-handle locks serializing I/O.
 	 */
-	WT_SPINLOCK lock;			/* Lock */
+	pthread_rwlock_t lock;			/* Lock */
 
 	int opened_file_count;
 	int opened_unique_file_count;
@@ -239,7 +253,7 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config)
 		goto err;
 	}
 
-	ALLOCATE_FILE_SYSTEM_LOCK(demo_fs);
+	allocate_file_system_lock(&demo_fs->lock);
 
 	/* Initialize the in-memory jump table. */
 	file_system->fs_directory_list = demo_fs_directory_list;
@@ -288,7 +302,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session,
 	demo_fh = NULL;
 	wtext = demo_fs->wtext;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	++demo_fs->opened_file_count;
 
 	/*
@@ -308,7 +322,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session,
 
 		*file_handlep = (WT_FILE_HANDLE *)demo_fh;
 
-		UNLOCK_FILE_SYSTEM(session, demo_fs);
+		unlock_file_system(&demo_fs->lock);
 		return (0);
 	}
 
@@ -366,7 +380,7 @@ err:		free(demo_fh->buf);
 		free(demo_fh);
 	}
 
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 	return (ret);
 }
 
@@ -386,6 +400,8 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system,
 	int ret = 0;
 	char *name, **entries;
 
+	(void)session;						/* Unused */
+
 	demo_fs = (DEMO_FILE_SYSTEM *)file_system;
 
 	*dirlistp = NULL;
@@ -396,7 +412,7 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system,
 	len = strlen(directory);
 	prefix_len = prefix == NULL ? 0 : strlen(prefix);
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	TAILQ_FOREACH(demo_fh, &demo_fs->fileq, q) {
 		name = demo_fh->iface.name;
 		if (strncmp(name, directory, len) != 0 ||
@@ -424,7 +440,7 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system,
 	*dirlistp = entries;
 	*countp = count;
 
-err:	UNLOCK_FILE_SYSTEM(session, demo_fs);
+err:	unlock_file_system(&demo_fs->lock);
 	if (ret == 0)
 		return (0);
 
@@ -466,11 +482,13 @@ demo_fs_exist(WT_FILE_SYSTEM *file_system,
 {
 	DEMO_FILE_SYSTEM *demo_fs;
 
+	(void)session;						/* Unused */
+
 	demo_fs = (DEMO_FILE_SYSTEM *)file_system;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	*existp = demo_handle_search(file_system, name) != NULL;
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 
 	return (0);
 }
@@ -487,15 +505,16 @@ demo_fs_remove(WT_FILE_SYSTEM *file_system,
 	DEMO_FILE_HANDLE *demo_fh;
 	int ret = 0;
 
-	(void)flags;					/* Unused */
+	(void)session;						/* Unused */
+	(void)flags;						/* Unused */
 
 	demo_fs = (DEMO_FILE_SYSTEM *)file_system;
 
 	ret = ENOENT;
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	if ((demo_fh = demo_handle_search(file_system, name)) != NULL)
 		ret = demo_handle_remove(session, demo_fh);
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 
 	return (ret);
 }
@@ -513,11 +532,12 @@ demo_fs_rename(WT_FILE_SYSTEM *file_system,
 	char *copy;
 	int ret = 0;
 
-	(void)flags;					/* Unused */
+	(void)session;						/* Unused */
+	(void)flags;						/* Unused */
 
 	demo_fs = (DEMO_FILE_SYSTEM *)file_system;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	if ((demo_fh = demo_handle_search(file_system, from)) == NULL)
 		ret = ENOENT;
 	else if ((copy = strdup(to)) == NULL)
@@ -526,7 +546,7 @@ demo_fs_rename(WT_FILE_SYSTEM *file_system,
 		free(demo_fh->iface.name);
 		demo_fh->iface.name = copy;
 	}
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 	return (ret);
 }
 
@@ -545,10 +565,10 @@ demo_fs_size(WT_FILE_SYSTEM *file_system,
 	demo_fs = (DEMO_FILE_SYSTEM *)file_system;
 
 	ret = ENOENT;
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	if ((demo_fh = demo_handle_search(file_system, name)) != NULL)
 		ret = demo_file_size((WT_FILE_HANDLE *)demo_fh, session, sizep);
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 
 	return (ret);
 }
@@ -578,7 +598,7 @@ demo_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session)
 	printf("\t%d reads, %d writes\n",
 	    demo_fs->read_ops, demo_fs->write_ops);
 
-	DESTROY_FILE_SYSTEM_LOCK(session, demo_fs);
+	destroy_file_system_lock(&demo_fs->lock);
 	free(demo_fs);
 
 	return (ret);
@@ -594,13 +614,15 @@ demo_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session)
 	DEMO_FILE_HANDLE *demo_fh;
 	DEMO_FILE_SYSTEM *demo_fs;
 
+	(void)session;						/* Unused */
+
 	demo_fh = (DEMO_FILE_HANDLE *)file_handle;
 	demo_fs = demo_fh->demo_fs;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	if (--demo_fh->ref == 0)
 		++demo_fs->closed_file_count;
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 
 	return (0);
 }
@@ -638,7 +660,7 @@ demo_file_read(WT_FILE_HANDLE *file_handle,
 	wtext = demo_fs->wtext;
 	off = (size_t)offset;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	++demo_fs->read_ops;
 	if (off < demo_fh->size) {
 		if (len > demo_fh->size - off)
@@ -646,7 +668,7 @@ demo_file_read(WT_FILE_HANDLE *file_handle,
 		memcpy(buf, (uint8_t *)demo_fh->buf + off, len);
 	} else
 		ret = EIO;		/* EOF */
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 	if (ret == 0)
 		return (0);
 
@@ -667,12 +689,14 @@ demo_file_size(
 	DEMO_FILE_HANDLE *demo_fh;
 	DEMO_FILE_SYSTEM *demo_fs;
 
+	(void)session;						/* Unused */
+
 	demo_fh = (DEMO_FILE_HANDLE *)file_handle;
 	demo_fs = demo_fh->demo_fs;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	*sizep = (wt_off_t)demo_fh->size;
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 	return (0);
 }
 
@@ -739,10 +763,10 @@ demo_file_truncate(
 	demo_fh = (DEMO_FILE_HANDLE *)file_handle;
 	demo_fs = demo_fh->demo_fs;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	if ((ret = demo_buffer_resize(session, demo_fh, offset)) == 0)
 		demo_fh->size = (size_t)offset;
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 	return (ret);
 }
 
@@ -765,7 +789,7 @@ demo_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session,
 	wtext = demo_fs->wtext;
 	off = (size_t)offset;
 
-	LOCK_FILE_SYSTEM(session, demo_fs);
+	lock_file_system(&demo_fs->lock);
 	++demo_fs->write_ops;
 	if ((ret = demo_buffer_resize(session, demo_fh,
 	    offset + (wt_off_t)(len + DEMO_FILE_SIZE_INCREMENT))) == 0) {
@@ -773,7 +797,7 @@ demo_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session,
 		if (off + len > demo_fh->size)
 			demo_fh->size = off + len;
 	}
-	UNLOCK_FILE_SYSTEM(session, demo_fs);
+	unlock_file_system(&demo_fs->lock);
 	if (ret == 0)
 		return (0);
 
diff --git a/src/third_party/wiredtiger/examples/c/ex_thread.c b/src/third_party/wiredtiger/examples/c/ex_thread.c
index 7c52d3b8189..fa82bd5f113 100644
--- a/src/third_party/wiredtiger/examples/c/ex_thread.c
+++ b/src/third_party/wiredtiger/examples/c/ex_thread.c
@@ -30,14 +30,13 @@
  *	table from multiple threads.
  */
 
-#ifndef _WIN32
-#include <pthread.h>
-#endif
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-#ifdef _WIN32
+#ifndef _WIN32
+#include <pthread.h>
+#else
 #include "windows_shim.h"
 #endif
 
diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java
index 03eab6b38b1..233ad1361d8 100644
--- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java
+++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java
@@ -35,7 +35,7 @@ import java.util.*;
 
 class Lsn {
     int file;
-    long offset;
+    int offset;
 }
 
 public class ex_log {
@@ -57,7 +57,7 @@ public class ex_log {
 
         conn = wiredtiger.open(home2, CONN_CONFIG);
         Session session = conn.open_session(null);
-        session.create(uri, "key_format=U,value_format=U");
+        session.create(uri, "key_format=S,value_format=S");
         return (session);
     }
 
@@ -72,17 +72,17 @@ public class ex_log {
 
         while ((ret = cursor.next()) == 0) {
             ret = curs_copy.next();
-            byte[] key = cursor.getKeyByteArray();
-            byte[] value = cursor.getValueByteArray();
-            byte[] key_copy = curs_copy.getKeyByteArray();
-            byte[] value_copy = curs_copy.getValueByteArray();
-            if (!Arrays.equals(key, key_copy) ||
-                !Arrays.equals(value, value_copy)) {
+            String key = cursor.getKeyString();
+            String value = cursor.getValueString();
+            String key_copy = curs_copy.getKeyString();
+            String value_copy = curs_copy.getValueString();
+            if (!key.equals(key_copy) ||
+                !value.equals(value_copy)) {
                 System.err.println(
-                    "Mismatched: key " + new String(key) +
-                    ", key_copy " + new String(key_copy) +
-                    ", value " + new String(value) +
-                    ", value_copy " + new String(value_copy));
+                    "Mismatched: key " + key + ", key_copy " + key_copy +
+                    ", value " + value + ", value_copy " + value_copy);
+                ret = cursor.close();
+                ret = curs_copy.close();
                 return (1);
             }
         }
@@ -121,7 +121,7 @@ public class ex_log {
      *	A simple walk of the log.
      */
     static int
-    simple_walk_log(Session session)
+    simple_walk_log(Session session, int count_min)
         throws WiredTigerException
     {
         Cursor cursor;
@@ -129,16 +129,18 @@ public class ex_log {
         byte[] logrec_key, logrec_value;
         long txnid;
         int fileid, opcount, optype, rectype;
-        int ret;
+        int count, ret;
 
         /*! [log cursor open] */
         cursor = session.open_cursor("log:", null, null);
         /*! [log cursor open] */
 
+        count = 0;
         while ((ret = cursor.next()) == 0) {
+            count++;
             /*! [log cursor get_key] */
             lsn.file = cursor.getKeyInt();
-            lsn.offset = cursor.getKeyLong();
+            lsn.offset = cursor.getKeyInt();
             opcount = cursor.getKeyInt();
             /*! [log cursor get_key] */
             /*! [log cursor get_value] */
@@ -156,6 +158,11 @@ public class ex_log {
         if (ret == wiredtiger.WT_NOTFOUND)
             ret = 0;
         ret = cursor.close();
+	if (count < count_min) {
+            System.err.println("Expected minimum " + count_min +
+                               " records, found " + count);
+            return (1);
+        }
         return (ret);
     }
     /*! [log cursor walk] */
@@ -185,7 +192,7 @@ public class ex_log {
         lsnsave = new Lsn();
         while ((ret = cursor.next()) == 0) {
             lsn.file = cursor.getKeyInt();
-            lsn.offset = cursor.getKeyLong();
+            lsn.offset = cursor.getKeyInt();
             opcount = cursor.getKeyInt();
 
             /*
@@ -194,8 +201,10 @@ public class ex_log {
              * that LSN to the end (where the multi-step transaction
              * was performed).  Just choose the record that is MAX_KEYS.
              */
-            if (++i == MAX_KEYS)
-                lsnsave = lsn;
+            if (++i == MAX_KEYS) {
+                lsnsave.file = lsn.file;
+                lsnsave.offset = lsn.offset;
+            }
             txnid = cursor.getValueLong();
             rectype = cursor.getValueInt();
             optype = cursor.getValueInt();
@@ -217,10 +226,10 @@ public class ex_log {
 
             /*
              * If the operation is a put, replay it here on the backup
-             * connection.  Note, we cheat by looking only for fileid 1
-             * in this example.  The metadata is fileid 0.
+             * connection.  Note, we cheat by looking at the fileid.
+             * The metadata is fileid 0, skip its records.
              */
-            if (fileid == 1 && rectype == wiredtiger.WT_LOGREC_COMMIT &&
+            if (fileid != 0 && rectype == wiredtiger.WT_LOGREC_COMMIT &&
                 optype == wiredtiger.WT_LOGOP_ROW_PUT) {
                 if (!in_txn) {
                     ret = session2.begin_transaction(null);
@@ -238,15 +247,20 @@ public class ex_log {
         /*
          * Compare the tables after replay.  They should be identical.
          */
-        if (compare_tables(session, session2) != 0)
-            System.out.println("compare failed");
+        if (compare_tables(session, session2) != 0) {
+            cursor.close();
+            session2.close(null);
+            wt_conn2.close(null);
+            return (ret);
+        }
         ret = session2.close(null);
         ret = wt_conn2.close(null);
 
         ret = cursor.reset();
         /*! [log cursor set_key] */
         cursor.putKeyInt(lsnsave.file);
-        cursor.putKeyLong(lsnsave.offset);
+        cursor.putKeyInt(lsnsave.offset);
+        cursor.putKeyInt(0);
         /*! [log cursor set_key] */
         /*! [log cursor search] */
         ret = cursor.search();
@@ -256,9 +270,9 @@ public class ex_log {
          * Walk all records starting with this key.
          */
         first = true;
-        while (ret == 0) {  /*TODO: not quite right*/
+        while (ret == 0) {
             lsn.file = cursor.getKeyInt();
-            lsn.offset = cursor.getKeyLong();
+            lsn.offset = cursor.getKeyInt();
             opcount = cursor.getKeyInt();
             if (first) {
                 first = false;
@@ -293,8 +307,9 @@ public class ex_log {
         Connection wt_conn;
         Cursor cursor;
         Session session;
-        int i, record_count, ret;
+        int count_min, i, record_count, ret;
 
+        count_min = 0;
         try {
             String command = "/bin/rm -rf " + home1 + " " + home2;
             Process proc = Runtime.getRuntime().exec(command);
@@ -317,6 +332,7 @@ public class ex_log {
 
         session = wt_conn.open_session(null);
         ret = session.create(uri, "key_format=S,value_format=S");
+        count_min++;
 
         cursor = session.open_cursor(uri, null, null);
         /*
@@ -328,6 +344,7 @@ public class ex_log {
             cursor.putKeyString(k);
             cursor.putValueString(v);
             ret = cursor.insert();
+            count_min++;
         }
         ret = session.begin_transaction(null);
         /*
@@ -341,10 +358,12 @@ public class ex_log {
             ret = cursor.insert();
         }
         ret = session.commit_transaction(null);
+        count_min++;
         ret = cursor.close();
 
         /*! [log cursor printf] */
         ret = session.log_printf("Wrote " + record_count + " records");
+        count_min++;
         /*! [log cursor printf] */
 
         session.close(null);
@@ -360,7 +379,7 @@ public class ex_log {
         }
 
         session = wt_conn.open_session(null);
-        ret = simple_walk_log(session);
+        ret = simple_walk_log(session, count_min);
         ret = walk_log(session);
         ret = session.close(null);
         ret = wt_conn.close(null);