Import wiredtiger: 315563f28850026673ebb146b6f3d727178e58bc from branch mongodb-4.2

ref: d9ec69f911..315563f288
for: 4.1.12

WT-3929       Fix "pip install" to work with Python3
WT-4669       Convert operation tracking logs to t2 format
WT-4750       Sweep can remove active lookaside records when files are closed and re-opened
WT-4765       Remove support for Helium/Levyx data sources in WiredTiger
WT-4766       Python3: fix wt_ckpt_decode.py to work with python2
WT-4767       Compatibility tests failed after 3.2.0 release

6 files changed, 102 insertions, 64 deletions

diff --git a/src/third_party/wiredtiger/lang/python/Makefile.am b/src/third_party/wiredtiger/lang/python/Makefile.am
index 3f29ea6aeea..ace11dfa5b9 100644
--- a/src/third_party/wiredtiger/lang/python/Makefile.am
+++ b/src/third_party/wiredtiger/lang/python/Makefile.am
@@ -2,7 +2,7 @@ PYSRC = $(top_srcdir)/lang/python
 PYDIRS = -t $(abs_builddir) -I $(abs_top_srcdir):$(abs_top_builddir) -L $(abs_top_builddir)/.libs
 PYDST = $(abs_builddir)/wiredtiger
 PYFILES = $(PYDST)/fpacking.py $(PYDST)/intpacking.py $(PYDST)/packing.py \
-	$(PYDST)/__init__.py
+	$(PYDST)/packutil.py $(PYDST)/__init__.py
 PY_MAJOR_VERSION := $$($(PYTHON) -c \
 	'import sys; print(int(sys.version_info.major))')
 
diff --git a/src/third_party/wiredtiger/lang/python/setup_pip.py b/src/third_party/wiredtiger/lang/python/setup_pip.py
index 9edd0b24a41..cafa05a9732 100644..100755
--- a/src/third_party/wiredtiger/lang/python/setup_pip.py
+++ b/src/third_party/wiredtiger/lang/python/setup_pip.py
@@ -98,7 +98,7 @@ def check_needed_dependencies(builtins, inc_paths, lib_paths):
 #   Locate an executable in the PATH.
 def find_executable(exename, path):
     p = subprocess.Popen(['which', exename ], stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE)
+                         stderr=subprocess.PIPE, universal_newlines=True)
     out, err = p.communicate('')
     out = str(out)  # needed for Python3
     if out == '':
@@ -146,7 +146,8 @@ def get_sources_curdir():
     DEVNULL = open(os.devnull, 'w')
     gitproc = subprocess.Popen(
         ['git', 'ls-tree', '-r', '--name-only', 'HEAD^{tree}'],
-        stdin=DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdin=DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        universal_newlines=True)
     sources = [line.rstrip() for line in gitproc.stdout.readlines()]
     err = gitproc.stderr.read()
     gitproc.wait()
@@ -176,6 +177,7 @@ def get_library_dirs():
     dirs.append("/usr/local/lib")
     dirs.append("/usr/local/lib64")
     dirs.append("/lib/x86_64-linux-gnu")
+    dirs.append("/usr/lib/x86_64-linux-gnu")
     dirs.append("/opt/local/lib")
     dirs.append("/usr/lib")
     dirs.append("/usr/lib64")
@@ -231,10 +233,6 @@ elif os.path.isfile(os.path.join(this_dir, 'LICENSE')):
 else:
     die('running from an unknown directory')
 
-python3 = (sys.version_info[0] > 2)
-if python3:
-    die('Python3 is not yet supported')
-
 # Ensure that Extensions won't be built for 32 bit,
 # that won't work with WiredTiger.
 if sys.maxsize < 2**32:
diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py
index ce28ea8ae2d..b20aee919a0 100755
--- a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py
+++ b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py
@@ -28,8 +28,12 @@
 #
 
 from __future__ import print_function
-from wiredtiger.packing import _chr, _ord
 import math, struct, sys
+try:
+    from wiredtiger.packutil import _chr, _ord, x00_entry, xff_entry
+except ImportError:
+    # When WiredTiger is installed as a package, python2 needs this
+    from .packutil import _chr, _ord, x00_entry, xff_entry
 
 # Variable-length integer packing
 # need: up to 64 bits, both signed and unsigned
@@ -64,14 +68,6 @@ POS_2BYTE_MAX = 2**13 + POS_1BYTE_MAX
 MINUS_BIT = -1 << 64
 UINT64_MASK = 0xffffffffffffffff
 
-_python3 = (sys.version_info >= (3, 0, 0))
-if _python3:
-    xff_entry = 0xff
-    x00_entry = 0x00
-else:
-    xff_entry = '\xff'
-    x00_entry = '\x00'
-
 def getbits(x, start, end=0):
     '''return the least significant bits of x, from start to end'''
     return (x & ((1 << start) - 1)) >> (end)
diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py
index 31b12a58126..47b6af1c786 100755
--- a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py
+++ b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py
@@ -49,50 +49,15 @@ Format  Python  Notes
   u     str     raw byte array
 """
 
-import sys
-
-# In the Python3 world, we pack into the bytes type, which like a list of ints.
-# In Python2, we pack into a string.  Create a set of constants and methods
-# to hide the differences from the main code.
-if sys.version_info[0] >= 3:
-    x00 = b'\x00'
-    xff = b'\xff'
-    empty_pack = b''
-    def _ord(b):
-        return b
-
-    def _chr(x, y=None):
-        a = [x]
-        if y != None:
-            a.append(y)
-        return bytes(a)
-
-    def _is_string(s):
-        return type(s) is str
-
-    def _string_result(s):
-        return s.decode()
-
-else:
-    x00 = '\x00'
-    xff = '\xff'
-    empty_pack = ''
-    def _ord(b):
-        return ord(b)
-
-    def _chr(x, y=None):
-        s = chr(x)
-        if y != None:
-            s += chr(y)
-        return s
-
-    def _is_string(s):
-        return type(s) is unicode
-
-    def _string_result(s):
-        return s
-
-from wiredtiger.intpacking import pack_int, unpack_int
+try:
+    from wiredtiger.packutil import _chr, _is_string, _ord, _string_result, \
+        empty_pack, x00
+    from wiredtiger.intpacking import pack_int, unpack_int
+except ImportError:
+    # When WiredTiger is installed as a package, python2 needs this
+    from .packutil import _chr, _is_string, _ord, _string_result, \
+        empty_pack, x00
+    from intpacking import pack_int, unpack_int
 
 def __get_type(fmt):
     if not fmt:
diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/packutil.py b/src/third_party/wiredtiger/lang/python/wiredtiger/packutil.py
new file mode 100755
index 00000000000..cbff2148361
--- /dev/null
+++ b/src/third_party/wiredtiger/lang/python/wiredtiger/packutil.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2019 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# WiredTiger packing and unpacking utility functions and constants
+
+import sys
+
+# In the Python3 world, we pack into the bytes type, which like a list of ints.
+# In Python2, we pack into a string.  Create a set of constants and methods
+# to hide the differences from the main code.
+
+# all bits on or off, expressed as a bytes type
+x00 = b'\x00'
+xff = b'\xff'
+x00_entry = x00[0]
+xff_entry = xff[0]
+empty_pack = b''
+
+_python3 = (sys.version_info >= (3, 0, 0))
+if _python3:
+    def _ord(b):
+        return b
+
+    def _chr(x, y=None):
+        a = [x]
+        if y != None:
+            a.append(y)
+        return bytes(a)
+
+    def _is_string(s):
+        return type(s) is str
+
+    def _string_result(s):
+        return s.decode()
+
+else:
+    def _ord(b):
+        return ord(b)
+
+    def _chr(x, y=None):
+        s = chr(x)
+        if y != None:
+            s += chr(y)
+        return s
+
+    def _is_string(s):
+        return type(s) is unicode
+
+    def _string_result(s):
+        return s
diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py b/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py
index 98f15bf88f4..187a21443b7 100644..100755
--- a/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py
+++ b/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py
@@ -41,8 +41,12 @@ if fname != '__init__.py' and fname != '__init__.pyc':
 # After importing the SWIG-generated file, copy all symbols from from it
 # to this module so they will appear in the wiredtiger namespace.
 me = sys.modules[__name__]
-sys.path.append(os.path.dirname(__file__))  # needed for Python3
-import wiredtiger
-for name in dir(wiredtiger):
-    value = getattr(wiredtiger, name)
+sys.path.append(os.path.dirname(__file__))
+try:
+    import wiredtiger.wiredtiger as swig_wiredtiger
+except ImportError:
+    # for Python2
+    import wiredtiger as swig_wiredtiger
+for name in dir(swig_wiredtiger):
+    value = getattr(swig_wiredtiger, name)
     setattr(me, name, value)