Merge: PEP8 fixup on previous patch, remove unused imports in test_email.

678 files changed, 77989 insertions, 14208 deletions

diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py
index ed50520ab3..13b1f26965 100644
--- a/Lib/_dummy_thread.py
+++ b/Lib/_dummy_thread.py
@@ -24,11 +24,7 @@ TIMEOUT_MAX = 2**31
 # imports are done when needed on a function-by-function basis.  Since threads
 # are disabled, the import lock should not be an issue anyway (??).
 
-class error(Exception):
-    """Dummy implementation of _thread.error."""
-
-    def __init__(self, *args):
-        self.args = args
+error = RuntimeError
 
 def start_new_thread(function, args, kwargs={}):
     """Dummy implementation of _thread.start_new_thread().
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index 2d376d8300..9cbb364dd1 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -5,7 +5,6 @@ Python implementation of the io module.
 import os
 import abc
 import codecs
-import warnings
 import errno
 # Import _thread instead of threading to reduce startup cost
 try:
@@ -15,7 +14,11 @@ except ImportError:
 
 import io
 from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
-from errno import EINTR
+
+valid_seek_flags = {0, 1, 2}  # Hardwired values
+if hasattr(os, 'SEEK_HOLE') :
+    valid_seek_flags.add(os.SEEK_HOLE)
+    valid_seek_flags.add(os.SEEK_DATA)
 
 # open() uses st_blksize whenever we can
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
@@ -24,20 +27,12 @@ DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 # defined in io.py. We don't use real inheritance though, because we don't
 # want to inherit the C implementations.
 
-
-class BlockingIOError(IOError):
-
-    """Exception raised when I/O would block on a non-blocking I/O stream."""
-
-    def __init__(self, errno, strerror, characters_written=0):
-        super().__init__(errno, strerror)
-        if not isinstance(characters_written, int):
-            raise TypeError("characters_written must be a integer")
-        self.characters_written = characters_written
+# Rebind for compatibility
+BlockingIOError = BlockingIOError
 
 
 def open(file, mode="r", buffering=-1, encoding=None, errors=None,
-         newline=None, closefd=True):
+         newline=None, closefd=True, opener=None):
 
     r"""Open file and return a stream.  Raise IOError upon failure.
 
@@ -47,21 +42,22 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     wrapped. (If a file descriptor is given, it is closed when the
     returned I/O object is closed, unless closefd is set to False.)
 
-    mode is an optional string that specifies the mode in which the file
-    is opened. It defaults to 'r' which means open for reading in text
-    mode.  Other common values are 'w' for writing (truncating the file if
-    it already exists), and 'a' for appending (which on some Unix systems,
-    means that all writes append to the end of the file regardless of the
-    current seek position). In text mode, if encoding is not specified the
-    encoding used is platform dependent. (For reading and writing raw
-    bytes use binary mode and leave encoding unspecified.) The available
-    modes are:
+    mode is an optional string that specifies the mode in which the file is
+    opened. It defaults to 'r' which means open for reading in text mode. Other
+    common values are 'w' for writing (truncating the file if it already
+    exists), 'x' for exclusive creation of a new file, and 'a' for appending
+    (which on some Unix systems, means that all writes append to the end of the
+    file regardless of the current seek position). In text mode, if encoding is
+    not specified the encoding used is platform dependent. (For reading and
+    writing raw bytes use binary mode and leave encoding unspecified.) The
+    available modes are:
 
     ========= ===============================================================
     Character Meaning
     --------- ---------------------------------------------------------------
     'r'       open for reading (default)
     'w'       open for writing, truncating the file first
+    'x'       create a new file and open it for writing
     'a'       open for writing, appending to the end of the file if it exists
     'b'       binary mode
     't'       text mode (default)
@@ -72,7 +68,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
 
     The default mode is 'rt' (open for reading text). For binary random
     access, the mode 'w+b' opens and truncates the file to 0 bytes, while
-    'r+b' opens the file without truncation.
+    'r+b' opens the file without truncation. The 'x' mode implies 'w' and
+    raises an `FileExistsError` if the file already exists.
 
     Python distinguishes between files opened in binary and text modes,
     even when the underlying operating system doesn't. Files opened in
@@ -132,6 +129,12 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     be kept open when the file is closed. This does not work when a file name is
     given and must be True in that case.
 
+    A custom opener can be used by passing a callable as *opener*. The
+    underlying file descriptor for the file object is then obtained by calling
+    *opener* with (*file*, *flags*). *opener* must return an open file
+    descriptor (passing os.open as *opener* results in functionality similar to
+    passing None).
+
     open() returns a file object whose type depends on the mode, and
     through which the standard file operations such as reading and writing
     are performed. When open() is used to open a file in a text mode ('w',
@@ -157,8 +160,9 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     if errors is not None and not isinstance(errors, str):
         raise TypeError("invalid errors: %r" % errors)
     modes = set(mode)
-    if modes - set("arwb+tU") or len(mode) > len(modes):
+    if modes - set("axrwb+tU") or len(mode) > len(modes):
         raise ValueError("invalid mode: %r" % mode)
+    creating = "x" in modes
     reading = "r" in modes
     writing = "w" in modes
     appending = "a" in modes
@@ -166,14 +170,14 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     text = "t" in modes
     binary = "b" in modes
     if "U" in modes:
-        if writing or appending:
+        if creating or writing or appending:
             raise ValueError("can't use U and writing mode at once")
         reading = True
     if text and binary:
         raise ValueError("can't have text and binary mode at once")
-    if reading + writing + appending > 1:
+    if creating + reading + writing + appending > 1:
         raise ValueError("can't have read/write/append mode at once")
-    if not (reading or writing or appending):
+    if not (creating or reading or writing or appending):
         raise ValueError("must have exactly one of read/write/append mode")
     if binary and encoding is not None:
         raise ValueError("binary mode doesn't take an encoding argument")
@@ -182,11 +186,12 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     if binary and newline is not None:
         raise ValueError("binary mode doesn't take a newline argument")
     raw = FileIO(file,
+                 (creating and "x" or "") +
                  (reading and "r" or "") +
                  (writing and "w" or "") +
                  (appending and "a" or "") +
                  (updating and "+" or ""),
-                 closefd)
+                 closefd, opener=opener)
     line_buffering = False
     if buffering == 1 or buffering < 0 and raw.isatty():
         buffering = -1
@@ -208,7 +213,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
         raise ValueError("can't have unbuffered text I/O")
     if updating:
         buffer = BufferedRandom(raw, buffering)
-    elif writing or appending:
+    elif creating or writing or appending:
         buffer = BufferedWriter(raw, buffering)
     elif reading:
         buffer = BufferedReader(raw, buffering)
@@ -305,6 +310,7 @@ class IOBase(metaclass=abc.ABCMeta):
         * 0 -- start of stream (the default); offset should be zero or positive
         * 1 -- current stream position; offset may be negative
         * 2 -- end of stream; offset is usually negative
+        Some operating systems / file systems could provide additional values.
 
         Return an int indicating the new absolute position.
         """
@@ -340,8 +346,10 @@ class IOBase(metaclass=abc.ABCMeta):
         This method has no effect if the file is already closed.
         """
         if not self.__closed:
-            self.flush()
-            self.__closed = True
+            try:
+                self.flush()
+            finally:
+                self.__closed = True
 
     def __del__(self):
         """Destructor.  Calls close()."""
@@ -865,7 +873,7 @@ class BytesIO(BufferedIOBase):
         elif whence == 2:
             self._pos = max(0, len(self._buffer) + pos)
         else:
-            raise ValueError("invalid whence value")
+            raise ValueError("unsupported whence value")
         return self._pos
 
     def tell(self):
@@ -954,15 +962,19 @@ class BufferedReader(_BufferedIOMixin):
         # Special case for when the number of bytes to read is unspecified.
         if n is None or n == -1:
             self._reset_read_buf()
+            if hasattr(self.raw, 'readall'):
+                chunk = self.raw.readall()
+                if chunk is None:
+                    return buf[pos:] or None
+                else:
+                    return buf[pos:] + chunk
             chunks = [buf[pos:]]  # Strip the consumed bytes.
             current_size = 0
             while True:
                 # Read until EOF or until read() would block.
                 try:
                     chunk = self.raw.read()
-                except IOError as e:
-                    if e.errno != EINTR:
-                        raise
+                except InterruptedError:
                     continue
                 if chunk in empty_values:
                     nodata_val = chunk
@@ -984,9 +996,7 @@ class BufferedReader(_BufferedIOMixin):
         while avail < n:
             try:
                 chunk = self.raw.read(wanted)
-            except IOError as e:
-                if e.errno != EINTR:
-                    raise
+            except InterruptedError:
                 continue
             if chunk in empty_values:
                 nodata_val = chunk
@@ -1019,9 +1029,7 @@ class BufferedReader(_BufferedIOMixin):
             while True:
                 try:
                     current = self.raw.read(to_read)
-                except IOError as e:
-                    if e.errno != EINTR:
-                        raise
+                except InterruptedError:
                     continue
                 break
             if current:
@@ -1046,7 +1054,7 @@ class BufferedReader(_BufferedIOMixin):
         return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
 
     def seek(self, pos, whence=0):
-        if not (0 <= whence <= 2):
+        if whence not in valid_seek_flags:
             raise ValueError("invalid whence value")
         with self._read_lock:
             if whence == 1:
@@ -1064,19 +1072,13 @@ class BufferedWriter(_BufferedIOMixin):
     DEFAULT_BUFFER_SIZE.
     """
 
-    _warning_stack_offset = 2
-
-    def __init__(self, raw,
-                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+    def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
         if not raw.writable():
             raise IOError('"raw" argument must be writable.')
 
         _BufferedIOMixin.__init__(self, raw)
         if buffer_size <= 0:
             raise ValueError("invalid buffer size")
-        if max_buffer_size is not None:
-            warnings.warn("max_buffer_size is deprecated", DeprecationWarning,
-                          self._warning_stack_offset)
         self.buffer_size = buffer_size
         self._write_buf = bytearray()
         self._write_lock = Lock()
@@ -1126,13 +1128,11 @@ class BufferedWriter(_BufferedIOMixin):
         while self._write_buf:
             try:
                 n = self.raw.write(self._write_buf)
+            except InterruptedError:
+                continue
             except BlockingIOError:
                 raise RuntimeError("self.raw should implement RawIOBase: it "
                                    "should not raise BlockingIOError")
-            except IOError as e:
-                if e.errno != EINTR:
-                    raise
-                continue
             if n is None:
                 raise BlockingIOError(
                     errno.EAGAIN,
@@ -1145,8 +1145,8 @@ class BufferedWriter(_BufferedIOMixin):
         return _BufferedIOMixin.tell(self) + len(self._write_buf)
 
     def seek(self, pos, whence=0):
-        if not (0 <= whence <= 2):
-            raise ValueError("invalid whence")
+        if whence not in valid_seek_flags:
+            raise ValueError("invalid whence value")
         with self._write_lock:
             self._flush_unlocked()
             return _BufferedIOMixin.seek(self, pos, whence)
@@ -1168,15 +1168,11 @@ class BufferedRWPair(BufferedIOBase):
     # XXX The usefulness of this (compared to having two separate IO
     # objects) is questionable.
 
-    def __init__(self, reader, writer,
-                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+    def __init__(self, reader, writer, buffer_size=DEFAULT_BUFFER_SIZE):
         """Constructor.
 
         The arguments are two RawIO instances.
         """
-        if max_buffer_size is not None:
-            warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2)
-
         if not reader.readable():
             raise IOError('"reader" argument must be readable.')
 
@@ -1233,17 +1229,14 @@ class BufferedRandom(BufferedWriter, BufferedReader):
     defaults to DEFAULT_BUFFER_SIZE.
     """
 
-    _warning_stack_offset = 3
-
-    def __init__(self, raw,
-                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+    def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
         raw._checkSeekable()
         BufferedReader.__init__(self, raw, buffer_size)
-        BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size)
+        BufferedWriter.__init__(self, raw, buffer_size)
 
     def seek(self, pos, whence=0):
-        if not (0 <= whence <= 2):
-            raise ValueError("invalid whence")
+        if whence not in valid_seek_flags:
+            raise ValueError("invalid whence value")
         self.flush()
         if self._read_buf:
             # Undo read ahead.
@@ -1455,7 +1448,7 @@ class TextIOWrapper(TextIOBase):
     r"""Character and line based layer over a BufferedIOBase object, buffer.
 
     encoding gives the name of the encoding that the stream will be
-    decoded or encoded with. It defaults to locale.getpreferredencoding.
+    decoded or encoded with. It defaults to locale.getpreferredencoding(False).
 
     errors determines the strictness of encoding and decoding (see the
     codecs.register) and defaults to "strict".
@@ -1476,6 +1469,9 @@ class TextIOWrapper(TextIOBase):
 
     _CHUNK_SIZE = 2048
 
+    # The write_through argument has no effect here since this
+    # implementation always writes through.  The argument is present only
+    # so that the signature can match the signature of the C version.
     def __init__(self, buffer, encoding=None, errors=None, newline=None,
                  line_buffering=False, write_through=False):
         if newline is not None and not isinstance(newline, str):
@@ -1494,7 +1490,7 @@ class TextIOWrapper(TextIOBase):
                     # Importing locale may fail if Python is being built
                     encoding = "ascii"
                 else:
-                    encoding = locale.getpreferredencoding()
+                    encoding = locale.getpreferredencoding(False)
 
         if not isinstance(encoding, str):
             raise ValueError("invalid encoding: %r" % encoding)
@@ -1521,6 +1517,7 @@ class TextIOWrapper(TextIOBase):
         self._snapshot = None  # info for reconstructing decoder state
         self._seekable = self._telling = self.buffer.seekable()
         self._has_read1 = hasattr(self.buffer, 'read1')
+        self._b2cratio = 0.0
 
         if self._seekable and self.writable():
             position = self.buffer.tell()
@@ -1589,8 +1586,10 @@ class TextIOWrapper(TextIOBase):
 
     def close(self):
         if self.buffer is not None and not self.closed:
-            self.flush()
-            self.buffer.close()
+            try:
+                self.flush()
+            finally:
+                self.buffer.close()
 
     @property
     def closed(self):
@@ -1693,7 +1692,12 @@ class TextIOWrapper(TextIOBase):
         else:
             input_chunk = self.buffer.read(self._CHUNK_SIZE)
         eof = not input_chunk
-        self._set_decoded_chars(self._decoder.decode(input_chunk, eof))
+        decoded_chars = self._decoder.decode(input_chunk, eof)
+        self._set_decoded_chars(decoded_chars)
+        if decoded_chars:
+            self._b2cratio = len(input_chunk) / len(self._decoded_chars)
+        else:
+            self._b2cratio = 0.0
 
         if self._telling:
             # At the snapshot point, len(dec_buffer) bytes before the read,
@@ -1747,20 +1751,56 @@ class TextIOWrapper(TextIOBase):
         # forward until it gives us enough decoded characters.
         saved_state = decoder.getstate()
         try:
+            # Fast search for an acceptable start point, close to our
+            # current pos.
+            # Rationale: calling decoder.decode() has a large overhead
+            # regardless of chunk size; we want the number of such calls to
+            # be O(1) in most situations (common decoders, non-crazy input).
+            # Actually, it will be exactly 1 for fixed-size codecs (all
+            # 8-bit codecs, also UTF-16 and UTF-32).
+            skip_bytes = int(self._b2cratio * chars_to_skip)
+            skip_back = 1
+            assert skip_bytes <= len(next_input)
+            while skip_bytes > 0:
+                decoder.setstate((b'', dec_flags))
+                # Decode up to temptative start point
+                n = len(decoder.decode(next_input[:skip_bytes]))
+                if n <= chars_to_skip:
+                    b, d = decoder.getstate()
+                    if not b:
+                        # Before pos and no bytes buffered in decoder => OK
+                        dec_flags = d
+                        chars_to_skip -= n
+                        break
+                    # Skip back by buffered amount and reset heuristic
+                    skip_bytes -= len(b)
+                    skip_back = 1
+                else:
+                    # We're too far ahead, skip back a bit
+                    skip_bytes -= skip_back
+                    skip_back = skip_back * 2
+            else:
+                skip_bytes = 0
+                decoder.setstate((b'', dec_flags))
+
             # Note our initial start point.
-            decoder.setstate((b'', dec_flags))
-            start_pos = position
-            start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
-            need_eof = 0
+            start_pos = position + skip_bytes
+            start_flags = dec_flags
+            if chars_to_skip == 0:
+                # We haven't moved from the start point.
+                return self._pack_cookie(start_pos, start_flags)
 
             # Feed the decoder one byte at a time.  As we go, note the
             # nearest "safe start point" before the current location
             # (a point where the decoder has nothing buffered, so seek()
             # can safely start from there and advance to this location).
-            next_byte = bytearray(1)
-            for next_byte[0] in next_input:
+            bytes_fed = 0
+            need_eof = 0
+            # Chars decoded since `start_pos`
+            chars_decoded = 0
+            for i in range(skip_bytes, len(next_input)):
                 bytes_fed += 1
-                chars_decoded += len(decoder.decode(next_byte))
+                chars_decoded += len(decoder.decode(next_input[i:i+1]))
                 dec_buffer, dec_flags = decoder.getstate()
                 if not dec_buffer and chars_decoded <= chars_to_skip:
                     # Decoder buffer is empty, so this is a safe start point.
@@ -1819,8 +1859,7 @@ class TextIOWrapper(TextIOBase):
                 self._decoder.reset()
             return position
         if whence != 0:
-            raise ValueError("invalid whence (%r, should be 0, 1 or 2)" %
-                             (whence,))
+            raise ValueError("unsupported whence (%r)" % (whence,))
         if cookie < 0:
             raise ValueError("negative seek position %r" % (cookie,))
         self.flush()
diff --git a/Lib/_strptime.py b/Lib/_strptime.py
index fa06376992..b0cd3d619e 100644
--- a/Lib/_strptime.py
+++ b/Lib/_strptime.py
@@ -486,19 +486,19 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
 
     return (year, month, day,
             hour, minute, second,
-            weekday, julian, tz, gmtoff, tzname), fraction
+            weekday, julian, tz, tzname, gmtoff), fraction
 
 def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
     """Return a time struct based on the input string and the
     format string."""
     tt = _strptime(data_string, format)[0]
-    return time.struct_time(tt[:9])
+    return time.struct_time(tt[:time._STRUCT_TM_ITEMS])
 
 def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
     """Return a class cls instance based on the input string and the
     format string."""
     tt, fraction = _strptime(data_string, format)
-    gmtoff, tzname = tt[-2:]
+    tzname, gmtoff = tt[-2:]
     args = tt[:6] + (fraction,)
     if gmtoff is not None:
         tzdelta = datetime_timedelta(seconds=gmtoff)
diff --git a/Lib/abc.py b/Lib/abc.py
index a6c2dc4877..09778e8609 100644
--- a/Lib/abc.py
+++ b/Lib/abc.py
@@ -26,7 +26,8 @@ def abstractmethod(funcobj):
 
 
 class abstractclassmethod(classmethod):
-    """A decorator indicating abstract classmethods.
+    """
+    A decorator indicating abstract classmethods.
 
     Similar to abstractmethod.
 
@@ -36,6 +37,9 @@ class abstractclassmethod(classmethod):
             @abstractclassmethod
             def my_abstract_classmethod(cls, ...):
                 ...
+
+    'abstractclassmethod' is deprecated. Use 'classmethod' with
+    'abstractmethod' instead.
     """
 
     __isabstractmethod__ = True
@@ -46,7 +50,8 @@ class abstractclassmethod(classmethod):
 
 
 class abstractstaticmethod(staticmethod):
-    """A decorator indicating abstract staticmethods.
+    """
+    A decorator indicating abstract staticmethods.
 
     Similar to abstractmethod.
 
@@ -56,6 +61,9 @@ class abstractstaticmethod(staticmethod):
             @abstractstaticmethod
             def my_abstract_staticmethod(...):
                 ...
+
+    'abstractstaticmethod' is deprecated. Use 'staticmethod' with
+    'abstractmethod' instead.
     """
 
     __isabstractmethod__ = True
@@ -66,7 +74,8 @@ class abstractstaticmethod(staticmethod):
 
 
 class abstractproperty(property):
-    """A decorator indicating abstract properties.
+    """
+    A decorator indicating abstract properties.
 
     Requires that the metaclass is ABCMeta or derived from it.  A
     class that has a metaclass derived from ABCMeta cannot be
@@ -88,7 +97,11 @@ class abstractproperty(property):
             def getx(self): ...
             def setx(self, value): ...
             x = abstractproperty(getx, setx)
+
+    'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
+    instead.
     """
+
     __isabstractmethod__ = True
 
 
@@ -133,11 +146,14 @@ class ABCMeta(type):
         return cls
 
     def register(cls, subclass):
-        """Register a virtual subclass of an ABC."""
+        """Register a virtual subclass of an ABC.
+
+        Returns the subclass, to allow usage as a class decorator.
+        """
         if not isinstance(subclass, type):
             raise TypeError("Can only register classes")
         if issubclass(subclass, cls):
-            return  # Already a subclass
+            return subclass  # Already a subclass
         # Subtle: test for cycles *after* testing for "already a subclass";
         # this means we allow X.register(X) and interpret it as a no-op.
         if issubclass(cls, subclass):
@@ -145,6 +161,7 @@ class ABCMeta(type):
             raise RuntimeError("Refusing to create an inheritance cycle")
         cls._abc_registry.add(subclass)
         ABCMeta._abc_invalidation_counter += 1  # Invalidate negative cache
+        return subclass
 
     def _dump_registry(cls, file=None):
         """Debug helper to print the ABC registry."""
diff --git a/Lib/aifc.py b/Lib/aifc.py
index 841f5ae287..a19b38f4aa 100644
--- a/Lib/aifc.py
+++ b/Lib/aifc.py
@@ -136,6 +136,7 @@ writeframesraw.
 
 import struct
 import builtins
+import warnings
 
 __all__ = ["Error", "open", "openfp"]
 
@@ -440,7 +441,7 @@ class Aifc_read:
             kludge = 0
             if chunk.chunksize == 18:
                 kludge = 1
-                print('Warning: bad COMM chunk size')
+                warnings.warn('Warning: bad COMM chunk size')
                 chunk.chunksize = 23
             #DEBUG end
             self._comptype = chunk.read(4)
@@ -484,11 +485,10 @@ class Aifc_read:
                     # a position 0 and name ''
                     self._markers.append((id, pos, name))
         except EOFError:
-            print('Warning: MARK chunk contains only', end=' ')
-            print(len(self._markers), end=' ')
-            if len(self._markers) == 1: print('marker', end=' ')
-            else: print('markers', end=' ')
-            print('instead of', nmarkers)
+            w = ('Warning: MARK chunk contains only %s marker%s instead of %s' %
+                 (len(self._markers), '' if len(self._markers) == 1 else 's',
+                  nmarkers))
+            warnings.warn(w)
 
 class Aifc_write:
     # Variables used in this class:
diff --git a/Lib/argparse.py b/Lib/argparse.py
index eb894caebc..f25b1b6610 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -71,6 +71,7 @@ __all__ = [
     'ArgumentDefaultsHelpFormatter',
     'RawDescriptionHelpFormatter',
     'RawTextHelpFormatter',
+    'MetavarTypeHelpFormatter',
     'Namespace',
     'Action',
     'ONE_OR_MORE',
@@ -419,7 +420,8 @@ class HelpFormatter(object):
 
             # produce all arg strings
             elif not action.option_strings:
-                part = self._format_args(action, action.dest)
+                default = self._get_default_metavar_for_positional(action)
+                part = self._format_args(action, default)
 
                 # if it's in a group, strip the outer []
                 if action in group_actions:
@@ -441,7 +443,7 @@ class HelpFormatter(object):
                 # if the Optional takes a value, format is:
                 #    -s ARGS or --long ARGS
                 else:
-                    default = action.dest.upper()
+                    default = self._get_default_metavar_for_optional(action)
                     args_string = self._format_args(action, default)
                     part = '%s %s' % (option_string, args_string)
 
@@ -527,7 +529,8 @@ class HelpFormatter(object):
 
     def _format_action_invocation(self, action):
         if not action.option_strings:
-            metavar, = self._metavar_formatter(action, action.dest)(1)
+            default = self._get_default_metavar_for_positional(action)
+            metavar, = self._metavar_formatter(action, default)(1)
             return metavar
 
         else:
@@ -541,7 +544,7 @@ class HelpFormatter(object):
             # if the Optional takes a value, format is:
             #    -s ARGS, --long ARGS
             else:
-                default = action.dest.upper()
+                default = self._get_default_metavar_for_optional(action)
                 args_string = self._format_args(action, default)
                 for option_string in action.option_strings:
                     parts.append('%s %s' % (option_string, args_string))
@@ -619,6 +622,12 @@ class HelpFormatter(object):
     def _get_help_string(self, action):
         return action.help
 
+    def _get_default_metavar_for_optional(self, action):
+        return action.dest.upper()
+
+    def _get_default_metavar_for_positional(self, action):
+        return action.dest
+
 
 class RawDescriptionHelpFormatter(HelpFormatter):
     """Help message formatter which retains any formatting in descriptions.
@@ -628,7 +637,7 @@ class RawDescriptionHelpFormatter(HelpFormatter):
     """
 
     def _fill_text(self, text, width, indent):
-        return ''.join([indent + line for line in text.splitlines(True)])
+        return ''.join(indent + line for line in text.splitlines(keepends=True))
 
 
 class RawTextHelpFormatter(RawDescriptionHelpFormatter):
@@ -659,6 +668,22 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
         return help
 
 
+class MetavarTypeHelpFormatter(HelpFormatter):
+    """Help message formatter which uses the argument 'type' as the default
+    metavar value (instead of the argument 'dest')
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _get_default_metavar_for_optional(self, action):
+        return action.type.__name__
+
+    def _get_default_metavar_for_positional(self, action):
+        return action.type.__name__
+
+
+
 # =====================
 # Options and Arguments
 # =====================
@@ -1554,7 +1579,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                  usage=None,
                  description=None,
                  epilog=None,
-                 version=None,
                  parents=[],
                  formatter_class=HelpFormatter,
                  prefix_chars='-',
@@ -1563,14 +1587,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                  conflict_handler='error',
                  add_help=True):
 
-        if version is not None:
-            import warnings
-            warnings.warn(
-                """The "version" argument to ArgumentParser is deprecated. """
-                """Please use """
-                """"add_argument(..., action='version', version="N", ...)" """
-                """instead""", DeprecationWarning)
-
         superinit = super(ArgumentParser, self).__init__
         superinit(description=description,
                   prefix_chars=prefix_chars,
@@ -1584,7 +1600,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         self.prog = prog
         self.usage = usage
         self.epilog = epilog
-        self.version = version
         self.formatter_class = formatter_class
         self.fromfile_prefix_chars = fromfile_prefix_chars
         self.add_help = add_help
@@ -1599,7 +1614,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             return string
         self.register('type', None, identity)
 
-        # add help and version arguments if necessary
+        # add help argument if necessary
         # (using explicit default to override global argument_default)
         default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
         if self.add_help:
@@ -1607,12 +1622,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 default_prefix+'h', default_prefix*2+'help',
                 action='help', default=SUPPRESS,
                 help=_('show this help message and exit'))
-        if self.version:
-            self.add_argument(
-                default_prefix+'v', default_prefix*2+'version',
-                action='version', default=SUPPRESS,
-                version=self.version,
-                help=_("show program's version number and exit"))
 
         # add parent arguments and defaults
         for parent in parents:
@@ -1632,7 +1641,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             'prog',
             'usage',
             'description',
-            'version',
             'formatter_class',
             'conflict_handler',
             'add_help',
@@ -1940,29 +1948,29 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # if we didn't consume all the argument strings, there were extras
         extras.extend(arg_strings[stop_index:])
 
-        # if we didn't use all the Positional objects, there were too few
-        # arg strings supplied.
-        if positionals:
-            self.error(_('too few arguments'))
-
-        # make sure all required actions were present, and convert defaults.
+        # make sure all required actions were present and also convert
+        # action defaults which were not given as arguments
+        required_actions = []
         for action in self._actions:
             if action not in seen_actions:
                 if action.required:
-                    name = _get_action_name(action)
-                    self.error(_('argument %s is required') % name)
+                    required_actions.append(_get_action_name(action))
                 else:
                     # Convert action default now instead of doing it before
                     # parsing arguments to avoid calling convert functions
                     # twice (which may fail) if the argument was given, but
                     # only if it was defined already in the namespace
                     if (action.default is not None and
-                            isinstance(action.default, str) and
-                            hasattr(namespace, action.dest) and
-                            action.default is getattr(namespace, action.dest)):
+                        isinstance(action.default, str) and
+                        hasattr(namespace, action.dest) and
+                        action.default is getattr(namespace, action.dest)):
                         setattr(namespace, action.dest,
                                 self._get_value(action, action.default))
 
+        if required_actions:
+            self.error(_('the following arguments are required: %s') %
+                       ', '.join(required_actions))
+
         # make sure all required groups had one option present
         for group in self._mutually_exclusive_groups:
             if group.required:
@@ -2314,16 +2322,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # determine help from format above
         return formatter.format_help()
 
-    def format_version(self):
-        import warnings
-        warnings.warn(
-            'The format_version method is deprecated -- the "version" '
-            'argument to ArgumentParser is no longer supported.',
-            DeprecationWarning)
-        formatter = self._get_formatter()
-        formatter.add_text(self.version)
-        return formatter.format_help()
-
     def _get_formatter(self):
         return self.formatter_class(prog=self.prog)
 
@@ -2340,14 +2338,6 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             file = _sys.stdout
         self._print_message(self.format_help(), file)
 
-    def print_version(self, file=None):
-        import warnings
-        warnings.warn(
-            'The print_version method is deprecated -- the "version" '
-            'argument to ArgumentParser is no longer supported.',
-            DeprecationWarning)
-        self._print_message(self.format_version(), file)
-
     def _print_message(self, message, file=None):
         if message:
             if file is None:
diff --git a/Lib/ast.py b/Lib/ast.py
index fb5adac8fd..13f59f9dfa 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -25,7 +25,6 @@
     :license: Python License.
 """
 from _ast import *
-from _ast import __version__
 
 
 def parse(source, filename='<unknown>', mode='exec'):
diff --git a/Lib/asynchat.py b/Lib/asynchat.py
index 65585129ce..4e26bb5856 100644
--- a/Lib/asynchat.py
+++ b/Lib/asynchat.py
@@ -49,18 +49,6 @@ import socket
 import asyncore
 from collections import deque
 
-def buffer(obj, start=None, stop=None):
-    # if memoryview objects gain slicing semantics,
-    # this function will change for the better
-    # memoryview used for the TypeError
-    memoryview(obj)
-    if start == None:
-        start = 0
-    if stop == None:
-        stop = len(obj)
-    x = obj[start:stop]
-    ## print("buffer type is: %s"%(type(x),))
-    return x
 
 class async_chat (asyncore.dispatcher):
     """This is an abstract class.  You must derive from this class, and add
@@ -75,7 +63,7 @@ class async_chat (asyncore.dispatcher):
     # sign of an application bug that we don't want to pass silently
 
     use_encoding            = 0
-    encoding                = 'latin1'
+    encoding                = 'latin-1'
 
     def __init__ (self, sock=None, map=None):
         # for string terminator matching
@@ -240,7 +228,7 @@ class async_chat (asyncore.dispatcher):
             # handle classic producer behavior
             obs = self.ac_out_buffer_size
             try:
-                data = buffer(first, 0, obs)
+                data = first[:obs]
             except TypeError:
                 data = first.more()
                 if data:
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index d379ba820c..909d9f605d 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -54,7 +54,7 @@ import warnings
 
 import os
 from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
-     ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
+     ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
      errorcode
 
 _DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
@@ -143,11 +143,8 @@ def poll(timeout=0.0, map=None):
 
         try:
             r, w, e = select.select(r, w, e, timeout)
-        except select.error as err:
-            if err.args[0] != EINTR:
-                raise
-            else:
-                return
+        except InterruptedError:
+            return
 
         for fd in r:
             obj = map.get(fd)
@@ -184,15 +181,10 @@ def poll2(timeout=0.0, map=None):
             if obj.writable() and not obj.accepting:
                 flags |= select.POLLOUT
             if flags:
-                # Only check for exceptions if object was either readable
-                # or writable.
-                flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL
                 pollster.register(fd, flags)
         try:
             r = pollster.poll(timeout)
-        except select.error as err:
-            if err.args[0] != EINTR:
-                raise
+        except InterruptedError:
             r = []
         for fd, flags in r:
             obj = map.get(fd)
@@ -292,7 +284,7 @@ class dispatcher:
             del map[fd]
         self._fileno = None
 
-    def create_socket(self, family, type):
+    def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
         self.family_and_type = family, type
         sock = socket.socket(family, type)
         sock.setblocking(0)
diff --git a/Lib/base64.py b/Lib/base64.py
index 895d813f7e..4042f004fd 100755
--- a/Lib/base64.py
+++ b/Lib/base64.py
@@ -29,14 +29,16 @@ __all__ = [
 
 bytes_types = (bytes, bytearray)  # Types acceptable as binary data
 
-
-def _translate(s, altchars):
-    if not isinstance(s, bytes_types):
-        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
-    translation = bytearray(range(256))
-    for k, v in altchars.items():
-        translation[ord(k)] = v[0]
-    return s.translate(translation)
+def _bytes_from_decode_data(s):
+    if isinstance(s, str):
+        try:
+            return s.encode('ascii')
+        except UnicodeEncodeError:
+            raise ValueError('string argument should contain only ASCII characters')
+    elif isinstance(s, bytes_types):
+        return s
+    else:
+        raise TypeError("argument should be bytes or ASCII string, not %s" % s.__class__.__name__)
 
 
 
@@ -61,7 +63,7 @@ def b64encode(s, altchars=None):
             raise TypeError("expected bytes, not %s"
                             % altchars.__class__.__name__)
         assert len(altchars) == 2, repr(altchars)
-        return _translate(encoded, {'+': altchars[0:1], '/': altchars[1:2]})
+        return encoded.translate(bytes.maketrans(b'+/', altchars))
     return encoded
 
 
@@ -79,14 +81,11 @@ def b64decode(s, altchars=None, validate=False):
     discarded prior to the padding check.  If validate is True,
     non-base64-alphabet characters in the input result in a binascii.Error.
     """
-    if not isinstance(s, bytes_types):
-        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    s = _bytes_from_decode_data(s)
     if altchars is not None:
-        if not isinstance(altchars, bytes_types):
-            raise TypeError("expected bytes, not %s"
-                            % altchars.__class__.__name__)
+        altchars = _bytes_from_decode_data(altchars)
         assert len(altchars) == 2, repr(altchars)
-        s = _translate(s, {chr(altchars[0]): b'+', chr(altchars[1]): b'/'})
+        s = s.translate(bytes.maketrans(altchars, b'+/'))
     if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
         raise binascii.Error('Non-base64 digit found')
     return binascii.a2b_base64(s)
@@ -109,6 +108,10 @@ def standard_b64decode(s):
     """
     return b64decode(s)
 
+
+_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_')
+_urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/')
+
 def urlsafe_b64encode(s):
     """Encode a byte string using a url-safe Base64 alphabet.
 
@@ -116,7 +119,7 @@ def urlsafe_b64encode(s):
     returned.  The alphabet uses '-' instead of '+' and '_' instead of
     '/'.
     """
-    return b64encode(s, b'-_')
+    return b64encode(s).translate(_urlsafe_encode_translation)
 
 def urlsafe_b64decode(s):
     """Decode a byte string encoded with the standard Base64 alphabet.
@@ -128,7 +131,9 @@ def urlsafe_b64decode(s):
 
     The alphabet uses '-' instead of '+' and '_' instead of '/'.
     """
-    return b64decode(s, b'-_')
+    s = _bytes_from_decode_data(s)
+    s = s.translate(_urlsafe_decode_translation)
+    return b64decode(s)
 
 
 
@@ -211,8 +216,7 @@ def b32decode(s, casefold=False, map01=None):
     the input is incorrectly padded or if there are non-alphabet
     characters present in the input.
     """
-    if not isinstance(s, bytes_types):
-        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    s = _bytes_from_decode_data(s)
     quanta, leftover = divmod(len(s), 8)
     if leftover:
         raise binascii.Error('Incorrect padding')
@@ -220,10 +224,9 @@ def b32decode(s, casefold=False, map01=None):
     # False, or the character to map the digit 1 (one) to.  It should be
     # either L (el) or I (eye).
     if map01 is not None:
-        if not isinstance(map01, bytes_types):
-            raise TypeError("expected bytes, not %s" % map01.__class__.__name__)
+        map01 = _bytes_from_decode_data(map01)
         assert len(map01) == 1, repr(map01)
-        s = _translate(s, {b'0': b'O', b'1': map01})
+        s = s.translate(bytes.maketrans(b'01', b'O' + map01))
     if casefold:
         s = s.upper()
     # Strip off pad characters from the right.  We need to count the pad
@@ -292,8 +295,7 @@ def b16decode(s, casefold=False):
     s were incorrectly padded or if there are non-alphabet characters
     present in the string.
     """
-    if not isinstance(s, bytes_types):
-        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    s = _bytes_from_decode_data(s)
     if casefold:
         s = s.upper()
     if re.search(b'[^0-9A-F]', s):
diff --git a/Lib/binhex.py b/Lib/binhex.py
index 999a675513..7bf9278430 100644
--- a/Lib/binhex.py
+++ b/Lib/binhex.py
@@ -23,7 +23,6 @@ hexbin(inputfilename, outputfilename)
 #
 import io
 import os
-import sys
 import struct
 import binascii
 
diff --git a/Lib/bz2.py b/Lib/bz2.py
new file mode 100644
index 0000000000..c3075073f0
--- /dev/null
+++ b/Lib/bz2.py
@@ -0,0 +1,504 @@
+"""Interface to the libbzip2 compression library.
+
+This module provides a file interface, classes for incremental
+(de)compression, and functions for one-shot (de)compression.
+"""
+
+__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
+           "open", "compress", "decompress"]
+
+__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
+
+import builtins
+import io
+import warnings
+
+try:
+    from threading import RLock
+except ImportError:
+    from dummy_threading import RLock
+
+from _bz2 import BZ2Compressor, BZ2Decompressor
+
+
+_MODE_CLOSED   = 0
+_MODE_READ     = 1
+_MODE_READ_EOF = 2
+_MODE_WRITE    = 3
+
+_BUFFER_SIZE = 8192
+
+
+class BZ2File(io.BufferedIOBase):
+
+    """A file object providing transparent bzip2 (de)compression.
+
+    A BZ2File can act as a wrapper for an existing file object, or refer
+    directly to a named file on disk.
+
+    Note that BZ2File provides a *binary* file interface - data read is
+    returned as bytes, and data to be written should be given as bytes.
+    """
+
+    def __init__(self, filename, mode="r", buffering=None, compresslevel=9):
+        """Open a bzip2-compressed file.
+
+        If filename is a str or bytes object, is gives the name of the file to
+        be opened. Otherwise, it should be a file object, which will be used to
+        read or write the compressed data.
+
+        mode can be 'r' for reading (default), 'w' for (over)writing, or 'a' for
+        appending. These can equivalently be given as 'rb', 'wb', and 'ab'.
+
+        buffering is ignored. Its use is deprecated.
+
+        If mode is 'w' or 'a', compresslevel can be a number between 1
+        and 9 specifying the level of compression: 1 produces the least
+        compression, and 9 (default) produces the most compression.
+
+        If mode is 'r', the input file may be the concatenation of
+        multiple compressed streams.
+        """
+        # This lock must be recursive, so that BufferedIOBase's
+        # readline(), readlines() and writelines() don't deadlock.
+        self._lock = RLock()
+        self._fp = None
+        self._closefp = False
+        self._mode = _MODE_CLOSED
+        self._pos = 0
+        self._size = -1
+
+        if buffering is not None:
+            warnings.warn("Use of 'buffering' argument is deprecated",
+                          DeprecationWarning)
+
+        if not (1 <= compresslevel <= 9):
+            raise ValueError("compresslevel must be between 1 and 9")
+
+        if mode in ("", "r", "rb"):
+            mode = "rb"
+            mode_code = _MODE_READ
+            self._decompressor = BZ2Decompressor()
+            self._buffer = b""
+            self._buffer_offset = 0
+        elif mode in ("w", "wb"):
+            mode = "wb"
+            mode_code = _MODE_WRITE
+            self._compressor = BZ2Compressor(compresslevel)
+        elif mode in ("a", "ab"):
+            mode = "ab"
+            mode_code = _MODE_WRITE
+            self._compressor = BZ2Compressor(compresslevel)
+        else:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if isinstance(filename, (str, bytes)):
+            self._fp = builtins.open(filename, mode)
+            self._closefp = True
+            self._mode = mode_code
+        elif hasattr(filename, "read") or hasattr(filename, "write"):
+            self._fp = filename
+            self._mode = mode_code
+        else:
+            raise TypeError("filename must be a str or bytes object, or a file")
+
+    def close(self):
+        """Flush and close the file.
+
+        May be called more than once without error. Once the file is
+        closed, any other operation on it will raise a ValueError.
+        """
+        with self._lock:
+            if self._mode == _MODE_CLOSED:
+                return
+            try:
+                if self._mode in (_MODE_READ, _MODE_READ_EOF):
+                    self._decompressor = None
+                elif self._mode == _MODE_WRITE:
+                    self._fp.write(self._compressor.flush())
+                    self._compressor = None
+            finally:
+                try:
+                    if self._closefp:
+                        self._fp.close()
+                finally:
+                    self._fp = None
+                    self._closefp = False
+                    self._mode = _MODE_CLOSED
+                    self._buffer = b""
+                    self._buffer_offset = 0
+
+    @property
+    def closed(self):
+        """True if this file is closed."""
+        return self._mode == _MODE_CLOSED
+
+    def fileno(self):
+        """Return the file descriptor for the underlying file."""
+        self._check_not_closed()
+        return self._fp.fileno()
+
+    def seekable(self):
+        """Return whether the file supports seeking."""
+        return self.readable() and self._fp.seekable()
+
+    def readable(self):
+        """Return whether the file was opened for reading."""
+        self._check_not_closed()
+        return self._mode in (_MODE_READ, _MODE_READ_EOF)
+
+    def writable(self):
+        """Return whether the file was opened for writing."""
+        self._check_not_closed()
+        return self._mode == _MODE_WRITE
+
+    # Mode-checking helper functions.
+
+    def _check_not_closed(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+    def _check_can_read(self):
+        if self._mode not in (_MODE_READ, _MODE_READ_EOF):
+            self._check_not_closed()
+            raise io.UnsupportedOperation("File not open for reading")
+
+    def _check_can_write(self):
+        if self._mode != _MODE_WRITE:
+            self._check_not_closed()
+            raise io.UnsupportedOperation("File not open for writing")
+
+    def _check_can_seek(self):
+        if self._mode not in (_MODE_READ, _MODE_READ_EOF):
+            self._check_not_closed()
+            raise io.UnsupportedOperation("Seeking is only supported "
+                                          "on files open for reading")
+        if not self._fp.seekable():
+            raise io.UnsupportedOperation("The underlying file object "
+                                          "does not support seeking")
+
+    # Fill the readahead buffer if it is empty. Returns False on EOF.
+    def _fill_buffer(self):
+        if self._mode == _MODE_READ_EOF:
+            return False
+        # Depending on the input data, our call to the decompressor may not
+        # return any data. In this case, try again after reading another block.
+        while self._buffer_offset == len(self._buffer):
+            rawblock = (self._decompressor.unused_data or
+                        self._fp.read(_BUFFER_SIZE))
+
+            if not rawblock:
+                if self._decompressor.eof:
+                    self._mode = _MODE_READ_EOF
+                    self._size = self._pos
+                    return False
+                else:
+                    raise EOFError("Compressed file ended before the "
+                                   "end-of-stream marker was reached")
+
+            # Continue to next stream.
+            if self._decompressor.eof:
+                self._decompressor = BZ2Decompressor()
+
+            self._buffer = self._decompressor.decompress(rawblock)
+            self._buffer_offset = 0
+        return True
+
+    # Read data until EOF.
+    # If return_data is false, consume the data without returning it.
+    def _read_all(self, return_data=True):
+        # The loop assumes that _buffer_offset is 0. Ensure that this is true.
+        self._buffer = self._buffer[self._buffer_offset:]
+        self._buffer_offset = 0
+
+        blocks = []
+        while self._fill_buffer():
+            if return_data:
+                blocks.append(self._buffer)
+            self._pos += len(self._buffer)
+            self._buffer = b""
+        if return_data:
+            return b"".join(blocks)
+
+    # Read a block of up to n bytes.
+    # If return_data is false, consume the data without returning it.
+    def _read_block(self, n, return_data=True):
+        # If we have enough data buffered, return immediately.
+        end = self._buffer_offset + n
+        if end <= len(self._buffer):
+            data = self._buffer[self._buffer_offset : end]
+            self._buffer_offset = end
+            self._pos += len(data)
+            return data if return_data else None
+
+        # The loop assumes that _buffer_offset is 0. Ensure that this is true.
+        self._buffer = self._buffer[self._buffer_offset:]
+        self._buffer_offset = 0
+
+        blocks = []
+        while n > 0 and self._fill_buffer():
+            if n < len(self._buffer):
+                data = self._buffer[:n]
+                self._buffer_offset = n
+            else:
+                data = self._buffer
+                self._buffer = b""
+            if return_data:
+                blocks.append(data)
+            self._pos += len(data)
+            n -= len(data)
+        if return_data:
+            return b"".join(blocks)
+
+    def peek(self, n=0):
+        """Return buffered data without advancing the file position.
+
+        Always returns at least one byte of data, unless at EOF.
+        The exact number of bytes returned is unspecified.
+        """
+        with self._lock:
+            self._check_can_read()
+            if not self._fill_buffer():
+                return b""
+            return self._buffer[self._buffer_offset:]
+
+    def read(self, size=-1):
+        """Read up to size uncompressed bytes from the file.
+
+        If size is negative or omitted, read until EOF is reached.
+        Returns b'' if the file is already at EOF.
+        """
+        with self._lock:
+            self._check_can_read()
+            if size == 0:
+                return b""
+            elif size < 0:
+                return self._read_all()
+            else:
+                return self._read_block(size)
+
+    def read1(self, size=-1):
+        """Read up to size uncompressed bytes, while trying to avoid
+        making multiple reads from the underlying stream.
+
+        Returns b'' if the file is at EOF.
+        """
+        # Usually, read1() calls _fp.read() at most once. However, sometimes
+        # this does not give enough data for the decompressor to make progress.
+        # In this case we make multiple reads, to avoid returning b"".
+        with self._lock:
+            self._check_can_read()
+            if (size == 0 or
+                # Only call _fill_buffer() if the buffer is actually empty.
+                # This gives a significant speedup if *size* is small.
+                (self._buffer_offset == len(self._buffer) and not self._fill_buffer())):
+                return b""
+            if size > 0:
+                data = self._buffer[self._buffer_offset :
+                                    self._buffer_offset + size]
+                self._buffer_offset += len(data)
+            else:
+                data = self._buffer[self._buffer_offset:]
+                self._buffer = b""
+                self._buffer_offset = 0
+            self._pos += len(data)
+            return data
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into b.
+
+        Returns the number of bytes read (0 for EOF).
+        """
+        with self._lock:
+            return io.BufferedIOBase.readinto(self, b)
+
+    def readline(self, size=-1):
+        """Read a line of uncompressed bytes from the file.
+
+        The terminating newline (if present) is retained. If size is
+        non-negative, no more than size bytes will be read (in which
+        case the line may be incomplete). Returns b'' if already at EOF.
+        """
+        if not isinstance(size, int):
+            if not hasattr(size, "__index__"):
+                raise TypeError("Integer argument expected")
+            size = size.__index__()
+        with self._lock:
+            self._check_can_read()
+            # Shortcut for the common case - the whole line is in the buffer.
+            if size < 0:
+                end = self._buffer.find(b"\n", self._buffer_offset) + 1
+                if end > 0:
+                    line = self._buffer[self._buffer_offset : end]
+                    self._buffer_offset = end
+                    self._pos += len(line)
+                    return line
+            return io.BufferedIOBase.readline(self, size)
+
+    def readlines(self, size=-1):
+        """Read a list of lines of uncompressed bytes from the file.
+
+        size can be specified to control the number of lines read: no
+        further lines will be read once the total size of the lines read
+        so far equals or exceeds size.
+        """
+        if not isinstance(size, int):
+            if not hasattr(size, "__index__"):
+                raise TypeError("Integer argument expected")
+            size = size.__index__()
+        with self._lock:
+            return io.BufferedIOBase.readlines(self, size)
+
+    def write(self, data):
+        """Write a byte string to the file.
+
+        Returns the number of uncompressed bytes written, which is
+        always len(data). Note that due to buffering, the file on disk
+        may not reflect the data written until close() is called.
+        """
+        with self._lock:
+            self._check_can_write()
+            compressed = self._compressor.compress(data)
+            self._fp.write(compressed)
+            self._pos += len(data)
+            return len(data)
+
+    def writelines(self, seq):
+        """Write a sequence of byte strings to the file.
+
+        Returns the number of uncompressed bytes written.
+        seq can be any iterable yielding byte strings.
+
+        Line separators are not added between the written byte strings.
+        """
+        with self._lock:
+            return io.BufferedIOBase.writelines(self, seq)
+
+    # Rewind the file to the beginning of the data stream.
+    def _rewind(self):
+        self._fp.seek(0, 0)
+        self._mode = _MODE_READ
+        self._pos = 0
+        self._decompressor = BZ2Decompressor()
+        self._buffer = b""
+        self._buffer_offset = 0
+
+    def seek(self, offset, whence=0):
+        """Change the file position.
+
+        The new position is specified by offset, relative to the
+        position indicated by whence. Values for whence are:
+
+            0: start of stream (default); offset must not be negative
+            1: current stream position
+            2: end of stream; offset must not be positive
+
+        Returns the new file position.
+
+        Note that seeking is emulated, so depending on the parameters,
+        this operation may be extremely slow.
+        """
+        with self._lock:
+            self._check_can_seek()
+
+            # Recalculate offset as an absolute file position.
+            if whence == 0:
+                pass
+            elif whence == 1:
+                offset = self._pos + offset
+            elif whence == 2:
+                # Seeking relative to EOF - we need to know the file's size.
+                if self._size < 0:
+                    self._read_all(return_data=False)
+                offset = self._size + offset
+            else:
+                raise ValueError("Invalid value for whence: {}".format(whence))
+
+            # Make it so that offset is the number of bytes to skip forward.
+            if offset < self._pos:
+                self._rewind()
+            else:
+                offset -= self._pos
+
+            # Read and discard data until we reach the desired position.
+            self._read_block(offset, return_data=False)
+
+            return self._pos
+
+    def tell(self):
+        """Return the current file position."""
+        with self._lock:
+            self._check_not_closed()
+            return self._pos
+
+
+def open(filename, mode="rb", compresslevel=9,
+         encoding=None, errors=None, newline=None):
+    """Open a bzip2-compressed file in binary or text mode.
+
+    The filename argument can be an actual filename (a str or bytes object), or
+    an existing file object to read from or write to.
+
+    The mode argument can be "r", "rb", "w", "wb", "a" or "ab" for binary mode,
+    or "rt", "wt" or "at" for text mode. The default mode is "rb", and the
+    default compresslevel is 9.
+
+    For binary mode, this function is equivalent to the BZ2File constructor:
+    BZ2File(filename, mode, compresslevel). In this case, the encoding, errors
+    and newline arguments must not be provided.
+
+    For text mode, a BZ2File object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
+
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    bz_mode = mode.replace("t", "")
+    binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
+
+    if "t" in mode:
+        return io.TextIOWrapper(binary_file, encoding, errors, newline)
+    else:
+        return binary_file
+
+
+def compress(data, compresslevel=9):
+    """Compress a block of data.
+
+    compresslevel, if given, must be a number between 1 and 9.
+
+    For incremental compression, use a BZ2Compressor object instead.
+    """
+    comp = BZ2Compressor(compresslevel)
+    return comp.compress(data) + comp.flush()
+
+
+def decompress(data):
+    """Decompress a block of data.
+
+    For incremental decompression, use a BZ2Decompressor object instead.
+    """
+    if len(data) == 0:
+        return b""
+
+    results = []
+    while True:
+        decomp = BZ2Decompressor()
+        results.append(decomp.decompress(data))
+        if not decomp.eof:
+            raise ValueError("Compressed data ended before the "
+                             "end-of-stream marker was reached")
+        if not decomp.unused_data:
+            return b"".join(results)
+        # There is unused data left over. Proceed to next stream.
+        data = decomp.unused_data
diff --git a/Lib/cgi.py b/Lib/cgi.py
index ad6b1f88a9..96b1f5721d 100755
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -76,7 +76,7 @@ def initlog(*allargs):
     send an error message).
 
     """
-    global logfp, log
+    global log, logfile, logfp
     if logfile and not logfp:
         try:
             logfp = open(logfile, "a")
@@ -96,6 +96,15 @@ def nolog(*allargs):
     """Dummy function, assigned to log when logging is disabled."""
     pass
 
+def closelog():
+    """Close the log file."""
+    global log, logfile, logfp
+    logfile = ''
+    if logfp:
+        logfp.close()
+        logfp = None
+    log = initlog
+
 log = initlog           # The current logging function
 
 
@@ -1003,7 +1012,7 @@ environment as well.  Here are some common variable names:
 def escape(s, quote=None):
     """Deprecated API."""
     warn("cgi.escape is deprecated, use html.escape instead",
-         PendingDeprecationWarning, stacklevel=2)
+         DeprecationWarning, stacklevel=2)
     s = s.replace("&", "&amp;") # Must be done first!
     s = s.replace("<", "&lt;")
     s = s.replace(">", "&gt;")
diff --git a/Lib/cgitb.py b/Lib/cgitb.py
index 6da40e82ec..6eb52e764e 100644
--- a/Lib/cgitb.py
+++ b/Lib/cgitb.py
@@ -31,7 +31,6 @@ import tempfile
 import time
 import tokenize
 import traceback
-import types
 
 def reset():
     """Return a string that resets the CGI and browser to a known state."""
diff --git a/Lib/code.py b/Lib/code.py
index 605aede5ef..9020aab701 100644
--- a/Lib/code.py
+++ b/Lib/code.py
@@ -105,9 +105,10 @@ class InteractiveInterpreter:
         The output is written by self.write(), below.
 
         """
-        type, value, sys.last_traceback = sys.exc_info()
+        type, value, tb = sys.exc_info()
         sys.last_type = type
         sys.last_value = value
+        sys.last_traceback = tb
         if filename and type is SyntaxError:
             # Work hard to stuff the correct filename in the exception
             try:
@@ -119,8 +120,13 @@ class InteractiveInterpreter:
                 # Stuff in the right filename
                 value = SyntaxError(msg, (filename, lineno, offset, line))
                 sys.last_value = value
-        lines = traceback.format_exception_only(type, value)
-        self.write(''.join(lines))
+        if sys.excepthook is sys.__excepthook__:
+            lines = traceback.format_exception_only(type, value)
+            self.write(''.join(lines))
+        else:
+            # If someone has set sys.excepthook, we let that take precedence
+            # over self.write
+            sys.excepthook(type, value, tb)
 
     def showtraceback(self):
         """Display the exception that just occurred.
@@ -143,7 +149,12 @@ class InteractiveInterpreter:
             lines.extend(traceback.format_exception_only(type, value))
         finally:
             tblist = tb = None
-        self.write(''.join(lines))
+        if sys.excepthook is sys.__excepthook__:
+            self.write(''.join(lines))
+        else:
+            # If someone has set sys.excepthook, we let that take precedence
+            # over self.write
+            sys.excepthook(type, value, tb)
 
     def write(self, data):
         """Write a string.
diff --git a/Lib/codecs.py b/Lib/codecs.py
index b150d64d53..48d4c9c739 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -461,7 +461,7 @@ class StreamReader(Codec):
 
         # read until we get the required number of characters (if available)
         while True:
-            # can the request can be satisfied from the character buffer?
+            # can the request be satisfied from the character buffer?
             if chars < 0:
                 if size < 0:
                     if self.charbuffer:
@@ -484,7 +484,7 @@ class StreamReader(Codec):
                 if firstline:
                     newchars, decodedbytes = \
                         self.decode(data[:exc.start], self.errors)
-                    lines = newchars.splitlines(True)
+                    lines = newchars.splitlines(keepends=True)
                     if len(lines)<=1:
                         raise
                 else:
@@ -526,7 +526,7 @@ class StreamReader(Codec):
                 self.charbuffer = self.linebuffer[0]
                 self.linebuffer = None
             if not keepends:
-                line = line.splitlines(False)[0]
+                line = line.splitlines(keepends=False)[0]
             return line
 
         readsize = size or 72
@@ -543,7 +543,7 @@ class StreamReader(Codec):
                     data += self.read(size=1, chars=1)
 
             line += data
-            lines = line.splitlines(True)
+            lines = line.splitlines(keepends=True)
             if lines:
                 if len(lines) > 1:
                     # More than one line result; the first line is a full line
@@ -559,10 +559,10 @@ class StreamReader(Codec):
                         # only one remaining line, put it back into charbuffer
                         self.charbuffer = lines[0] + self.charbuffer
                     if not keepends:
-                        line = line.splitlines(False)[0]
+                        line = line.splitlines(keepends=False)[0]
                     break
                 line0withend = lines[0]
-                line0withoutend = lines[0].splitlines(False)[0]
+                line0withoutend = lines[0].splitlines(keepends=False)[0]
                 if line0withend != line0withoutend: # We really have a line end
                     # Put the rest back together and keep it until the next call
                     self.charbuffer = self._empty_charbuffer.join(lines[1:]) + \
@@ -575,7 +575,7 @@ class StreamReader(Codec):
             # we didn't get anything or this was our only try
             if not data or size is not None:
                 if line and not keepends:
-                    line = line.splitlines(False)[0]
+                    line = line.splitlines(keepends=False)[0]
                 break
             if readsize < 8000:
                 readsize *= 2
@@ -803,7 +803,7 @@ class StreamRecoder:
 
         data = self.reader.read()
         data, bytesencoded = self.encode(data, self.errors)
-        return data.splitlines(1)
+        return data.splitlines(keepends=True)
 
     def __next__(self):
 
@@ -1042,10 +1042,7 @@ def make_identity_dict(rng):
         mapped to themselves.
 
     """
-    res = {}
-    for i in rng:
-        res[i]=i
-    return res
+    return {i:i for i in rng}
 
 def make_encoding_map(decoding_map):
 
diff --git a/Lib/collections.py b/Lib/collections/__init__.py
index eb2024352d..2dcc395702 100644
--- a/Lib/collections.py
+++ b/Lib/collections/__init__.py
@@ -1,13 +1,14 @@
 __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList',
-            'UserString', 'Counter', 'OrderedDict']
-# For bootstrapping reasons, the collection ABCs are defined in _abcoll.py.
-# They should however be considered an integral part of collections.py.
-from _abcoll import *
-import _abcoll
-__all__ += _abcoll.__all__
+            'UserString', 'Counter', 'OrderedDict', 'ChainMap']
+
+# For backwards compatibility, continue to make the collections ABCs
+# available through the collections module.
+from collections.abc import *
+import collections.abc
+__all__ += collections.abc.__all__
 
 from _collections import deque, defaultdict
-from operator import itemgetter as _itemgetter
+from operator import itemgetter as _itemgetter, eq as _eq
 from keyword import iskeyword as _iskeyword
 import sys as _sys
 import heapq as _heapq
@@ -228,7 +229,7 @@ class OrderedDict(dict):
         '''
         if isinstance(other, OrderedDict):
             return len(self)==len(other) and \
-                   all(p==q for p, q in zip(self.items(), other.items()))
+                   all(map(_eq, self.items(), other.items()))
         return dict.__eq__(self, other)
 
 
@@ -314,33 +315,32 @@ def namedtuple(typename, field_names, verbose=False, rename=False):
 
     """
 
-    # Parse and validate the field names.  Validation serves two purposes,
-    # generating informative error messages and preventing template injection attacks.
+    # Validate the field names.  At the user's option, either generate an error
+    # message or automatically replace the field name with a valid name.
     if isinstance(field_names, str):
-        field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas
+        field_names = field_names.replace(',', ' ').split()
     field_names = list(map(str, field_names))
     if rename:
         seen = set()
         for index, name in enumerate(field_names):
-            if (not all(c.isalnum() or c=='_' for c in name)
+            if (not name.isidentifier()
                 or _iskeyword(name)
-                or not name
-                or name[0].isdigit()
                 or name.startswith('_')
                 or name in seen):
                 field_names[index] = '_%d' % index
             seen.add(name)
     for name in [typename] + field_names:
-        if not all(c.isalnum() or c=='_' for c in name):
-            raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name)
+        if not name.isidentifier():
+            raise ValueError('Type names and field names must be valid '
+                             'identifiers: %r' % name)
         if _iskeyword(name):
-            raise ValueError('Type names and field names cannot be a keyword: %r' % name)
-        if name[0].isdigit():
-            raise ValueError('Type names and field names cannot start with a number: %r' % name)
+            raise ValueError('Type names and field names cannot be a '
+                             'keyword: %r' % name)
     seen = set()
     for name in field_names:
         if name.startswith('_') and not rename:
-            raise ValueError('Field names cannot start with an underscore: %r' % name)
+            raise ValueError('Field names cannot start with an underscore: '
+                             '%r' % name)
         if name in seen:
             raise ValueError('Encountered duplicate field name: %r' % name)
         seen.add(name)
@@ -351,21 +351,23 @@ def namedtuple(typename, field_names, verbose=False, rename=False):
         field_names = tuple(field_names),
         num_fields = len(field_names),
         arg_list = repr(tuple(field_names)).replace("'", "")[1:-1],
-        repr_fmt = ', '.join(_repr_template.format(name=name) for name in field_names),
+        repr_fmt = ', '.join(_repr_template.format(name=name)
+                             for name in field_names),
         field_defs = '\n'.join(_field_template.format(index=index, name=name)
                                for index, name in enumerate(field_names))
     )
 
-    # Execute the template string in a temporary namespace and
-    # support tracing utilities by setting a value for frame.f_globals['__name__']
+    # Execute the template string in a temporary namespace and support
+    # tracing utilities by setting a value for frame.f_globals['__name__']
     namespace = dict(__name__='namedtuple_%s' % typename)
     try:
         exec(class_definition, namespace)
     except SyntaxError as e:
         raise SyntaxError(e.msg + ':\n\n' + class_definition)
     result = namespace[typename]
+    result._source = class_definition
     if verbose:
-        print(class_definition)
+        print(result._source)
 
     # For pickling to work, the __module__ variable needs to be set to the frame
     # where the named tuple is created.  Bypass this step in enviroments where
@@ -674,12 +676,86 @@ class Counter(dict):
                 result[elem] = newcount
         return result
 
+    def __pos__(self):
+        'Adds an empty counter, effectively stripping negative and zero counts'
+        return self + Counter()
+
+    def __neg__(self):
+        '''Subtracts from an empty counter.  Strips positive and zero counts,
+        and flips the sign on negative counts.
+
+        '''
+        return Counter() - self
+
+    def _keep_positive(self):
+        '''Internal method to strip elements with a negative or zero count'''
+        nonpositive = [elem for elem, count in self.items() if not count > 0]
+        for elem in nonpositive:
+            del self[elem]
+        return self
+
+    def __iadd__(self, other):
+        '''Inplace add from another counter, keeping only positive counts.
+
+        >>> c = Counter('abbb')
+        >>> c += Counter('bcc')
+        >>> c
+        Counter({'b': 4, 'c': 2, 'a': 1})
+
+        '''
+        for elem, count in other.items():
+            self[elem] += count
+        return self._keep_positive()
+
+    def __isub__(self, other):
+        '''Inplace subtract counter, but keep only results with positive counts.
+
+        >>> c = Counter('abbbc')
+        >>> c -= Counter('bccd')
+        >>> c
+        Counter({'b': 2, 'a': 1})
+
+        '''
+        for elem, count in other.items():
+            self[elem] -= count
+        return self._keep_positive()
+
+    def __ior__(self, other):
+        '''Inplace union is the maximum of value from either counter.
+
+        >>> c = Counter('abbb')
+        >>> c |= Counter('bcc')
+        >>> c
+        Counter({'b': 3, 'c': 2, 'a': 1})
+
+        '''
+        for elem, other_count in other.items():
+            count = self[elem]
+            if other_count > count:
+                self[elem] = other_count
+        return self._keep_positive()
+
+    def __iand__(self, other):
+        '''Inplace intersection is the minimum of corresponding counts.
+
+        >>> c = Counter('abbb')
+        >>> c &= Counter('bcc')
+        >>> c
+        Counter({'b': 1})
+
+        '''
+        for elem, count in self.items():
+            other_count = other[elem]
+            if other_count < count:
+                self[elem] = other_count
+        return self._keep_positive()
+
 
 ########################################################################
-###  ChainMap (helper for configparser)
+###  ChainMap (helper for configparser and string.Template)
 ########################################################################
 
-class _ChainMap(MutableMapping):
+class ChainMap(MutableMapping):
     ''' A ChainMap groups multiple dicts (or other mappings) together
     to create a single, updateable view.
 
@@ -890,6 +966,8 @@ class UserList(MutableSequence):
     def insert(self, i, item): self.data.insert(i, item)
     def pop(self, i=-1): return self.data.pop(i)
     def remove(self, item): self.data.remove(item)
+    def clear(self): self.data.clear()
+    def copy(self): return self.__class__(self)
     def count(self, item): return self.data.count(item)
     def index(self, item, *args): return self.data.index(item, *args)
     def reverse(self): self.data.reverse()
@@ -1034,7 +1112,7 @@ class UserString(Sequence):
         return self.data.split(sep, maxsplit)
     def rsplit(self, sep=None, maxsplit=-1):
         return self.data.rsplit(sep, maxsplit)
-    def splitlines(self, keepends=0): return self.data.splitlines(keepends)
+    def splitlines(self, keepends=False): return self.data.splitlines(keepends)
     def startswith(self, prefix, start=0, end=_sys.maxsize):
         return self.data.startswith(prefix, start, end)
     def strip(self, chars=None): return self.__class__(self.data.strip(chars))
@@ -1044,44 +1122,3 @@ class UserString(Sequence):
         return self.__class__(self.data.translate(*args))
     def upper(self): return self.__class__(self.data.upper())
     def zfill(self, width): return self.__class__(self.data.zfill(width))
-
-
-
-################################################################################
-### Simple tests
-################################################################################
-
-if __name__ == '__main__':
-    # verify that instances can be pickled
-    from pickle import loads, dumps
-    Point = namedtuple('Point', 'x, y', True)
-    p = Point(x=10, y=20)
-    assert p == loads(dumps(p))
-
-    # test and demonstrate ability to override methods
-    class Point(namedtuple('Point', 'x y')):
-        __slots__ = ()
-        @property
-        def hypot(self):
-            return (self.x ** 2 + self.y ** 2) ** 0.5
-        def __str__(self):
-            return 'Point: x=%6.3f  y=%6.3f  hypot=%6.3f' % (self.x, self.y, self.hypot)
-
-    for p in Point(3, 4), Point(14, 5/7.):
-        print (p)
-
-    class Point(namedtuple('Point', 'x y')):
-        'Point class with optimized _make() and _replace() without error-checking'
-        __slots__ = ()
-        _make = classmethod(tuple.__new__)
-        def _replace(self, _map=map, **kwds):
-            return self._make(_map(kwds.get, ('x', 'y'), self))
-
-    print(Point(11, 22)._replace(x=100))
-
-    Point3D = namedtuple('Point3D', Point._fields + ('z',))
-    print(Point3D.__doc__)
-
-    import doctest
-    TestResults = namedtuple('TestResults', 'failed attempted')
-    print(TestResults(*doctest.testmod()))
diff --git a/Lib/collections/__main__.py b/Lib/collections/__main__.py
new file mode 100644
index 0000000000..763e38e0c4
--- /dev/null
+++ b/Lib/collections/__main__.py
@@ -0,0 +1,38 @@
+################################################################################
+### Simple tests
+################################################################################
+
+# verify that instances can be pickled
+from collections import namedtuple
+from pickle import loads, dumps
+Point = namedtuple('Point', 'x, y', True)
+p = Point(x=10, y=20)
+assert p == loads(dumps(p))
+
+# test and demonstrate ability to override methods
+class Point(namedtuple('Point', 'x y')):
+    __slots__ = ()
+    @property
+    def hypot(self):
+        return (self.x ** 2 + self.y ** 2) ** 0.5
+    def __str__(self):
+        return 'Point: x=%6.3f  y=%6.3f  hypot=%6.3f' % (self.x, self.y, self.hypot)
+
+for p in Point(3, 4), Point(14, 5/7.):
+    print (p)
+
+class Point(namedtuple('Point', 'x y')):
+    'Point class with optimized _make() and _replace() without error-checking'
+    __slots__ = ()
+    _make = classmethod(tuple.__new__)
+    def _replace(self, _map=map, **kwds):
+        return self._make(_map(kwds.get, ('x', 'y'), self))
+
+print(Point(11, 22)._replace(x=100))
+
+Point3D = namedtuple('Point3D', Point._fields + ('z',))
+print(Point3D.__doc__)
+
+import doctest, collections
+TestResults = namedtuple('TestResults', 'failed attempted')
+print(TestResults(*doctest.testmod(collections)))
diff --git a/Lib/_abcoll.py b/Lib/collections/abc.py
index 5ddcea3a30..c23b7ddef0 100644
--- a/Lib/_abcoll.py
+++ b/Lib/collections/abc.py
@@ -3,9 +3,7 @@
 
 """Abstract Base Classes (ABCs) for collections, according to PEP 3119.
 
-DON'T USE THIS MODULE DIRECTLY!  The classes here should be imported
-via collections; they are defined here only to alleviate certain
-bootstrapping issues.  Unit tests are in test_collections.
+Unit tests are in test_collections.
 """
 
 from abc import ABCMeta, abstractmethod
@@ -20,9 +18,13 @@ __all__ = ["Hashable", "Iterable", "Iterator",
            "ByteString",
            ]
 
-
-### collection related types which are not exposed through builtin ###
-## iterators ##
+# Private list of types that we want to register with the various ABCs
+# so that they will pass tests like:
+#       it = iter(somebytearray)
+#       assert isinstance(it, Iterable)
+# Note:  in other implementations, these types many not be distinct
+# and they make have their own implementation specific types that
+# are not included on this list.
 bytes_iterator = type(iter(b''))
 bytearray_iterator = type(iter(bytearray()))
 #callable_iterator = ???
@@ -41,13 +43,15 @@ dict_keys = type({}.keys())
 dict_values = type({}.values())
 dict_items = type({}.items())
 ## misc ##
-dict_proxy = type(type.__dict__)
+mappingproxy = type(type.__dict__)
 
 
 ### ONE-TRICK PONIES ###
 
 class Hashable(metaclass=ABCMeta):
 
+    __slots__ = ()
+
     @abstractmethod
     def __hash__(self):
         return 0
@@ -65,6 +69,8 @@ class Hashable(metaclass=ABCMeta):
 
 class Iterable(metaclass=ABCMeta):
 
+    __slots__ = ()
+
     @abstractmethod
     def __iter__(self):
         while False:
@@ -80,6 +86,8 @@ class Iterable(metaclass=ABCMeta):
 
 class Iterator(Iterable):
 
+    __slots__ = ()
+
     @abstractmethod
     def __next__(self):
         raise StopIteration
@@ -111,6 +119,8 @@ Iterator.register(zip_iterator)
 
 class Sized(metaclass=ABCMeta):
 
+    __slots__ = ()
+
     @abstractmethod
     def __len__(self):
         return 0
@@ -125,6 +135,8 @@ class Sized(metaclass=ABCMeta):
 
 class Container(metaclass=ABCMeta):
 
+    __slots__ = ()
+
     @abstractmethod
     def __contains__(self, x):
         return False
@@ -139,6 +151,8 @@ class Container(metaclass=ABCMeta):
 
 class Callable(metaclass=ABCMeta):
 
+    __slots__ = ()
+
     @abstractmethod
     def __call__(self, *args, **kwds):
         return False
@@ -166,6 +180,8 @@ class Set(Sized, Iterable, Container):
     then the other operations will automatically follow suit.
     """
 
+    __slots__ = ()
+
     def __le__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
@@ -277,6 +293,8 @@ Set.register(frozenset)
 
 class MutableSet(Set):
 
+    __slots__ = ()
+
     @abstractmethod
     def add(self, value):
         """Add an element."""
@@ -350,6 +368,8 @@ MutableSet.register(set)
 
 class Mapping(Sized, Iterable, Container):
 
+    __slots__ = ()
+
     @abstractmethod
     def __getitem__(self, key):
         raise KeyError
@@ -385,6 +405,8 @@ class Mapping(Sized, Iterable, Container):
     def __ne__(self, other):
         return not (self == other)
 
+Mapping.register(mappingproxy)
+
 
 class MappingView(Sized):
 
@@ -453,6 +475,8 @@ ValuesView.register(dict_values)
 
 class MutableMapping(Mapping):
 
+    __slots__ = ()
+
     @abstractmethod
     def __setitem__(self, key, value):
         raise KeyError
@@ -532,6 +556,8 @@ class Sequence(Sized, Iterable, Container):
     __getitem__, and __len__.
     """
 
+    __slots__ = ()
+
     @abstractmethod
     def __getitem__(self, index):
         raise IndexError
@@ -577,12 +603,16 @@ class ByteString(Sequence):
     XXX Should add all their methods.
     """
 
+    __slots__ = ()
+
 ByteString.register(bytes)
 ByteString.register(bytearray)
 
 
 class MutableSequence(Sequence):
 
+    __slots__ = ()
+
     @abstractmethod
     def __setitem__(self, index, value):
         raise IndexError
@@ -598,6 +628,13 @@ class MutableSequence(Sequence):
     def append(self, value):
         self.insert(len(self), value)
 
+    def clear(self):
+        try:
+            while True:
+                self.pop()
+        except IndexError:
+            pass
+
     def reverse(self):
         n = len(self)
         for i in range(n//2):
diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py
index 9f11f6977f..e997c02b43 100644
--- a/Lib/concurrent/futures/_base.py
+++ b/Lib/concurrent/futures/_base.py
@@ -4,7 +4,6 @@
 __author__ = 'Brian Quinlan (brian@sweetapp.com)'
 
 import collections
-import functools
 import logging
 import threading
 import time
@@ -333,7 +332,7 @@ class Future(object):
         return True
 
     def cancelled(self):
-        """Return True if the future has cancelled."""
+        """Return True if the future was cancelled."""
         with self._condition:
             return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]
 
@@ -471,8 +470,8 @@ class Future(object):
                 return True
             else:
                 LOGGER.critical('Future %s in unexpected state: %s',
-                                id(self.future),
-                                self.future._state)
+                                id(self),
+                                self._state)
                 raise RuntimeError('Future in unexpected state')
 
     def set_result(self, result):
@@ -538,15 +537,19 @@ class Executor(object):
 
         fs = [self.submit(fn, *args) for args in zip(*iterables)]
 
-        try:
-            for future in fs:
-                if timeout is None:
-                    yield future.result()
-                else:
-                    yield future.result(end_time - time.time())
-        finally:
-            for future in fs:
-                future.cancel()
+        # Yield must be hidden in closure so that the futures are submitted
+        # before the first iterator value is required.
+        def result_iterator():
+            try:
+                for future in fs:
+                    if timeout is None:
+                        yield future.result()
+                    else:
+                        yield future.result(end_time - time.time())
+            finally:
+                for future in fs:
+                    future.cancel()
+        return result_iterator()
 
     def shutdown(self, wait=True):
         """Clean-up the resources associated with the Executor.
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index d3bbe2c5e6..04238a7ace 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -46,9 +46,12 @@ Process #1..n:
 __author__ = 'Brian Quinlan (brian@sweetapp.com)'
 
 import atexit
+import os
 from concurrent.futures import _base
 import queue
 import multiprocessing
+from multiprocessing.queues import SimpleQueue, Full
+from multiprocessing.connection import wait
 import threading
 import weakref
 
@@ -121,7 +124,7 @@ def _process_worker(call_queue, result_queue):
         call_item = call_queue.get(block=True)
         if call_item is None:
             # Wake up queue management thread
-            result_queue.put(None)
+            result_queue.put(os.getpid())
             return
         try:
             r = call_item.fn(*call_item.args, **call_item.kwargs)
@@ -193,46 +196,92 @@ def _queue_management_worker(executor_reference,
         result_queue: A multiprocessing.Queue of _ResultItems generated by the
             process workers.
     """
-    nb_shutdown_processes = 0
-    def shutdown_one_process():
-        """Tell a worker to terminate, which will in turn wake us again"""
-        nonlocal nb_shutdown_processes
-        call_queue.put(None)
-        nb_shutdown_processes += 1
+    executor = None
+
+    def shutting_down():
+        return _shutdown or executor is None or executor._shutdown_thread
+
+    def shutdown_worker():
+        # This is an upper bound
+        nb_children_alive = sum(p.is_alive() for p in processes.values())
+        for i in range(0, nb_children_alive):
+            call_queue.put_nowait(None)
+        # Release the queue's resources as soon as possible.
+        call_queue.close()
+        # If .join() is not called on the created processes then
+        # some multiprocessing.Queue methods may deadlock on Mac OS X.
+        for p in processes.values():
+            p.join()
+
+    reader = result_queue._reader
+
     while True:
         _add_call_item_to_queue(pending_work_items,
                                 work_ids_queue,
                                 call_queue)
 
-        result_item = result_queue.get(block=True)
-        if result_item is not None:
-            work_item = pending_work_items[result_item.work_id]
-            del pending_work_items[result_item.work_id]
-
-            if result_item.exception:
-                work_item.future.set_exception(result_item.exception)
-            else:
-                work_item.future.set_result(result_item.result)
+        sentinels = [p.sentinel for p in processes.values()]
+        assert sentinels
+        ready = wait([reader] + sentinels)
+        if reader in ready:
+            result_item = reader.recv()
+        else:
+            # Mark the process pool broken so that submits fail right now.
+            executor = executor_reference()
+            if executor is not None:
+                executor._broken = True
+                executor._shutdown_thread = True
+                executor = None
+            # All futures in flight must be marked failed
+            for work_id, work_item in pending_work_items.items():
+                work_item.future.set_exception(
+                    BrokenProcessPool(
+                        "A process in the process pool was "
+                        "terminated abruptly while the future was "
+                        "running or pending."
+                    ))
+            pending_work_items.clear()
+            # Terminate remaining workers forcibly: the queues or their
+            # locks may be in a dirty state and block forever.
+            for p in processes.values():
+                p.terminate()
+            shutdown_worker()
+            return
+        if isinstance(result_item, int):
+            # Clean shutdown of a worker using its PID
+            # (avoids marking the executor broken)
+            assert shutting_down()
+            p = processes.pop(result_item)
+            p.join()
+            if not processes:
+                shutdown_worker()
+                return
+        elif result_item is not None:
+            work_item = pending_work_items.pop(result_item.work_id, None)
+            # work_item can be None if another process terminated (see above)
+            if work_item is not None:
+                if result_item.exception:
+                    work_item.future.set_exception(result_item.exception)
+                else:
+                    work_item.future.set_result(result_item.result)
         # Check whether we should start shutting down.
         executor = executor_reference()
         # No more work items can be added if:
         #   - The interpreter is shutting down OR
         #   - The executor that owns this worker has been collected OR
         #   - The executor that owns this worker has been shutdown.
-        if _shutdown or executor is None or executor._shutdown_thread:
-            # Since no new work items can be added, it is safe to shutdown
-            # this thread if there are no pending work items.
-            if not pending_work_items:
-                while nb_shutdown_processes < len(processes):
-                    shutdown_one_process()
-                # If .join() is not called on the created processes then
-                # some multiprocessing.Queue methods may deadlock on Mac OS
-                # X.
-                for p in processes:
-                    p.join()
-                call_queue.close()
-                return
-        del executor
+        if shutting_down():
+            try:
+                # Since no new work items can be added, it is safe to shutdown
+                # this thread if there are no pending work items.
+                if not pending_work_items:
+                    shutdown_worker()
+                    return
+            except Full:
+                # This is not a problem: we will eventually be woken up (in
+                # result_queue.get()) and be able to send a sentinel again.
+                pass
+        executor = None
 
 _system_limits_checked = False
 _system_limited = None
@@ -243,7 +292,6 @@ def _check_system_limits():
             raise NotImplementedError(_system_limited)
     _system_limits_checked = True
     try:
-        import os
         nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")
     except (AttributeError, ValueError):
         # sysconf not available or setting not available
@@ -259,6 +307,14 @@ def _check_system_limits():
     _system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max
     raise NotImplementedError(_system_limited)
 
+
+class BrokenProcessPool(RuntimeError):
+    """
+    Raised when a process in a ProcessPoolExecutor terminated abruptly
+    while a future was in the running state.
+    """
+
+
 class ProcessPoolExecutor(_base.Executor):
     def __init__(self, max_workers=None):
         """Initializes a new ProcessPoolExecutor instance.
@@ -280,14 +336,20 @@ class ProcessPoolExecutor(_base.Executor):
         # because futures in the call queue cannot be cancelled.
         self._call_queue = multiprocessing.Queue(self._max_workers +
                                                  EXTRA_QUEUED_CALLS)
-        self._result_queue = multiprocessing.Queue()
+        # Killed worker processes can produce spurious "broken pipe"
+        # tracebacks in the queue's own worker thread. But we detect killed
+        # processes anyway, so silence the tracebacks.
+        self._call_queue._ignore_epipe = True
+        self._result_queue = SimpleQueue()
         self._work_ids = queue.Queue()
         self._queue_management_thread = None
-        self._processes = set()
+        # Map of pids to processes
+        self._processes = {}
 
         # Shutdown is a two-step process.
         self._shutdown_thread = False
         self._shutdown_lock = threading.Lock()
+        self._broken = False
         self._queue_count = 0
         self._pending_work_items = {}
 
@@ -297,6 +359,8 @@ class ProcessPoolExecutor(_base.Executor):
         def weakref_cb(_, q=self._result_queue):
             q.put(None)
         if self._queue_management_thread is None:
+            # Start the processes so that their sentinels are known.
+            self._adjust_process_count()
             self._queue_management_thread = threading.Thread(
                     target=_queue_management_worker,
                     args=(weakref.ref(self, weakref_cb),
@@ -316,10 +380,13 @@ class ProcessPoolExecutor(_base.Executor):
                     args=(self._call_queue,
                           self._result_queue))
             p.start()
-            self._processes.add(p)
+            self._processes[p.pid] = p
 
     def submit(self, fn, *args, **kwargs):
         with self._shutdown_lock:
+            if self._broken:
+                raise BrokenProcessPool('A child process terminated '
+                    'abruptly, the process pool is not usable anymore')
             if self._shutdown_thread:
                 raise RuntimeError('cannot schedule new futures after shutdown')
 
@@ -333,7 +400,6 @@ class ProcessPoolExecutor(_base.Executor):
             self._result_queue.put(None)
 
             self._start_queue_management_thread()
-            self._adjust_process_count()
             return f
     submit.__doc__ = _base.Executor.submit.__doc__
 
diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py
index fbac0887a5..95bb682565 100644
--- a/Lib/concurrent/futures/thread.py
+++ b/Lib/concurrent/futures/thread.py
@@ -74,7 +74,7 @@ def _worker(executor_reference, work_queue):
                 work_queue.put(None)
                 return
             del executor
-    except BaseException as e:
+    except BaseException:
         _base.LOGGER.critical('Exception in worker', exc_info=True)
 
 class ThreadPoolExecutor(_base.Executor):
diff --git a/Lib/configparser.py b/Lib/configparser.py
index e5536a0024..c7bee6bf17 100644
--- a/Lib/configparser.py
+++ b/Lib/configparser.py
@@ -118,7 +118,8 @@ ConfigParser -- responsible for parsing a list of
         between keys and values are surrounded by spaces.
 """
 
-from collections import MutableMapping, OrderedDict as _default_dict, _ChainMap
+from collections.abc import MutableMapping
+from collections import OrderedDict as _default_dict, ChainMap as _ChainMap
 import functools
 import io
 import itertools
@@ -959,7 +960,9 @@ class RawConfigParser(MutableMapping):
 
         # XXX this is not atomic if read_dict fails at any point. Then again,
         # no update method in configparser is atomic in this implementation.
-        if key in self._sections:
+        if key == self.default_section:
+            self._defaults.clear()
+        elif key in self._sections:
             self._sections[key].clear()
         self.read_dict({key: value})
 
@@ -1005,18 +1008,26 @@ class RawConfigParser(MutableMapping):
         indent_level = 0
         e = None                              # None, or an exception
         for lineno, line in enumerate(fp, start=1):
-            comment_start = None
+            comment_start = sys.maxsize
             # strip inline comments
-            for prefix in self._inline_comment_prefixes:
-                index = line.find(prefix)
-                if index == 0 or (index > 0 and line[index-1].isspace()):
-                    comment_start = index
-                    break
+            inline_prefixes = {p: -1 for p in self._inline_comment_prefixes}
+            while comment_start == sys.maxsize and inline_prefixes:
+                next_prefixes = {}
+                for prefix, index in inline_prefixes.items():
+                    index = line.find(prefix, index+1)
+                    if index == -1:
+                        continue
+                    next_prefixes[prefix] = index
+                    if index == 0 or (index > 0 and line[index-1].isspace()):
+                        comment_start = min(comment_start, index)
+                inline_prefixes = next_prefixes
             # strip full line comments
             for prefix in self._comment_prefixes:
                 if line.strip().startswith(prefix):
                     comment_start = 0
                     break
+            if comment_start == sys.maxsize:
+                comment_start = None
             value = line[:comment_start].strip()
             if not value:
                 if self._empty_lines_in_values:
diff --git a/Lib/contextlib.py b/Lib/contextlib.py
index 5ebbbc6583..0b6bf71b08 100644
--- a/Lib/contextlib.py
+++ b/Lib/contextlib.py
@@ -1,10 +1,10 @@
 """Utilities for with-statement contexts.  See PEP 343."""
 
 import sys
+from collections import deque
 from functools import wraps
-from warnings import warn
 
-__all__ = ["contextmanager", "closing", "ContextDecorator"]
+__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack"]
 
 
 class ContextDecorator(object):
@@ -13,12 +13,12 @@ class ContextDecorator(object):
     def _recreate_cm(self):
         """Return a recreated instance of self.
 
-        Allows otherwise one-shot context managers like
+        Allows an otherwise one-shot context manager like
         _GeneratorContextManager to support use as
-        decorators via implicit recreation.
+        a decorator via implicit recreation.
 
-        Note: this is a private interface just for _GCM in 3.2 but will be
-        renamed and documented for third party use in 3.3
+        This is a private interface just for _GeneratorContextManager.
+        See issue #11647 for details.
         """
         return self
 
@@ -139,3 +139,117 @@ class closing(object):
         return self.thing
     def __exit__(self, *exc_info):
         self.thing.close()
+
+
+# Inspired by discussions on http://bugs.python.org/issue13585
+class ExitStack(object):
+    """Context manager for dynamic management of a stack of exit callbacks
+
+    For example:
+
+        with ExitStack() as stack:
+            files = [stack.enter_context(open(fname)) for fname in filenames]
+            # All opened files will automatically be closed at the end of
+            # the with statement, even if attempts to open files later
+            # in the list raise an exception
+
+    """
+    def __init__(self):
+        self._exit_callbacks = deque()
+
+    def pop_all(self):
+        """Preserve the context stack by transferring it to a new instance"""
+        new_stack = type(self)()
+        new_stack._exit_callbacks = self._exit_callbacks
+        self._exit_callbacks = deque()
+        return new_stack
+
+    def _push_cm_exit(self, cm, cm_exit):
+        """Helper to correctly register callbacks to __exit__ methods"""
+        def _exit_wrapper(*exc_details):
+            return cm_exit(cm, *exc_details)
+        _exit_wrapper.__self__ = cm
+        self.push(_exit_wrapper)
+
+    def push(self, exit):
+        """Registers a callback with the standard __exit__ method signature
+
+        Can suppress exceptions the same way __exit__ methods can.
+
+        Also accepts any object with an __exit__ method (registering a call
+        to the method instead of the object itself)
+        """
+        # We use an unbound method rather than a bound method to follow
+        # the standard lookup behaviour for special methods
+        _cb_type = type(exit)
+        try:
+            exit_method = _cb_type.__exit__
+        except AttributeError:
+            # Not a context manager, so assume its a callable
+            self._exit_callbacks.append(exit)
+        else:
+            self._push_cm_exit(exit, exit_method)
+        return exit # Allow use as a decorator
+
+    def callback(self, callback, *args, **kwds):
+        """Registers an arbitrary callback and arguments.
+
+        Cannot suppress exceptions.
+        """
+        def _exit_wrapper(exc_type, exc, tb):
+            callback(*args, **kwds)
+        # We changed the signature, so using @wraps is not appropriate, but
+        # setting __wrapped__ may still help with introspection
+        _exit_wrapper.__wrapped__ = callback
+        self.push(_exit_wrapper)
+        return callback # Allow use as a decorator
+
+    def enter_context(self, cm):
+        """Enters the supplied context manager
+
+        If successful, also pushes its __exit__ method as a callback and
+        returns the result of the __enter__ method.
+        """
+        # We look up the special methods on the type to match the with statement
+        _cm_type = type(cm)
+        _exit = _cm_type.__exit__
+        result = _cm_type.__enter__(cm)
+        self._push_cm_exit(cm, _exit)
+        return result
+
+    def close(self):
+        """Immediately unwind the context stack"""
+        self.__exit__(None, None, None)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_details):
+        # We manipulate the exception state so it behaves as though
+        # we were actually nesting multiple with statements
+        frame_exc = sys.exc_info()[1]
+        def _fix_exception_context(new_exc, old_exc):
+            while 1:
+                exc_context = new_exc.__context__
+                if exc_context in (None, frame_exc):
+                    break
+                new_exc = exc_context
+            new_exc.__context__ = old_exc
+
+        # Callbacks are invoked in LIFO order to match the behaviour of
+        # nested context managers
+        suppressed_exc = False
+        while self._exit_callbacks:
+            cb = self._exit_callbacks.pop()
+            try:
+                if cb(*exc_details):
+                    suppressed_exc = True
+                    exc_details = (None, None, None)
+            except:
+                new_exc_details = sys.exc_info()
+                # simulate the stack of exceptions by setting the context
+                _fix_exception_context(new_exc_details[1], exc_details[1])
+                if not self._exit_callbacks:
+                    raise
+                exc_details = new_exc_details
+        return suppressed_exc
diff --git a/Lib/copy.py b/Lib/copy.py
index 089d101c7c..d96201ea98 100644
--- a/Lib/copy.py
+++ b/Lib/copy.py
@@ -173,8 +173,10 @@ def deepcopy(x, memo=None, _nil=[]):
                                 "un(deep)copyable object of type %s" % cls)
                 y = _reconstruct(x, rv, 1, memo)
 
-    memo[d] = y
-    _keep_alive(x, memo) # Make sure x lives at least as long as d
+    # If is its own copy, don't memoize.
+    if y is not x:
+        memo[d] = y
+        _keep_alive(x, memo) # Make sure x lives at least as long as d
     return y
 
 _deepcopy_dispatch = d = {}
@@ -214,9 +216,10 @@ def _deepcopy_tuple(x, memo):
     y = []
     for a in x:
         y.append(deepcopy(a, memo))
-    d = id(x)
+    # We're not going to put the tuple in the memo, but it's still important we
+    # check for it, in case the tuple contains recursive mutable structures.
     try:
-        return memo[d]
+        return memo[id(x)]
     except KeyError:
         pass
     for i in range(len(x)):
@@ -225,7 +228,6 @@ def _deepcopy_tuple(x, memo):
             break
     else:
         y = x
-    memo[d] = y
     return y
 d[tuple] = _deepcopy_tuple
 
@@ -321,68 +323,3 @@ del types
 # Helper for instance creation without calling __init__
 class _EmptyClass:
     pass
-
-def _test():
-    l = [None, 1, 2, 3.14, 'xyzzy', (1, 2), [3.14, 'abc'],
-         {'abc': 'ABC'}, (), [], {}]
-    l1 = copy(l)
-    print(l1==l)
-    l1 = map(copy, l)
-    print(l1==l)
-    l1 = deepcopy(l)
-    print(l1==l)
-    class C:
-        def __init__(self, arg=None):
-            self.a = 1
-            self.arg = arg
-            if __name__ == '__main__':
-                import sys
-                file = sys.argv[0]
-            else:
-                file = __file__
-            self.fp = open(file)
-            self.fp.close()
-        def __getstate__(self):
-            return {'a': self.a, 'arg': self.arg}
-        def __setstate__(self, state):
-            for key, value in state.items():
-                setattr(self, key, value)
-        def __deepcopy__(self, memo=None):
-            new = self.__class__(deepcopy(self.arg, memo))
-            new.a = self.a
-            return new
-    c = C('argument sketch')
-    l.append(c)
-    l2 = copy(l)
-    print(l == l2)
-    print(l)
-    print(l2)
-    l2 = deepcopy(l)
-    print(l == l2)
-    print(l)
-    print(l2)
-    l.append({l[1]: l, 'xyz': l[2]})
-    l3 = copy(l)
-    import reprlib
-    print(map(reprlib.repr, l))
-    print(map(reprlib.repr, l1))
-    print(map(reprlib.repr, l2))
-    print(map(reprlib.repr, l3))
-    l3 = deepcopy(l)
-    print(map(reprlib.repr, l))
-    print(map(reprlib.repr, l1))
-    print(map(reprlib.repr, l2))
-    print(map(reprlib.repr, l3))
-    class odict(dict):
-        def __init__(self, d = {}):
-            self.a = 99
-            dict.__init__(self, d)
-        def __setitem__(self, k, i):
-            dict.__setitem__(self, k, i)
-            self.a
-    o = odict({"A" : "B"})
-    x = deepcopy(o)
-    print(o, x)
-
-if __name__ == '__main__':
-    _test()
diff --git a/Lib/crypt.py b/Lib/crypt.py
new file mode 100644
index 0000000000..b90c81cc40
--- /dev/null
+++ b/Lib/crypt.py
@@ -0,0 +1,62 @@
+"""Wrapper to the POSIX crypt library call and associated functionality."""
+
+import _crypt
+import string as _string
+from random import SystemRandom as _SystemRandom
+from collections import namedtuple as _namedtuple
+
+
+_saltchars = _string.ascii_letters + _string.digits + './'
+_sr = _SystemRandom()
+
+
+class _Method(_namedtuple('_Method', 'name ident salt_chars total_size')):
+
+    """Class representing a salt method per the Modular Crypt Format or the
+    legacy 2-character crypt method."""
+
+    def __repr__(self):
+        return '<crypt.METHOD_{}>'.format(self.name)
+
+
+def mksalt(method=None):
+    """Generate a salt for the specified method.
+
+    If not specified, the strongest available method will be used.
+
+    """
+    if method is None:
+        method = methods[0]
+    s = '${}$'.format(method.ident) if method.ident else ''
+    s += ''.join(_sr.sample(_saltchars, method.salt_chars))
+    return s
+
+
+def crypt(word, salt=None):
+    """Return a string representing the one-way hash of a password, with a salt
+    prepended.
+
+    If ``salt`` is not specified or is ``None``, the strongest
+    available method will be selected and a salt generated.  Otherwise,
+    ``salt`` may be one of the ``crypt.METHOD_*`` values, or a string as
+    returned by ``crypt.mksalt()``.
+
+    """
+    if salt is None or isinstance(salt, _Method):
+        salt = mksalt(salt)
+    return _crypt.crypt(word, salt)
+
+
+#  available salting/crypto methods
+METHOD_CRYPT = _Method('CRYPT', None, 2, 13)
+METHOD_MD5 = _Method('MD5', '1', 8, 34)
+METHOD_SHA256 = _Method('SHA256', '5', 16, 63)
+METHOD_SHA512 = _Method('SHA512', '6', 16, 106)
+
+methods = []
+for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5):
+    _result = crypt('', _method)
+    if _result and len(_result) == _method.total_size:
+        methods.append(_method)
+methods.append(METHOD_CRYPT)
+del _result, _method
diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py
index 111209a953..c92e130976 100644
--- a/Lib/ctypes/__init__.py
+++ b/Lib/ctypes/__init__.py
@@ -26,7 +26,7 @@ if _os.name == "posix" and _sys.platform == "darwin":
     # libraries.  OS X 10.3 is Darwin 7, so we check for
     # that.
 
-    if int(_os.uname()[2].split('.')[0]) < 8:
+    if int(_os.uname().release.split('.')[0]) < 8:
         DEFAULT_MODE = RTLD_GLOBAL
 
 from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
@@ -456,7 +456,7 @@ if _os.name in ("nt", "ce"):
             code = GetLastError()
         if descr is None:
             descr = FormatError(code).strip()
-        return WindowsError(code, descr)
+        return WindowsError(None, descr, None, code)
 
 if sizeof(c_uint) == sizeof(c_void_p):
     c_size_t = c_uint
diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py
index c7207eab9d..5600b437c1 100644
--- a/Lib/ctypes/test/test_callbacks.py
+++ b/Lib/ctypes/test/test_callbacks.py
@@ -140,7 +140,7 @@ class Callbacks(unittest.TestCase):
             def __del__(self):
                 gc.collect()
         CFUNCTYPE(None)(lambda x=Nasty(): None)
-        
+
 
 try:
     WINFUNCTYPE
diff --git a/Lib/ctypes/test/test_memfunctions.py b/Lib/ctypes/test/test_memfunctions.py
index aa2113b885..aec4aaadac 100644
--- a/Lib/ctypes/test/test_memfunctions.py
+++ b/Lib/ctypes/test/test_memfunctions.py
@@ -1,4 +1,5 @@
 import sys
+from test import support
 import unittest
 from ctypes import *
 
@@ -49,6 +50,7 @@ class MemFunctionsTest(unittest.TestCase):
         self.assertEqual(cast(a, POINTER(c_byte))[:7:7],
                              [97])
 
+    @support.refcount_test
     def test_string_at(self):
         s = string_at(b"foo bar")
         # XXX The following may be wrong, depending on how Python
diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py
index e83fd9a6fc..9762fb915d 100644
--- a/Lib/ctypes/test/test_parameters.py
+++ b/Lib/ctypes/test/test_parameters.py
@@ -73,13 +73,10 @@ class SimpleTypesTestCase(unittest.TestCase):
         except ImportError:
 ##            print "(No c_wchar_p)"
             return
-        s = "123"
-        if sys.platform == "win32":
-            self.assertTrue(c_wchar_p.from_param(s)._obj is s)
-            self.assertRaises(TypeError, c_wchar_p.from_param, 42)
 
-            # new in 0.9.1: convert (decode) ascii to unicode
-            self.assertEqual(c_wchar_p.from_param("123")._obj, "123")
+        c_wchar_p.from_param("123")
+
+        self.assertRaises(TypeError, c_wchar_p.from_param, 42)
         self.assertRaises(TypeError, c_wchar_p.from_param, b"123\377")
 
         pa = c_wchar_p.from_param(c_wchar_p("123"))
diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py
index fa6461f546..ad13b016e7 100644
--- a/Lib/ctypes/test/test_pep3118.py
+++ b/Lib/ctypes/test/test_pep3118.py
@@ -25,14 +25,17 @@ class Test(unittest.TestCase):
             v = memoryview(ob)
             try:
                 self.assertEqual(normalize(v.format), normalize(fmt))
-                if shape is not None:
+                if shape:
                     self.assertEqual(len(v), shape[0])
                 else:
                     self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob))
                 self.assertEqual(v.itemsize, sizeof(itemtp))
                 self.assertEqual(v.shape, shape)
-                # ctypes object always have a non-strided memory block
-                self.assertEqual(v.strides, None)
+                # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides
+                #     if requested. memoryview currently reconstructs missing
+                #     stride information, so this assert will fail.
+                # self.assertEqual(v.strides, ())
+
                 # they are always read/write
                 self.assertFalse(v.readonly)
 
@@ -52,14 +55,15 @@ class Test(unittest.TestCase):
             v = memoryview(ob)
             try:
                 self.assertEqual(v.format, fmt)
-                if shape is not None:
+                if shape:
                     self.assertEqual(len(v), shape[0])
                 else:
                     self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob))
                 self.assertEqual(v.itemsize, sizeof(itemtp))
                 self.assertEqual(v.shape, shape)
-                # ctypes object always have a non-strided memory block
-                self.assertEqual(v.strides, None)
+                # XXX Issue #12851
+                # self.assertEqual(v.strides, ())
+
                 # they are always read/write
                 self.assertFalse(v.readonly)
 
@@ -110,34 +114,34 @@ native_types = [
 
     ## simple types
 
-    (c_char,                    "<c",                   None,           c_char),
-    (c_byte,                    "<b",                   None,           c_byte),
-    (c_ubyte,                   "<B",                   None,           c_ubyte),
-    (c_short,                   "<h",                   None,           c_short),
-    (c_ushort,                  "<H",                   None,           c_ushort),
+    (c_char,                    "<c",                   (),           c_char),
+    (c_byte,                    "<b",                   (),           c_byte),
+    (c_ubyte,                   "<B",                   (),           c_ubyte),
+    (c_short,                   "<h",                   (),           c_short),
+    (c_ushort,                  "<H",                   (),           c_ushort),
 
     # c_int and c_uint may be aliases to c_long
-    #(c_int,                     "<i",                   None,           c_int),
-    #(c_uint,                    "<I",                   None,           c_uint),
+    #(c_int,                     "<i",                   (),           c_int),
+    #(c_uint,                    "<I",                   (),           c_uint),
 
-    (c_long,                    "<l",                   None,           c_long),
-    (c_ulong,                   "<L",                   None,           c_ulong),
+    (c_long,                    "<l",                   (),           c_long),
+    (c_ulong,                   "<L",                   (),           c_ulong),
 
     # c_longlong and c_ulonglong are aliases on 64-bit platforms
     #(c_longlong,                "<q",                   None,           c_longlong),
     #(c_ulonglong,               "<Q",                   None,           c_ulonglong),
 
-    (c_float,                   "<f",                   None,           c_float),
-    (c_double,                  "<d",                   None,           c_double),
+    (c_float,                   "<f",                   (),           c_float),
+    (c_double,                  "<d",                   (),           c_double),
     # c_longdouble may be an alias to c_double
 
-    (c_bool,                    "<?",                   None,           c_bool),
-    (py_object,                 "<O",                   None,           py_object),
+    (c_bool,                    "<?",                   (),           c_bool),
+    (py_object,                 "<O",                   (),           py_object),
 
     ## pointers
 
-    (POINTER(c_byte),           "&<b",                  None,           POINTER(c_byte)),
-    (POINTER(POINTER(c_long)),  "&&<l",                 None,           POINTER(POINTER(c_long))),
+    (POINTER(c_byte),           "&<b",                  (),           POINTER(c_byte)),
+    (POINTER(POINTER(c_long)),  "&&<l",                 (),           POINTER(POINTER(c_long))),
 
     ## arrays and pointers
 
@@ -145,32 +149,32 @@ native_types = [
     (c_float * 4 * 3 * 2,       "(2,3,4)<f",            (2,3,4),        c_float),
     (POINTER(c_short) * 2,      "(2)&<h",               (2,),           POINTER(c_short)),
     (POINTER(c_short) * 2 * 3,  "(3,2)&<h",             (3,2,),         POINTER(c_short)),
-    (POINTER(c_short * 2),      "&(2)<h",               None,           POINTER(c_short)),
+    (POINTER(c_short * 2),      "&(2)<h",               (),           POINTER(c_short)),
 
     ## structures and unions
 
-    (Point,                     "T{<l:x:<l:y:}",        None,           Point),
+    (Point,                     "T{<l:x:<l:y:}",        (),           Point),
     # packed structures do not implement the pep
-    (PackedPoint,               "B",                    None,           PackedPoint),
-    (Point2,                    "T{<l:x:<l:y:}",        None,           Point2),
-    (EmptyStruct,               "T{}",                  None,           EmptyStruct),
+    (PackedPoint,               "B",                    (),           PackedPoint),
+    (Point2,                    "T{<l:x:<l:y:}",        (),           Point2),
+    (EmptyStruct,               "T{}",                  (),           EmptyStruct),
     # the pep does't support unions
-    (aUnion,                    "B",                    None,           aUnion),
+    (aUnion,                    "B",                    (),           aUnion),
 
     ## pointer to incomplete structure
-    (Incomplete,                "B",                    None,           Incomplete),
-    (POINTER(Incomplete),       "&B",                   None,           POINTER(Incomplete)),
+    (Incomplete,                "B",                    (),           Incomplete),
+    (POINTER(Incomplete),       "&B",                   (),           POINTER(Incomplete)),
 
     # 'Complete' is a structure that starts incomplete, but is completed after the
     # pointer type to it has been created.
-    (Complete,                  "T{<l:a:}",             None,           Complete),
+    (Complete,                  "T{<l:a:}",             (),           Complete),
     # Unfortunately the pointer format string is not fixed...
-    (POINTER(Complete),         "&B",                   None,           POINTER(Complete)),
+    (POINTER(Complete),         "&B",                   (),           POINTER(Complete)),
 
     ## other
 
     # function signatures are not implemented
-    (CFUNCTYPE(None),           "X{}",                  None,           CFUNCTYPE(None)),
+    (CFUNCTYPE(None),           "X{}",                  (),           CFUNCTYPE(None)),
 
     ]
 
@@ -186,10 +190,10 @@ class LEPoint(LittleEndianStructure):
 # and little endian machines.
 #
 endian_types = [
-    (BEPoint,                   "T{>l:x:>l:y:}",        None,           BEPoint),
-    (LEPoint,                   "T{<l:x:<l:y:}",        None,           LEPoint),
-    (POINTER(BEPoint),          "&T{>l:x:>l:y:}",       None,           POINTER(BEPoint)),
-    (POINTER(LEPoint),          "&T{<l:x:<l:y:}",       None,           POINTER(LEPoint)),
+    (BEPoint,                   "T{>l:x:>l:y:}",        (),           BEPoint),
+    (LEPoint,                   "T{<l:x:<l:y:}",        (),           LEPoint),
+    (POINTER(BEPoint),          "&T{>l:x:>l:y:}",       (),           POINTER(BEPoint)),
+    (POINTER(LEPoint),          "&T{<l:x:<l:y:}",       (),           POINTER(LEPoint)),
     ]
 
 if __name__ == "__main__":
diff --git a/Lib/ctypes/test/test_python_api.py b/Lib/ctypes/test/test_python_api.py
index 1f4c6039dc..9de3980ed4 100644
--- a/Lib/ctypes/test/test_python_api.py
+++ b/Lib/ctypes/test/test_python_api.py
@@ -1,5 +1,6 @@
 from ctypes import *
 import unittest, sys
+from test import support
 from ctypes.test import is_resource_enabled
 
 ################################################################
@@ -25,6 +26,7 @@ class PythonAPITestCase(unittest.TestCase):
 
         self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc")
 
+    @support.refcount_test
     def test_PyString_FromString(self):
         pythonapi.PyBytes_FromString.restype = py_object
         pythonapi.PyBytes_FromString.argtypes = (c_char_p,)
@@ -56,6 +58,7 @@ class PythonAPITestCase(unittest.TestCase):
             del res
             self.assertEqual(grc(42), ref42)
 
+    @support.refcount_test
     def test_PyObj_FromPtr(self):
         s = "abc def ghi jkl"
         ref = grc(s)
diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/ctypes/test/test_refcounts.py
index 35a81aa40b..5613e7a387 100644
--- a/Lib/ctypes/test/test_refcounts.py
+++ b/Lib/ctypes/test/test_refcounts.py
@@ -1,4 +1,5 @@
 import unittest
+from test import support
 import ctypes
 import gc
 
@@ -10,6 +11,7 @@ dll = ctypes.CDLL(_ctypes_test.__file__)
 
 class RefcountTestCase(unittest.TestCase):
 
+    @support.refcount_test
     def test_1(self):
         from sys import getrefcount as grc
 
@@ -34,6 +36,7 @@ class RefcountTestCase(unittest.TestCase):
         self.assertEqual(grc(callback), 2)
 
 
+    @support.refcount_test
     def test_refcount(self):
         from sys import getrefcount as grc
         def func(*args):
diff --git a/Lib/ctypes/test/test_stringptr.py b/Lib/ctypes/test/test_stringptr.py
index 3d25fa5360..95cd1614c6 100644
--- a/Lib/ctypes/test/test_stringptr.py
+++ b/Lib/ctypes/test/test_stringptr.py
@@ -1,4 +1,5 @@
 import unittest
+from test import support
 from ctypes import *
 
 import _ctypes_test
@@ -7,6 +8,7 @@ lib = CDLL(_ctypes_test.__file__)
 
 class StringPtrTestCase(unittest.TestCase):
 
+    @support.refcount_test
     def test__POINTER_c_char(self):
         class X(Structure):
             _fields_ = [("str", POINTER(c_char))]
diff --git a/Lib/ctypes/test/test_win32.py b/Lib/ctypes/test/test_win32.py
index 7e6fe3fba0..da21336682 100644
--- a/Lib/ctypes/test/test_win32.py
+++ b/Lib/ctypes/test/test_win32.py
@@ -70,6 +70,28 @@ if sys.platform == "win32":
             self.assertEqual(ex.text, "text")
             self.assertEqual(ex.details, ("details",))
 
+    class TestWinError(unittest.TestCase):
+        def test_winerror(self):
+            # see Issue 16169
+            import errno
+            ERROR_INVALID_PARAMETER = 87
+            msg = FormatError(ERROR_INVALID_PARAMETER).strip()
+            args = (errno.EINVAL, msg, None, ERROR_INVALID_PARAMETER)
+
+            e = WinError(ERROR_INVALID_PARAMETER)
+            self.assertEqual(e.args, args)
+            self.assertEqual(e.errno, errno.EINVAL)
+            self.assertEqual(e.winerror, ERROR_INVALID_PARAMETER)
+
+            windll.kernel32.SetLastError(ERROR_INVALID_PARAMETER)
+            try:
+                raise WinError()
+            except OSError as exc:
+                e = exc
+            self.assertEqual(e.args, args)
+            self.assertEqual(e.errno, errno.EINVAL)
+            self.assertEqual(e.winerror, ERROR_INVALID_PARAMETER)
+
 class Structures(unittest.TestCase):
 
     def test_struct_by_value(self):
diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
index 1bb7d1de7e..1515604272 100644
--- a/Lib/ctypes/util.py
+++ b/Lib/ctypes/util.py
@@ -1,5 +1,6 @@
 import sys, os
 import contextlib
+import subprocess
 
 # find_library(name) returns the pathname of a library, or None.
 if os.name == "nt":
@@ -39,8 +40,8 @@ if os.name == "nt":
             clibname = 'msvcr%d' % (version * 10)
 
         # If python was built with in debug mode
-        import imp
-        if imp.get_suffixes()[0][0] == '_d.pyd':
+        import importlib.machinery
+        if '_d.pyd' in importlib.machinery.EXTENSION_SUFFIXES:
             clibname += 'd'
         return clibname+'.dll'
 
@@ -136,16 +137,12 @@ elif os.name == "posix":
             rv = f.close()
             if rv == 10:
                 raise OSError('objdump command not found')
-            with contextlib.closing(os.popen(cmd)) as f:
-                data = f.read()
-            res = re.search(r'\sSONAME\s+([^\s]+)', data)
+            res = re.search(r'\sSONAME\s+([^\s]+)', dump)
             if not res:
                 return None
             return res.group(1)
 
-    if (sys.platform.startswith("freebsd")
-        or sys.platform.startswith("openbsd")
-        or sys.platform.startswith("dragonfly")):
+    if sys.platform.startswith(("freebsd", "openbsd", "dragonfly")):
 
         def _num_version(libname):
             # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ]
@@ -169,14 +166,43 @@ elif os.name == "posix":
             res.sort(key=_num_version)
             return res[-1]
 
+    elif sys.platform == "sunos5":
+
+        def _findLib_crle(name, is64):
+            if not os.path.exists('/usr/bin/crle'):
+                return None
+
+            if is64:
+                cmd = 'env LC_ALL=C /usr/bin/crle -64 2>/dev/null'
+            else:
+                cmd = 'env LC_ALL=C /usr/bin/crle 2>/dev/null'
+
+            for line in os.popen(cmd).readlines():
+                line = line.strip()
+                if line.startswith('Default Library Path (ELF):'):
+                    paths = line.split()[4]
+
+            if not paths:
+                return None
+
+            for dir in paths.split(":"):
+                libfile = os.path.join(dir, "lib%s.so" % name)
+                if os.path.exists(libfile):
+                    return libfile
+
+            return None
+
+        def find_library(name, is64 = False):
+            return _get_soname(_findLib_crle(name, is64) or _findLib_gcc(name))
+
     else:
 
         def _findSoname_ldconfig(name):
             import struct
             if struct.calcsize('l') == 4:
-                machine = os.uname()[4] + '-32'
+                machine = os.uname().machine + '-32'
             else:
-                machine = os.uname()[4] + '-64'
+                machine = os.uname().machine + '-64'
             mach_map = {
                 'x86_64-64': 'libc6,x86-64',
                 'ppc64-64': 'libc6,64bit',
@@ -187,13 +213,19 @@ elif os.name == "posix":
             abi_type = mach_map.get(machine, 'libc6')
 
             # XXX assuming GLIBC's ldconfig (with option -p)
-            expr = r'\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type)
-            with contextlib.closing(os.popen('LC_ALL=C LANG=C /sbin/ldconfig -p 2>/dev/null')) as f:
-                data = f.read()
-            res = re.search(expr, data)
-            if not res:
-                return None
-            return res.group(1)
+            regex = os.fsencode(
+                '\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type))
+            try:
+                with subprocess.Popen(['/sbin/ldconfig', '-p'],
+                                      stdin=subprocess.DEVNULL,
+                                      stderr=subprocess.DEVNULL,
+                                      stdout=subprocess.PIPE,
+                                      env={'LC_ALL': 'C', 'LANG': 'C'}) as p:
+                    res = re.search(regex, p.stdout.read())
+                    if res:
+                        return os.fsdecode(res.group(1))
+            except OSError:
+                pass
 
         def find_library(name):
             return _findSoname_ldconfig(name) or _get_soname(_findLib_gcc(name))
diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py
index 8f09b2459f..47378741ac 100644
--- a/Lib/curses/__init__.py
+++ b/Lib/curses/__init__.py
@@ -11,7 +11,6 @@ the package, and perhaps a particular module inside it.
 """
 
 from _curses import *
-from curses.wrapper import wrapper
 import os as _os
 import sys as _sys
 
@@ -55,3 +54,48 @@ try:
     has_key
 except NameError:
     from .has_key import has_key
+
+# Wrapper for the entire curses-based application.  Runs a function which
+# should be the rest of your curses-based application.  If the application
+# raises an exception, wrapper() will restore the terminal to a sane state so
+# you can read the resulting traceback.
+
+def wrapper(func, *args, **kwds):
+    """Wrapper function that initializes curses and calls another function,
+    restoring normal keyboard/screen behavior on error.
+    The callable object 'func' is then passed the main window 'stdscr'
+    as its first argument, followed by any other arguments passed to
+    wrapper().
+    """
+
+    try:
+        # Initialize curses
+        stdscr = initscr()
+
+        # Turn off echoing of keys, and enter cbreak mode,
+        # where no buffering is performed on keyboard input
+        noecho()
+        cbreak()
+
+        # In keypad mode, escape sequences for special keys
+        # (like the cursor keys) will be interpreted and
+        # a special value like curses.KEY_LEFT will be returned
+        stdscr.keypad(1)
+
+        # Start color, too.  Harmless if the terminal doesn't have
+        # color; user can test with has_color() later on.  The try/catch
+        # works around a minor bit of over-conscientiousness in the curses
+        # module -- the error return from C start_color() is ignorable.
+        try:
+            start_color()
+        except:
+            pass
+
+        return func(stdscr, *args, **kwds)
+    finally:
+        # Set everything back to normal
+        if 'stdscr' in locals():
+            stdscr.keypad(0)
+            echo()
+            nocbreak()
+            endwin()
diff --git a/Lib/curses/wrapper.py b/Lib/curses/wrapper.py
deleted file mode 100644
index 5183ce741f..0000000000
--- a/Lib/curses/wrapper.py
+++ /dev/null
@@ -1,50 +0,0 @@
-"""curses.wrapper
-
-Contains one function, wrapper(), which runs another function which
-should be the rest of your curses-based application.  If the
-application raises an exception, wrapper() will restore the terminal
-to a sane state so you can read the resulting traceback.
-
-"""
-
-import curses
-
-def wrapper(func, *args, **kwds):
-    """Wrapper function that initializes curses and calls another function,
-    restoring normal keyboard/screen behavior on error.
-    The callable object 'func' is then passed the main window 'stdscr'
-    as its first argument, followed by any other arguments passed to
-    wrapper().
-    """
-
-    try:
-        # Initialize curses
-        stdscr = curses.initscr()
-
-        # Turn off echoing of keys, and enter cbreak mode,
-        # where no buffering is performed on keyboard input
-        curses.noecho()
-        curses.cbreak()
-
-        # In keypad mode, escape sequences for special keys
-        # (like the cursor keys) will be interpreted and
-        # a special value like curses.KEY_LEFT will be returned
-        stdscr.keypad(1)
-
-        # Start color, too.  Harmless if the terminal doesn't have
-        # color; user can test with has_color() later on.  The try/catch
-        # works around a minor bit of over-conscientiousness in the curses
-        # module -- the error return from C start_color() is ignorable.
-        try:
-            curses.start_color()
-        except:
-            pass
-
-        return func(stdscr, *args, **kwds)
-    finally:
-        # Set everything back to normal
-        if 'stdscr' in locals():
-            stdscr.keypad(0)
-            curses.echo()
-            curses.nocbreak()
-            curses.endwin()
diff --git a/Lib/datetime.py b/Lib/datetime.py
index bf23e5002d..f506e9ab22 100644
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -172,10 +172,6 @@ def _format_time(hh, mm, ss, us):
 
 # Correctly substitute for %z and %Z escapes in strftime formats.
 def _wrap_strftime(object, format, timetuple):
-    year = timetuple[0]
-    if year < 1000:
-        raise ValueError("year=%d is before 1000; the datetime strftime() "
-                         "methods require year >= 1000" % year)
     # Don't call utcoffset() or tzname() unless actually needed.
     freplace = None # the string to use for %f
     zreplace = None # the string to use for %z
@@ -1069,13 +1065,13 @@ class time:
 
     def __eq__(self, other):
         if isinstance(other, time):
-            return self._cmp(other) == 0
+            return self._cmp(other, allow_mixed=True) == 0
         else:
             return False
 
     def __ne__(self, other):
         if isinstance(other, time):
-            return self._cmp(other) != 0
+            return self._cmp(other, allow_mixed=True) != 0
         else:
             return True
 
@@ -1103,7 +1099,7 @@ class time:
         else:
             _cmperror(self, other)
 
-    def _cmp(self, other):
+    def _cmp(self, other, allow_mixed=False):
         assert isinstance(other, time)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1122,7 +1118,10 @@ class time:
                        (other._hour, other._minute, other._second,
                         other._microsecond))
         if myoff is None or otoff is None:
-            raise TypeError("cannot compare naive and aware times")
+            if allow_mixed:
+                return 2 # arbitrary non-zero value
+            else:
+                raise TypeError("cannot compare naive and aware times")
         myhhmm = self._hour * 60 + self._minute - myoff//timedelta(minutes=1)
         othhmm = other._hour * 60 + other._minute - otoff//timedelta(minutes=1)
         return _cmp((myhhmm, self._second, self._microsecond),
@@ -1364,7 +1363,7 @@ class datetime(date):
         converter = _time.localtime if tz is None else _time.gmtime
 
         t, frac = divmod(t, 1.0)
-        us = round(frac * 1e6)
+        us = int(frac * 1e6)
 
         # If timestamp is less than one microsecond smaller than a
         # full second, us can be rounded up to 1000000.  In this case,
@@ -1384,7 +1383,7 @@ class datetime(date):
     def utcfromtimestamp(cls, t):
         "Construct a UTC datetime from a POSIX timestamp (like time.time())."
         t, frac = divmod(t, 1.0)
-        us = round(frac * 1e6)
+        us = int(frac * 1e6)
 
         # If timestamp is less than one microsecond smaller than a
         # full second, us can be rounded up to 1000000.  In this case,
@@ -1438,6 +1437,15 @@ class datetime(date):
                                   self.hour, self.minute, self.second,
                                   dst)
 
+    def timestamp(self):
+        "Return POSIX timestamp as float"
+        if self._tzinfo is None:
+            return _time.mktime((self.year, self.month, self.day,
+                                 self.hour, self.minute, self.second,
+                                 -1, -1, -1)) + self.microsecond / 1e6
+        else:
+            return (self - _EPOCH).total_seconds()
+
     def utctimetuple(self):
         "Return UTC time tuple compatible with time.gmtime()."
         offset = self.utcoffset()
@@ -1485,8 +1493,32 @@ class datetime(date):
         return datetime(year, month, day, hour, minute, second,
                           microsecond, tzinfo)
 
-    def astimezone(self, tz):
-        if not isinstance(tz, tzinfo):
+    def astimezone(self, tz=None):
+        if tz is None:
+            if self.tzinfo is None:
+                raise ValueError("astimezone() requires an aware datetime")
+            ts = (self - _EPOCH) // timedelta(seconds=1)
+            localtm = _time.localtime(ts)
+            local = datetime(*localtm[:6])
+            try:
+                # Extract TZ data if available
+                gmtoff = localtm.tm_gmtoff
+                zone = localtm.tm_zone
+            except AttributeError:
+                # Compute UTC offset and compare with the value implied
+                # by tm_isdst.  If the values match, use the zone name
+                # implied by tm_isdst.
+                delta = local - datetime(*_time.gmtime(ts)[:6])
+                dst = _time.daylight and localtm.tm_isdst > 0
+                gmtoff = -(_time.altzone if dst else _time.timezone)
+                if delta == timedelta(seconds=gmtoff):
+                    tz = timezone(delta, _time.tzname[dst])
+                else:
+                    tz = timezone(delta)
+            else:
+                tz = timezone(timedelta(seconds=gmtoff), zone)
+
+        elif not isinstance(tz, tzinfo):
             raise TypeError("tz argument must be an instance of tzinfo")
 
         mytz = self.tzinfo
@@ -1610,7 +1642,7 @@ class datetime(date):
 
     def __eq__(self, other):
         if isinstance(other, datetime):
-            return self._cmp(other) == 0
+            return self._cmp(other, allow_mixed=True) == 0
         elif not isinstance(other, date):
             return NotImplemented
         else:
@@ -1618,7 +1650,7 @@ class datetime(date):
 
     def __ne__(self, other):
         if isinstance(other, datetime):
-            return self._cmp(other) != 0
+            return self._cmp(other, allow_mixed=True) != 0
         elif not isinstance(other, date):
             return NotImplemented
         else:
@@ -1656,7 +1688,7 @@ class datetime(date):
         else:
             _cmperror(self, other)
 
-    def _cmp(self, other):
+    def _cmp(self, other, allow_mixed=False):
         assert isinstance(other, datetime)
         mytz = self._tzinfo
         ottz = other._tzinfo
@@ -1665,10 +1697,8 @@ class datetime(date):
         if mytz is ottz:
             base_compare = True
         else:
-            if mytz is not None:
-                myoff = self.utcoffset()
-            if ottz is not None:
-                otoff = other.utcoffset()
+            myoff = self.utcoffset()
+            otoff = other.utcoffset()
             base_compare = myoff == otoff
 
         if base_compare:
@@ -1679,7 +1709,10 @@ class datetime(date):
                         other._hour, other._minute, other._second,
                         other._microsecond))
         if myoff is None or otoff is None:
-            raise TypeError("cannot compare naive and aware datetimes")
+            if allow_mixed:
+                return 2 # arbitrary non-zero value
+            else:
+                raise TypeError("cannot compare naive and aware datetimes")
         # XXX What follows could be done more efficiently...
         diff = self - other     # this will take offsets into account
         if diff.days < 0:
@@ -1895,7 +1928,7 @@ class timezone(tzinfo):
 timezone.utc = timezone._create(timedelta(0))
 timezone.min = timezone._create(timezone._minoffset)
 timezone.max = timezone._create(timezone._maxoffset)
-
+_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
 """
 Some time zone algebra.  For a datetime x, let
     x.n = x stripped of its timezone -- its naive time.
diff --git a/Lib/decimal.py b/Lib/decimal.py
index 49de53592f..746b34a894 100644
--- a/Lib/decimal.py
+++ b/Lib/decimal.py
@@ -46,8 +46,8 @@ Decimal('1')
 Decimal('-0.0123')
 >>> Decimal(123456)
 Decimal('123456')
->>> Decimal('123.45e12345678901234567890')
-Decimal('1.2345E+12345678901234567892')
+>>> Decimal('123.45e12345678')
+Decimal('1.2345E+12345680')
 >>> Decimal('1.33') + Decimal('1.27')
 Decimal('2.60')
 >>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
@@ -122,13 +122,20 @@ __all__ = [
     # Exceptions
     'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero',
     'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow',
+    'FloatOperation',
 
     # Constants for use in setting up contexts
     'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING',
     'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP',
 
     # Functions for manipulating contexts
-    'setcontext', 'getcontext', 'localcontext'
+    'setcontext', 'getcontext', 'localcontext',
+
+    # Limits for the C version for compatibility
+    'MAX_PREC',  'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
+
+    # C version: compile time choice that enables the thread local context
+    'HAVE_THREADS'
 ]
 
 __version__ = '1.70'    # Highest version of the spec this complies with
@@ -137,6 +144,7 @@ __version__ = '1.70'    # Highest version of the spec this complies with
 import copy as _copy
 import math as _math
 import numbers as _numbers
+import sys
 
 try:
     from collections import namedtuple as _namedtuple
@@ -154,6 +162,19 @@ ROUND_UP = 'ROUND_UP'
 ROUND_HALF_DOWN = 'ROUND_HALF_DOWN'
 ROUND_05UP = 'ROUND_05UP'
 
+# Compatibility with the C version
+HAVE_THREADS = True
+if sys.maxsize == 2**63-1:
+    MAX_PREC = 999999999999999999
+    MAX_EMAX = 999999999999999999
+    MIN_EMIN = -999999999999999999
+else:
+    MAX_PREC = 425000000
+    MAX_EMAX = 425000000
+    MIN_EMIN = -425000000
+
+MIN_ETINY = MIN_EMIN - (MAX_PREC-1)
+
 # Errors
 
 class DecimalException(ArithmeticError):
@@ -370,9 +391,24 @@ class Underflow(Inexact, Rounded, Subnormal):
     In all cases, Inexact, Rounded, and Subnormal will also be raised.
     """
 
+class FloatOperation(DecimalException, TypeError):
+    """Enable stricter semantics for mixing floats and Decimals.
+
+    If the signal is not trapped (default), mixing floats and Decimals is
+    permitted in the Decimal() constructor, context.create_decimal() and
+    all comparison operators. Both conversion and comparisons are exact.
+    Any occurrence of a mixed operation is silently recorded by setting
+    FloatOperation in the context flags.  Explicit conversions with
+    Decimal.from_float() or context.create_decimal_from_float() do not
+    set the flag.
+
+    Otherwise (the signal is trapped), only equality comparisons and explicit
+    conversions are silent. All other mixed operations raise FloatOperation.
+    """
+
 # List of public traps and flags
 _signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded,
-           Underflow, InvalidOperation, Subnormal]
+            Underflow, InvalidOperation, Subnormal, FloatOperation]
 
 # Map conditions (per the spec) to signals
 _condition_map = {ConversionSyntax:InvalidOperation,
@@ -380,6 +416,10 @@ _condition_map = {ConversionSyntax:InvalidOperation,
                   DivisionUndefined:InvalidOperation,
                   InvalidContext:InvalidOperation}
 
+# Valid rounding modes
+_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,
+                   ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP)
+
 ##### Context Functions ##################################################
 
 # The getcontext() and setcontext() function manage access to a thread-local
@@ -392,12 +432,11 @@ try:
     import threading
 except ImportError:
     # Python was compiled without threads; create a mock object instead
-    import sys
     class MockThreading(object):
         def local(self, sys=sys):
             return sys.modules[__name__]
     threading = MockThreading()
-    del sys, MockThreading
+    del MockThreading
 
 try:
     threading.local
@@ -650,6 +689,11 @@ class Decimal(object):
             return self
 
         if isinstance(value, float):
+            if context is None:
+                context = getcontext()
+            context._raise_error(FloatOperation,
+                "strict semantics for mixing floats and Decimals are "
+                "enabled")
             value = Decimal.from_float(value)
             self._exp  = value._exp
             self._sign = value._sign
@@ -684,7 +728,9 @@ class Decimal(object):
         """
         if isinstance(f, int):                # handle integer inputs
             return cls(f)
-        if _math.isinf(f) or _math.isnan(f):  # raises TypeError if not a float
+        if not isinstance(f, float):
+            raise TypeError("argument must be int or float.")
+        if _math.isinf(f) or _math.isnan(f):
             return cls(repr(f))
         if _math.copysign(1.0, f) == 1.0:
             sign = 0
@@ -1877,6 +1923,7 @@ class Decimal(object):
         """
 
         other = _convert_other(other, raiseit=True)
+        third = _convert_other(third, raiseit=True)
 
         # compute product; raise InvalidOperation if either operand is
         # a signaling NaN or if the product is zero times infinity.
@@ -1906,17 +1953,17 @@ class Decimal(object):
                                        str(int(self._int) * int(other._int)),
                                        self._exp + other._exp)
 
-        third = _convert_other(third, raiseit=True)
         return product.__add__(third, context)
 
     def _power_modulo(self, other, modulo, context=None):
         """Three argument version of __pow__"""
 
-        # if can't convert other and modulo to Decimal, raise
-        # TypeError; there's no point returning NotImplemented (no
-        # equivalent of __rpow__ for three argument pow)
-        other = _convert_other(other, raiseit=True)
-        modulo = _convert_other(modulo, raiseit=True)
+        other = _convert_other(other)
+        if other is NotImplemented:
+            return other
+        modulo = _convert_other(modulo)
+        if modulo is NotImplemented:
+            return modulo
 
         if context is None:
             context = getcontext()
@@ -2007,9 +2054,9 @@ class Decimal(object):
         nonzero.  For efficiency, other._exp should not be too large,
         so that 10**abs(other._exp) is a feasible calculation."""
 
-        # In the comments below, we write x for the value of self and
-        # y for the value of other.  Write x = xc*10**xe and y =
-        # yc*10**ye.
+        # In the comments below, we write x for the value of self and y for the
+        # value of other.  Write x = xc*10**xe and abs(y) = yc*10**ye, with xc
+        # and yc positive integers not divisible by 10.
 
         # The main purpose of this method is to identify the *failure*
         # of x**y to be exactly representable with as little effort as
@@ -2017,13 +2064,12 @@ class Decimal(object):
         # eliminate the possibility of x**y being exact.  Only if all
         # these tests are passed do we go on to actually compute x**y.
 
-        # Here's the main idea.  First normalize both x and y.  We
-        # express y as a rational m/n, with m and n relatively prime
-        # and n>0.  Then for x**y to be exactly representable (at
-        # *any* precision), xc must be the nth power of a positive
-        # integer and xe must be divisible by n.  If m is negative
-        # then additionally xc must be a power of either 2 or 5, hence
-        # a power of 2**n or 5**n.
+        # Here's the main idea.  Express y as a rational number m/n, with m and
+        # n relatively prime and n>0.  Then for x**y to be exactly
+        # representable (at *any* precision), xc must be the nth power of a
+        # positive integer and xe must be divisible by n.  If y is negative
+        # then additionally xc must be a power of either 2 or 5, hence a power
+        # of 2**n or 5**n.
         #
         # There's a limit to how small |y| can be: if y=m/n as above
         # then:
@@ -2095,21 +2141,43 @@ class Decimal(object):
                     return None
                 # now xc is a power of 2; e is its exponent
                 e = _nbits(xc)-1
-                # find e*y and xe*y; both must be integers
-                if ye >= 0:
-                    y_as_int = yc*10**ye
-                    e = e*y_as_int
-                    xe = xe*y_as_int
-                else:
-                    ten_pow = 10**-ye
-                    e, remainder = divmod(e*yc, ten_pow)
-                    if remainder:
-                        return None
-                    xe, remainder = divmod(xe*yc, ten_pow)
-                    if remainder:
-                        return None
-
-                if e*65 >= p*93: # 93/65 > log(10)/log(5)
+
+                # We now have:
+                #
+                #   x = 2**e * 10**xe, e > 0, and y < 0.
+                #
+                # The exact result is:
+                #
+                #   x**y = 5**(-e*y) * 10**(e*y + xe*y)
+                #
+                # provided that both e*y and xe*y are integers.  Note that if
+                # 5**(-e*y) >= 10**p, then the result can't be expressed
+                # exactly with p digits of precision.
+                #
+                # Using the above, we can guard against large values of ye.
+                # 93/65 is an upper bound for log(10)/log(5), so if
+                #
+                #   ye >= len(str(93*p//65))
+                #
+                # then
+                #
+                #   -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5),
+                #
+                # so 5**(-e*y) >= 10**p, and the coefficient of the result
+                # can't be expressed in p digits.
+
+                # emax >= largest e such that 5**e < 10**p.
+                emax = p*93//65
+                if ye >= len(str(emax)):
+                    return None
+
+                # Find -e*y and -xe*y; both must be integers
+                e = _decimal_lshift_exact(e * yc, ye)
+                xe = _decimal_lshift_exact(xe * yc, ye)
+                if e is None or xe is None:
+                    return None
+
+                if e > emax:
                     return None
                 xc = 5**e
 
@@ -2123,19 +2191,20 @@ class Decimal(object):
                 while xc % 5 == 0:
                     xc //= 5
                     e -= 1
-                if ye >= 0:
-                    y_as_integer = yc*10**ye
-                    e = e*y_as_integer
-                    xe = xe*y_as_integer
-                else:
-                    ten_pow = 10**-ye
-                    e, remainder = divmod(e*yc, ten_pow)
-                    if remainder:
-                        return None
-                    xe, remainder = divmod(xe*yc, ten_pow)
-                    if remainder:
-                        return None
-                if e*3 >= p*10: # 10/3 > log(10)/log(2)
+
+                # Guard against large values of ye, using the same logic as in
+                # the 'xc is a power of 2' branch.  10/3 is an upper bound for
+                # log(10)/log(2).
+                emax = p*10//3
+                if ye >= len(str(emax)):
+                    return None
+
+                e = _decimal_lshift_exact(e * yc, ye)
+                xe = _decimal_lshift_exact(xe * yc, ye)
+                if e is None or xe is None:
+                    return None
+
+                if e > emax:
                     return None
                 xc = 2**e
             else:
@@ -2527,7 +2596,7 @@ class Decimal(object):
         ans = ans._fix(context)
         return ans
 
-    def same_quantum(self, other):
+    def same_quantum(self, other, context=None):
         """Return True if self and other have the same exponent; otherwise
         return False.
 
@@ -2845,7 +2914,7 @@ class Decimal(object):
         except TypeError:
             return 0
 
-    def canonical(self, context=None):
+    def canonical(self):
         """Returns the same Decimal object.
 
         As we do not have different encodings for the same number, the
@@ -2865,7 +2934,7 @@ class Decimal(object):
             return ans
         return self.compare(other, context=context)
 
-    def compare_total(self, other):
+    def compare_total(self, other, context=None):
         """Compares self to other using the abstract representations.
 
         This is not like the standard compare, which use their numerical
@@ -2938,7 +3007,7 @@ class Decimal(object):
         return _Zero
 
 
-    def compare_total_mag(self, other):
+    def compare_total_mag(self, other, context=None):
         """Compares self to other using abstract repr., ignoring sign.
 
         Like compare_total, but with operand's sign ignored and assumed to be 0.
@@ -2960,7 +3029,7 @@ class Decimal(object):
         else:
             return _dec_from_triple(1, self._int, self._exp, self._is_special)
 
-    def copy_sign(self, other):
+    def copy_sign(self, other, context=None):
         """Returns self with the sign of other."""
         other = _convert_other(other, raiseit=True)
         return _dec_from_triple(other._sign, self._int,
@@ -3816,11 +3885,9 @@ class Context(object):
     clamp -  If 1, change exponents if too high (Default 0)
     """
 
-    def __init__(self, prec=None, rounding=None,
-                 traps=None, flags=None,
-                 Emin=None, Emax=None,
-                 capitals=None, clamp=None,
-                 _ignored_flags=None):
+    def __init__(self, prec=None, rounding=None, Emin=None, Emax=None,
+                       capitals=None, clamp=None, flags=None, traps=None,
+                       _ignored_flags=None):
         # Set defaults; for everything except flags and _ignored_flags,
         # inherit from DefaultContext.
         try:
@@ -3843,17 +3910,78 @@ class Context(object):
         if traps is None:
             self.traps = dc.traps.copy()
         elif not isinstance(traps, dict):
-            self.traps = dict((s, int(s in traps)) for s in _signals)
+            self.traps = dict((s, int(s in traps)) for s in _signals + traps)
         else:
             self.traps = traps
 
         if flags is None:
             self.flags = dict.fromkeys(_signals, 0)
         elif not isinstance(flags, dict):
-            self.flags = dict((s, int(s in flags)) for s in _signals)
+            self.flags = dict((s, int(s in flags)) for s in _signals + flags)
         else:
             self.flags = flags
 
+    def _set_integer_check(self, name, value, vmin, vmax):
+        if not isinstance(value, int):
+            raise TypeError("%s must be an integer" % name)
+        if vmin == '-inf':
+            if value > vmax:
+                raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value))
+        elif vmax == 'inf':
+            if value < vmin:
+                raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value))
+        else:
+            if value < vmin or value > vmax:
+                raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value))
+        return object.__setattr__(self, name, value)
+
+    def _set_signal_dict(self, name, d):
+        if not isinstance(d, dict):
+            raise TypeError("%s must be a signal dict" % d)
+        for key in d:
+            if not key in _signals:
+                raise KeyError("%s is not a valid signal dict" % d)
+        for key in _signals:
+            if not key in d:
+                raise KeyError("%s is not a valid signal dict" % d)
+        return object.__setattr__(self, name, d)
+
+    def __setattr__(self, name, value):
+        if name == 'prec':
+            return self._set_integer_check(name, value, 1, 'inf')
+        elif name == 'Emin':
+            return self._set_integer_check(name, value, '-inf', 0)
+        elif name == 'Emax':
+            return self._set_integer_check(name, value, 0, 'inf')
+        elif name == 'capitals':
+            return self._set_integer_check(name, value, 0, 1)
+        elif name == 'clamp':
+            return self._set_integer_check(name, value, 0, 1)
+        elif name == 'rounding':
+            if not value in _rounding_modes:
+                # raise TypeError even for strings to have consistency
+                # among various implementations.
+                raise TypeError("%s: invalid rounding mode" % value)
+            return object.__setattr__(self, name, value)
+        elif name == 'flags' or name == 'traps':
+            return self._set_signal_dict(name, value)
+        elif name == '_ignored_flags':
+            return object.__setattr__(self, name, value)
+        else:
+            raise AttributeError(
+                "'decimal.Context' object has no attribute '%s'" % name)
+
+    def __delattr__(self, name):
+        raise AttributeError("%s cannot be deleted" % name)
+
+    # Support for pickling, copy, and deepcopy
+    def __reduce__(self):
+        flags = [sig for sig, v in self.flags.items() if v]
+        traps = [sig for sig, v in self.traps.items() if v]
+        return (self.__class__,
+                (self.prec, self.rounding, self.Emin, self.Emax,
+                 self.capitals, self.clamp, flags, traps))
+
     def __repr__(self):
         """Show the current context."""
         s = []
@@ -3872,43 +4000,27 @@ class Context(object):
         for flag in self.flags:
             self.flags[flag] = 0
 
+    def clear_traps(self):
+        """Reset all traps to zero"""
+        for flag in self.traps:
+            self.traps[flag] = 0
+
     def _shallow_copy(self):
         """Returns a shallow copy from self."""
-        nc = Context(self.prec, self.rounding, self.traps,
-                     self.flags, self.Emin, self.Emax,
-                     self.capitals, self.clamp, self._ignored_flags)
+        nc = Context(self.prec, self.rounding, self.Emin, self.Emax,
+                     self.capitals, self.clamp, self.flags, self.traps,
+                     self._ignored_flags)
         return nc
 
     def copy(self):
         """Returns a deep copy from self."""
-        nc = Context(self.prec, self.rounding, self.traps.copy(),
-                     self.flags.copy(), self.Emin, self.Emax,
-                     self.capitals, self.clamp, self._ignored_flags)
+        nc = Context(self.prec, self.rounding, self.Emin, self.Emax,
+                     self.capitals, self.clamp,
+                     self.flags.copy(), self.traps.copy(),
+                     self._ignored_flags)
         return nc
     __copy__ = copy
 
-    # _clamp is provided for backwards compatibility with third-party
-    # code.  May be removed in Python >= 3.3.
-    def _get_clamp(self):
-        "_clamp mirrors the clamp attribute.  Its use is deprecated."
-        import warnings
-        warnings.warn('Use of the _clamp attribute is deprecated. '
-                      'Please use clamp instead.',
-                      DeprecationWarning)
-        return self.clamp
-
-    def _set_clamp(self, clamp):
-        "_clamp mirrors the clamp attribute.  Its use is deprecated."
-        import warnings
-        warnings.warn('Use of the _clamp attribute is deprecated. '
-                      'Please use clamp instead.',
-                      DeprecationWarning)
-        self.clamp = clamp
-
-    # don't bother with _del_clamp;  no sane 3rd party code should
-    # be deleting the _clamp attribute
-    _clamp = property(_get_clamp, _set_clamp)
-
     def _raise_error(self, condition, explanation = None, *args):
         """Handles an error
 
@@ -4068,7 +4180,9 @@ class Context(object):
         >>> ExtendedContext.canonical(Decimal('2.50'))
         Decimal('2.50')
         """
-        return a.canonical(context=self)
+        if not isinstance(a, Decimal):
+            raise TypeError("canonical requires a Decimal as an argument.")
+        return a.canonical()
 
     def compare(self, a, b):
         """Compares values numerically.
@@ -4378,6 +4492,8 @@ class Context(object):
         >>> ExtendedContext.is_canonical(Decimal('2.50'))
         True
         """
+        if not isinstance(a, Decimal):
+            raise TypeError("is_canonical requires a Decimal as an argument.")
         return a.is_canonical()
 
     def is_finite(self, a):
@@ -4970,7 +5086,7 @@ class Context(object):
           +Normal
           +Infinity
 
-        >>> c = Context(ExtendedContext)
+        >>> c = ExtendedContext.copy()
         >>> c.Emin = -999
         >>> c.Emax = 999
         >>> c.number_class(Decimal('Infinity'))
@@ -5535,6 +5651,27 @@ def _normalize(op1, op2, prec = 0):
 
 _nbits = int.bit_length
 
+def _decimal_lshift_exact(n, e):
+    """ Given integers n and e, return n * 10**e if it's an integer, else None.
+
+    The computation is designed to avoid computing large powers of 10
+    unnecessarily.
+
+    >>> _decimal_lshift_exact(3, 4)
+    30000
+    >>> _decimal_lshift_exact(300, -999999999)  # returns None
+
+    """
+    if n == 0:
+        return 0
+    elif e >= 0:
+        return n * 10**e
+    else:
+        # val_n = largest power of 10 dividing n.
+        str_n = str(abs(n))
+        val_n = len(str_n) - len(str_n.rstrip('0'))
+        return None if val_n < -e else n // 10**-e
+
 def _sqrt_nearest(n, a):
     """Closest integer to the square root of the positive integer n.  a is
     an initial approximation to the square root.  Any positive integer
@@ -5901,6 +6038,12 @@ def _convert_for_comparison(self, other, equality_op=False):
     if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0:
         other = other.real
     if isinstance(other, float):
+        context = getcontext()
+        if equality_op:
+            context.flags[FloatOperation] = 1
+        else:
+            context._raise_error(FloatOperation,
+                "strict semantics for mixing floats and Decimals are enabled")
         return self, Decimal.from_float(other)
     return NotImplemented, NotImplemented
 
@@ -5914,8 +6057,8 @@ DefaultContext = Context(
         prec=28, rounding=ROUND_HALF_EVEN,
         traps=[DivisionByZero, Overflow, InvalidOperation],
         flags=[],
-        Emax=999999999,
-        Emin=-999999999,
+        Emax=999999,
+        Emin=-999999,
         capitals=1,
         clamp=0
 )
@@ -6065,7 +6208,7 @@ def _parse_format_specifier(format_spec, _localeconv=None):
     # if format type is 'g' or 'G' then a precision of 0 makes little
     # sense; convert it to 1.  Same if format type is unspecified.
     if format_dict['precision'] == 0:
-        if format_dict['type'] is None or format_dict['type'] in 'gG':
+        if format_dict['type'] is None or format_dict['type'] in 'gGn':
             format_dict['precision'] = 1
 
     # determine thousands separator, grouping, and decimal separator, and
@@ -6239,16 +6382,26 @@ _SignedInfinity = (_Infinity, _NegativeInfinity)
 
 # Constants related to the hash implementation;  hash(x) is based
 # on the reduction of x modulo _PyHASH_MODULUS
-import sys
 _PyHASH_MODULUS = sys.hash_info.modulus
 # hash values to use for positive and negative infinities, and nans
 _PyHASH_INF = sys.hash_info.inf
 _PyHASH_NAN = sys.hash_info.nan
-del sys
 
 # _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS
 _PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+del sys
 
+try:
+    import _decimal
+except ImportError:
+    pass
+else:
+    s1 = set(dir())
+    s2 = set(dir(_decimal))
+    for name in s1 - s2:
+        del globals()[name]
+    del s1, s2, name
+    from _decimal import *
 
 if __name__ == '__main__':
     import doctest, decimal
diff --git a/Lib/difflib.py b/Lib/difflib.py
index e6cc6ee442..ae377d745d 100644
--- a/Lib/difflib.py
+++ b/Lib/difflib.py
@@ -204,7 +204,7 @@ class SequenceMatcher:
         #      returning true iff the element is "junk" -- this has
         #      subtle but helpful effects on the algorithm, which I'll
         #      get around to writing up someday <0.9 wink>.
-        #      DON'T USE!  Only __chain_b uses this.  Use isbjunk.
+        #      DON'T USE!  Only __chain_b uses this.  Use "in self.bjunk".
         # bjunk
         #      the items in b for which isjunk is True.
         # bpopular
@@ -287,7 +287,6 @@ class SequenceMatcher:
     # when self.isjunk is defined, junk elements don't show up in this
     # map at all, which stops the central find_longest_match method
     # from starting any matching block at a junk element ...
-    # also creates the fast isbjunk function ...
     # b2j also does not contain entries for "popular" elements, meaning
     # elements that account for more than 1 + 1% of the total elements, and
     # when the sequence is reasonably large (>= 200 elements); this can
@@ -800,7 +799,7 @@ class Differ:
     ...   2. Explicit is better than implicit.
     ...   3. Simple is better than complex.
     ...   4. Complex is better than complicated.
-    ... '''.splitlines(1)
+    ... '''.splitlines(keepends=True)
     >>> len(text1)
     4
     >>> text1[0][-1]
@@ -809,7 +808,7 @@ class Differ:
     ...   3.   Simple is better than complex.
     ...   4. Complicated is better than complex.
     ...   5. Flat is better than nested.
-    ... '''.splitlines(1)
+    ... '''.splitlines(keepends=True)
 
     Next we instantiate a Differ object:
 
@@ -896,8 +895,8 @@ class Differ:
 
         Example:
 
-        >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(1),
-        ...                                'ore\ntree\nemu\n'.splitlines(1))),
+        >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(True),
+        ...                                'ore\ntree\nemu\n'.splitlines(True))),
         ...       end="")
         - one
         ?  ^
@@ -1269,8 +1268,8 @@ def context_diff(a, b, fromfile='', tofile='',
 
     Example:
 
-    >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(1),
-    ...       'zero\none\ntree\nfour\n'.splitlines(1), 'Original', 'Current')),
+    >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True),
+    ...       'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')),
     ...       end="")
     *** Original
     --- Current
@@ -1339,8 +1338,8 @@ def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK):
 
     Example:
 
-    >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1),
-    ...              'ore\ntree\nemu\n'.splitlines(1))
+    >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True),
+    ...              'ore\ntree\nemu\n'.splitlines(keepends=True))
     >>> print(''.join(diff), end="")
     - one
     ?  ^
@@ -2034,8 +2033,8 @@ def restore(delta, which):
 
     Examples:
 
-    >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1),
-    ...              'ore\ntree\nemu\n'.splitlines(1))
+    >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True),
+    ...              'ore\ntree\nemu\n'.splitlines(keepends=True))
     >>> diff = list(diff)
     >>> print(''.join(restore(diff, 1)), end="")
     one
diff --git a/Lib/dis.py b/Lib/dis.py
index f64bae66fb..543fdc7ed0 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -190,6 +190,9 @@ def disassemble(co, lasti=-1):
                 if free is None:
                     free = co.co_cellvars + co.co_freevars
                 print('(' + free[oparg] + ')', end=' ')
+            elif op in hasnargs:
+                print('(%d positional, %d keyword pair)'
+                      % (code[i-2], code[i-1]), end=' ')
         print()
 
 def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
@@ -229,6 +232,9 @@ def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
                     print('(%d)' % oparg, end=' ')
             elif op in hascompare:
                 print('(' + cmp_op[oparg] + ')', end=' ')
+            elif op in hasnargs:
+                print('(%d positional, %d keyword pair)'
+                      % (code[i-2], code[i-1]), end=' ')
         print()
 
 def _disassemble_str(source):
diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py
index b52a9fe6c4..345ac4f8dd 100644
--- a/Lib/distutils/__init__.py
+++ b/Lib/distutils/__init__.py
@@ -13,5 +13,5 @@ used from a setup script as
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "3.2.3"
+__version__ = "3.3.0"
 #--end constants--
diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py
index e3ed3ad82c..959a8bf62e 100644
--- a/Lib/distutils/command/bdist_wininst.py
+++ b/Lib/distutils/command/bdist_wininst.py
@@ -265,11 +265,11 @@ class bdist_wininst(Command):
         cfgdata = cfgdata + b"\0"
         if self.pre_install_script:
             # We need to normalize newlines, so we open in text mode and
-            # convert back to bytes. "latin1" simply avoids any possible
+            # convert back to bytes. "latin-1" simply avoids any possible
             # failures.
             with open(self.pre_install_script, "r",
-                encoding="latin1") as script:
-                script_data = script.read().encode("latin1")
+                encoding="latin-1") as script:
+                script_data = script.read().encode("latin-1")
             cfgdata = cfgdata + script_data + b"\n\0"
         else:
             # empty pre-install script
diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
index 34b61bdb82..b1d951e6f8 100644
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -8,6 +8,7 @@ import sys, os, re
 from distutils.core import Command
 from distutils.errors import *
 from distutils.sysconfig import customize_compiler, get_python_version
+from distutils.sysconfig import get_config_h_filename
 from distutils.dep_util import newer_group
 from distutils.extension import Extension
 from distutils.util import get_platform
@@ -159,6 +160,11 @@ class build_ext(Command):
         if isinstance(self.include_dirs, str):
             self.include_dirs = self.include_dirs.split(os.pathsep)
 
+        # If in a virtualenv, add its include directory
+        # Issue 16116
+        if sys.exec_prefix != sys.base_exec_prefix:
+            self.include_dirs.append(os.path.join(sys.exec_prefix, 'include'))
+
         # Put the Python "system" include dir at the end, so that
         # any local include dirs take precedence.
         self.include_dirs.append(py_include)
@@ -189,6 +195,8 @@ class build_ext(Command):
             # must be the *native* platform.  But we don't really support
             # cross-compiling via a binary install anyway, so we let it go.
             self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs'))
+            if sys.base_exec_prefix != sys.prefix:  # Issue 16116
+                self.library_dirs.append(os.path.join(sys.base_exec_prefix, 'libs'))
             if self.debug:
                 self.build_temp = os.path.join(self.build_temp, "Debug")
             else:
@@ -196,8 +204,11 @@ class build_ext(Command):
 
             # Append the source distribution include and library directories,
             # this allows distutils on windows to work in the source tree
-            self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
-            if MSVC_VERSION == 9:
+            self.include_dirs.append(os.path.dirname(get_config_h_filename()))
+            _sys_home = getattr(sys, '_home', None)
+            if _sys_home:
+                self.library_dirs.append(_sys_home)
+            if MSVC_VERSION >= 9:
                 # Use the .lib files for the correct architecture
                 if self.plat_name == 'win32':
                     suffix = ''
@@ -239,8 +250,7 @@ class build_ext(Command):
         # for extensions under Linux or Solaris with a shared Python library,
         # Python's library directory must be appended to library_dirs
         sysconfig.get_config_var('Py_ENABLE_SHARED')
-        if ((sys.platform.startswith('linux') or sys.platform.startswith('gnu')
-             or sys.platform.startswith('sunos'))
+        if (sys.platform.startswith(('linux', 'gnu', 'sunos'))
             and sysconfig.get_config_var('Py_ENABLE_SHARED')):
             if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
                 # building third party extensions
diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py
index ec43477061..4b5b22ec20 100644
--- a/Lib/distutils/command/build_scripts.py
+++ b/Lib/distutils/command/build_scripts.py
@@ -126,10 +126,9 @@ class build_scripts(Command):
                             "The shebang ({!r}) is not decodable "
                             "from the script encoding ({})"
                             .format(shebang, encoding))
-                    outf = open(outfile, "wb")
-                    outf.write(shebang)
-                    outf.writelines(f.readlines())
-                    outf.close()
+                    with open(outfile, "wb") as outf:
+                        outf.write(shebang)
+                        outf.writelines(f.readlines())
                 if f:
                     f.close()
             else:
diff --git a/Lib/distutils/command/wininst-10.0-amd64.exe b/Lib/distutils/command/wininst-10.0-amd64.exe
new file mode 100644
index 0000000000..6fa0dce163
--- /dev/null
+++ b/Lib/distutils/command/wininst-10.0-amd64.exe
diff --git a/Lib/distutils/command/wininst-10.0.exe b/Lib/distutils/command/wininst-10.0.exe
new file mode 100644
index 0000000000..afc3bc6c14
--- /dev/null
+++ b/Lib/distutils/command/wininst-10.0.exe
diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py
index 819e1a97be..0bdd539c37 100644
--- a/Lib/distutils/cygwinccompiler.py
+++ b/Lib/distutils/cygwinccompiler.py
@@ -78,6 +78,9 @@ def get_msvcr():
         elif msc_ver == '1500':
             # VS2008 / MSVC 9.0
             return ['msvcr90']
+        elif msc_ver == '1600':
+            # VS2010 / MSVC 10.0
+            return ['msvcr100']
         else:
             raise ValueError("Unknown MS Compiler version %s " % msc_ver)
 
diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
index b6007a904f..d125e0b44a 100644
--- a/Lib/distutils/sysconfig.py
+++ b/Lib/distutils/sysconfig.py
@@ -18,11 +18,17 @@ from .errors import DistutilsPlatformError
 # These are needed in a couple of spots, so just compute them once.
 PREFIX = os.path.normpath(sys.prefix)
 EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+BASE_PREFIX = os.path.normpath(sys.base_prefix)
+BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
 
 # Path to the base directory of the project. On Windows the binary may
 # live in project/PCBuild9.  If we're dealing with an x64 Windows build,
 # it'll live in project/PCbuild/amd64.
-project_base = os.path.dirname(os.path.abspath(sys.executable))
+# set for cross builds
+if "_PYTHON_PROJECT_BASE" in os.environ:
+    project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"])
+else:
+    project_base = os.path.dirname(os.path.abspath(sys.executable))
 if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
     project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
 # PC/VS7.1
@@ -39,11 +45,21 @@ if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
 # different (hard-wired) directories.
 # Setup.local is available for Makefile builds including VPATH builds,
 # Setup.dist is available on Windows
-def _python_build():
+def _is_python_source_dir(d):
     for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
+        if os.path.isfile(os.path.join(d, "Modules", fn)):
             return True
     return False
+_sys_home = getattr(sys, '_home', None)
+if _sys_home and os.name == 'nt' and \
+    _sys_home.lower().endswith(('pcbuild', 'pcbuild\\amd64')):
+    _sys_home = os.path.dirname(_sys_home)
+    if _sys_home.endswith('pcbuild'):   # must be amd64
+        _sys_home = os.path.dirname(_sys_home)
+def _python_build():
+    if _sys_home:
+        return _is_python_source_dir(_sys_home)
+    return _is_python_source_dir(project_base)
 python_build = _python_build()
 
 # Calculate the build qualifier flags if they are defined.  Adding the flags
@@ -74,11 +90,11 @@ def get_python_inc(plat_specific=0, prefix=None):
     otherwise, this is the path to platform-specific header files
     (namely pyconfig.h).
 
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
+    If 'prefix' is supplied, use it instead of sys.base_prefix or
+    sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
     """
     if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
+        prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
     if os.name == "posix":
         if python_build:
             # Assume the executable is in the build directory.  The
@@ -86,12 +102,14 @@ def get_python_inc(plat_specific=0, prefix=None):
             # the build directory may not be the source directory, we
             # must use "srcdir" from the makefile to find the "Include"
             # directory.
-            base = os.path.dirname(os.path.abspath(sys.executable))
+            base = _sys_home or project_base
             if plat_specific:
                 return base
+            if _sys_home:
+                incdir = os.path.join(_sys_home, get_config_var('AST_H_DIR'))
             else:
                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
-                return os.path.normpath(incdir)
+            return os.path.normpath(incdir)
         python_dir = 'python' + get_python_version() + build_flags
         return os.path.join(prefix, "include", python_dir)
     elif os.name == "nt":
@@ -115,11 +133,14 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
     containing standard Python library modules; otherwise, return the
     directory for site-specific modules.
 
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
+    If 'prefix' is supplied, use it instead of sys.base_prefix or
+    sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
     """
     if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
+        if standard_lib:
+            prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
+        else:
+            prefix = plat_specific and EXEC_PREFIX or PREFIX
 
     if os.name == "posix":
         libpython = os.path.join(prefix,
@@ -218,9 +239,9 @@ def get_config_h_filename():
     """Return full pathname of installed pyconfig.h file."""
     if python_build:
         if os.name == "nt":
-            inc_dir = os.path.join(project_base, "PC")
+            inc_dir = os.path.join(_sys_home or project_base, "PC")
         else:
-            inc_dir = project_base
+            inc_dir = _sys_home or project_base
     else:
         inc_dir = get_python_inc(plat_specific=1)
     if get_python_version() < '2.2':
@@ -234,7 +255,7 @@ def get_config_h_filename():
 def get_makefile_filename():
     """Return full pathname of installed Makefile from the Python build."""
     if python_build:
-        return os.path.join(os.path.dirname(sys.executable), "Makefile")
+        return os.path.join(_sys_home or project_base, "Makefile")
     lib_dir = get_python_lib(plat_specific=0, standard_lib=1)
     config_file = 'config-{}{}'.format(get_python_version(), build_flags)
     return os.path.join(lib_dir, config_file, 'Makefile')
@@ -496,7 +517,7 @@ def get_config_vars(*args):
     variables relevant for the current platform.  Generally this includes
     everything needed to build extensions and install both pure modules and
     extensions.  On Unix, this means every variable defined in Python's
-    installed Makefile; on Windows and Mac OS it's a much smaller set.
+    installed Makefile; on Windows it's a much smaller set.
 
     With arguments, return a list of values that result from looking up
     each argument in the configuration variable dictionary.
@@ -515,12 +536,29 @@ def get_config_vars(*args):
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # Always convert srcdir to an absolute path
+        srcdir = _config_vars.get('srcdir', project_base)
+        if os.name == 'posix':
+            if python_build:
+                # If srcdir is a relative path (typically '.' or '..')
+                # then it should be interpreted relative to the directory
+                # containing Makefile.
+                base = os.path.dirname(get_makefile_filename())
+                srcdir = os.path.join(base, srcdir)
+            else:
+                # srcdir is not meaningful since the installation is
+                # spread about the filesystem.  We choose the
+                # directory containing the Makefile since we know it
+                # exists.
+                srcdir = os.path.dirname(get_makefile_filename())
+        _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir))
+
         # Convert srcdir into an absolute path if it appears necessary.
         # Normally it is relative to the build directory.  However, during
         # testing, for example, we might be running a non-installed python
         # from a different directory.
         if python_build and os.name == "posix":
-            base = os.path.dirname(os.path.abspath(sys.executable))
+            base = project_base
             if (not os.path.isabs(_config_vars['srcdir']) and
                 base != os.getcwd()):
                 # srcdir is relative and we are not in the same directory
diff --git a/Lib/distutils/tests/test_archive_util.py b/Lib/distutils/tests/test_archive_util.py
index 8edfab49f8..1afdd46225 100644
--- a/Lib/distutils/tests/test_archive_util.py
+++ b/Lib/distutils/tests/test_archive_util.py
@@ -1,6 +1,8 @@
+# -*- coding: utf-8 -*-
 """Tests for distutils.archive_util."""
 import unittest
 import os
+import sys
 import tarfile
 from os.path import splitdrive
 import warnings
@@ -25,6 +27,18 @@ try:
 except ImportError:
     ZLIB_SUPPORT = False
 
+def can_fs_encode(filename):
+    """
+    Return True if the filename can be saved in the file system.
+    """
+    if os.path.supports_unicode_filenames:
+        return True
+    try:
+        filename.encode(sys.getfilesystemencoding())
+    except UnicodeEncodeError:
+        return False
+    return True
+
 
 class ArchiveUtilTestCase(support.TempdirManager,
                           support.LoggingSilencer,
@@ -32,6 +46,28 @@ class ArchiveUtilTestCase(support.TempdirManager,
 
     @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
     def test_make_tarball(self):
+        self._make_tarball('archive')
+
+    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
+    @unittest.skipUnless(can_fs_encode('Ã¥rchiv'),
+        'File system cannot handle this filename')
+    def test_make_tarball_latin1(self):
+        """
+        Mirror test_make_tarball, except filename contains latin characters.
+        """
+        self._make_tarball('Ã¥rchiv') # note this isn't a real word
+
+    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
+    @unittest.skipUnless(can_fs_encode('�アーカイブ'),
+        'File system cannot handle this filename')
+    def test_make_tarball_extended(self):
+        """
+        Mirror test_make_tarball, except filename contains extended
+        characters outside the latin charset.
+        """
+        self._make_tarball('�アーカイブ') # japanese for archive
+
+    def _make_tarball(self, target_name):
         # creating something to tar
         tmpdir = self.mkdtemp()
         self.write_file([tmpdir, 'file1'], 'xxx')
@@ -43,7 +79,7 @@ class ArchiveUtilTestCase(support.TempdirManager,
         unittest.skipUnless(splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0],
                             "Source and target should be on same drive")
 
-        base_name = os.path.join(tmpdir2, 'archive')
+        base_name = os.path.join(tmpdir2, target_name)
 
         # working with relative paths to avoid tar warnings
         old_dir = os.getcwd()
@@ -58,7 +94,7 @@ class ArchiveUtilTestCase(support.TempdirManager,
         self.assertTrue(os.path.exists(tarball))
 
         # trying an uncompressed one
-        base_name = os.path.join(tmpdir2, 'archive')
+        base_name = os.path.join(tmpdir2, target_name)
         old_dir = os.getcwd()
         os.chdir(tmpdir)
         try:
diff --git a/Lib/distutils/tests/test_bdist_rpm.py b/Lib/distutils/tests/test_bdist_rpm.py
index ab7a1bf24e..b090b79e0c 100644
--- a/Lib/distutils/tests/test_bdist_rpm.py
+++ b/Lib/distutils/tests/test_bdist_rpm.py
@@ -28,6 +28,11 @@ class BuildRpmTestCase(support.TempdirManager,
                        unittest.TestCase):
 
     def setUp(self):
+        try:
+            sys.executable.encode("UTF-8")
+        except UnicodeEncodeError:
+            raise unittest.SkipTest("sys.executable is not encodable to UTF-8")
+
         super(BuildRpmTestCase, self).setUp()
         self.old_location = os.getcwd()
         self.old_sys_argv = sys.argv, sys.argv[:]
@@ -42,7 +47,7 @@ class BuildRpmTestCase(support.TempdirManager,
 
         # XXX I am unable yet to make this test work without
         # spurious sdtout/stderr output under Mac OS X
-        if sys.platform != 'linux2':
+        if not sys.platform.startswith('linux'):
             return
 
         # this test will run only if the rpm commands are found
@@ -86,7 +91,7 @@ class BuildRpmTestCase(support.TempdirManager,
 
         # XXX I am unable yet to make this test work without
         # spurious sdtout/stderr output under Mac OS X
-        if sys.platform != 'linux2':
+        if not sys.platform.startswith('linux'):
             return
 
         # http://bugs.python.org/issue1533164
diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py
index 545ef3b548..826ea4247d 100644
--- a/Lib/distutils/tests/test_sysconfig.py
+++ b/Lib/distutils/tests/test_sysconfig.py
@@ -53,6 +53,35 @@ class SysconfigTestCase(support.EnvironGuard,
         self.assertTrue(isinstance(cvars, dict))
         self.assertTrue(cvars)
 
+    def test_srcdir(self):
+        # See Issues #15322, #15364.
+        srcdir = sysconfig.get_config_var('srcdir')
+
+        self.assertTrue(os.path.isabs(srcdir), srcdir)
+        self.assertTrue(os.path.isdir(srcdir), srcdir)
+
+        if sysconfig.python_build:
+            # The python executable has not been installed so srcdir
+            # should be a full source checkout.
+            Python_h = os.path.join(srcdir, 'Include', 'Python.h')
+            self.assertTrue(os.path.exists(Python_h), Python_h)
+            self.assertTrue(sysconfig._is_python_source_dir(srcdir))
+        elif os.name == 'posix':
+            self.assertEqual(os.path.dirname(sysconfig.get_makefile_filename()),
+                                 srcdir)
+
+    def test_srcdir_independent_of_cwd(self):
+        # srcdir should be independent of the current working directory
+        # See Issues #15322, #15364.
+        srcdir = sysconfig.get_config_var('srcdir')
+        cwd = os.getcwd()
+        try:
+            os.chdir('..')
+            srcdir2 = sysconfig.get_config_var('srcdir')
+        finally:
+            os.chdir(cwd)
+        self.assertEqual(srcdir, srcdir2)
+
     def test_customize_compiler(self):
 
         # not testing if default compiler is not unix
diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py
index 52280db0b2..67d8166349 100644
--- a/Lib/distutils/util.py
+++ b/Lib/distutils/util.py
@@ -53,6 +53,10 @@ def get_platform ():
             return 'win-ia64'
         return sys.platform
 
+    # Set for cross builds explicitly
+    if "_PYTHON_HOST_PLATFORM" in os.environ:
+        return os.environ["_PYTHON_HOST_PLATFORM"]
+
     if os.name != "posix" or not hasattr(os, 'uname'):
         # XXX what about the architecture? NT is Intel or Alpha,
         # Mac OS is M68k or PPC, etc.
diff --git a/Lib/doctest.py b/Lib/doctest.py
index e189c8feba..3af05fb87e 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -458,7 +458,6 @@ class Example:
         return hash((self.source, self.want, self.lineno, self.indent,
                      self.exc_msg))
 
-
 class DocTest:
     """
     A collection of doctest examples that should be run in a single
@@ -1367,7 +1366,7 @@ class DocTestRunner:
         m = self.__LINECACHE_FILENAME_RE.match(filename)
         if m and m.group('name') == self.test.name:
             example = self.test.examples[int(m.group('examplenum'))]
-            return example.source.splitlines(True)
+            return example.source.splitlines(keepends=True)
         else:
             return self.save_linecache_getlines(filename, module_globals)
 
@@ -1413,6 +1412,7 @@ class DocTestRunner:
         # Note that the interactive output will go to *our*
         # save_stdout, even if that's not the real sys.stdout; this
         # allows us to write test cases for the set_trace behavior.
+        save_trace = sys.gettrace()
         save_set_trace = pdb.set_trace
         self.debugger = _OutputRedirectingPdb(save_stdout)
         self.debugger.reset()
@@ -1432,6 +1432,7 @@ class DocTestRunner:
         finally:
             sys.stdout = save_stdout
             pdb.set_trace = save_set_trace
+            sys.settrace(save_trace)
             linecache.getlines = self.save_linecache_getlines
             sys.displayhook = save_displayhook
             if clear_globs:
@@ -1628,8 +1629,8 @@ class OutputChecker:
         # Check if we should use diff.
         if self._do_a_fancy_diff(want, got, optionflags):
             # Split want & got into lines.
-            want_lines = want.splitlines(True)  # True == keep line ends
-            got_lines = got.splitlines(True)
+            want_lines = want.splitlines(keepends=True)
+            got_lines = got.splitlines(keepends=True)
             # Use difflib to find their differences.
             if optionflags & REPORT_UDIFF:
                 diff = difflib.unified_diff(want_lines, got_lines, n=2)
diff --git a/Lib/email/_encoded_words.py b/Lib/email/_encoded_words.py
new file mode 100644
index 0000000000..e9f6e20bd7
--- /dev/null
+++ b/Lib/email/_encoded_words.py
@@ -0,0 +1,221 @@
+""" Routines for manipulating RFC2047 encoded words.
+
+This is currently a package-private API, but will be considered for promotion
+to a public API if there is demand.
+
+"""
+
+# An ecoded word looks like this:
+#
+#        =?charset[*lang]?cte?encoded_string?=
+#
+# for more information about charset see the charset module.  Here it is one
+# of the preferred MIME charset names (hopefully; you never know when parsing).
+# cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case).  In
+# theory other letters could be used for other encodings, but in practice this
+# (almost?) never happens.  There could be a public API for adding entries
+# to to the CTE tables, but YAGNI for now.  'q' is Quoted Printable, 'b' is
+# Base64.  The meaning of encoded_string should be obvious.  'lang' is optional
+# as indicated by the brackets (they are not part of the syntax) but is almost
+# never encountered in practice.
+#
+# The general interface for a CTE decoder is that it takes the encoded_string
+# as its argument, and returns a tuple (cte_decoded_string, defects).  The
+# cte_decoded_string is the original binary that was encoded using the
+# specified cte.  'defects' is a list of MessageDefect instances indicating any
+# problems encountered during conversion.  'charset' and 'lang' are the
+# corresponding strings extracted from the EW, case preserved.
+#
+# The general interface for a CTE encoder is that it takes a binary sequence
+# as input and returns the cte_encoded_string, which is an ascii-only string.
+#
+# Each decoder must also supply a length function that takes the binary
+# sequence as its argument and returns the length of the resulting encoded
+# string.
+#
+# The main API functions for the module are decode, which calls the decoder
+# referenced by the cte specifier, and encode, which adds the appropriate
+# RFC 2047 "chrome" to the encoded string, and can optionally automatically
+# select the shortest possible encoding.  See their docstrings below for
+# details.
+
+import re
+import base64
+import binascii
+import functools
+from string import ascii_letters, digits
+from email import errors
+
+__all__ = ['decode_q',
+           'encode_q',
+           'decode_b',
+           'encode_b',
+           'len_q',
+           'len_b',
+           'decode',
+           'encode',
+           ]
+
+#
+# Quoted Printable
+#
+
+# regex based decoder.
+_q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub,
+        lambda m: bytes([int(m.group(1), 16)]))
+
+def decode_q(encoded):
+    encoded = encoded.replace(b'_', b' ')
+    return _q_byte_subber(encoded), []
+
+
+# dict mapping bytes to their encoded form
+class _QByteMap(dict):
+
+    safe = b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii')
+
+    def __missing__(self, key):
+        if key in self.safe:
+            self[key] = chr(key)
+        else:
+            self[key] = "={:02X}".format(key)
+        return self[key]
+
+_q_byte_map = _QByteMap()
+
+# In headers spaces are mapped to '_'.
+_q_byte_map[ord(' ')] = '_'
+
+def encode_q(bstring):
+    return ''.join(_q_byte_map[x] for x in bstring)
+
+def len_q(bstring):
+    return sum(len(_q_byte_map[x]) for x in bstring)
+
+
+#
+# Base64
+#
+
+def decode_b(encoded):
+    defects = []
+    pad_err = len(encoded) % 4
+    if pad_err:
+        defects.append(errors.InvalidBase64PaddingDefect())
+        padded_encoded = encoded + b'==='[:4-pad_err]
+    else:
+        padded_encoded = encoded
+    try:
+        return base64.b64decode(padded_encoded, validate=True), defects
+    except binascii.Error:
+        # Since we had correct padding, this must an invalid char error.
+        defects = [errors.InvalidBase64CharactersDefect()]
+        # The non-alphabet characters are ignored as far as padding
+        # goes, but we don't know how many there are.  So we'll just
+        # try various padding lengths until something works.
+        for i in 0, 1, 2, 3:
+            try:
+                return base64.b64decode(encoded+b'='*i, validate=False), defects
+            except binascii.Error:
+                if i==0:
+                    defects.append(errors.InvalidBase64PaddingDefect())
+        else:
+            # This should never happen.
+            raise AssertionError("unexpected binascii.Error")
+
+def encode_b(bstring):
+    return base64.b64encode(bstring).decode('ascii')
+
+def len_b(bstring):
+    groups_of_3, leftover = divmod(len(bstring), 3)
+    # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in.
+    return groups_of_3 * 4 + (4 if leftover else 0)
+
+
+_cte_decoders = {
+    'q': decode_q,
+    'b': decode_b,
+    }
+
+def decode(ew):
+    """Decode encoded word and return (string, charset, lang, defects) tuple.
+
+    An RFC 2047/2243 encoded word has the form:
+
+        =?charset*lang?cte?encoded_string?=
+
+    where '*lang' may be omitted but the other parts may not be.
+
+    This function expects exactly such a string (that is, it does not check the
+    syntax and may raise errors if the string is not well formed), and returns
+    the encoded_string decoded first from its Content Transfer Encoding and
+    then from the resulting bytes into unicode using the specified charset.  If
+    the cte-decoded string does not successfully decode using the specified
+    character set, a defect is added to the defects list and the unknown octets
+    are replaced by the unicode 'unknown' character \uFDFF.
+
+    The specified charset and language are returned.  The default for language,
+    which is rarely if ever encountered, is the empty string.
+
+    """
+    _, charset, cte, cte_string, _ = ew.split('?')
+    charset, _, lang = charset.partition('*')
+    cte = cte.lower()
+    # Recover the original bytes and do CTE decoding.
+    bstring = cte_string.encode('ascii', 'surrogateescape')
+    bstring, defects = _cte_decoders[cte](bstring)
+    # Turn the CTE decoded bytes into unicode.
+    try:
+        string = bstring.decode(charset)
+    except UnicodeError:
+        defects.append(errors.UndecodableBytesDefect("Encoded word "
+            "contains bytes not decodable using {} charset".format(charset)))
+        string = bstring.decode(charset, 'surrogateescape')
+    except LookupError:
+        string = bstring.decode('ascii', 'surrogateescape')
+        if charset.lower() != 'unknown-8bit':
+            defects.append(errors.CharsetError("Unknown charset {} "
+                "in encoded word; decoded as unknown bytes".format(charset)))
+    return string, charset, lang, defects
+
+
+_cte_encoders = {
+    'q': encode_q,
+    'b': encode_b,
+    }
+
+_cte_encode_length = {
+    'q': len_q,
+    'b': len_b,
+    }
+
+def encode(string, charset='utf-8', encoding=None, lang=''):
+    """Encode string using the CTE encoding that produces the shorter result.
+
+    Produces an RFC 2047/2243 encoded word of the form:
+
+        =?charset*lang?cte?encoded_string?=
+
+    where '*lang' is omitted unless the 'lang' parameter is given a value.
+    Optional argument charset (defaults to utf-8) specifies the charset to use
+    to encode the string to binary before CTE encoding it.  Optional argument
+    'encoding' is the cte specifier for the encoding that should be used ('q'
+    or 'b'); if it is None (the default) the encoding which produces the
+    shortest encoded sequence is used, except that 'q' is preferred if it is up
+    to five characters longer.  Optional argument 'lang' (default '') gives the
+    RFC 2243 language string to specify in the encoded word.
+
+    """
+    if charset == 'unknown-8bit':
+        bstring = string.encode('ascii', 'surrogateescape')
+    else:
+        bstring = string.encode(charset)
+    if encoding is None:
+        qlen = _cte_encode_length['q'](bstring)
+        blen = _cte_encode_length['b'](bstring)
+        # Bias toward q.  5 is arbitrary.
+        encoding = 'q' if qlen - blen < 5 else 'b'
+    encoded = _cte_encoders[encoding](bstring)
+    if lang:
+        lang = '*' + lang
+    return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded)
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
new file mode 100644
index 0000000000..1924ed1588
--- /dev/null
+++ b/Lib/email/_header_value_parser.py
@@ -0,0 +1,2953 @@
+"""Header value parser implementing various email-related RFC parsing rules.
+
+The parsing methods defined in this module implement various email related
+parsing rules.  Principal among them is RFC 5322, which is the followon
+to RFC 2822 and primarily a clarification of the former.  It also implements
+RFC 2047 encoded word decoding.
+
+RFC 5322 goes to considerable trouble to maintain backward compatibility with
+RFC 822 in the parse phase, while cleaning up the structure on the generation
+phase.  This parser supports correct RFC 5322 generation by tagging white space
+as folding white space only when folding is allowed in the non-obsolete rule
+sets.  Actually, the parser is even more generous when accepting input than RFC
+5322 mandates, following the spirit of Postel's Law, which RFC 5322 encourages.
+Where possible deviations from the standard are annotated on the 'defects'
+attribute of tokens that deviate.
+
+The general structure of the parser follows RFC 5322, and uses its terminology
+where there is a direct correspondence.  Where the implementation requires a
+somewhat different structure than that used by the formal grammar, new terms
+that mimic the closest existing terms are used.  Thus, it really helps to have
+a copy of RFC 5322 handy when studying this code.
+
+Input to the parser is a string that has already been unfolded according to
+RFC 5322 rules.  According to the RFC this unfolding is the very first step, and
+this parser leaves the unfolding step to a higher level message parser, which
+will have already detected the line breaks that need unfolding while
+determining the beginning and end of each header.
+
+The output of the parser is a TokenList object, which is a list subclass.  A
+TokenList is a recursive data structure.  The terminal nodes of the structure
+are Terminal objects, which are subclasses of str.  These do not correspond
+directly to terminal objects in the formal grammar, but are instead more
+practical higher level combinations of true terminals.
+
+All TokenList and Terminal objects have a 'value' attribute, which produces the
+semantically meaningful value of that part of the parse subtree.  The value of
+all whitespace tokens (no matter how many sub-tokens they may contain) is a
+single space, as per the RFC rules.  This includes 'CFWS', which is herein
+included in the general class of whitespace tokens.  There is one exception to
+the rule that whitespace tokens are collapsed into single spaces in values: in
+the value of a 'bare-quoted-string' (a quoted-string with no leading or
+trailing whitespace), any whitespace that appeared between the quotation marks
+is preserved in the returned value.  Note that in all Terminal strings quoted
+pairs are turned into their unquoted values.
+
+All TokenList and Terminal objects also have a string value, which attempts to
+be a "canonical" representation of the RFC-compliant form of the substring that
+produced the parsed subtree, including minimal use of quoted pair quoting.
+Whitespace runs are not collapsed.
+
+Comment tokens also have a 'content' attribute providing the string found
+between the parens (including any nested comments) with whitespace preserved.
+
+All TokenList and Terminal objects have a 'defects' attribute which is a
+possibly empty list all of the defects found while creating the token.  Defects
+may appear on any token in the tree, and a composite list of all defects in the
+subtree is available through the 'all_defects' attribute of any node.  (For
+Terminal notes x.defects == x.all_defects.)
+
+Each object in a parse tree is called a 'token', and each has a 'token_type'
+attribute that gives the name from the RFC 5322 grammar that it represents.
+Not all RFC 5322 nodes are produced, and there is one non-RFC 5322 node that
+may be produced: 'ptext'.  A 'ptext' is a string of printable ascii characters.
+It is returned in place of lists of (ctext/quoted-pair) and
+(qtext/quoted-pair).
+
+XXX: provide complete list of token types.
+"""
+
+import re
+import urllib   # For urllib.parse.unquote
+from collections import namedtuple, OrderedDict
+from email import _encoded_words as _ew
+from email import errors
+from email import utils
+
+#
+# Useful constants and functions
+#
+
+WSP = set(' \t')
+CFWS_LEADER = WSP | set('(')
+SPECIALS = set(r'()<>@,:;.\"[]')
+ATOM_ENDS = SPECIALS | WSP
+DOT_ATOM_ENDS = ATOM_ENDS - set('.')
+# '.', '"', and '(' do not end phrases in order to support obs-phrase
+PHRASE_ENDS = SPECIALS - set('."(')
+TSPECIALS = (SPECIALS | set('/?=')) - set('.')
+TOKEN_ENDS = TSPECIALS | WSP
+ASPECIALS = TSPECIALS | set("*'%")
+ATTRIBUTE_ENDS = ASPECIALS | WSP
+EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%')
+
+def quote_string(value):
+    return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"'
+
+#
+# Accumulator for header folding
+#
+
+class _Folded:
+
+    def __init__(self, maxlen, policy):
+        self.maxlen = maxlen
+        self.policy = policy
+        self.lastlen = 0
+        self.stickyspace = None
+        self.firstline = True
+        self.done = []
+        self.current = []
+
+    def newline(self):
+        self.done.extend(self.current)
+        self.done.append(self.policy.linesep)
+        self.current.clear()
+        self.lastlen = 0
+
+    def finalize(self):
+        if self.current:
+            self.newline()
+
+    def __str__(self):
+        return ''.join(self.done)
+
+    def append(self, stoken):
+        self.current.append(stoken)
+
+    def append_if_fits(self, token, stoken=None):
+        if stoken is None:
+            stoken = str(token)
+        l = len(stoken)
+        if self.stickyspace is not None:
+            stickyspace_len = len(self.stickyspace)
+            if self.lastlen + stickyspace_len + l <= self.maxlen:
+                self.current.append(self.stickyspace)
+                self.lastlen += stickyspace_len
+                self.current.append(stoken)
+                self.lastlen += l
+                self.stickyspace = None
+                self.firstline = False
+                return True
+            if token.has_fws:
+                ws = token.pop_leading_fws()
+                if ws is not None:
+                    self.stickyspace += str(ws)
+                    stickyspace_len += len(ws)
+                token._fold(self)
+                return True
+            if stickyspace_len and l + 1 <= self.maxlen:
+                margin = self.maxlen - l
+                if 0 < margin < stickyspace_len:
+                    trim = stickyspace_len - margin
+                    self.current.append(self.stickyspace[:trim])
+                    self.stickyspace = self.stickyspace[trim:]
+                    stickyspace_len = trim
+                self.newline()
+                self.current.append(self.stickyspace)
+                self.current.append(stoken)
+                self.lastlen = l + stickyspace_len
+                self.stickyspace = None
+                self.firstline = False
+                return True
+            if not self.firstline:
+                self.newline()
+            self.current.append(self.stickyspace)
+            self.current.append(stoken)
+            self.stickyspace = None
+            self.firstline = False
+            return True
+        if self.lastlen + l <= self.maxlen:
+            self.current.append(stoken)
+            self.lastlen += l
+            return True
+        if l < self.maxlen:
+            self.newline()
+            self.current.append(stoken)
+            self.lastlen = l
+            return True
+        return False
+
+#
+# TokenList and its subclasses
+#
+
+class TokenList(list):
+
+    token_type = None
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self.defects = []
+
+    def __str__(self):
+        return ''.join(str(x) for x in self)
+
+    def __repr__(self):
+        return '{}({})'.format(self.__class__.__name__,
+                             super().__repr__())
+
+    @property
+    def value(self):
+        return ''.join(x.value for x in self if x.value)
+
+    @property
+    def all_defects(self):
+        return sum((x.all_defects for x in self), self.defects)
+
+    #
+    # Folding API
+    #
+    # parts():
+    #
+    # return a list of objects that constitute the "higher level syntactic
+    # objects" specified by the RFC as the best places to fold a header line.
+    # The returned objects must include leading folding white space, even if
+    # this means mutating the underlying parse tree of the object.  Each object
+    # is only responsible for returning *its* parts, and should not drill down
+    # to any lower level except as required to meet the leading folding white
+    # space constraint.
+    #
+    # _fold(folded):
+    #
+    #   folded: the result accumulator.  This is an instance of _Folded.
+    #       (XXX: I haven't finished factoring this out yet, the folding code
+    #       pretty much uses this as a state object.) When the folded.current
+    #       contains as much text as will fit, the _fold method should call
+    #       folded.newline.
+    #  folded.lastlen: the current length of the test stored in folded.current.
+    #  folded.maxlen: The maximum number of characters that may appear on a
+    #       folded line.  Differs from the policy setting in that "no limit" is
+    #       represented by +inf, which means it can be used in the trivially
+    #       logical fashion in comparisons.
+    #
+    # Currently no subclasses implement parts, and I think this will remain
+    # true.  A subclass only needs to implement _fold when the generic version
+    # isn't sufficient.  _fold will need to be implemented primarily when it is
+    # possible for encoded words to appear in the specialized token-list, since
+    # there is no generic algorithm that can know where exactly the encoded
+    # words are allowed.  A _fold implementation is responsible for filling
+    # lines in the same general way that the top level _fold does. It may, and
+    # should, call the _fold method of sub-objects in a similar fashion to that
+    # of the top level _fold.
+    #
+    # XXX: I'm hoping it will be possible to factor the existing code further
+    # to reduce redundancy and make the logic clearer.
+
+    @property
+    def parts(self):
+        klass = self.__class__
+        this = []
+        for token in self:
+            if token.startswith_fws():
+                if this:
+                    yield this[0] if len(this)==1 else klass(this)
+                    this.clear()
+            end_ws = token.pop_trailing_ws()
+            this.append(token)
+            if end_ws:
+                yield klass(this)
+                this = [end_ws]
+        if this:
+            yield this[0] if len(this)==1 else klass(this)
+
+    def startswith_fws(self):
+        return self[0].startswith_fws()
+
+    def pop_leading_fws(self):
+        if self[0].token_type == 'fws':
+            return self.pop(0)
+        return self[0].pop_leading_fws()
+
+    def pop_trailing_ws(self):
+        if self[-1].token_type == 'cfws':
+            return self.pop(-1)
+        return self[-1].pop_trailing_ws()
+
+    @property
+    def has_fws(self):
+        for part in self:
+            if part.has_fws:
+                return True
+        return False
+
+    def has_leading_comment(self):
+        return self[0].has_leading_comment()
+
+    @property
+    def comments(self):
+        comments = []
+        for token in self:
+            comments.extend(token.comments)
+        return comments
+
+    def fold(self, *, policy):
+        # max_line_length 0/None means no limit, ie: infinitely long.
+        maxlen = policy.max_line_length or float("+inf")
+        folded = _Folded(maxlen, policy)
+        self._fold(folded)
+        folded.finalize()
+        return str(folded)
+
+    def as_encoded_word(self, charset):
+        # This works only for things returned by 'parts', which include
+        # the leading fws, if any, that should be used.
+        res = []
+        ws = self.pop_leading_fws()
+        if ws:
+            res.append(ws)
+        trailer = self.pop(-1) if self[-1].token_type=='fws' else ''
+        res.append(_ew.encode(str(self), charset))
+        res.append(trailer)
+        return ''.join(res)
+
+    def cte_encode(self, charset, policy):
+        res = []
+        for part in self:
+            res.append(part.cte_encode(charset, policy))
+        return ''.join(res)
+
+    def _fold(self, folded):
+        for part in self.parts:
+            tstr = str(part)
+            tlen = len(tstr)
+            try:
+                str(part).encode('us-ascii')
+            except UnicodeEncodeError:
+                if any(isinstance(x, errors.UndecodableBytesDefect)
+                        for x in part.all_defects):
+                    charset = 'unknown-8bit'
+                else:
+                    # XXX: this should be a policy setting
+                    charset = 'utf-8'
+                tstr = part.cte_encode(charset, folded.policy)
+                tlen = len(tstr)
+            if folded.append_if_fits(part, tstr):
+                continue
+            # Peel off the leading whitespace if any and make it sticky, to
+            # avoid infinite recursion.
+            ws = part.pop_leading_fws()
+            if ws is not None:
+                # Peel off the leading whitespace and make it sticky, to
+                # avoid infinite recursion.
+                folded.stickyspace = str(part.pop(0))
+                if folded.append_if_fits(part):
+                    continue
+            if part.has_fws:
+                part._fold(folded)
+                continue
+            # There are no fold points in this one; it is too long for a single
+            # line and can't be split...we just have to put it on its own line.
+            folded.append(tstr)
+            folded.newline()
+
+    def pprint(self, indent=''):
+        print('\n'.join(self._pp(indent='')))
+
+    def ppstr(self, indent=''):
+        return '\n'.join(self._pp(indent=''))
+
+    def _pp(self, indent=''):
+        yield '{}{}/{}('.format(
+            indent,
+            self.__class__.__name__,
+            self.token_type)
+        for token in self:
+            if not hasattr(token, '_pp'):
+                yield (indent + '    !! invalid element in token '
+                                        'list: {!r}'.format(token))
+            else:
+                for line in token._pp(indent+'    '):
+                    yield line
+        if self.defects:
+            extra = ' Defects: {}'.format(self.defects)
+        else:
+            extra = ''
+        yield '{}){}'.format(indent, extra)
+
+
+class WhiteSpaceTokenList(TokenList):
+
+    @property
+    def value(self):
+        return ' '
+
+    @property
+    def comments(self):
+        return [x.content for x in self if x.token_type=='comment']
+
+
+class UnstructuredTokenList(TokenList):
+
+    token_type = 'unstructured'
+
+    def _fold(self, folded):
+        if any(x.token_type=='encoded-word' for x in self):
+            return self._fold_encoded(folded)
+        # Here we can have either a pure ASCII string that may or may not
+        # have surrogateescape encoded bytes, or a unicode string.
+        last_ew = None
+        for part in self.parts:
+            tstr = str(part)
+            is_ew = False
+            try:
+                str(part).encode('us-ascii')
+            except UnicodeEncodeError:
+                if any(isinstance(x, errors.UndecodableBytesDefect)
+                       for x in part.all_defects):
+                    charset = 'unknown-8bit'
+                else:
+                    charset = 'utf-8'
+                if last_ew is not None:
+                    # We've already done an EW, combine this one with it
+                    # if there's room.
+                    chunk = get_unstructured(
+                        ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset)
+                    oldlastlen = sum(len(x) for x in folded.current[:last_ew])
+                    schunk = str(chunk)
+                    lchunk = len(schunk)
+                    if oldlastlen + lchunk <= folded.maxlen:
+                        del folded.current[last_ew:]
+                        folded.append(schunk)
+                        folded.lastlen = oldlastlen + lchunk
+                        continue
+                tstr = part.as_encoded_word(charset)
+                is_ew = True
+            if folded.append_if_fits(part, tstr):
+                if is_ew:
+                    last_ew = len(folded.current) - 1
+                continue
+            if is_ew or last_ew:
+                # It's too big to fit on the line, but since we've
+                # got encoded words we can use encoded word folding.
+                part._fold_as_ew(folded)
+                continue
+            # Peel off the leading whitespace if any and make it sticky, to
+            # avoid infinite recursion.
+            ws = part.pop_leading_fws()
+            if ws is not None:
+                folded.stickyspace = str(ws)
+                if folded.append_if_fits(part):
+                    continue
+            if part.has_fws:
+                part.fold(folded)
+                continue
+            # It can't be split...we just have to put it on its own line.
+            folded.append(tstr)
+            folded.newline()
+            last_ew = None
+
+    def cte_encode(self, charset, policy):
+        res = []
+        last_ew = None
+        for part in self:
+            spart = str(part)
+            try:
+                spart.encode('us-ascii')
+                res.append(spart)
+            except UnicodeEncodeError:
+                if last_ew is None:
+                    res.append(part.cte_encode(charset, policy))
+                    last_ew = len(res)
+                else:
+                    tl = get_unstructured(''.join(res[last_ew:] + [spart]))
+                    res.append(tl.as_encoded_word())
+        return ''.join(res)
+
+
+class Phrase(TokenList):
+
+    token_type = 'phrase'
+
+    def _fold(self, folded):
+        # As with Unstructured, we can have pure ASCII with or without
+        # surrogateescape encoded bytes, or we could have unicode.  But this
+        # case is more complicated, since we have to deal with the various
+        # sub-token types and how they can be composed in the face of
+        # unicode-that-needs-CTE-encoding, and the fact that if a token a
+        # comment that becomes a barrier across which we can't compose encoded
+        # words.
+        last_ew = None
+        for part in self.parts:
+            tstr = str(part)
+            tlen = len(tstr)
+            has_ew = False
+            try:
+                str(part).encode('us-ascii')
+            except UnicodeEncodeError:
+                if any(isinstance(x, errors.UndecodableBytesDefect)
+                        for x in part.all_defects):
+                    charset = 'unknown-8bit'
+                else:
+                    charset = 'utf-8'
+                if last_ew is not None and not part.has_leading_comment():
+                    # We've already done an EW, let's see if we can combine
+                    # this one with it.  The last_ew logic ensures that all we
+                    # have at this point is atoms, no comments or quoted
+                    # strings.  So we can treat the text between the last
+                    # encoded word and the content of this token as
+                    # unstructured text, and things will work correctly.  But
+                    # we have to strip off any trailing comment on this token
+                    # first, and if it is a quoted string we have to pull out
+                    # the content (we're encoding it, so it no longer needs to
+                    # be quoted).
+                    if part[-1].token_type == 'cfws' and part.comments:
+                        remainder = part.pop(-1)
+                    else:
+                        remainder = ''
+                    for i, token in enumerate(part):
+                        if token.token_type == 'bare-quoted-string':
+                            part[i] = UnstructuredTokenList(token[:])
+                    chunk = get_unstructured(
+                        ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset)
+                    schunk = str(chunk)
+                    lchunk = len(schunk)
+                    if last_ew + lchunk <= folded.maxlen:
+                        del folded.current[last_ew:]
+                        folded.append(schunk)
+                        folded.lastlen = sum(len(x) for x in folded.current)
+                        continue
+                tstr = part.as_encoded_word(charset)
+                tlen = len(tstr)
+                has_ew = True
+            if folded.append_if_fits(part, tstr):
+                if has_ew and not part.comments:
+                    last_ew = len(folded.current) - 1
+                elif part.comments or part.token_type == 'quoted-string':
+                    # If a comment is involved we can't combine EWs.  And if a
+                    # quoted string is involved, it's not worth the effort to
+                    # try to combine them.
+                    last_ew = None
+                continue
+            part._fold(folded)
+
+    def cte_encode(self, charset, policy):
+        res = []
+        last_ew = None
+        is_ew = False
+        for part in self:
+            spart = str(part)
+            try:
+                spart.encode('us-ascii')
+                res.append(spart)
+            except UnicodeEncodeError:
+                is_ew = True
+                if last_ew is None:
+                    if not part.comments:
+                        last_ew = len(res)
+                    res.append(part.cte_encode(charset, policy))
+                elif not part.has_leading_comment():
+                    if part[-1].token_type == 'cfws' and part.comments:
+                        remainder = part.pop(-1)
+                    else:
+                        remainder = ''
+                    for i, token in enumerate(part):
+                        if token.token_type == 'bare-quoted-string':
+                            part[i] = UnstructuredTokenList(token[:])
+                    tl = get_unstructured(''.join(res[last_ew:] + [spart]))
+                    res[last_ew:] = [tl.as_encoded_word(charset)]
+            if part.comments or (not is_ew and part.token_type == 'quoted-string'):
+                last_ew = None
+        return ''.join(res)
+
+class Word(TokenList):
+
+    token_type = 'word'
+
+
+class CFWSList(WhiteSpaceTokenList):
+
+    token_type = 'cfws'
+
+    def has_leading_comment(self):
+        return bool(self.comments)
+
+
+class Atom(TokenList):
+
+    token_type = 'atom'
+
+
+class Token(TokenList):
+
+    token_type = 'token'
+
+
+class EncodedWord(TokenList):
+
+    token_type = 'encoded-word'
+    cte = None
+    charset = None
+    lang = None
+
+    @property
+    def encoded(self):
+        if self.cte is not None:
+            return self.cte
+        _ew.encode(str(self), self.charset)
+
+
+
+class QuotedString(TokenList):
+
+    token_type = 'quoted-string'
+
+    @property
+    def content(self):
+        for x in self:
+            if x.token_type == 'bare-quoted-string':
+                return x.value
+
+    @property
+    def quoted_value(self):
+        res = []
+        for x in self:
+            if x.token_type == 'bare-quoted-string':
+                res.append(str(x))
+            else:
+                res.append(x.value)
+        return ''.join(res)
+
+    @property
+    def stripped_value(self):
+        for token in self:
+            if token.token_type == 'bare-quoted-string':
+                return token.value
+
+
+class BareQuotedString(QuotedString):
+
+    token_type = 'bare-quoted-string'
+
+    def __str__(self):
+        return quote_string(''.join(str(x) for x in self))
+
+    @property
+    def value(self):
+        return ''.join(str(x) for x in self)
+
+
+class Comment(WhiteSpaceTokenList):
+
+    token_type = 'comment'
+
+    def __str__(self):
+        return ''.join(sum([
+                            ["("],
+                            [self.quote(x) for x in self],
+                            [")"],
+                            ], []))
+
+    def quote(self, value):
+        if value.token_type == 'comment':
+            return str(value)
+        return str(value).replace('\\', '\\\\').replace(
+                                  '(', '\(').replace(
+                                  ')', '\)')
+
+    @property
+    def content(self):
+        return ''.join(str(x) for x in self)
+
+    @property
+    def comments(self):
+        return [self.content]
+
+class AddressList(TokenList):
+
+    token_type = 'address-list'
+
+    @property
+    def addresses(self):
+        return [x for x in self if x.token_type=='address']
+
+    @property
+    def mailboxes(self):
+        return sum((x.mailboxes
+                    for x in self if x.token_type=='address'), [])
+
+    @property
+    def all_mailboxes(self):
+        return sum((x.all_mailboxes
+                    for x in self if x.token_type=='address'), [])
+
+
+class Address(TokenList):
+
+    token_type = 'address'
+
+    @property
+    def display_name(self):
+        if self[0].token_type == 'group':
+            return self[0].display_name
+
+    @property
+    def mailboxes(self):
+        if self[0].token_type == 'mailbox':
+            return [self[0]]
+        elif self[0].token_type == 'invalid-mailbox':
+            return []
+        return self[0].mailboxes
+
+    @property
+    def all_mailboxes(self):
+        if self[0].token_type == 'mailbox':
+            return [self[0]]
+        elif self[0].token_type == 'invalid-mailbox':
+            return [self[0]]
+        return self[0].all_mailboxes
+
+class MailboxList(TokenList):
+
+    token_type = 'mailbox-list'
+
+    @property
+    def mailboxes(self):
+        return [x for x in self if x.token_type=='mailbox']
+
+    @property
+    def all_mailboxes(self):
+        return [x for x in self
+            if x.token_type in ('mailbox', 'invalid-mailbox')]
+
+
+class GroupList(TokenList):
+
+    token_type = 'group-list'
+
+    @property
+    def mailboxes(self):
+        if not self or self[0].token_type != 'mailbox-list':
+            return []
+        return self[0].mailboxes
+
+    @property
+    def all_mailboxes(self):
+        if not self or self[0].token_type != 'mailbox-list':
+            return []
+        return self[0].all_mailboxes
+
+
+class Group(TokenList):
+
+    token_type = "group"
+
+    @property
+    def mailboxes(self):
+        if self[2].token_type != 'group-list':
+            return []
+        return self[2].mailboxes
+
+    @property
+    def all_mailboxes(self):
+        if self[2].token_type != 'group-list':
+            return []
+        return self[2].all_mailboxes
+
+    @property
+    def display_name(self):
+        return self[0].display_name
+
+
+class NameAddr(TokenList):
+
+    token_type = 'name-addr'
+
+    @property
+    def display_name(self):
+        if len(self) == 1:
+            return None
+        return self[0].display_name
+
+    @property
+    def local_part(self):
+        return self[-1].local_part
+
+    @property
+    def domain(self):
+        return self[-1].domain
+
+    @property
+    def route(self):
+        return self[-1].route
+
+    @property
+    def addr_spec(self):
+        return self[-1].addr_spec
+
+
+class AngleAddr(TokenList):
+
+    token_type = 'angle-addr'
+
+    @property
+    def local_part(self):
+        for x in self:
+            if x.token_type == 'addr-spec':
+                return x.local_part
+
+    @property
+    def domain(self):
+        for x in self:
+            if x.token_type == 'addr-spec':
+                return x.domain
+
+    @property
+    def route(self):
+        for x in self:
+            if x.token_type == 'obs-route':
+                return x.domains
+
+    @property
+    def addr_spec(self):
+        for x in self:
+            if x.token_type == 'addr-spec':
+                return x.addr_spec
+        else:
+            return '<>'
+
+
+class ObsRoute(TokenList):
+
+    token_type = 'obs-route'
+
+    @property
+    def domains(self):
+        return [x.domain for x in self if x.token_type == 'domain']
+
+
+class Mailbox(TokenList):
+
+    token_type = 'mailbox'
+
+    @property
+    def display_name(self):
+        if self[0].token_type == 'name-addr':
+            return self[0].display_name
+
+    @property
+    def local_part(self):
+        return self[0].local_part
+
+    @property
+    def domain(self):
+        return self[0].domain
+
+    @property
+    def route(self):
+        if self[0].token_type == 'name-addr':
+            return self[0].route
+
+    @property
+    def addr_spec(self):
+        return self[0].addr_spec
+
+
+class InvalidMailbox(TokenList):
+
+    token_type = 'invalid-mailbox'
+
+    @property
+    def display_name(self):
+        return None
+
+    local_part = domain = route = addr_spec = display_name
+
+
+class Domain(TokenList):
+
+    token_type = 'domain'
+
+    @property
+    def domain(self):
+        return ''.join(super().value.split())
+
+
+class DotAtom(TokenList):
+
+    token_type = 'dot-atom'
+
+
+class DotAtomText(TokenList):
+
+    token_type = 'dot-atom-text'
+
+
+class AddrSpec(TokenList):
+
+    token_type = 'addr-spec'
+
+    @property
+    def local_part(self):
+        return self[0].local_part
+
+    @property
+    def domain(self):
+        if len(self) < 3:
+            return None
+        return self[-1].domain
+
+    @property
+    def value(self):
+        if len(self) < 3:
+            return self[0].value
+        return self[0].value.rstrip()+self[1].value+self[2].value.lstrip()
+
+    @property
+    def addr_spec(self):
+        nameset = set(self.local_part)
+        if len(nameset) > len(nameset-DOT_ATOM_ENDS):
+            lp = quote_string(self.local_part)
+        else:
+            lp = self.local_part
+        if self.domain is not None:
+            return lp + '@' + self.domain
+        return lp
+
+
+class ObsLocalPart(TokenList):
+
+    token_type = 'obs-local-part'
+
+
+class DisplayName(Phrase):
+
+    token_type = 'display-name'
+
+    @property
+    def display_name(self):
+        res = TokenList(self)
+        if res[0].token_type == 'cfws':
+            res.pop(0)
+        else:
+            if res[0][0].token_type == 'cfws':
+                res[0] = TokenList(res[0][1:])
+        if res[-1].token_type == 'cfws':
+            res.pop()
+        else:
+            if res[-1][-1].token_type == 'cfws':
+                res[-1] = TokenList(res[-1][:-1])
+        return res.value
+
+    @property
+    def value(self):
+        quote = False
+        if self.defects:
+            quote = True
+        else:
+            for x in self:
+                if x.token_type == 'quoted-string':
+                    quote = True
+        if quote:
+            pre = post = ''
+            if self[0].token_type=='cfws' or self[0][0].token_type=='cfws':
+                pre = ' '
+            if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws':
+                post = ' '
+            return pre+quote_string(self.display_name)+post
+        else:
+            return super().value
+
+
+class LocalPart(TokenList):
+
+    token_type = 'local-part'
+
+    @property
+    def value(self):
+        if self[0].token_type == "quoted-string":
+            return self[0].quoted_value
+        else:
+            return self[0].value
+
+    @property
+    def local_part(self):
+        # Strip whitespace from front, back, and around dots.
+        res = [DOT]
+        last = DOT
+        last_is_tl = False
+        for tok in self[0] + [DOT]:
+            if tok.token_type == 'cfws':
+                continue
+            if (last_is_tl and tok.token_type == 'dot' and
+                    last[-1].token_type == 'cfws'):
+                res[-1] = TokenList(last[:-1])
+            is_tl = isinstance(tok, TokenList)
+            if (is_tl and last.token_type == 'dot' and
+                    tok[0].token_type == 'cfws'):
+                res.append(TokenList(tok[1:]))
+            else:
+                res.append(tok)
+            last = res[-1]
+            last_is_tl = is_tl
+        res = TokenList(res[1:-1])
+        return res.value
+
+
+class DomainLiteral(TokenList):
+
+    token_type = 'domain-literal'
+
+    @property
+    def domain(self):
+        return ''.join(super().value.split())
+
+    @property
+    def ip(self):
+        for x in self:
+            if x.token_type == 'ptext':
+                return x.value
+
+
+class MIMEVersion(TokenList):
+
+    token_type = 'mime-version'
+    major = None
+    minor = None
+
+
+class Parameter(TokenList):
+
+    token_type = 'parameter'
+    sectioned = False
+    extended = False
+    charset = 'us-ascii'
+
+    @property
+    def section_number(self):
+        # Because the first token, the attribute (name) eats CFWS, the second
+        # token is always the section if there is one.
+        return self[1].number if self.sectioned else 0
+
+    @property
+    def param_value(self):
+        # This is part of the "handle quoted extended parameters" hack.
+        for token in self:
+            if token.token_type == 'value':
+                return token.stripped_value
+            if token.token_type == 'quoted-string':
+                for token in token:
+                    if token.token_type == 'bare-quoted-string':
+                        for token in token:
+                            if token.token_type == 'value':
+                                return token.stripped_value
+        return ''
+
+
+class InvalidParameter(Parameter):
+
+    token_type = 'invalid-parameter'
+
+
+class Attribute(TokenList):
+
+    token_type = 'attribute'
+
+    @property
+    def stripped_value(self):
+        for token in self:
+            if token.token_type.endswith('attrtext'):
+                return token.value
+
+class Section(TokenList):
+
+    token_type = 'section'
+    number = None
+
+
+class Value(TokenList):
+
+    token_type = 'value'
+
+    @property
+    def stripped_value(self):
+        token = self[0]
+        if token.token_type == 'cfws':
+            token = self[1]
+        if token.token_type.endswith(
+                ('quoted-string', 'attribute', 'extended-attribute')):
+            return token.stripped_value
+        return self.value
+
+
+class MimeParameters(TokenList):
+
+    token_type = 'mime-parameters'
+
+    @property
+    def params(self):
+        # The RFC specifically states that the ordering of parameters is not
+        # guaranteed and may be reordered by the transport layer.  So we have
+        # to assume the RFC 2231 pieces can come in any order.  However, we
+        # output them in the order that we first see a given name, which gives
+        # us a stable __str__.
+        params = OrderedDict()
+        for token in self:
+            if not token.token_type.endswith('parameter'):
+                continue
+            if token[0].token_type != 'attribute':
+                continue
+            name = token[0].value.strip()
+            if name not in params:
+                params[name] = []
+            params[name].append((token.section_number, token))
+        for name, parts in params.items():
+            parts = sorted(parts)
+            # XXX: there might be more recovery we could do here if, for
+            # example, this is really a case of a duplicate attribute name.
+            value_parts = []
+            charset = parts[0][1].charset
+            for i, (section_number, param) in enumerate(parts):
+                if section_number != i:
+                    param.defects.append(errors.InvalidHeaderDefect(
+                        "inconsistent multipart parameter numbering"))
+                value = param.param_value
+                if param.extended:
+                    try:
+                        value = urllib.parse.unquote_to_bytes(value)
+                    except UnicodeEncodeError:
+                        # source had surrogate escaped bytes.  What we do now
+                        # is a bit of an open question.  I'm not sure this is
+                        # the best choice, but it is what the old algorithm did
+                        value = urllib.parse.unquote(value, encoding='latin-1')
+                    else:
+                        try:
+                            value = value.decode(charset, 'surrogateescape')
+                        except LookupError:
+                            # XXX: there should really be a custom defect for
+                            # unknown character set to make it easy to find,
+                            # because otherwise unknown charset is a silent
+                            # failure.
+                            value = value.decode('us-ascii', 'surrogateescape')
+                        if utils._has_surrogates(value):
+                            param.defects.append(errors.UndecodableBytesDefect())
+                value_parts.append(value)
+            value = ''.join(value_parts)
+            yield name, value
+
+    def __str__(self):
+        params = []
+        for name, value in self.params:
+            if value:
+                params.append('{}={}'.format(name, quote_string(value)))
+            else:
+                params.append(name)
+        params = '; '.join(params)
+        return ' ' + params if params else ''
+
+
+class ParameterizedHeaderValue(TokenList):
+
+    @property
+    def params(self):
+        for token in reversed(self):
+            if token.token_type == 'mime-parameters':
+                return token.params
+        return {}
+
+    @property
+    def parts(self):
+        if self and self[-1].token_type == 'mime-parameters':
+            # We don't want to start a new line if all of the params don't fit
+            # after the value, so unwrap the parameter list.
+            return TokenList(self[:-1] + self[-1])
+        return TokenList(self).parts
+
+
+class ContentType(ParameterizedHeaderValue):
+
+    token_type = 'content-type'
+    maintype = 'text'
+    subtype = 'plain'
+
+
+class ContentDisposition(ParameterizedHeaderValue):
+
+    token_type = 'content-disposition'
+    content_disposition = None
+
+
+class ContentTransferEncoding(TokenList):
+
+    token_type = 'content-transfer-encoding'
+    cte = '7bit'
+
+
+class HeaderLabel(TokenList):
+
+    token_type = 'header-label'
+
+
+class Header(TokenList):
+
+    token_type = 'header'
+
+    def _fold(self, folded):
+        folded.append(str(self.pop(0)))
+        folded.lastlen = len(folded.current[0])
+        # The first line of the header is different from all others: we don't
+        # want to start a new object on a new line if it has any fold points in
+        # it that would allow part of it to be on the first header line.
+        # Further, if the first fold point would fit on the new line, we want
+        # to do that, but if it doesn't we want to put it on the first line.
+        # Folded supports this via the stickyspace attribute.  If this
+        # attribute is not None, it does the special handling.
+        folded.stickyspace = str(self.pop(0)) if self[0].token_type == 'cfws' else ''
+        rest = self.pop(0)
+        if self:
+            raise ValueError("Malformed Header token list")
+        rest._fold(folded)
+
+
+#
+# Terminal classes and instances
+#
+
+class Terminal(str):
+
+    def __new__(cls, value, token_type):
+        self = super().__new__(cls, value)
+        self.token_type = token_type
+        self.defects = []
+        return self
+
+    def __repr__(self):
+        return "{}({})".format(self.__class__.__name__, super().__repr__())
+
+    @property
+    def all_defects(self):
+        return list(self.defects)
+
+    def _pp(self, indent=''):
+        return ["{}{}/{}({}){}".format(
+            indent,
+            self.__class__.__name__,
+            self.token_type,
+            super().__repr__(),
+            '' if not self.defects else ' {}'.format(self.defects),
+            )]
+
+    def cte_encode(self, charset, policy):
+        value = str(self)
+        try:
+            value.encode('us-ascii')
+            return value
+        except UnicodeEncodeError:
+            return _ew.encode(value, charset)
+
+    def pop_trailing_ws(self):
+        # This terminates the recursion.
+        return None
+
+    def pop_leading_fws(self):
+        # This terminates the recursion.
+        return None
+
+    @property
+    def comments(self):
+        return []
+
+    def has_leading_comment(self):
+        return False
+
+    def __getnewargs__(self):
+        return(str(self), self.token_type)
+
+
+class WhiteSpaceTerminal(Terminal):
+
+    @property
+    def value(self):
+        return ' '
+
+    def startswith_fws(self):
+        return True
+
+    has_fws = True
+
+
+class ValueTerminal(Terminal):
+
+    @property
+    def value(self):
+        return self
+
+    def startswith_fws(self):
+        return False
+
+    has_fws = False
+
+    def as_encoded_word(self, charset):
+        return _ew.encode(str(self), charset)
+
+
+class EWWhiteSpaceTerminal(WhiteSpaceTerminal):
+
+    @property
+    def value(self):
+        return ''
+
+    @property
+    def encoded(self):
+        return self[:]
+
+    def __str__(self):
+        return ''
+
+    has_fws = True
+
+
+# XXX these need to become classes and used as instances so
+# that a program can't change them in a parse tree and screw
+# up other parse trees.  Maybe should have  tests for that, too.
+DOT = ValueTerminal('.', 'dot')
+ListSeparator = ValueTerminal(',', 'list-separator')
+RouteComponentMarker = ValueTerminal('@', 'route-component-marker')
+
+#
+# Parser
+#
+
+"""Parse strings according to RFC822/2047/2822/5322 rules.
+
+This is a stateless parser.  Each get_XXX function accepts a string and
+returns either a Terminal or a TokenList representing the RFC object named
+by the method and a string containing the remaining unparsed characters
+from the input.  Thus a parser method consumes the next syntactic construct
+of a given type and returns a token representing the construct plus the
+unparsed remainder of the input string.
+
+For example, if the first element of a structured header is a 'phrase',
+then:
+
+    phrase, value = get_phrase(value)
+
+returns the complete phrase from the start of the string value, plus any
+characters left in the string after the phrase is removed.
+
+"""
+
+_wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split
+_non_atom_end_matcher = re.compile(r"[^{}]+".format(
+    ''.join(ATOM_ENDS).replace('\\','\\\\').replace(']','\]'))).match
+_non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall
+_non_token_end_matcher = re.compile(r"[^{}]+".format(
+    ''.join(TOKEN_ENDS).replace('\\','\\\\').replace(']','\]'))).match
+_non_attribute_end_matcher = re.compile(r"[^{}]+".format(
+    ''.join(ATTRIBUTE_ENDS).replace('\\','\\\\').replace(']','\]'))).match
+_non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format(
+    ''.join(EXTENDED_ATTRIBUTE_ENDS).replace(
+                                    '\\','\\\\').replace(']','\]'))).match
+
+def _validate_xtext(xtext):
+    """If input token contains ASCII non-printables, register a defect."""
+
+    non_printables = _non_printable_finder(xtext)
+    if non_printables:
+        xtext.defects.append(errors.NonPrintableDefect(non_printables))
+    if utils._has_surrogates(xtext):
+        xtext.defects.append(errors.UndecodableBytesDefect(
+            "Non-ASCII characters found in header token"))
+
+def _get_ptext_to_endchars(value, endchars):
+    """Scan printables/quoted-pairs until endchars and return unquoted ptext.
+
+    This function turns a run of qcontent, ccontent-without-comments, or
+    dtext-with-quoted-printables into a single string by unquoting any
+    quoted printables.  It returns the string, the remaining value, and
+    a flag that is True iff there were any quoted printables decoded.
+
+    """
+    fragment, *remainder = _wsp_splitter(value, 1)
+    vchars = []
+    escape = False
+    had_qp = False
+    for pos in range(len(fragment)):
+        if fragment[pos] == '\\':
+            if escape:
+                escape = False
+                had_qp = True
+            else:
+                escape = True
+                continue
+        if escape:
+            escape = False
+        elif fragment[pos] in endchars:
+            break
+        vchars.append(fragment[pos])
+    else:
+        pos = pos + 1
+    return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp
+
+def _decode_ew_run(value):
+    """ Decode a run of RFC2047 encoded words.
+
+        _decode_ew_run(value) -> (text, value, defects)
+
+    Scans the supplied value for a run of tokens that look like they are RFC
+    2047 encoded words, decodes those words into text according to RFC 2047
+    rules (whitespace between encoded words is discarded), and returns the text
+    and the remaining value (including any leading whitespace on the remaining
+    value), as well as a list of any defects encountered while decoding.  The
+    input value may not have any leading whitespace.
+
+    """
+    res = []
+    defects = []
+    last_ws = ''
+    while value:
+        try:
+            tok, ws, value = _wsp_splitter(value, 1)
+        except ValueError:
+            tok, ws, value = value, '', ''
+        if not (tok.startswith('=?') and tok.endswith('?=')):
+            return ''.join(res), last_ws + tok + ws + value, defects
+        text, charset, lang, new_defects = _ew.decode(tok)
+        res.append(text)
+        defects.extend(new_defects)
+        last_ws = ws
+    return ''.join(res), last_ws, defects
+
+def get_fws(value):
+    """FWS = 1*WSP
+
+    This isn't the RFC definition.  We're using fws to represent tokens where
+    folding can be done, but when we are parsing the *un*folding has already
+    been done so we don't need to watch out for CRLF.
+
+    """
+    newvalue = value.lstrip()
+    fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws')
+    return fws, newvalue
+
+def get_encoded_word(value):
+    """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?="
+
+    """
+    ew = EncodedWord()
+    if not value.startswith('=?'):
+        raise errors.HeaderParseError(
+            "expected encoded word but found {}".format(value))
+    tok, *remainder = value[2:].split('?=', 1)
+    if tok == value[2:]:
+        raise errors.HeaderParseError(
+            "expected encoded word but found {}".format(value))
+    remstr = ''.join(remainder)
+    if remstr[:2].isdigit():
+        rest, *remainder = remstr.split('?=', 1)
+        tok = tok + '?=' + rest
+    if len(tok.split()) > 1:
+        ew.defects.append(errors.InvalidHeaderDefect(
+            "whitespace inside encoded word"))
+    ew.cte = value
+    value = ''.join(remainder)
+    try:
+        text, charset, lang, defects = _ew.decode('=?' + tok + '?=')
+    except ValueError:
+        raise errors.HeaderParseError(
+            "encoded word format invalid: '{}'".format(ew.cte))
+    ew.charset = charset
+    ew.lang = lang
+    ew.defects.extend(defects)
+    while text:
+        if text[0] in WSP:
+            token, text = get_fws(text)
+            ew.append(token)
+            continue
+        chars, *remainder = _wsp_splitter(text, 1)
+        vtext = ValueTerminal(chars, 'vtext')
+        _validate_xtext(vtext)
+        ew.append(vtext)
+        text = ''.join(remainder)
+    return ew, value
+
+def get_unstructured(value):
+    """unstructured = (*([FWS] vchar) *WSP) / obs-unstruct
+       obs-unstruct = *((*LF *CR *(obs-utext) *LF *CR)) / FWS)
+       obs-utext = %d0 / obs-NO-WS-CTL / LF / CR
+
+       obs-NO-WS-CTL is control characters except WSP/CR/LF.
+
+    So, basically, we have printable runs, plus control characters or nulls in
+    the obsolete syntax, separated by whitespace.  Since RFC 2047 uses the
+    obsolete syntax in its specification, but requires whitespace on either
+    side of the encoded words, I can see no reason to need to separate the
+    non-printable-non-whitespace from the printable runs if they occur, so we
+    parse this into xtext tokens separated by WSP tokens.
+
+    Because an 'unstructured' value must by definition constitute the entire
+    value, this 'get' routine does not return a remaining value, only the
+    parsed TokenList.
+
+    """
+    # XXX: but what about bare CR and LF?  They might signal the start or
+    # end of an encoded word.  YAGNI for now, since out current parsers
+    # will never send us strings with bard CR or LF.
+
+    unstructured = UnstructuredTokenList()
+    while value:
+        if value[0] in WSP:
+            token, value = get_fws(value)
+            unstructured.append(token)
+            continue
+        if value.startswith('=?'):
+            try:
+                token, value = get_encoded_word(value)
+            except errors.HeaderParseError:
+                pass
+            else:
+                have_ws = True
+                if len(unstructured) > 0:
+                    if unstructured[-1].token_type != 'fws':
+                        unstructured.defects.append(errors.InvalidHeaderDefect(
+                            "missing whitespace before encoded word"))
+                        have_ws = False
+                if have_ws and len(unstructured) > 1:
+                    if unstructured[-2].token_type == 'encoded-word':
+                        unstructured[-1] = EWWhiteSpaceTerminal(
+                            unstructured[-1], 'fws')
+                unstructured.append(token)
+                continue
+        tok, *remainder = _wsp_splitter(value, 1)
+        vtext = ValueTerminal(tok, 'vtext')
+        _validate_xtext(vtext)
+        unstructured.append(vtext)
+        value = ''.join(remainder)
+    return unstructured
+
+def get_qp_ctext(value):
+    """ctext = <printable ascii except \ ( )>
+
+    This is not the RFC ctext, since we are handling nested comments in comment
+    and unquoting quoted-pairs here.  We allow anything except the '()'
+    characters, but if we find any ASCII other than the RFC defined printable
+    ASCII an NonPrintableDefect is added to the token's defects list.  Since
+    quoted pairs are converted to their unquoted values, what is returned is
+    a 'ptext' token.  In this case it is a WhiteSpaceTerminal, so it's value
+    is ' '.
+
+    """
+    ptext, value, _ = _get_ptext_to_endchars(value, '()')
+    ptext = WhiteSpaceTerminal(ptext, 'ptext')
+    _validate_xtext(ptext)
+    return ptext, value
+
+def get_qcontent(value):
+    """qcontent = qtext / quoted-pair
+
+    We allow anything except the DQUOTE character, but if we find any ASCII
+    other than the RFC defined printable ASCII an NonPrintableDefect is
+    added to the token's defects list.  Any quoted pairs are converted to their
+    unquoted values, so what is returned is a 'ptext' token.  In this case it
+    is a ValueTerminal.
+
+    """
+    ptext, value, _ = _get_ptext_to_endchars(value, '"')
+    ptext = ValueTerminal(ptext, 'ptext')
+    _validate_xtext(ptext)
+    return ptext, value
+
+def get_atext(value):
+    """atext = <matches _atext_matcher>
+
+    We allow any non-ATOM_ENDS in atext, but add an InvalidATextDefect to
+    the token's defects list if we find non-atext characters.
+    """
+    m = _non_atom_end_matcher(value)
+    if not m:
+        raise errors.HeaderParseError(
+            "expected atext but found '{}'".format(value))
+    atext = m.group()
+    value = value[len(atext):]
+    atext = ValueTerminal(atext, 'atext')
+    _validate_xtext(atext)
+    return atext, value
+
+def get_bare_quoted_string(value):
+    """bare-quoted-string = DQUOTE *([FWS] qcontent) [FWS] DQUOTE
+
+    A quoted-string without the leading or trailing white space.  Its
+    value is the text between the quote marks, with whitespace
+    preserved and quoted pairs decoded.
+    """
+    if value[0] != '"':
+        raise errors.HeaderParseError(
+            "expected '\"' but found '{}'".format(value))
+    bare_quoted_string = BareQuotedString()
+    value = value[1:]
+    while value and value[0] != '"':
+        if value[0] in WSP:
+            token, value = get_fws(value)
+        else:
+            token, value = get_qcontent(value)
+        bare_quoted_string.append(token)
+    if not value:
+        bare_quoted_string.defects.append(errors.InvalidHeaderDefect(
+            "end of header inside quoted string"))
+        return bare_quoted_string, value
+    return bare_quoted_string, value[1:]
+
+def get_comment(value):
+    """comment = "(" *([FWS] ccontent) [FWS] ")"
+       ccontent = ctext / quoted-pair / comment
+
+    We handle nested comments here, and quoted-pair in our qp-ctext routine.
+    """
+    if value and value[0] != '(':
+        raise errors.HeaderParseError(
+            "expected '(' but found '{}'".format(value))
+    comment = Comment()
+    value = value[1:]
+    while value and value[0] != ")":
+        if value[0] in WSP:
+            token, value = get_fws(value)
+        elif value[0] == '(':
+            token, value = get_comment(value)
+        else:
+            token, value = get_qp_ctext(value)
+        comment.append(token)
+    if not value:
+        comment.defects.append(errors.InvalidHeaderDefect(
+            "end of header inside comment"))
+        return comment, value
+    return comment, value[1:]
+
+def get_cfws(value):
+    """CFWS = (1*([FWS] comment) [FWS]) / FWS
+
+    """
+    cfws = CFWSList()
+    while value and value[0] in CFWS_LEADER:
+        if value[0] in WSP:
+            token, value = get_fws(value)
+        else:
+            token, value = get_comment(value)
+        cfws.append(token)
+    return cfws, value
+
+def get_quoted_string(value):
+    """quoted-string = [CFWS] <bare-quoted-string> [CFWS]
+
+    'bare-quoted-string' is an intermediate class defined by this
+    parser and not by the RFC grammar.  It is the quoted string
+    without any attached CFWS.
+    """
+    quoted_string = QuotedString()
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        quoted_string.append(token)
+    token, value = get_bare_quoted_string(value)
+    quoted_string.append(token)
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        quoted_string.append(token)
+    return quoted_string, value
+
+def get_atom(value):
+    """atom = [CFWS] 1*atext [CFWS]
+
+    """
+    atom = Atom()
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        atom.append(token)
+    if value and value[0] in ATOM_ENDS:
+        raise errors.HeaderParseError(
+            "expected atom but found '{}'".format(value))
+    token, value = get_atext(value)
+    atom.append(token)
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        atom.append(token)
+    return atom, value
+
+def get_dot_atom_text(value):
+    """ dot-text = 1*atext *("." 1*atext)
+
+    """
+    dot_atom_text = DotAtomText()
+    if not value or value[0] in ATOM_ENDS:
+        raise errors.HeaderParseError("expected atom at a start of "
+            "dot-atom-text but found '{}'".format(value))
+    while value and value[0] not in ATOM_ENDS:
+        token, value = get_atext(value)
+        dot_atom_text.append(token)
+        if value and value[0] == '.':
+            dot_atom_text.append(DOT)
+            value = value[1:]
+    if dot_atom_text[-1] is DOT:
+        raise errors.HeaderParseError("expected atom at end of dot-atom-text "
+            "but found '{}'".format('.'+value))
+    return dot_atom_text, value
+
+def get_dot_atom(value):
+    """ dot-atom = [CFWS] dot-atom-text [CFWS]
+
+    """
+    dot_atom = DotAtom()
+    if value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        dot_atom.append(token)
+    token, value = get_dot_atom_text(value)
+    dot_atom.append(token)
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        dot_atom.append(token)
+    return dot_atom, value
+
+def get_word(value):
+    """word = atom / quoted-string
+
+    Either atom or quoted-string may start with CFWS.  We have to peel off this
+    CFWS first to determine which type of word to parse.  Afterward we splice
+    the leading CFWS, if any, into the parsed sub-token.
+
+    If neither an atom or a quoted-string is found before the next special, a
+    HeaderParseError is raised.
+
+    The token returned is either an Atom or a QuotedString, as appropriate.
+    This means the 'word' level of the formal grammar is not represented in the
+    parse tree; this is because having that extra layer when manipulating the
+    parse tree is more confusing than it is helpful.
+
+    """
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+    else:
+        leader = None
+    if value[0]=='"':
+        token, value = get_quoted_string(value)
+    elif value[0] in SPECIALS:
+        raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' "
+                                      "but found '{}'".format(value))
+    else:
+        token, value = get_atom(value)
+    if leader is not None:
+        token[:0] = [leader]
+    return token, value
+
+def get_phrase(value):
+    """ phrase = 1*word / obs-phrase
+        obs-phrase = word *(word / "." / CFWS)
+
+    This means a phrase can be a sequence of words, periods, and CFWS in any
+    order as long as it starts with at least one word.  If anything other than
+    words is detected, an ObsoleteHeaderDefect is added to the token's defect
+    list.  We also accept a phrase that starts with CFWS followed by a dot;
+    this is registered as an InvalidHeaderDefect, since it is not supported by
+    even the obsolete grammar.
+
+    """
+    phrase = Phrase()
+    try:
+        token, value = get_word(value)
+        phrase.append(token)
+    except errors.HeaderParseError:
+        phrase.defects.append(errors.InvalidHeaderDefect(
+            "phrase does not start with word"))
+    while value and value[0] not in PHRASE_ENDS:
+        if value[0]=='.':
+            phrase.append(DOT)
+            phrase.defects.append(errors.ObsoleteHeaderDefect(
+                "period in 'phrase'"))
+            value = value[1:]
+        else:
+            try:
+                token, value = get_word(value)
+            except errors.HeaderParseError:
+                if value[0] in CFWS_LEADER:
+                    token, value = get_cfws(value)
+                    phrase.defects.append(errors.ObsoleteHeaderDefect(
+                        "comment found without atom"))
+                else:
+                    raise
+            phrase.append(token)
+    return phrase, value
+
+def get_local_part(value):
+    """ local-part = dot-atom / quoted-string / obs-local-part
+
+    """
+    local_part = LocalPart()
+    leader = None
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+    if not value:
+        raise errors.HeaderParseError(
+            "expected local-part but found '{}'".format(value))
+    try:
+        token, value = get_dot_atom(value)
+    except errors.HeaderParseError:
+        try:
+            token, value = get_word(value)
+        except errors.HeaderParseError:
+            if value[0] != '\\' and value[0] in PHRASE_ENDS:
+                raise
+            token = TokenList()
+    if leader is not None:
+        token[:0] = [leader]
+    local_part.append(token)
+    if value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
+        obs_local_part, value = get_obs_local_part(str(local_part) + value)
+        if obs_local_part.token_type == 'invalid-obs-local-part':
+            local_part.defects.append(errors.InvalidHeaderDefect(
+                "local-part is not dot-atom, quoted-string, or obs-local-part"))
+        else:
+            local_part.defects.append(errors.ObsoleteHeaderDefect(
+                "local-part is not a dot-atom (contains CFWS)"))
+        local_part[0] = obs_local_part
+    try:
+        local_part.value.encode('ascii')
+    except UnicodeEncodeError:
+        local_part.defects.append(errors.NonASCIILocalPartDefect(
+                "local-part contains non-ASCII characters)"))
+    return local_part, value
+
+def get_obs_local_part(value):
+    """ obs-local-part = word *("." word)
+    """
+    obs_local_part = ObsLocalPart()
+    last_non_ws_was_dot = False
+    while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
+        if value[0] == '.':
+            if last_non_ws_was_dot:
+                obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                    "invalid repeated '.'"))
+            obs_local_part.append(DOT)
+            last_non_ws_was_dot = True
+            value = value[1:]
+            continue
+        elif value[0]=='\\':
+            obs_local_part.append(ValueTerminal(value[0],
+                                                'misplaced-special'))
+            value = value[1:]
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "'\\' character outside of quoted-string/ccontent"))
+            last_non_ws_was_dot = False
+            continue
+        if obs_local_part and obs_local_part[-1].token_type != 'dot':
+            obs_local_part.defects.append(errors.InvalidHeaderDefect(
+                "missing '.' between words"))
+        try:
+            token, value = get_word(value)
+            last_non_ws_was_dot = False
+        except errors.HeaderParseError:
+            if value[0] not in CFWS_LEADER:
+                raise
+            token, value = get_cfws(value)
+        obs_local_part.append(token)
+    if (obs_local_part[0].token_type == 'dot' or
+            obs_local_part[0].token_type=='cfws' and
+            obs_local_part[1].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid leading '.' in local part"))
+    if (obs_local_part[-1].token_type == 'dot' or
+            obs_local_part[-1].token_type=='cfws' and
+            obs_local_part[-2].token_type=='dot'):
+        obs_local_part.defects.append(errors.InvalidHeaderDefect(
+            "Invalid trailing '.' in local part"))
+    if obs_local_part.defects:
+        obs_local_part.token_type = 'invalid-obs-local-part'
+    return obs_local_part, value
+
+def get_dtext(value):
+    """ dtext = <printable ascii except \ [ ]> / obs-dtext
+        obs-dtext = obs-NO-WS-CTL / quoted-pair
+
+    We allow anything except the excluded characters, but but if we find any
+    ASCII other than the RFC defined printable ASCII an NonPrintableDefect is
+    added to the token's defects list.  Quoted pairs are converted to their
+    unquoted values, so what is returned is a ptext token, in this case a
+    ValueTerminal.  If there were quoted-printables, an ObsoleteHeaderDefect is
+    added to the returned token's defect list.
+
+    """
+    ptext, value, had_qp = _get_ptext_to_endchars(value, '[]')
+    ptext = ValueTerminal(ptext, 'ptext')
+    if had_qp:
+        ptext.defects.append(errors.ObsoleteHeaderDefect(
+            "quoted printable found in domain-literal"))
+    _validate_xtext(ptext)
+    return ptext, value
+
+def _check_for_early_dl_end(value, domain_literal):
+    if value:
+        return False
+    domain_literal.append(errors.InvalidHeaderDefect(
+        "end of input inside domain-literal"))
+    domain_literal.append(ValueTerminal(']', 'domain-literal-end'))
+    return True
+
+def get_domain_literal(value):
+    """ domain-literal = [CFWS] "[" *([FWS] dtext) [FWS] "]" [CFWS]
+
+    """
+    domain_literal = DomainLiteral()
+    if value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        domain_literal.append(token)
+    if not value:
+        raise errors.HeaderParseError("expected domain-literal")
+    if value[0] != '[':
+        raise errors.HeaderParseError("expected '[' at start of domain-literal "
+                "but found '{}'".format(value))
+    value = value[1:]
+    if _check_for_early_dl_end(value, domain_literal):
+        return domain_literal, value
+    domain_literal.append(ValueTerminal('[', 'domain-literal-start'))
+    if value[0] in WSP:
+        token, value = get_fws(value)
+        domain_literal.append(token)
+    token, value = get_dtext(value)
+    domain_literal.append(token)
+    if _check_for_early_dl_end(value, domain_literal):
+        return domain_literal, value
+    if value[0] in WSP:
+        token, value = get_fws(value)
+        domain_literal.append(token)
+    if _check_for_early_dl_end(value, domain_literal):
+        return domain_literal, value
+    if value[0] != ']':
+        raise errors.HeaderParseError("expected ']' at end of domain-literal "
+                "but found '{}'".format(value))
+    domain_literal.append(ValueTerminal(']', 'domain-literal-end'))
+    value = value[1:]
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        domain_literal.append(token)
+    return domain_literal, value
+
+def get_domain(value):
+    """ domain = dot-atom / domain-literal / obs-domain
+        obs-domain = atom *("." atom))
+
+    """
+    domain = Domain()
+    leader = None
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+    if not value:
+        raise errors.HeaderParseError(
+            "expected domain but found '{}'".format(value))
+    if value[0] == '[':
+        token, value = get_domain_literal(value)
+        if leader is not None:
+            token[:0] = [leader]
+        domain.append(token)
+        return domain, value
+    try:
+        token, value = get_dot_atom(value)
+    except errors.HeaderParseError:
+        token, value = get_atom(value)
+    if leader is not None:
+        token[:0] = [leader]
+    domain.append(token)
+    if value and value[0] == '.':
+        domain.defects.append(errors.ObsoleteHeaderDefect(
+            "domain is not a dot-atom (contains CFWS)"))
+        if domain[0].token_type == 'dot-atom':
+            domain[:] = domain[0]
+        while value and value[0] == '.':
+            domain.append(DOT)
+            token, value = get_atom(value[1:])
+            domain.append(token)
+    return domain, value
+
+def get_addr_spec(value):
+    """ addr-spec = local-part "@" domain
+
+    """
+    addr_spec = AddrSpec()
+    token, value = get_local_part(value)
+    addr_spec.append(token)
+    if not value or value[0] != '@':
+        addr_spec.defects.append(errors.InvalidHeaderDefect(
+            "add-spec local part with no domain"))
+        return addr_spec, value
+    addr_spec.append(ValueTerminal('@', 'address-at-symbol'))
+    token, value = get_domain(value[1:])
+    addr_spec.append(token)
+    return addr_spec, value
+
+def get_obs_route(value):
+    """ obs-route = obs-domain-list ":"
+        obs-domain-list = *(CFWS / ",") "@" domain *("," [CFWS] ["@" domain])
+
+        Returns an obs-route token with the appropriate sub-tokens (that is,
+        there is no obs-domain-list in the parse tree).
+    """
+    obs_route = ObsRoute()
+    while value and (value[0]==',' or value[0] in CFWS_LEADER):
+        if value[0] in CFWS_LEADER:
+            token, value = get_cfws(value)
+            obs_route.append(token)
+        elif value[0] == ',':
+            obs_route.append(ListSeparator)
+            value = value[1:]
+    if not value or value[0] != '@':
+        raise errors.HeaderParseError(
+            "expected obs-route domain but found '{}'".format(value))
+    obs_route.append(RouteComponentMarker)
+    token, value = get_domain(value[1:])
+    obs_route.append(token)
+    while value and value[0]==',':
+        obs_route.append(ListSeparator)
+        value = value[1:]
+        if not value:
+            break
+        if value[0] in CFWS_LEADER:
+            token, value = get_cfws(value)
+            obs_route.append(token)
+        if value[0] == '@':
+            obs_route.append(RouteComponentMarker)
+            token, value = get_domain(value[1:])
+            obs_route.append(token)
+    if not value:
+        raise errors.HeaderParseError("end of header while parsing obs-route")
+    if value[0] != ':':
+        raise errors.HeaderParseError( "expected ':' marking end of "
+            "obs-route but found '{}'".format(value))
+    obs_route.append(ValueTerminal(':', 'end-of-obs-route-marker'))
+    return obs_route, value[1:]
+
+def get_angle_addr(value):
+    """ angle-addr = [CFWS] "<" addr-spec ">" [CFWS] / obs-angle-addr
+        obs-angle-addr = [CFWS] "<" obs-route addr-spec ">" [CFWS]
+
+    """
+    angle_addr = AngleAddr()
+    if value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        angle_addr.append(token)
+    if not value or value[0] != '<':
+        raise errors.HeaderParseError(
+            "expected angle-addr but found '{}'".format(value))
+    angle_addr.append(ValueTerminal('<', 'angle-addr-start'))
+    value = value[1:]
+    # Although it is not legal per RFC5322, SMTP uses '<>' in certain
+    # circumstances.
+    if value[0] == '>':
+        angle_addr.append(ValueTerminal('>', 'angle-addr-end'))
+        angle_addr.defects.append(errors.InvalidHeaderDefect(
+            "null addr-spec in angle-addr"))
+        value = value[1:]
+        return angle_addr, value
+    try:
+        token, value = get_addr_spec(value)
+    except errors.HeaderParseError:
+        try:
+            token, value = get_obs_route(value)
+            angle_addr.defects.append(errors.ObsoleteHeaderDefect(
+                "obsolete route specification in angle-addr"))
+        except errors.HeaderParseError:
+            raise errors.HeaderParseError(
+                "expected addr-spec or obs-route but found '{}'".format(value))
+        angle_addr.append(token)
+        token, value = get_addr_spec(value)
+    angle_addr.append(token)
+    if value and value[0] == '>':
+        value = value[1:]
+    else:
+        angle_addr.defects.append(errors.InvalidHeaderDefect(
+            "missing trailing '>' on angle-addr"))
+    angle_addr.append(ValueTerminal('>', 'angle-addr-end'))
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        angle_addr.append(token)
+    return angle_addr, value
+
+def get_display_name(value):
+    """ display-name = phrase
+
+    Because this is simply a name-rule, we don't return a display-name
+    token containing a phrase, but rather a display-name token with
+    the content of the phrase.
+
+    """
+    display_name = DisplayName()
+    token, value = get_phrase(value)
+    display_name.extend(token[:])
+    display_name.defects = token.defects[:]
+    return display_name, value
+
+
+def get_name_addr(value):
+    """ name-addr = [display-name] angle-addr
+
+    """
+    name_addr = NameAddr()
+    # Both the optional display name and the angle-addr can start with cfws.
+    leader = None
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(leader))
+    if value[0] != '<':
+        if value[0] in PHRASE_ENDS:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(value))
+        token, value = get_display_name(value)
+        if not value:
+            raise errors.HeaderParseError(
+                "expected name-addr but found '{}'".format(token))
+        if leader is not None:
+            token[0][:0] = [leader]
+            leader = None
+        name_addr.append(token)
+    token, value = get_angle_addr(value)
+    if leader is not None:
+        token[:0] = [leader]
+    name_addr.append(token)
+    return name_addr, value
+
+def get_mailbox(value):
+    """ mailbox = name-addr / addr-spec
+
+    """
+    # The only way to figure out if we are dealing with a name-addr or an
+    # addr-spec is to try parsing each one.
+    mailbox = Mailbox()
+    try:
+        token, value = get_name_addr(value)
+    except errors.HeaderParseError:
+        try:
+            token, value = get_addr_spec(value)
+        except errors.HeaderParseError:
+            raise errors.HeaderParseError(
+                "expected mailbox but found '{}'".format(value))
+    if any(isinstance(x, errors.InvalidHeaderDefect)
+                       for x in token.all_defects):
+        mailbox.token_type = 'invalid-mailbox'
+    mailbox.append(token)
+    return mailbox, value
+
+def get_invalid_mailbox(value, endchars):
+    """ Read everything up to one of the chars in endchars.
+
+    This is outside the formal grammar.  The InvalidMailbox TokenList that is
+    returned acts like a Mailbox, but the data attributes are None.
+
+    """
+    invalid_mailbox = InvalidMailbox()
+    while value and value[0] not in endchars:
+        if value[0] in PHRASE_ENDS:
+            invalid_mailbox.append(ValueTerminal(value[0],
+                                                 'misplaced-special'))
+            value = value[1:]
+        else:
+            token, value = get_phrase(value)
+            invalid_mailbox.append(token)
+    return invalid_mailbox, value
+
+def get_mailbox_list(value):
+    """ mailbox-list = (mailbox *("," mailbox)) / obs-mbox-list
+        obs-mbox-list = *([CFWS] ",") mailbox *("," [mailbox / CFWS])
+
+    For this routine we go outside the formal grammar in order to improve error
+    handling.  We recognize the end of the mailbox list only at the end of the
+    value or at a ';' (the group terminator).  This is so that we can turn
+    invalid mailboxes into InvalidMailbox tokens and continue parsing any
+    remaining valid mailboxes.  We also allow all mailbox entries to be null,
+    and this condition is handled appropriately at a higher level.
+
+    """
+    mailbox_list = MailboxList()
+    while value and value[0] != ';':
+        try:
+            token, value = get_mailbox(value)
+            mailbox_list.append(token)
+        except errors.HeaderParseError:
+            leader = None
+            if value[0] in CFWS_LEADER:
+                leader, value = get_cfws(value)
+                if not value or value[0] in ',;':
+                    mailbox_list.append(leader)
+                    mailbox_list.defects.append(errors.ObsoleteHeaderDefect(
+                        "empty element in mailbox-list"))
+                else:
+                    token, value = get_invalid_mailbox(value, ',;')
+                    if leader is not None:
+                        token[:0] = [leader]
+                    mailbox_list.append(token)
+                    mailbox_list.defects.append(errors.InvalidHeaderDefect(
+                        "invalid mailbox in mailbox-list"))
+            elif value[0] == ',':
+                mailbox_list.defects.append(errors.ObsoleteHeaderDefect(
+                    "empty element in mailbox-list"))
+            else:
+                token, value = get_invalid_mailbox(value, ',;')
+                if leader is not None:
+                    token[:0] = [leader]
+                mailbox_list.append(token)
+                mailbox_list.defects.append(errors.InvalidHeaderDefect(
+                    "invalid mailbox in mailbox-list"))
+        if value and value[0] not in ',;':
+            # Crap after mailbox; treat it as an invalid mailbox.
+            # The mailbox info will still be available.
+            mailbox = mailbox_list[-1]
+            mailbox.token_type = 'invalid-mailbox'
+            token, value = get_invalid_mailbox(value, ',;')
+            mailbox.extend(token)
+            mailbox_list.defects.append(errors.InvalidHeaderDefect(
+                "invalid mailbox in mailbox-list"))
+        if value and value[0] == ',':
+            mailbox_list.append(ListSeparator)
+            value = value[1:]
+    return mailbox_list, value
+
+
+def get_group_list(value):
+    """ group-list = mailbox-list / CFWS / obs-group-list
+        obs-group-list = 1*([CFWS] ",") [CFWS]
+
+    """
+    group_list = GroupList()
+    if not value:
+        group_list.defects.append(errors.InvalidHeaderDefect(
+            "end of header before group-list"))
+        return group_list, value
+    leader = None
+    if value and value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+        if not value:
+            # This should never happen in email parsing, since CFWS-only is a
+            # legal alternative to group-list in a group, which is the only
+            # place group-list appears.
+            group_list.defects.append(errors.InvalidHeaderDefect(
+                "end of header in group-list"))
+            group_list.append(leader)
+            return group_list, value
+        if value[0] == ';':
+            group_list.append(leader)
+            return group_list, value
+    token, value = get_mailbox_list(value)
+    if len(token.all_mailboxes)==0:
+        if leader is not None:
+            group_list.append(leader)
+        group_list.extend(token)
+        group_list.defects.append(errors.ObsoleteHeaderDefect(
+            "group-list with empty entries"))
+        return group_list, value
+    if leader is not None:
+        token[:0] = [leader]
+    group_list.append(token)
+    return group_list, value
+
+def get_group(value):
+    """ group = display-name ":" [group-list] ";" [CFWS]
+
+    """
+    group = Group()
+    token, value = get_display_name(value)
+    if not value or value[0] != ':':
+        raise errors.HeaderParseError("expected ':' at end of group "
+            "display name but found '{}'".format(value))
+    group.append(token)
+    group.append(ValueTerminal(':', 'group-display-name-terminator'))
+    value = value[1:]
+    if value and value[0] == ';':
+        group.append(ValueTerminal(';', 'group-terminator'))
+        return group, value[1:]
+    token, value = get_group_list(value)
+    group.append(token)
+    if not value:
+        group.defects.append(errors.InvalidHeaderDefect(
+            "end of header in group"))
+    if value[0] != ';':
+        raise errors.HeaderParseError(
+            "expected ';' at end of group but found {}".format(value))
+    group.append(ValueTerminal(';', 'group-terminator'))
+    value = value[1:]
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        group.append(token)
+    return group, value
+
+def get_address(value):
+    """ address = mailbox / group
+
+    Note that counter-intuitively, an address can be either a single address or
+    a list of addresses (a group).  This is why the returned Address object has
+    a 'mailboxes' attribute which treats a single address as a list of length
+    one.  When you need to differentiate between to two cases, extract the single
+    element, which is either a mailbox or a group token.
+
+    """
+    # The formal grammar isn't very helpful when parsing an address.  mailbox
+    # and group, especially when allowing for obsolete forms, start off very
+    # similarly.  It is only when you reach one of @, <, or : that you know
+    # what you've got.  So, we try each one in turn, starting with the more
+    # likely of the two.  We could perhaps make this more efficient by looking
+    # for a phrase and then branching based on the next character, but that
+    # would be a premature optimization.
+    address = Address()
+    try:
+        token, value = get_group(value)
+    except errors.HeaderParseError:
+        try:
+            token, value = get_mailbox(value)
+        except errors.HeaderParseError:
+            raise errors.HeaderParseError(
+                "expected address but found '{}'".format(value))
+    address.append(token)
+    return address, value
+
+def get_address_list(value):
+    """ address_list = (address *("," address)) / obs-addr-list
+        obs-addr-list = *([CFWS] ",") address *("," [address / CFWS])
+
+    We depart from the formal grammar here by continuing to parse until the end
+    of the input, assuming the input to be entirely composed of an
+    address-list.  This is always true in email parsing, and allows us
+    to skip invalid addresses to parse additional valid ones.
+
+    """
+    address_list = AddressList()
+    while value:
+        try:
+            token, value = get_address(value)
+            address_list.append(token)
+        except errors.HeaderParseError as err:
+            leader = None
+            if value[0] in CFWS_LEADER:
+                leader, value = get_cfws(value)
+                if not value or value[0] == ',':
+                    address_list.append(leader)
+                    address_list.defects.append(errors.ObsoleteHeaderDefect(
+                        "address-list entry with no content"))
+                else:
+                    token, value = get_invalid_mailbox(value, ',')
+                    if leader is not None:
+                        token[:0] = [leader]
+                    address_list.append(Address([token]))
+                    address_list.defects.append(errors.InvalidHeaderDefect(
+                        "invalid address in address-list"))
+            elif value[0] == ',':
+                address_list.defects.append(errors.ObsoleteHeaderDefect(
+                    "empty element in address-list"))
+            else:
+                token, value = get_invalid_mailbox(value, ',')
+                if leader is not None:
+                    token[:0] = [leader]
+                address_list.append(Address([token]))
+                address_list.defects.append(errors.InvalidHeaderDefect(
+                    "invalid address in address-list"))
+        if value and value[0] != ',':
+            # Crap after address; treat it as an invalid mailbox.
+            # The mailbox info will still be available.
+            mailbox = address_list[-1][0]
+            mailbox.token_type = 'invalid-mailbox'
+            token, value = get_invalid_mailbox(value, ',')
+            mailbox.extend(token)
+            address_list.defects.append(errors.InvalidHeaderDefect(
+                "invalid address in address-list"))
+        if value:  # Must be a , at this point.
+            address_list.append(ValueTerminal(',', 'list-separator'))
+            value = value[1:]
+    return address_list, value
+
+#
+# XXX: As I begin to add additional header parsers, I'm realizing we probably
+# have two level of parser routines: the get_XXX methods that get a token in
+# the grammar, and parse_XXX methods that parse an entire field value.  So
+# get_address_list above should really be a parse_ method, as probably should
+# be get_unstructured.
+#
+
+def parse_mime_version(value):
+    """ mime-version = [CFWS] 1*digit [CFWS] "." [CFWS] 1*digit [CFWS]
+
+    """
+    # The [CFWS] is implicit in the RFC 2045 BNF.
+    # XXX: This routine is a bit verbose, should factor out a get_int method.
+    mime_version = MIMEVersion()
+    if not value:
+        mime_version.defects.append(errors.HeaderMissingRequiredValue(
+            "Missing MIME version number (eg: 1.0)"))
+        return mime_version
+    if value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        mime_version.append(token)
+        if not value:
+            mime_version.defects.append(errors.HeaderMissingRequiredValue(
+                "Expected MIME version number but found only CFWS"))
+    digits = ''
+    while value and value[0] != '.' and value[0] not in CFWS_LEADER:
+        digits += value[0]
+        value = value[1:]
+    if not digits.isdigit():
+        mime_version.defects.append(errors.InvalidHeaderDefect(
+            "Expected MIME major version number but found {!r}".format(digits)))
+        mime_version.append(ValueTerminal(digits, 'xtext'))
+    else:
+        mime_version.major = int(digits)
+        mime_version.append(ValueTerminal(digits, 'digits'))
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        mime_version.append(token)
+    if not value or value[0] != '.':
+        if mime_version.major is not None:
+            mime_version.defects.append(errors.InvalidHeaderDefect(
+                "Incomplete MIME version; found only major number"))
+        if value:
+            mime_version.append(ValueTerminal(value, 'xtext'))
+        return mime_version
+    mime_version.append(ValueTerminal('.', 'version-separator'))
+    value = value[1:]
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        mime_version.append(token)
+    if not value:
+        if mime_version.major is not None:
+            mime_version.defects.append(errors.InvalidHeaderDefect(
+                "Incomplete MIME version; found only major number"))
+        return mime_version
+    digits = ''
+    while value and value[0] not in CFWS_LEADER:
+        digits += value[0]
+        value = value[1:]
+    if not digits.isdigit():
+        mime_version.defects.append(errors.InvalidHeaderDefect(
+            "Expected MIME minor version number but found {!r}".format(digits)))
+        mime_version.append(ValueTerminal(digits, 'xtext'))
+    else:
+        mime_version.minor = int(digits)
+        mime_version.append(ValueTerminal(digits, 'digits'))
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        mime_version.append(token)
+    if value:
+        mime_version.defects.append(errors.InvalidHeaderDefect(
+            "Excess non-CFWS text after MIME version"))
+        mime_version.append(ValueTerminal(value, 'xtext'))
+    return mime_version
+
+def get_invalid_parameter(value):
+    """ Read everything up to the next ';'.
+
+    This is outside the formal grammar.  The InvalidParameter TokenList that is
+    returned acts like a Parameter, but the data attributes are None.
+
+    """
+    invalid_parameter = InvalidParameter()
+    while value and value[0] != ';':
+        if value[0] in PHRASE_ENDS:
+            invalid_parameter.append(ValueTerminal(value[0],
+                                                   'misplaced-special'))
+            value = value[1:]
+        else:
+            token, value = get_phrase(value)
+            invalid_parameter.append(token)
+    return invalid_parameter, value
+
+def get_ttext(value):
+    """ttext = <matches _ttext_matcher>
+
+    We allow any non-TOKEN_ENDS in ttext, but add defects to the token's
+    defects list if we find non-ttext characters.  We also register defects for
+    *any* non-printables even though the RFC doesn't exclude all of them,
+    because we follow the spirit of RFC 5322.
+
+    """
+    m = _non_token_end_matcher(value)
+    if not m:
+        raise errors.HeaderParseError(
+            "expected ttext but found '{}'".format(value))
+    ttext = m.group()
+    value = value[len(ttext):]
+    ttext = ValueTerminal(ttext, 'ttext')
+    _validate_xtext(ttext)
+    return ttext, value
+
+def get_token(value):
+    """token = [CFWS] 1*ttext [CFWS]
+
+    The RFC equivalent of ttext is any US-ASCII chars except space, ctls, or
+    tspecials.  We also exclude tabs even though the RFC doesn't.
+
+    The RFC implies the CFWS but is not explicit about it in the BNF.
+
+    """
+    mtoken = Token()
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        mtoken.append(token)
+    if value and value[0] in TOKEN_ENDS:
+        raise errors.HeaderParseError(
+            "expected token but found '{}'".format(value))
+    token, value = get_ttext(value)
+    mtoken.append(token)
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        mtoken.append(token)
+    return mtoken, value
+
+def get_attrtext(value):
+    """attrtext = 1*(any non-ATTRIBUTE_ENDS character)
+
+    We allow any non-ATTRIBUTE_ENDS in attrtext, but add defects to the
+    token's defects list if we find non-attrtext characters.  We also register
+    defects for *any* non-printables even though the RFC doesn't exclude all of
+    them, because we follow the spirit of RFC 5322.
+
+    """
+    m = _non_attribute_end_matcher(value)
+    if not m:
+        raise errors.HeaderParseError(
+            "expected attrtext but found {!r}".format(value))
+    attrtext = m.group()
+    value = value[len(attrtext):]
+    attrtext = ValueTerminal(attrtext, 'attrtext')
+    _validate_xtext(attrtext)
+    return attrtext, value
+
+def get_attribute(value):
+    """ [CFWS] 1*attrtext [CFWS]
+
+    This version of the BNF makes the CFWS explicit, and as usual we use a
+    value terminal for the actual run of characters.  The RFC equivalent of
+    attrtext is the token characters, with the subtraction of '*', "'", and '%'.
+    We include tab in the excluded set just as we do for token.
+
+    """
+    attribute = Attribute()
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        attribute.append(token)
+    if value and value[0] in ATTRIBUTE_ENDS:
+        raise errors.HeaderParseError(
+            "expected token but found '{}'".format(value))
+    token, value = get_attrtext(value)
+    attribute.append(token)
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        attribute.append(token)
+    return attribute, value
+
+def get_extended_attrtext(value):
+    """attrtext = 1*(any non-ATTRIBUTE_ENDS character plus '%')
+
+    This is a special parsing routine so that we get a value that
+    includes % escapes as a single string (which we decode as a single
+    string later).
+
+    """
+    m = _non_extended_attribute_end_matcher(value)
+    if not m:
+        raise errors.HeaderParseError(
+            "expected extended attrtext but found {!r}".format(value))
+    attrtext = m.group()
+    value = value[len(attrtext):]
+    attrtext = ValueTerminal(attrtext, 'extended-attrtext')
+    _validate_xtext(attrtext)
+    return attrtext, value
+
+def get_extended_attribute(value):
+    """ [CFWS] 1*extended_attrtext [CFWS]
+
+    This is like the non-extended version except we allow % characters, so that
+    we can pick up an encoded value as a single string.
+
+    """
+    # XXX: should we have an ExtendedAttribute TokenList?
+    attribute = Attribute()
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        attribute.append(token)
+    if value and value[0] in EXTENDED_ATTRIBUTE_ENDS:
+        raise errors.HeaderParseError(
+            "expected token but found '{}'".format(value))
+    token, value = get_extended_attrtext(value)
+    attribute.append(token)
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        attribute.append(token)
+    return attribute, value
+
+def get_section(value):
+    """ '*' digits
+
+    The formal BNF is more complicated because leading 0s are not allowed.  We
+    check for that and add a defect.  We also assume no CFWS is allowed between
+    the '*' and the digits, though the RFC is not crystal clear on that.
+    The caller should already have dealt with leading CFWS.
+
+    """
+    section = Section()
+    if not value or value[0] != '*':
+        raise errors.HeaderParseError("Expected section but found {}".format(
+                                        value))
+    section.append(ValueTerminal('*', 'section-marker'))
+    value = value[1:]
+    if not value or not value[0].isdigit():
+        raise errors.HeaderParseError("Expected section number but "
+                                      "found {}".format(value))
+    digits = ''
+    while value and value[0].isdigit():
+        digits += value[0]
+        value = value[1:]
+    if digits[0] == '0' and digits != '0':
+        section.defects.append(errors.InvalidHeaderError("section number"
+            "has an invalid leading 0"))
+    section.number = int(digits)
+    section.append(ValueTerminal(digits, 'digits'))
+    return section, value
+
+
+def get_value(value):
+    """ quoted-string / attribute
+
+    """
+    v = Value()
+    if not value:
+        raise errors.HeaderParseError("Expected value but found end of string")
+    leader = None
+    if value[0] in CFWS_LEADER:
+        leader, value = get_cfws(value)
+    if not value:
+        raise errors.HeaderParseError("Expected value but found "
+                                      "only {}".format(leader))
+    if value[0] == '"':
+        token, value = get_quoted_string(value)
+    else:
+        token, value = get_extended_attribute(value)
+    if leader is not None:
+        token[:0] = [leader]
+    v.append(token)
+    return v, value
+
+def get_parameter(value):
+    """ attribute [section] ["*"] [CFWS] "=" value
+
+    The CFWS is implied by the RFC but not made explicit in the BNF.  This
+    simplified form of the BNF from the RFC is made to conform with the RFC BNF
+    through some extra checks.  We do it this way because it makes both error
+    recovery and working with the resulting parse tree easier.
+    """
+    # It is possible CFWS would also be implicitly allowed between the section
+    # and the 'extended-attribute' marker (the '*') , but we've never seen that
+    # in the wild and we will therefore ignore the possibility.
+    param = Parameter()
+    token, value = get_attribute(value)
+    param.append(token)
+    if not value or value[0] == ';':
+        param.defects.append(errors.InvalidHeaderDefect("Parameter contains "
+            "name ({}) but no value".format(token)))
+        return param, value
+    if value[0] == '*':
+        try:
+            token, value = get_section(value)
+            param.sectioned = True
+            param.append(token)
+        except errors.HeaderParseError:
+            pass
+        if not value:
+            raise errors.HeaderParseError("Incomplete parameter")
+        if value[0] == '*':
+            param.append(ValueTerminal('*', 'extended-parameter-marker'))
+            value = value[1:]
+            param.extended = True
+    if value[0] != '=':
+        raise errors.HeaderParseError("Parameter not followed by '='")
+    param.append(ValueTerminal('=', 'parameter-separator'))
+    value = value[1:]
+    leader = None
+    if value and value[0] in CFWS_LEADER:
+        token, value = get_cfws(value)
+        param.append(token)
+    remainder = None
+    appendto = param
+    if param.extended and value and value[0] == '"':
+        # Now for some serious hackery to handle the common invalid case of
+        # double quotes around an extended value.  We also accept (with defect)
+        # a value marked as encoded that isn't really.
+        qstring, remainder = get_quoted_string(value)
+        inner_value = qstring.stripped_value
+        semi_valid = False
+        if param.section_number == 0:
+            if inner_value and inner_value[0] == "'":
+                semi_valid = True
+            else:
+                token, rest = get_attrtext(inner_value)
+                if rest and rest[0] == "'":
+                    semi_valid = True
+        else:
+            try:
+                token, rest = get_extended_attrtext(inner_value)
+            except:
+                pass
+            else:
+                if not rest:
+                    semi_valid = True
+        if semi_valid:
+            param.defects.append(errors.InvalidHeaderDefect(
+                "Quoted string value for extended parameter is invalid"))
+            param.append(qstring)
+            for t in qstring:
+                if t.token_type == 'bare-quoted-string':
+                    t[:] = []
+                    appendto = t
+                    break
+            value = inner_value
+        else:
+            remainder = None
+            param.defects.append(errors.InvalidHeaderDefect(
+                "Parameter marked as extended but appears to have a "
+                "quoted string value that is non-encoded"))
+    if value and value[0] == "'":
+        token = None
+    else:
+        token, value = get_value(value)
+    if not param.extended or param.section_number > 0:
+        if not value or value[0] != "'":
+            appendto.append(token)
+            if remainder is not None:
+                assert not value, value
+                value = remainder
+            return param, value
+        param.defects.append(errors.InvalidHeaderDefect(
+            "Apparent initial-extended-value but attribute "
+            "was not marked as extended or was not initial section"))
+    if not value:
+        # Assume the charset/lang is missing and the token is the value.
+        param.defects.append(errors.InvalidHeaderDefect(
+            "Missing required charset/lang delimiters"))
+        appendto.append(token)
+        if remainder is None:
+            return param, value
+    else:
+        if token is not None:
+            for t in token:
+                if t.token_type == 'extended-attrtext':
+                    break
+            t.token_type == 'attrtext'
+            appendto.append(t)
+            param.charset = t.value
+        if value[0] != "'":
+            raise errors.HeaderParseError("Expected RFC2231 char/lang encoding "
+                                          "delimiter, but found {!r}".format(value))
+        appendto.append(ValueTerminal("'", 'RFC2231 delimiter'))
+        value = value[1:]
+        if value and value[0] != "'":
+            token, value = get_attrtext(value)
+            appendto.append(token)
+            param.lang = token.value
+            if not value or value[0] != "'":
+                raise errors.HeaderParseError("Expected RFC2231 char/lang encoding "
+                                  "delimiter, but found {}".format(value))
+        appendto.append(ValueTerminal("'", 'RFC2231 delimiter'))
+        value = value[1:]
+    if remainder is not None:
+        # Treat the rest of value as bare quoted string content.
+        v = Value()
+        while value:
+            if value[0] in WSP:
+                token, value = get_fws(value)
+            else:
+                token, value = get_qcontent(value)
+            v.append(token)
+        token = v
+    else:
+        token, value = get_value(value)
+    appendto.append(token)
+    if remainder is not None:
+        assert not value, value
+        value = remainder
+    return param, value
+
+def parse_mime_parameters(value):
+    """ parameter *( ";" parameter )
+
+    That BNF is meant to indicate this routine should only be called after
+    finding and handling the leading ';'.  There is no corresponding rule in
+    the formal RFC grammar, but it is more convenient for us for the set of
+    parameters to be treated as its own TokenList.
+
+    This is 'parse' routine because it consumes the reminaing value, but it
+    would never be called to parse a full header.  Instead it is called to
+    parse everything after the non-parameter value of a specific MIME header.
+
+    """
+    mime_parameters = MimeParameters()
+    while value:
+        try:
+            token, value = get_parameter(value)
+            mime_parameters.append(token)
+        except errors.HeaderParseError as err:
+            leader = None
+            if value[0] in CFWS_LEADER:
+                leader, value = get_cfws(value)
+            if not value:
+                mime_parameters.append(leader)
+                return mime_parameters
+            if value[0] == ';':
+                if leader is not None:
+                    mime_parameters.append(leader)
+                mime_parameters.defects.append(errors.InvalidHeaderDefect(
+                    "parameter entry with no content"))
+            else:
+                token, value = get_invalid_parameter(value)
+                if leader:
+                    token[:0] = [leader]
+                mime_parameters.append(token)
+                mime_parameters.defects.append(errors.InvalidHeaderDefect(
+                    "invalid parameter {!r}".format(token)))
+        if value and value[0] != ';':
+            # Junk after the otherwise valid parameter.  Mark it as
+            # invalid, but it will have a value.
+            param = mime_parameters[-1]
+            param.token_type = 'invalid-parameter'
+            token, value = get_invalid_parameter(value)
+            param.extend(token)
+            mime_parameters.defects.append(errors.InvalidHeaderDefect(
+                "parameter with invalid trailing text {!r}".format(token)))
+        if value:
+            # Must be a ';' at this point.
+            mime_parameters.append(ValueTerminal(';', 'parameter-separator'))
+            value = value[1:]
+    return mime_parameters
+
+def _find_mime_parameters(tokenlist, value):
+    """Do our best to find the parameters in an invalid MIME header
+
+    """
+    while value and value[0] != ';':
+        if value[0] in PHRASE_ENDS:
+            tokenlist.append(ValueTerminal(value[0], 'misplaced-special'))
+            value = value[1:]
+        else:
+            token, value = get_phrase(value)
+            tokenlist.append(token)
+    if not value:
+        return
+    tokenlist.append(ValueTerminal(';', 'parameter-separator'))
+    tokenlist.append(parse_mime_parameters(value[1:]))
+
+def parse_content_type_header(value):
+    """ maintype "/" subtype *( ";" parameter )
+
+    The maintype and substype are tokens.  Theoretically they could
+    be checked against the official IANA list + x-token, but we
+    don't do that.
+    """
+    ctype = ContentType()
+    recover = False
+    if not value:
+        ctype.defects.append(errors.HeaderMissingRequiredValue(
+            "Missing content type specification"))
+        return ctype
+    try:
+        token, value = get_token(value)
+    except errors.HeaderParseError:
+        ctype.defects.append(errors.InvalidHeaderDefect(
+            "Expected content maintype but found {!r}".format(value)))
+        _find_mime_parameters(ctype, value)
+        return ctype
+    ctype.append(token)
+    # XXX: If we really want to follow the formal grammer we should make
+    # mantype and subtype specialized TokenLists here.  Probably not worth it.
+    if not value or value[0] != '/':
+        ctype.defects.append(errors.InvalidHeaderDefect(
+            "Invalid content type"))
+        if value:
+            _find_mime_parameters(ctype, value)
+        return ctype
+    ctype.maintype = token.value.strip().lower()
+    ctype.append(ValueTerminal('/', 'content-type-separator'))
+    value = value[1:]
+    try:
+        token, value = get_token(value)
+    except errors.HeaderParseError:
+        ctype.defects.append(errors.InvalidHeaderDefect(
+            "Expected content subtype but found {!r}".format(value)))
+        _find_mime_parameters(ctype, value)
+        return ctype
+    ctype.append(token)
+    ctype.subtype = token.value.strip().lower()
+    if not value:
+        return ctype
+    if value[0] != ';':
+        ctype.defects.append(errors.InvalidHeaderDefect(
+            "Only parameters are valid after content type, but "
+            "found {!r}".format(value)))
+        # The RFC requires that a syntactically invalid content-type be treated
+        # as text/plain.  Perhaps we should postel this, but we should probably
+        # only do that if we were checking the subtype value against IANA.
+        del ctype.maintype, ctype.subtype
+        _find_mime_parameters(ctype, value)
+        return ctype
+    ctype.append(ValueTerminal(';', 'parameter-separator'))
+    ctype.append(parse_mime_parameters(value[1:]))
+    return ctype
+
+def parse_content_disposition_header(value):
+    """ disposition-type *( ";" parameter )
+
+    """
+    disp_header = ContentDisposition()
+    if not value:
+        disp_header.defects.append(errors.HeaderMissingRequiredValue(
+            "Missing content disposition"))
+        return disp_header
+    try:
+        token, value = get_token(value)
+    except errors.HeaderParseError:
+        ctype.defects.append(errors.InvalidHeaderDefect(
+            "Expected content disposition but found {!r}".format(value)))
+        _find_mime_parameters(disp_header, value)
+        return disp_header
+    disp_header.append(token)
+    disp_header.content_disposition = token.value.strip().lower()
+    if not value:
+        return disp_header
+    if value[0] != ';':
+        disp_header.defects.append(errors.InvalidHeaderDefect(
+            "Only parameters are valid after content disposition, but "
+            "found {!r}".format(value)))
+        _find_mime_parameters(disp_header, value)
+        return disp_header
+    disp_header.append(ValueTerminal(';', 'parameter-separator'))
+    disp_header.append(parse_mime_parameters(value[1:]))
+    return disp_header
+
+def parse_content_transfer_encoding_header(value):
+    """ mechanism
+
+    """
+    # We should probably validate the values, since the list is fixed.
+    cte_header = ContentTransferEncoding()
+    if not value:
+        cte_header.defects.append(errors.HeaderMissingRequiredValue(
+            "Missing content transfer encoding"))
+        return cte_header
+    try:
+        token, value = get_token(value)
+    except errors.HeaderParseError:
+        ctype.defects.append(errors.InvalidHeaderDefect(
+            "Expected content trnasfer encoding but found {!r}".format(value)))
+    else:
+        cte_header.append(token)
+        cte_header.cte = token.value.strip().lower()
+    if not value:
+        return cte_header
+    while value:
+        cte_header.defects.append(errors.InvalidHeaderDefect(
+            "Extra text after content transfer encoding"))
+        if value[0] in PHRASE_ENDS:
+            cte_header.append(ValueTerminal(value[0], 'misplaced-special'))
+            value = value[1:]
+        else:
+            token, value = get_phrase(value)
+            cte_header.append(token)
+    return cte_header
diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py
index 79573c6177..cdfa3729ad 100644
--- a/Lib/email/_parseaddr.py
+++ b/Lib/email/_parseaddr.py
@@ -47,6 +47,25 @@ def parsedate_tz(data):
 
     Accounts for military timezones.
     """
+    res = _parsedate_tz(data)
+    if not res:
+        return
+    if res[9] is None:
+        res[9] = 0
+    return tuple(res)
+
+def _parsedate_tz(data):
+    """Convert date to extended time tuple.
+
+    The last (additional) element is the time zone offset in seconds, except if
+    the timezone was specified as -0000.  In that case the last element is
+    None.  This indicates a UTC timestamp that explicitly declaims knowledge of
+    the source timezone, as opposed to a +0000 timestamp that indicates the
+    source timezone really was UTC.
+
+    """
+    if not data:
+        return
     data = data.split()
     # The FWS after the comma after the day-of-week is optional, so search and
     # adjust for this.
@@ -99,6 +118,14 @@ def parsedate_tz(data):
         tss = '0'
     elif len(tm) == 3:
         [thh, tmm, tss] = tm
+    elif len(tm) == 1 and '.' in tm[0]:
+        # Some non-compliant MUAs use '.' to separate time elements.
+        tm = tm[0].split('.')
+        if len(tm) == 2:
+            [thh, tmm] = tm
+            tss = 0
+        elif len(tm) == 3:
+            [thh, tmm, tss] = tm
     else:
         return None
     try:
@@ -130,6 +157,8 @@ def parsedate_tz(data):
             tzoffset = int(tz)
         except ValueError:
             pass
+        if tzoffset==0 and tz.startswith('-'):
+            tzoffset = None
     # Convert a timezone offset into seconds ; -0500 -> -18000
     if tzoffset:
         if tzoffset < 0:
@@ -139,7 +168,7 @@ def parsedate_tz(data):
             tzsign = 1
         tzoffset = tzsign * ( (tzoffset//100)*3600 + (tzoffset % 100)*60)
     # Daylight Saving Time flag is set to -1, since DST is unknown.
-    return yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset
+    return [yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset]
 
 
 def parsedate(data):
diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py
new file mode 100644
index 0000000000..81061149c3
--- /dev/null
+++ b/Lib/email/_policybase.py
@@ -0,0 +1,358 @@
+"""Policy framework for the email package.
+
+Allows fine grained feature control of how the package parses and emits data.
+"""
+
+import abc
+from email import header
+from email import charset as _charset
+from email.utils import _has_surrogates
+
+__all__ = [
+    'Policy',
+    'Compat32',
+    'compat32',
+    ]
+
+
+class _PolicyBase:
+
+    """Policy Object basic framework.
+
+    This class is useless unless subclassed.  A subclass should define
+    class attributes with defaults for any values that are to be
+    managed by the Policy object.  The constructor will then allow
+    non-default values to be set for these attributes at instance
+    creation time.  The instance will be callable, taking these same
+    attributes keyword arguments, and returning a new instance
+    identical to the called instance except for those values changed
+    by the keyword arguments.  Instances may be added, yielding new
+    instances with any non-default values from the right hand
+    operand overriding those in the left hand operand.  That is,
+
+        A + B == A(<non-default values of B>)
+
+    The repr of an instance can be used to reconstruct the object
+    if and only if the repr of the values can be used to reconstruct
+    those values.
+
+    """
+
+    def __init__(self, **kw):
+        """Create new Policy, possibly overriding some defaults.
+
+        See class docstring for a list of overridable attributes.
+
+        """
+        for name, value in kw.items():
+            if hasattr(self, name):
+                super(_PolicyBase,self).__setattr__(name, value)
+            else:
+                raise TypeError(
+                    "{!r} is an invalid keyword argument for {}".format(
+                        name, self.__class__.__name__))
+
+    def __repr__(self):
+        args = [ "{}={!r}".format(name, value)
+                 for name, value in self.__dict__.items() ]
+        return "{}({})".format(self.__class__.__name__, ', '.join(args))
+
+    def clone(self, **kw):
+        """Return a new instance with specified attributes changed.
+
+        The new instance has the same attribute values as the current object,
+        except for the changes passed in as keyword arguments.
+
+        """
+        newpolicy = self.__class__.__new__(self.__class__)
+        for attr, value in self.__dict__.items():
+            object.__setattr__(newpolicy, attr, value)
+        for attr, value in kw.items():
+            if not hasattr(self, attr):
+                raise TypeError(
+                    "{!r} is an invalid keyword argument for {}".format(
+                        attr, self.__class__.__name__))
+            object.__setattr__(newpolicy, attr, value)
+        return newpolicy
+
+    def __setattr__(self, name, value):
+        if hasattr(self, name):
+            msg = "{!r} object attribute {!r} is read-only"
+        else:
+            msg = "{!r} object has no attribute {!r}"
+        raise AttributeError(msg.format(self.__class__.__name__, name))
+
+    def __add__(self, other):
+        """Non-default values from right operand override those from left.
+
+        The object returned is a new instance of the subclass.
+
+        """
+        return self.clone(**other.__dict__)
+
+
+def _append_doc(doc, added_doc):
+    doc = doc.rsplit('\n', 1)[0]
+    added_doc = added_doc.split('\n', 1)[1]
+    return doc + '\n' + added_doc
+
+def _extend_docstrings(cls):
+    if cls.__doc__ and cls.__doc__.startswith('+'):
+        cls.__doc__ = _append_doc(cls.__bases__[0].__doc__, cls.__doc__)
+    for name, attr in cls.__dict__.items():
+        if attr.__doc__ and attr.__doc__.startswith('+'):
+            for c in (c for base in cls.__bases__ for c in base.mro()):
+                doc = getattr(getattr(c, name), '__doc__')
+                if doc:
+                    attr.__doc__ = _append_doc(doc, attr.__doc__)
+                    break
+    return cls
+
+
+class Policy(_PolicyBase, metaclass=abc.ABCMeta):
+
+    r"""Controls for how messages are interpreted and formatted.
+
+    Most of the classes and many of the methods in the email package accept
+    Policy objects as parameters.  A Policy object contains a set of values and
+    functions that control how input is interpreted and how output is rendered.
+    For example, the parameter 'raise_on_defect' controls whether or not an RFC
+    violation results in an error being raised or not, while 'max_line_length'
+    controls the maximum length of output lines when a Message is serialized.
+
+    Any valid attribute may be overridden when a Policy is created by passing
+    it as a keyword argument to the constructor.  Policy objects are immutable,
+    but a new Policy object can be created with only certain values changed by
+    calling the Policy instance with keyword arguments.  Policy objects can
+    also be added, producing a new Policy object in which the non-default
+    attributes set in the right hand operand overwrite those specified in the
+    left operand.
+
+    Settable attributes:
+
+    raise_on_defect     -- If true, then defects should be raised as errors.
+                           Default: False.
+
+    linesep             -- string containing the value to use as separation
+                           between output lines.  Default '\n'.
+
+    cte_type            -- Type of allowed content transfer encodings
+
+                           7bit  -- ASCII only
+                           8bit  -- Content-Transfer-Encoding: 8bit is allowed
+
+                           Default: 8bit.  Also controls the disposition of
+                           (RFC invalid) binary data in headers; see the
+                           documentation of the binary_fold method.
+
+    max_line_length     -- maximum length of lines, excluding 'linesep',
+                           during serialization.  None or 0 means no line
+                           wrapping is done.  Default is 78.
+
+    """
+
+    raise_on_defect = False
+    linesep = '\n'
+    cte_type = '8bit'
+    max_line_length = 78
+
+    def handle_defect(self, obj, defect):
+        """Based on policy, either raise defect or call register_defect.
+
+            handle_defect(obj, defect)
+
+        defect should be a Defect subclass, but in any case must be an
+        Exception subclass.  obj is the object on which the defect should be
+        registered if it is not raised.  If the raise_on_defect is True, the
+        defect is raised as an error, otherwise the object and the defect are
+        passed to register_defect.
+
+        This method is intended to be called by parsers that discover defects.
+        The email package parsers always call it with Defect instances.
+
+        """
+        if self.raise_on_defect:
+            raise defect
+        self.register_defect(obj, defect)
+
+    def register_defect(self, obj, defect):
+        """Record 'defect' on 'obj'.
+
+        Called by handle_defect if raise_on_defect is False.  This method is
+        part of the Policy API so that Policy subclasses can implement custom
+        defect handling.  The default implementation calls the append method of
+        the defects attribute of obj.  The objects used by the email package by
+        default that get passed to this method will always have a defects
+        attribute with an append method.
+
+        """
+        obj.defects.append(defect)
+
+    def header_max_count(self, name):
+        """Return the maximum allowed number of headers named 'name'.
+
+        Called when a header is added to a Message object.  If the returned
+        value is not 0 or None, and there are already a number of headers with
+        the name 'name' equal to the value returned, a ValueError is raised.
+
+        Because the default behavior of Message's __setitem__ is to append the
+        value to the list of headers, it is easy to create duplicate headers
+        without realizing it.  This method allows certain headers to be limited
+        in the number of instances of that header that may be added to a
+        Message programmatically.  (The limit is not observed by the parser,
+        which will faithfully produce as many headers as exist in the message
+        being parsed.)
+
+        The default implementation returns None for all header names.
+        """
+        return None
+
+    @abc.abstractmethod
+    def header_source_parse(self, sourcelines):
+        """Given a list of linesep terminated strings constituting the lines of
+        a single header, return the (name, value) tuple that should be stored
+        in the model.  The input lines should retain their terminating linesep
+        characters.  The lines passed in by the email package may contain
+        surrogateescaped binary data.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def header_store_parse(self, name, value):
+        """Given the header name and the value provided by the application
+        program, return the (name, value) that should be stored in the model.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def header_fetch_parse(self, name, value):
+        """Given the header name and the value from the model, return the value
+        to be returned to the application program that is requesting that
+        header.  The value passed in by the email package may contain
+        surrogateescaped binary data if the lines were parsed by a BytesParser.
+        The returned value should not contain any surrogateescaped data.
+
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def fold(self, name, value):
+        """Given the header name and the value from the model, return a string
+        containing linesep characters that implement the folding of the header
+        according to the policy controls.  The value passed in by the email
+        package may contain surrogateescaped binary data if the lines were
+        parsed by a BytesParser.  The returned value should not contain any
+        surrogateescaped data.
+
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def fold_binary(self, name, value):
+        """Given the header name and the value from the model, return binary
+        data containing linesep characters that implement the folding of the
+        header according to the policy controls.  The value passed in by the
+        email package may contain surrogateescaped binary data.
+
+        """
+        raise NotImplementedError
+
+
+@_extend_docstrings
+class Compat32(Policy):
+
+    """+
+    This particular policy is the backward compatibility Policy.  It
+    replicates the behavior of the email package version 5.1.
+    """
+
+    def _sanitize_header(self, name, value):
+        # If the header value contains surrogates, return a Header using
+        # the unknown-8bit charset to encode the bytes as encoded words.
+        if not isinstance(value, str):
+            # Assume it is already a header object
+            return value
+        if _has_surrogates(value):
+            return header.Header(value, charset=_charset.UNKNOWN8BIT,
+                                 header_name=name)
+        else:
+            return value
+
+    def header_source_parse(self, sourcelines):
+        """+
+        The name is parsed as everything up to the ':' and returned unmodified.
+        The value is determined by stripping leading whitespace off the
+        remainder of the first line, joining all subsequent lines together, and
+        stripping any trailing carriage return or linefeed characters.
+
+        """
+        name, value = sourcelines[0].split(':', 1)
+        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        return (name, value.rstrip('\r\n'))
+
+    def header_store_parse(self, name, value):
+        """+
+        The name and value are returned unmodified.
+        """
+        return (name, value)
+
+    def header_fetch_parse(self, name, value):
+        """+
+        If the value contains binary data, it is converted into a Header object
+        using the unknown-8bit charset.  Otherwise it is returned unmodified.
+        """
+        return self._sanitize_header(name, value)
+
+    def fold(self, name, value):
+        """+
+        Headers are folded using the Header folding algorithm, which preserves
+        existing line breaks in the value, and wraps each resulting line to the
+        max_line_length.  Non-ASCII binary data are CTE encoded using the
+        unknown-8bit charset.
+
+        """
+        return self._fold(name, value, sanitize=True)
+
+    def fold_binary(self, name, value):
+        """+
+        Headers are folded using the Header folding algorithm, which preserves
+        existing line breaks in the value, and wraps each resulting line to the
+        max_line_length.  If cte_type is 7bit, non-ascii binary data is CTE
+        encoded using the unknown-8bit charset.  Otherwise the original source
+        header is used, with its existing line breaks and/or binary data.
+
+        """
+        folded = self._fold(name, value, sanitize=self.cte_type=='7bit')
+        return folded.encode('ascii', 'surrogateescape')
+
+    def _fold(self, name, value, sanitize):
+        parts = []
+        parts.append('%s: ' % name)
+        if isinstance(value, str):
+            if _has_surrogates(value):
+                if sanitize:
+                    h = header.Header(value,
+                                      charset=_charset.UNKNOWN8BIT,
+                                      header_name=name)
+                else:
+                    # If we have raw 8bit data in a byte string, we have no idea
+                    # what the encoding is.  There is no safe way to split this
+                    # string.  If it's ascii-subset, then we could do a normal
+                    # ascii split, but if it's multibyte then we could break the
+                    # string.  There's no way to know so the least harm seems to
+                    # be to not split the string and risk it being too long.
+                    parts.append(value)
+                    h = None
+            else:
+                h = header.Header(value, header_name=name)
+        else:
+            # Assume it is a Header-like object.
+            h = value
+        if h is not None:
+            parts.append(h.encode(linesep=self.linesep,
+                                  maxlinelen=self.max_line_length))
+        parts.append(self.linesep)
+        return ''.join(parts)
+
+
+compat32 = Compat32()
diff --git a/Lib/email/architecture.rst b/Lib/email/architecture.rst
new file mode 100644
index 0000000000..80d24fe5b9
--- /dev/null
+++ b/Lib/email/architecture.rst
@@ -0,0 +1,216 @@
+:mod:`email` Package Architecture
+=================================
+
+Overview
+--------
+
+The email package consists of three major components:
+
+    Model
+        An object structure that represents an email message, and provides an
+        API for creating, querying, and modifying a message.
+
+    Parser
+        Takes a sequence of characters or bytes and produces a model of the
+        email message represented by those characters or bytes.
+
+    Generator
+        Takes a model and turns it into a sequence of characters or bytes.  The
+        sequence can either be intended for human consumption (a printable
+        unicode string) or bytes suitable for transmission over the wire.  In
+        the latter case all data is properly encoded using the content transfer
+        encodings specified by the relevant RFCs.
+
+Conceptually the package is organized around the model.  The model provides both
+"external" APIs intended for use by application programs using the library,
+and "internal" APIs intended for use by the Parser and Generator components.
+This division is intentionally a bit fuzy; the API described by this documentation
+is all a public, stable API.  This allows for an application with special needs
+to implement its own parser and/or generator.
+
+In addition to the three major functional components, there is a third key
+component to the architecture:
+
+    Policy
+        An object that specifies various behavioral settings and carries
+        implementations of various behavior-controlling methods.
+
+The Policy framework provides a simple and convenient way to control the
+behavior of the library, making it possible for the library to be used in a
+very flexible fashion while leveraging the common code required to parse,
+represent, and generate message-like objects.  For example, in addition to the
+default :rfc:`5322` email message policy, we also have a policy that manages
+HTTP headers in a fashion compliant with :rfc:`2616`.  Individual policy
+controls, such as the maximum line length produced by the generator, can also
+be controlled individually to meet specialized application requirements.
+
+
+The Model
+---------
+
+The message model is implemented by the :class:`~email.message.Message` class.
+The model divides a message into the two fundamental parts discussed by the
+RFC: the header section and the body.  The `Message` object acts as a
+pseudo-dictionary of named headers.  Its dictionary interface provides
+convenient access to individual headers by name.  However, all headers are kept
+internally in an ordered list, so that the information about the order of the
+headers in the original message is preserved.
+
+The `Message` object also has a `payload` that holds the body.  A `payload` can
+be one of two things: data, or a list of `Message` objects.  The latter is used
+to represent a multipart MIME message.  Lists can be nested arbitrarily deeply
+in order to represent the message, with all terminal leaves having non-list
+data payloads.
+
+
+Message Lifecycle
+-----------------
+
+The general lifecyle of a message is:
+
+    Creation
+        A `Message` object can be created by a Parser, or it can be
+        instantiated as an empty message by an application.
+
+    Manipulation
+        The application may examine one or more headers, and/or the
+        payload, and it may modify one or more headers and/or
+        the payload.  This may be done on the top level `Message`
+        object, or on any sub-object.
+
+    Finalization
+        The Model is converted into a unicode or binary stream,
+        or the model is discarded.
+
+
+
+Header Policy Control During Lifecycle
+--------------------------------------
+
+One of the major controls exerted by the Policy is the management of headers
+during the `Message` lifecycle.  Most applications don't need to be aware of
+this.
+
+A header enters the model in one of two ways: via a Parser, or by being set to
+a specific value by an application program after the Model already exists.
+Similarly, a header exits the model in one of two ways: by being serialized by
+a Generator, or by being retrieved from a Model by an application program.  The
+Policy object provides hooks for all four of these pathways.
+
+The model storage for headers is a list of (name, value) tuples.
+
+The Parser identifies headers during parsing, and passes them to the
+:meth:`~email.policy.Policy.header_source_parse` method of the Policy.  The
+result of that method is the (name, value) tuple to be stored in the model.
+
+When an application program supplies a header value (for example, through the
+`Message` object `__setitem__` interface), the name and the value are passed to
+the :meth:`~email.policy.Policy.header_store_parse` method of the Policy, which
+returns the (name, value) tuple to be stored in the model.
+
+When an application program retrieves a header (through any of the dict or list
+interfaces of `Message`), the name and value are passed to the
+:meth:`~email.policy.Policy.header_fetch_parse` method of the Policy to
+obtain the value returned to the application.
+
+When a Generator requests a header during serialization, the name and value are
+passed to the :meth:`~email.policy.Policy.fold` method of the Policy, which
+returns a string containing line breaks in the appropriate places.  The
+:meth:`~email.policy.Policy.cte_type` Policy control determines whether or
+not Content Transfer Encoding is performed on the data in the header.  There is
+also a :meth:`~email.policy.Policy.binary_fold` method for use by generators
+that produce binary output, which returns the folded header as binary data,
+possibly folded at different places than the corresponding string would be.
+
+
+Handling Binary Data
+--------------------
+
+In an ideal world all message data would conform to the RFCs, meaning that the
+parser could decode the message into the idealized unicode message that the
+sender originally wrote.  In the real world, the email package must also be
+able to deal with badly formatted messages, including messages containing
+non-ASCII characters that either have no indicated character set or are not
+valid characters in the indicated character set.
+
+Since email messages are *primarily* text data, and operations on message data
+are primarily text operations (except for binary payloads of course), the model
+stores all text data as unicode strings.  Un-decodable binary inside text
+data is handled by using the `surrogateescape` error handler of the ASCII
+codec.  As with the binary filenames the error handler was introduced to
+handle, this allows the email package to "carry" the binary data received
+during parsing along until the output stage, at which time it is regenerated
+in its original form.
+
+This carried binary data is almost entirely an implementation detail.  The one
+place where it is visible in the API is in the "internal" API.  A Parser must
+do the `surrogateescape` encoding of binary input data, and pass that data to
+the appropriate Policy method.  The "internal" interface used by the Generator
+to access header values preserves the `surrogateescaped` bytes.  All other
+interfaces convert the binary data either back into bytes or into a safe form
+(losing information in some cases).
+
+
+Backward Compatibility
+----------------------
+
+The :class:`~email.policy.Policy.Compat32` Policy provides backward
+compatibility with version 5.1 of the email package.  It does this via the
+following implementation of the four+1 Policy methods described above:
+
+header_source_parse
+    Splits the first line on the colon to obtain the name, discards any spaces
+    after the colon, and joins the remainder of the line with all of the
+    remaining lines, preserving the linesep characters to obtain the value.
+    Trailing carriage return and/or linefeed characters are stripped from the
+    resulting value string.
+
+header_store_parse
+    Returns the name and value exactly as received from the application.
+
+header_fetch_parse
+    If the value contains any `surrogateescaped` binary data, return the value
+    as a :class:`~email.header.Header` object, using the character set
+    `unknown-8bit`.  Otherwise just returns the value.
+
+fold
+    Uses :class:`~email.header.Header`'s folding to fold headers in the
+    same way the email5.1 generator did.
+
+binary_fold
+    Same as fold, but encodes to 'ascii'.
+
+
+New Algorithm
+-------------
+
+header_source_parse
+    Same as legacy behavior.
+
+header_store_parse
+    Same as legacy behavior.
+
+header_fetch_parse
+    If the value is already a header object, returns it.  Otherwise, parses the
+    value using the new parser, and returns the resulting object as the value.
+    `surrogateescaped` bytes get turned into unicode unknown character code
+    points.
+
+fold
+    Uses the new header folding algorithm, respecting the policy settings.
+    surrogateescaped bytes are encoded using the ``unknown-8bit`` charset for
+    ``cte_type=7bit`` or ``8bit``.  Returns a string.
+
+    At some point there will also be a ``cte_type=unicode``, and for that
+    policy fold will serialize the idealized unicode message with RFC-like
+    folding, converting any surrogateescaped bytes into the unicode
+    unknown character glyph.
+
+binary_fold
+    Uses the new header folding algorithm, respecting the policy settings.
+    surrogateescaped bytes are encoded using the `unknown-8bit` charset for
+    ``cte_type=7bit``, and get turned back into bytes for ``cte_type=8bit``.
+    Returns bytes.
+
+    At some point there will also be a ``cte_type=unicode``, and for that
+    policy binary_fold will serialize the message according to :rfc:``5335``.
diff --git a/Lib/email/errors.py b/Lib/email/errors.py
index d52a624601..791239fa6a 100644
--- a/Lib/email/errors.py
+++ b/Lib/email/errors.py
@@ -5,7 +5,6 @@
 """email package exception classes."""
 
 
-
 class MessageError(Exception):
     """Base class for errors in the email package."""
 
@@ -30,12 +29,13 @@ class CharsetError(MessageError):
     """An illegal charset was given."""
 
 
-
 # These are parsing defects which the parser was able to work around.
-class MessageDefect:
+class MessageDefect(ValueError):
     """Base class for a message defect."""
 
     def __init__(self, line=None):
+        if line is not None:
+            super().__init__(line)
         self.line = line
 
 class NoBoundaryInMultipartDefect(MessageDefect):
@@ -44,14 +44,64 @@ class NoBoundaryInMultipartDefect(MessageDefect):
 class StartBoundaryNotFoundDefect(MessageDefect):
     """The claimed start boundary was never found."""
 
+class CloseBoundaryNotFoundDefect(MessageDefect):
+    """A start boundary was found, but not the corresponding close boundary."""
+
 class FirstHeaderLineIsContinuationDefect(MessageDefect):
     """A message had a continuation line as its first header line."""
 
 class MisplacedEnvelopeHeaderDefect(MessageDefect):
     """A 'Unix-from' header was found in the middle of a header block."""
 
-class MalformedHeaderDefect(MessageDefect):
-    """Found a header that was missing a colon, or was otherwise malformed."""
+class MissingHeaderBodySeparatorDefect(MessageDefect):
+    """Found line with no leading whitespace and no colon before blank line."""
+# XXX: backward compatibility, just in case (it was never emitted).
+MalformedHeaderDefect = MissingHeaderBodySeparatorDefect
 
 class MultipartInvariantViolationDefect(MessageDefect):
     """A message claimed to be a multipart but no subparts were found."""
+
+class InvalidMultipartContentTransferEncodingDefect(MessageDefect):
+    """An invalid content transfer encoding was set on the multipart itself."""
+
+class UndecodableBytesDefect(MessageDefect):
+    """Header contained bytes that could not be decoded"""
+
+class InvalidBase64PaddingDefect(MessageDefect):
+    """base64 encoded sequence had an incorrect length"""
+
+class InvalidBase64CharactersDefect(MessageDefect):
+    """base64 encoded sequence had characters not in base64 alphabet"""
+
+# These errors are specific to header parsing.
+
+class HeaderDefect(MessageDefect):
+    """Base class for a header defect."""
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+
+class InvalidHeaderDefect(HeaderDefect):
+    """Header is not valid, message gives details."""
+
+class HeaderMissingRequiredValue(HeaderDefect):
+    """A header that must have a value had none"""
+
+class NonPrintableDefect(HeaderDefect):
+    """ASCII characters outside the ascii-printable range found"""
+
+    def __init__(self, non_printables):
+        super().__init__(non_printables)
+        self.non_printables = non_printables
+
+    def __str__(self):
+        return ("the following ASCII non-printables found in header: "
+            "{}".format(self.non_printables))
+
+class ObsoleteHeaderDefect(HeaderDefect):
+    """Header uses syntax declared obsolete by RFC 5322"""
+
+class NonASCIILocalPartDefect(HeaderDefect):
+    """local_part contains non-ASCII characters"""
+    # This defect only occurs during unicode parsing, not when
+    # parsing messages decoded from binary.
diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py
index 16ed2885d7..ea41e9571d 100644
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -25,6 +25,7 @@ import re
 
 from email import errors
 from email import message
+from email._policybase import compat32
 
 NLCRE = re.compile('\r\n|\r|\n')
 NLCRE_bol = re.compile('(\r\n|\r|\n)')
@@ -120,9 +121,6 @@ class BufferedSubFile(object):
         # Reverse and insert at the front of the lines.
         self._lines[:0] = lines[::-1]
 
-    def is_closed(self):
-        return self._closed
-
     def __iter__(self):
         return self
 
@@ -137,9 +135,22 @@ class BufferedSubFile(object):
 class FeedParser:
     """A feed-style parser of email."""
 
-    def __init__(self, _factory=message.Message):
-        """_factory is called with no arguments to create a new message obj"""
+    def __init__(self, _factory=message.Message, *, policy=compat32):
+        """_factory is called with no arguments to create a new message obj
+
+        The policy keyword specifies a policy object that controls a number of
+        aspects of the parser's operation.  The default policy maintains
+        backward compatibility.
+
+        """
         self._factory = _factory
+        self.policy = policy
+        try:
+            _factory(policy=self.policy)
+            self._factory_kwds = lambda: {'policy': self.policy}
+        except TypeError:
+            # Assume this is an old-style factory
+            self._factory_kwds = lambda: {}
         self._input = BufferedSubFile()
         self._msgstack = []
         self._parse = self._parsegen().__next__
@@ -171,11 +182,12 @@ class FeedParser:
         # Look for final set of defects
         if root.get_content_maintype() == 'multipart' \
                and not root.is_multipart():
-            root.defects.append(errors.MultipartInvariantViolationDefect())
+            defect = errors.MultipartInvariantViolationDefect()
+            self.policy.handle_defect(root, defect)
         return root
 
     def _new_message(self):
-        msg = self._factory()
+        msg = self._factory(**self._factory_kwds())
         if self._cur and self._cur.get_content_type() == 'multipart/digest':
             msg.set_default_type('message/rfc822')
         if self._msgstack:
@@ -207,6 +219,8 @@ class FeedParser:
                 # (i.e. newline), just throw it away. Otherwise the line is
                 # part of the body so push it back.
                 if not NLCRE.match(line):
+                    defect = errors.MissingHeaderBodySeparatorDefect()
+                    self.policy.handle_defect(self._cur, defect)
                     self._input.unreadline(line)
                 break
             headers.append(line)
@@ -284,7 +298,8 @@ class FeedParser:
                 # defined a boundary.  That's a problem which we'll handle by
                 # reading everything until the EOF and marking the message as
                 # defective.
-                self._cur.defects.append(errors.NoBoundaryInMultipartDefect())
+                defect = errors.NoBoundaryInMultipartDefect()
+                self.policy.handle_defect(self._cur, defect)
                 lines = []
                 for line in self._input:
                     if line is NeedMoreData:
@@ -293,6 +308,11 @@ class FeedParser:
                     lines.append(line)
                 self._cur.set_payload(EMPTYSTRING.join(lines))
                 return
+            # Make sure a valid content type was specified per RFC 2045:6.4.
+            if (self._cur.get('content-transfer-encoding', '8bit').lower()
+                    not in ('7bit', '8bit', 'binary')):
+                defect = errors.InvalidMultipartContentTransferEncodingDefect()
+                self.policy.handle_defect(self._cur, defect)
             # Create a line match predicate which matches the inter-part
             # boundary as well as the end-of-multipart boundary.  Don't push
             # this onto the input stream until we've scanned past the
@@ -304,6 +324,7 @@ class FeedParser:
             capturing_preamble = True
             preamble = []
             linesep = False
+            close_boundary_seen = False
             while True:
                 line = self._input.readline()
                 if line is NeedMoreData:
@@ -318,6 +339,7 @@ class FeedParser:
                     # the closing boundary, then we need to initialize the
                     # epilogue with the empty string (see below).
                     if mo.group('end'):
+                        close_boundary_seen = True
                         linesep = mo.group('linesep')
                         break
                     # We saw an inter-part boundary.  Were we in the preamble?
@@ -386,9 +408,9 @@ class FeedParser:
             # We've seen either the EOF or the end boundary.  If we're still
             # capturing the preamble, we never saw the start boundary.  Note
             # that as a defect and store the captured text as the payload.
-            # Everything from here to the EOF is epilogue.
             if capturing_preamble:
-                self._cur.defects.append(errors.StartBoundaryNotFoundDefect())
+                defect = errors.StartBoundaryNotFoundDefect()
+                self.policy.handle_defect(self._cur, defect)
                 self._cur.set_payload(EMPTYSTRING.join(preamble))
                 epilogue = []
                 for line in self._input:
@@ -397,8 +419,15 @@ class FeedParser:
                         continue
                 self._cur.epilogue = EMPTYSTRING.join(epilogue)
                 return
-            # If the end boundary ended in a newline, we'll need to make sure
-            # the epilogue isn't None
+            # If we're not processing the preamble, then we might have seen
+            # EOF without seeing that end boundary...that is also a defect.
+            if not close_boundary_seen:
+                defect = errors.CloseBoundaryNotFoundDefect()
+                self.policy.handle_defect(self._cur, defect)
+                return
+            # Everything from here to the EOF is epilogue.  If the end boundary
+            # ended in a newline, we'll need to make sure the epilogue isn't
+            # None
             if linesep:
                 epilogue = ['']
             else:
@@ -440,14 +469,12 @@ class FeedParser:
                     # is illegal, so let's note the defect, store the illegal
                     # line, and ignore it for purposes of headers.
                     defect = errors.FirstHeaderLineIsContinuationDefect(line)
-                    self._cur.defects.append(defect)
+                    self.policy.handle_defect(self._cur, defect)
                     continue
                 lastvalue.append(line)
                 continue
             if lastheader:
-                # XXX reconsider the joining of folded lines
-                lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n')
-                self._cur[lastheader] = lhdr
+                self._cur.set_raw(*self.policy.header_source_parse(lastvalue))
                 lastheader, lastvalue = '', []
             # Check for envelope header, i.e. unix-from
             if line.startswith('From '):
@@ -471,19 +498,17 @@ class FeedParser:
                     self._cur.defects.append(defect)
                     continue
             # Split the line on the colon separating field name from value.
+            # There will always be a colon, because if there wasn't the part of
+            # the parser that calls us would have started parsing the body.
             i = line.find(':')
-            if i < 0:
-                defect = errors.MalformedHeaderDefect(line)
-                self._cur.defects.append(defect)
-                continue
+            assert i>0, "_parse_headers fed line with no : and no leading WS"
             lastheader = line[:i]
-            lastvalue = [line[i+1:].lstrip()]
+            lastvalue = [line]
         # Done with all the lines, so handle the last header.
         if lastheader:
-            # XXX reconsider the joining of folded lines
-            self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n')
+            self._cur.set_raw(*self.policy.header_source_parse(lastvalue))
+
 
-
 class BytesFeedParser(FeedParser):
     """Like FeedParser, but feed accepts bytes."""
 
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index ab37e94dbe..24f2abf210 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -13,8 +13,10 @@ import random
 import warnings
 
 from io import StringIO, BytesIO
+from email._policybase import compat32
 from email.header import Header
-from email.message import _has_surrogates
+from email.utils import _has_surrogates
+import email.charset as _charset
 
 UNDERSCORE = '_'
 NL = '\n'  # XXX: no longer used by the code below.
@@ -33,7 +35,8 @@ class Generator:
     # Public interface
     #
 
-    def __init__(self, outfp, mangle_from_=True, maxheaderlen=78):
+    def __init__(self, outfp, mangle_from_=True, maxheaderlen=None, *,
+                 policy=None):
         """Create the generator for message flattening.
 
         outfp is the output file-like object for writing the message to.  It
@@ -49,16 +52,22 @@ class Generator:
         defined in the Header class.  Set maxheaderlen to zero to disable
         header wrapping.  The default is 78, as recommended (but not required)
         by RFC 2822.
+
+        The policy keyword specifies a policy object that controls a number of
+        aspects of the generator's operation.  The default policy maintains
+        backward compatibility.
+
         """
         self._fp = outfp
         self._mangle_from_ = mangle_from_
-        self._maxheaderlen = maxheaderlen
+        self.maxheaderlen = maxheaderlen
+        self.policy = policy
 
     def write(self, s):
         # Just delegate to the file object
         self._fp.write(s)
 
-    def flatten(self, msg, unixfrom=False, linesep='\n'):
+    def flatten(self, msg, unixfrom=False, linesep=None):
         r"""Print the message object tree rooted at msg to the output file
         specified when the Generator instance was created.
 
@@ -70,29 +79,47 @@ class Generator:
         Note that for subobjects, no From_ line is printed.
 
         linesep specifies the characters used to indicate a new line in
-        the output.  The default value is the most useful for typical
-        Python applications, but it can be set to \r\n to produce RFC-compliant
-        line separators when needed.
+        the output.  The default value is determined by the policy.
 
         """
         # We use the _XXX constants for operating on data that comes directly
         # from the msg, and _encoded_XXX constants for operating on data that
         # has already been converted (to bytes in the BytesGenerator) and
         # inserted into a temporary buffer.
-        self._NL = linesep
-        self._encoded_NL = self._encode(linesep)
+        policy = msg.policy if self.policy is None else self.policy
+        if linesep is not None:
+            policy = policy.clone(linesep=linesep)
+        if self.maxheaderlen is not None:
+            policy = policy.clone(max_line_length=self.maxheaderlen)
+        self._NL = policy.linesep
+        self._encoded_NL = self._encode(self._NL)
         self._EMPTY = ''
         self._encoded_EMTPY = self._encode('')
-        if unixfrom:
-            ufrom = msg.get_unixfrom()
-            if not ufrom:
-                ufrom = 'From nobody ' + time.ctime(time.time())
-            self.write(ufrom + self._NL)
-        self._write(msg)
+        # Because we use clone (below) when we recursively process message
+        # subparts, and because clone uses the computed policy (not None),
+        # submessages will automatically get set to the computed policy when
+        # they are processed by this code.
+        old_gen_policy = self.policy
+        old_msg_policy = msg.policy
+        try:
+            self.policy = policy
+            msg.policy = policy
+            if unixfrom:
+                ufrom = msg.get_unixfrom()
+                if not ufrom:
+                    ufrom = 'From nobody ' + time.ctime(time.time())
+                self.write(ufrom + self._NL)
+            self._write(msg)
+        finally:
+            self.policy = old_gen_policy
+            msg.policy = old_msg_policy
 
     def clone(self, fp):
         """Clone this generator with the exact same options."""
-        return self.__class__(fp, self._mangle_from_, self._maxheaderlen)
+        return self.__class__(fp,
+                              self._mangle_from_,
+                              None, # Use policy setting, which we've adjusted
+                              policy=self.policy)
 
     #
     # Protected interface - undocumented ;/
@@ -180,16 +207,8 @@ class Generator:
     #
 
     def _write_headers(self, msg):
-        for h, v in msg.items():
-            self.write('%s: ' % h)
-            if isinstance(v, Header):
-                self.write(v.encode(
-                    maxlinelen=self._maxheaderlen, linesep=self._NL)+self._NL)
-            else:
-                # Header's got lots of smarts, so use it.
-                header = Header(v, maxlinelen=self._maxheaderlen,
-                                header_name=h)
-                self.write(header.encode(linesep=self._NL)+self._NL)
+        for h, v in msg.raw_items():
+            self.write(self.policy.fold(h, v))
         # A blank line always separates headers from body
         self.write(self._NL)
 
@@ -279,12 +298,12 @@ class Generator:
         # The contents of signed parts has to stay unmodified in order to keep
         # the signature intact per RFC1847 2.1, so we disable header wrapping.
         # RDM: This isn't enough to completely preserve the part, but it helps.
-        old_maxheaderlen = self._maxheaderlen
+        p = self.policy
+        self.policy = p.clone(max_line_length=0)
         try:
-            self._maxheaderlen = 0
             self._handle_multipart(msg)
         finally:
-            self._maxheaderlen = old_maxheaderlen
+            self.policy = p
 
     def _handle_message_delivery_status(self, msg):
         # We can't just write the headers directly to self's file object
@@ -319,10 +338,12 @@ class Generator:
         # message/rfc822.  Such messages are generated by, for example,
         # Groupwise when forwarding unadorned messages.  (Issue 7970.)  So
         # in that case we just emit the string body.
-        payload = msg.get_payload()
+        payload = msg._payload
         if isinstance(payload, list):
             g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL)
             payload = s.getvalue()
+        else:
+            payload = self._encode(payload)
         self._fp.write(payload)
 
     # This used to be a module level function; we use a classmethod for this
@@ -358,7 +379,10 @@ class BytesGenerator(Generator):
 
     Functionally identical to the base Generator except that the output is
     bytes and not string.  When surrogates were used in the input to encode
-    bytes, these are decoded back to bytes for output.
+    bytes, these are decoded back to bytes for output.  If the policy has
+    cte_type set to 7bit, then the message is transformed such that the
+    non-ASCII bytes are properly content transfer encoded, using the charset
+    unknown-8bit.
 
     The outfp object must accept bytes in its write method.
     """
@@ -379,23 +403,8 @@ class BytesGenerator(Generator):
     def _write_headers(self, msg):
         # This is almost the same as the string version, except for handling
         # strings with 8bit bytes.
-        for h, v in msg._headers:
-            self.write('%s: ' % h)
-            if isinstance(v, Header):
-                self.write(v.encode(maxlinelen=self._maxheaderlen)+self._NL)
-            elif _has_surrogates(v):
-                # If we have raw 8bit data in a byte string, we have no idea
-                # what the encoding is.  There is no safe way to split this
-                # string.  If it's ascii-subset, then we could do a normal
-                # ascii split, but if it's multibyte then we could break the
-                # string.  There's no way to know so the least harm seems to
-                # be to not split the string and risk it being too long.
-                self.write(v+NL)
-            else:
-                # Header's got lots of smarts and this string is safe...
-                header = Header(v, maxlinelen=self._maxheaderlen,
-                                header_name=h)
-                self.write(header.encode(linesep=self._NL)+self._NL)
+        for h, v in msg.raw_items():
+            self._fp.write(self.policy.fold_binary(h, v))
         # A blank line always separates headers from body
         self.write(self._NL)
 
@@ -404,7 +413,7 @@ class BytesGenerator(Generator):
         # just write it back out.
         if msg._payload is None:
             return
-        if _has_surrogates(msg._payload):
+        if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit':
             if self._mangle_from_:
                 msg._payload = fcre.sub(">From ", msg._payload)
             self._write_lines(msg._payload)
diff --git a/Lib/email/header.py b/Lib/email/header.py
index e33324ad38..5bd06380ce 100644
--- a/Lib/email/header.py
+++ b/Lib/email/header.py
@@ -40,7 +40,6 @@ ecre = re.compile(r'''
   \?                    # literal ?
   (?P<encoded>.*?)      # non-greedy up to the next ?= is the encoded string
   \?=                   # literal ?=
-  (?=[ \t]|$)           # whitespace or the end of the string
   ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE)
 
 # Field name regexp, including trailing colon, but not separating whitespace,
@@ -86,8 +85,12 @@ def decode_header(header):
     words = []
     for line in header.splitlines():
         parts = ecre.split(line)
+        first = True
         while parts:
-            unencoded = parts.pop(0).strip()
+            unencoded = parts.pop(0)
+            if first:
+                unencoded = unencoded.lstrip()
+                first = False
             if unencoded:
                 words.append((unencoded, None, None))
             if parts:
@@ -95,6 +98,16 @@ def decode_header(header):
                 encoding = parts.pop(0).lower()
                 encoded = parts.pop(0)
                 words.append((encoded, encoding, charset))
+    # Now loop over words and remove words that consist of whitespace
+    # between two encoded strings.
+    import sys
+    droplist = []
+    for n, w in enumerate(words):
+        if n>1 and w[1] and words[n-2][1] and words[n-1][0].isspace():
+            droplist.append(n-1)
+    for d in reversed(droplist):
+        del words[d]
+
     # The next step is to decode each encoded word by applying the reverse
     # base64 or quopri transformation.  decoded_words is now a list of the
     # form (decoded_word, charset).
@@ -217,22 +230,27 @@ class Header:
         self._normalize()
         uchunks = []
         lastcs = None
+        lastspace = None
         for string, charset in self._chunks:
             # We must preserve spaces between encoded and non-encoded word
             # boundaries, which means for us we need to add a space when we go
             # from a charset to None/us-ascii, or from None/us-ascii to a
             # charset.  Only do this for the second and subsequent chunks.
+            # Don't add a space if the None/us-ascii string already has
+            # a space (trailing or leading depending on transition)
             nextcs = charset
             if nextcs == _charset.UNKNOWN8BIT:
                 original_bytes = string.encode('ascii', 'surrogateescape')
                 string = original_bytes.decode('ascii', 'replace')
             if uchunks:
+                hasspace = string and self._nonctext(string[0])
                 if lastcs not in (None, 'us-ascii'):
-                    if nextcs in (None, 'us-ascii'):
+                    if nextcs in (None, 'us-ascii') and not hasspace:
                         uchunks.append(SPACE)
                         nextcs = None
-                elif nextcs not in (None, 'us-ascii'):
+                elif nextcs not in (None, 'us-ascii') and not lastspace:
                     uchunks.append(SPACE)
+            lastspace = string and self._nonctext(string[-1])
             lastcs = nextcs
             uchunks.append(string)
         return EMPTYSTRING.join(uchunks)
@@ -291,6 +309,11 @@ class Header:
                 charset = UTF8
         self._chunks.append((s, charset))
 
+    def _nonctext(self, s):
+        """True if string s is not a ctext character of RFC822.
+        """
+        return s.isspace() or s in ('(', ')', '\\')
+
     def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'):
         r"""Encode a message header into an RFC-compliant format.
 
@@ -334,7 +357,20 @@ class Header:
             maxlinelen = 1000000
         formatter = _ValueFormatter(self._headerlen, maxlinelen,
                                     self._continuation_ws, splitchars)
+        lastcs = None
+        hasspace = lastspace = None
         for string, charset in self._chunks:
+            if hasspace is not None:
+                hasspace = string and self._nonctext(string[0])
+                import sys
+                if lastcs not in (None, 'us-ascii'):
+                    if not hasspace or charset not in (None, 'us-ascii'):
+                        formatter.add_transition()
+                elif charset not in (None, 'us-ascii') and not lastspace:
+                    formatter.add_transition()
+            lastspace = string and self._nonctext(string[-1])
+            lastcs = charset
+            hasspace = False
             lines = string.splitlines()
             if lines:
                 formatter.feed('', lines[0], charset)
@@ -351,6 +387,7 @@ class Header:
                     formatter.feed(fws, sline, charset)
             if len(lines) > 1:
                 formatter.newline()
+        if self._chunks:
             formatter.add_transition()
         value = formatter._str(linesep)
         if _embeded_header.search(value):
diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py
new file mode 100644
index 0000000000..1fae950820
--- /dev/null
+++ b/Lib/email/headerregistry.py
@@ -0,0 +1,583 @@
+"""Representing and manipulating email headers via custom objects.
+
+This module provides an implementation of the HeaderRegistry API.
+The implementation is designed to flexibly follow RFC5322 rules.
+
+Eventually HeaderRegistry will be a public API, but it isn't yet,
+and will probably change some before that happens.
+
+"""
+
+from email import utils
+from email import errors
+from email import _header_value_parser as parser
+
+class Address:
+
+    def __init__(self, display_name='', username='', domain='', addr_spec=None):
+        """Create an object represeting a full email address.
+
+        An address can have a 'display_name', a 'username', and a 'domain'.  In
+        addition to specifying the username and domain separately, they may be
+        specified together by using the addr_spec keyword *instead of* the
+        username and domain keywords.  If an addr_spec string is specified it
+        must be properly quoted according to RFC 5322 rules; an error will be
+        raised if it is not.
+
+        An Address object has display_name, username, domain, and addr_spec
+        attributes, all of which are read-only.  The addr_spec and the string
+        value of the object are both quoted according to RFC5322 rules, but
+        without any Content Transfer Encoding.
+
+        """
+        # This clause with its potential 'raise' may only happen when an
+        # application program creates an Address object using an addr_spec
+        # keyword.  The email library code itself must always supply username
+        # and domain.
+        if addr_spec is not None:
+            if username or domain:
+                raise TypeError("addrspec specified when username and/or "
+                                "domain also specified")
+            a_s, rest = parser.get_addr_spec(addr_spec)
+            if rest:
+                raise ValueError("Invalid addr_spec; only '{}' "
+                                 "could be parsed from '{}'".format(
+                                    a_s, addr_spec))
+            if a_s.all_defects:
+                raise a_s.all_defects[0]
+            username = a_s.local_part
+            domain = a_s.domain
+        self._display_name = display_name
+        self._username = username
+        self._domain = domain
+
+    @property
+    def display_name(self):
+        return self._display_name
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def domain(self):
+        return self._domain
+
+    @property
+    def addr_spec(self):
+        """The addr_spec (username@domain) portion of the address, quoted
+        according to RFC 5322 rules, but with no Content Transfer Encoding.
+        """
+        nameset = set(self.username)
+        if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS):
+            lp = parser.quote_string(self.username)
+        else:
+            lp = self.username
+        if self.domain:
+            return lp + '@' + self.domain
+        if not lp:
+            return '<>'
+        return lp
+
+    def __repr__(self):
+        return "Address(display_name={!r}, username={!r}, domain={!r})".format(
+                        self.display_name, self.username, self.domain)
+
+    def __str__(self):
+        nameset = set(self.display_name)
+        if len(nameset) > len(nameset-parser.SPECIALS):
+            disp = parser.quote_string(self.display_name)
+        else:
+            disp = self.display_name
+        if disp:
+            addr_spec = '' if self.addr_spec=='<>' else self.addr_spec
+            return "{} <{}>".format(disp, addr_spec)
+        return self.addr_spec
+
+    def __eq__(self, other):
+        if type(other) != type(self):
+            return False
+        return (self.display_name == other.display_name and
+                self.username == other.username and
+                self.domain == other.domain)
+
+
+class Group:
+
+    def __init__(self, display_name=None, addresses=None):
+        """Create an object representing an address group.
+
+        An address group consists of a display_name followed by colon and an
+        list of addresses (see Address) terminated by a semi-colon.  The Group
+        is created by specifying a display_name and a possibly empty list of
+        Address objects.  A Group can also be used to represent a single
+        address that is not in a group, which is convenient when manipulating
+        lists that are a combination of Groups and individual Addresses.  In
+        this case the display_name should be set to None.  In particular, the
+        string representation of a Group whose display_name is None is the same
+        as the Address object, if there is one and only one Address object in
+        the addresses list.
+
+        """
+        self._display_name = display_name
+        self._addresses = tuple(addresses) if addresses else tuple()
+
+    @property
+    def display_name(self):
+        return self._display_name
+
+    @property
+    def addresses(self):
+        return self._addresses
+
+    def __repr__(self):
+        return "Group(display_name={!r}, addresses={!r}".format(
+                 self.display_name, self.addresses)
+
+    def __str__(self):
+        if self.display_name is None and len(self.addresses)==1:
+            return str(self.addresses[0])
+        disp = self.display_name
+        if disp is not None:
+            nameset = set(disp)
+            if len(nameset) > len(nameset-parser.SPECIALS):
+                disp = parser.quote_string(disp)
+        adrstr = ", ".join(str(x) for x in self.addresses)
+        adrstr = ' ' + adrstr if adrstr else adrstr
+        return "{}:{};".format(disp, adrstr)
+
+    def __eq__(self, other):
+        if type(other) != type(self):
+            return False
+        return (self.display_name == other.display_name and
+                self.addresses == other.addresses)
+
+
+# Header Classes #
+
+class BaseHeader(str):
+
+    """Base class for message headers.
+
+    Implements generic behavior and provides tools for subclasses.
+
+    A subclass must define a classmethod named 'parse' that takes an unfolded
+    value string and a dictionary as its arguments.  The dictionary will
+    contain one key, 'defects', initialized to an empty list.  After the call
+    the dictionary must contain two additional keys: parse_tree, set to the
+    parse tree obtained from parsing the header, and 'decoded', set to the
+    string value of the idealized representation of the data from the value.
+    (That is, encoded words are decoded, and values that have canonical
+    representations are so represented.)
+
+    The defects key is intended to collect parsing defects, which the message
+    parser will subsequently dispose of as appropriate.  The parser should not,
+    insofar as practical, raise any errors.  Defects should be added to the
+    list instead.  The standard header parsers register defects for RFC
+    compliance issues, for obsolete RFC syntax, and for unrecoverable parsing
+    errors.
+
+    The parse method may add additional keys to the dictionary.  In this case
+    the subclass must define an 'init' method, which will be passed the
+    dictionary as its keyword arguments.  The method should use (usually by
+    setting them as the value of similarly named attributes) and remove all the
+    extra keys added by its parse method, and then use super to call its parent
+    class with the remaining arguments and keywords.
+
+    The subclass should also make sure that a 'max_count' attribute is defined
+    that is either None or 1. XXX: need to better define this API.
+
+    """
+
+    def __new__(cls, name, value):
+        kwds = {'defects': []}
+        cls.parse(value, kwds)
+        if utils._has_surrogates(kwds['decoded']):
+            kwds['decoded'] = utils._sanitize(kwds['decoded'])
+        self = str.__new__(cls, kwds['decoded'])
+        del kwds['decoded']
+        self.init(name, **kwds)
+        return self
+
+    def init(self, name, *, parse_tree, defects):
+        self._name = name
+        self._parse_tree = parse_tree
+        self._defects = defects
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def defects(self):
+        return tuple(self._defects)
+
+    def __reduce__(self):
+        return (
+            _reconstruct_header,
+            (
+                self.__class__.__name__,
+                self.__class__.__bases__,
+                str(self),
+            ),
+            self.__dict__)
+
+    @classmethod
+    def _reconstruct(cls, value):
+        return str.__new__(cls, value)
+
+    def fold(self, *, policy):
+        """Fold header according to policy.
+
+        The parsed representation of the header is folded according to
+        RFC5322 rules, as modified by the policy.  If the parse tree
+        contains surrogateescaped bytes, the bytes are CTE encoded using
+        the charset 'unknown-8bit".
+
+        Any non-ASCII characters in the parse tree are CTE encoded using
+        charset utf-8. XXX: make this a policy setting.
+
+        The returned value is an ASCII-only string possibly containing linesep
+        characters, and ending with a linesep character.  The string includes
+        the header name and the ': ' separator.
+
+        """
+        # At some point we need to only put fws here if it was in the source.
+        header = parser.Header([
+            parser.HeaderLabel([
+                parser.ValueTerminal(self.name, 'header-name'),
+                parser.ValueTerminal(':', 'header-sep')]),
+            parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]),
+                             self._parse_tree])
+        return header.fold(policy=policy)
+
+
+def _reconstruct_header(cls_name, bases, value):
+    return type(cls_name, bases, {})._reconstruct(value)
+
+
+class UnstructuredHeader:
+
+    max_count = None
+    value_parser = staticmethod(parser.get_unstructured)
+
+    @classmethod
+    def parse(cls, value, kwds):
+        kwds['parse_tree'] = cls.value_parser(value)
+        kwds['decoded'] = str(kwds['parse_tree'])
+
+
+class UniqueUnstructuredHeader(UnstructuredHeader):
+
+    max_count = 1
+
+
+class DateHeader:
+
+    """Header whose value consists of a single timestamp.
+
+    Provides an additional attribute, datetime, which is either an aware
+    datetime using a timezone, or a naive datetime if the timezone
+    in the input string is -0000.  Also accepts a datetime as input.
+    The 'value' attribute is the normalized form of the timestamp,
+    which means it is the output of format_datetime on the datetime.
+    """
+
+    max_count = None
+
+    # This is used only for folding, not for creating 'decoded'.
+    value_parser = staticmethod(parser.get_unstructured)
+
+    @classmethod
+    def parse(cls, value, kwds):
+        if not value:
+            kwds['defects'].append(errors.HeaderMissingRequiredValue())
+            kwds['datetime'] = None
+            kwds['decoded'] = ''
+            kwds['parse_tree'] = parser.TokenList()
+            return
+        if isinstance(value, str):
+            value = utils.parsedate_to_datetime(value)
+        kwds['datetime'] = value
+        kwds['decoded'] = utils.format_datetime(kwds['datetime'])
+        kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
+
+    def init(self, *args, **kw):
+        self._datetime = kw.pop('datetime')
+        super().init(*args, **kw)
+
+    @property
+    def datetime(self):
+        return self._datetime
+
+
+class UniqueDateHeader(DateHeader):
+
+    max_count = 1
+
+
+class AddressHeader:
+
+    max_count = None
+
+    @staticmethod
+    def value_parser(value):
+        address_list, value = parser.get_address_list(value)
+        assert not value, 'this should not happen'
+        return address_list
+
+    @classmethod
+    def parse(cls, value, kwds):
+        if isinstance(value, str):
+            # We are translating here from the RFC language (address/mailbox)
+            # to our API language (group/address).
+            kwds['parse_tree'] = address_list = cls.value_parser(value)
+            groups = []
+            for addr in address_list.addresses:
+                groups.append(Group(addr.display_name,
+                                    [Address(mb.display_name or '',
+                                             mb.local_part or '',
+                                             mb.domain or '')
+                                     for mb in addr.all_mailboxes]))
+            defects = list(address_list.all_defects)
+        else:
+            # Assume it is Address/Group stuff
+            if not hasattr(value, '__iter__'):
+                value = [value]
+            groups = [Group(None, [item]) if not hasattr(item, 'addresses')
+                                          else item
+                                    for item in value]
+            defects = []
+        kwds['groups'] = groups
+        kwds['defects'] = defects
+        kwds['decoded'] = ', '.join([str(item) for item in groups])
+        if 'parse_tree' not in kwds:
+            kwds['parse_tree'] = cls.value_parser(kwds['decoded'])
+
+    def init(self, *args, **kw):
+        self._groups = tuple(kw.pop('groups'))
+        self._addresses = None
+        super().init(*args, **kw)
+
+    @property
+    def groups(self):
+        return self._groups
+
+    @property
+    def addresses(self):
+        if self._addresses is None:
+            self._addresses = tuple([address for group in self._groups
+                                             for address in group.addresses])
+        return self._addresses
+
+
+class UniqueAddressHeader(AddressHeader):
+
+    max_count = 1
+
+
+class SingleAddressHeader(AddressHeader):
+
+    @property
+    def address(self):
+        if len(self.addresses)!=1:
+            raise ValueError(("value of single address header {} is not "
+                "a single address").format(self.name))
+        return self.addresses[0]
+
+
+class UniqueSingleAddressHeader(SingleAddressHeader):
+
+    max_count = 1
+
+
+class MIMEVersionHeader:
+
+    max_count = 1
+
+    value_parser = staticmethod(parser.parse_mime_version)
+
+    @classmethod
+    def parse(cls, value, kwds):
+        kwds['parse_tree'] = parse_tree = cls.value_parser(value)
+        kwds['decoded'] = str(parse_tree)
+        kwds['defects'].extend(parse_tree.all_defects)
+        kwds['major'] = None if parse_tree.minor is None else parse_tree.major
+        kwds['minor'] = parse_tree.minor
+        if parse_tree.minor is not None:
+            kwds['version'] = '{}.{}'.format(kwds['major'], kwds['minor'])
+        else:
+            kwds['version'] = None
+
+    def init(self, *args, **kw):
+        self._version = kw.pop('version')
+        self._major = kw.pop('major')
+        self._minor = kw.pop('minor')
+        super().init(*args, **kw)
+
+    @property
+    def major(self):
+        return self._major
+
+    @property
+    def minor(self):
+        return self._minor
+
+    @property
+    def version(self):
+        return self._version
+
+
+class ParameterizedMIMEHeader:
+
+    # Mixin that handles the params dict.  Must be subclassed and
+    # a property value_parser for the specific header provided.
+
+    max_count = 1
+
+    @classmethod
+    def parse(cls, value, kwds):
+        kwds['parse_tree'] = parse_tree = cls.value_parser(value)
+        kwds['decoded'] = str(parse_tree)
+        kwds['defects'].extend(parse_tree.all_defects)
+        if parse_tree.params is None:
+            kwds['params'] = {}
+        else:
+            # The MIME RFCs specify that parameter ordering is arbitrary.
+            kwds['params'] = {utils._sanitize(name).lower():
+                                    utils._sanitize(value)
+                               for name, value in parse_tree.params}
+
+    def init(self, *args, **kw):
+        self._params = kw.pop('params')
+        super().init(*args, **kw)
+
+    @property
+    def params(self):
+        return self._params.copy()
+
+
+class ContentTypeHeader(ParameterizedMIMEHeader):
+
+    value_parser = staticmethod(parser.parse_content_type_header)
+
+    def init(self, *args, **kw):
+        super().init(*args, **kw)
+        self._maintype = utils._sanitize(self._parse_tree.maintype)
+        self._subtype = utils._sanitize(self._parse_tree.subtype)
+
+    @property
+    def maintype(self):
+        return self._maintype
+
+    @property
+    def subtype(self):
+        return self._subtype
+
+    @property
+    def content_type(self):
+        return self.maintype + '/' + self.subtype
+
+
+class ContentDispositionHeader(ParameterizedMIMEHeader):
+
+    value_parser = staticmethod(parser.parse_content_disposition_header)
+
+    def init(self, *args, **kw):
+        super().init(*args, **kw)
+        cd = self._parse_tree.content_disposition
+        self._content_disposition = cd if cd is None else utils._sanitize(cd)
+
+    @property
+    def content_disposition(self):
+        return self._content_disposition
+
+
+class ContentTransferEncodingHeader:
+
+    max_count = 1
+
+    value_parser = staticmethod(parser.parse_content_transfer_encoding_header)
+
+    @classmethod
+    def parse(cls, value, kwds):
+        kwds['parse_tree'] = parse_tree = cls.value_parser(value)
+        kwds['decoded'] = str(parse_tree)
+        kwds['defects'].extend(parse_tree.all_defects)
+
+    def init(self, *args, **kw):
+        super().init(*args, **kw)
+        self._cte = utils._sanitize(self._parse_tree.cte)
+
+    @property
+    def cte(self):
+        return self._cte
+
+
+# The header factory #
+
+_default_header_map = {
+    'subject':                      UniqueUnstructuredHeader,
+    'date':                         UniqueDateHeader,
+    'resent-date':                  DateHeader,
+    'orig-date':                    UniqueDateHeader,
+    'sender':                       UniqueSingleAddressHeader,
+    'resent-sender':                SingleAddressHeader,
+    'to':                           UniqueAddressHeader,
+    'resent-to':                    AddressHeader,
+    'cc':                           UniqueAddressHeader,
+    'resent-cc':                    AddressHeader,
+    'bcc':                          UniqueAddressHeader,
+    'resent-bcc':                   AddressHeader,
+    'from':                         UniqueAddressHeader,
+    'resent-from':                  AddressHeader,
+    'reply-to':                     UniqueAddressHeader,
+    'mime-version':                 MIMEVersionHeader,
+    'content-type':                 ContentTypeHeader,
+    'content-disposition':          ContentDispositionHeader,
+    'content-transfer-encoding':    ContentTransferEncodingHeader,
+    }
+
+class HeaderRegistry:
+
+    """A header_factory and header registry."""
+
+    def __init__(self, base_class=BaseHeader, default_class=UnstructuredHeader,
+                       use_default_map=True):
+        """Create a header_factory that works with the Policy API.
+
+        base_class is the class that will be the last class in the created
+        header class's __bases__ list.  default_class is the class that will be
+        used if "name" (see __call__) does not appear in the registry.
+        use_default_map controls whether or not the default mapping of names to
+        specialized classes is copied in to the registry when the factory is
+        created.  The default is True.
+
+        """
+        self.registry = {}
+        self.base_class = base_class
+        self.default_class = default_class
+        if use_default_map:
+            self.registry.update(_default_header_map)
+
+    def map_to_type(self, name, cls):
+        """Register cls as the specialized class for handling "name" headers.
+
+        """
+        self.registry[name.lower()] = cls
+
+    def __getitem__(self, name):
+        cls = self.registry.get(name.lower(), self.default_class)
+        return type('_'+cls.__name__, (cls, self.base_class), {})
+
+    def __call__(self, name, value):
+        """Create a header instance for header 'name' from 'value'.
+
+        Creates a header instance by creating a specialized class for parsing
+        and representing the specified header by combining the factory
+        base_class with a specialized class from the registry or the
+        default_class, and passing the name and value to the constructed
+        class's constructor.
+
+        """
+        return self[name](name, value)
diff --git a/Lib/email/message.py b/Lib/email/message.py
index f1ffcdb4de..3feab52799 100644
--- a/Lib/email/message.py
+++ b/Lib/email/message.py
@@ -10,14 +10,14 @@ import re
 import uu
 import base64
 import binascii
-import warnings
 from io import BytesIO, StringIO
 
 # Intrapackage imports
 from email import utils
 from email import errors
-from email import header
+from email._policybase import compat32
 from email import charset as _charset
+from email._encoded_words import decode_b
 Charset = _charset.Charset
 
 SEMISPACE = '; '
@@ -26,24 +26,6 @@ SEMISPACE = '; '
 # existence of which force quoting of the parameter value.
 tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]')
 
-# How to figure out if we are processing strings that come from a byte
-# source with undecodable characters.
-_has_surrogates = re.compile(
-    '([^\ud800-\udbff]|\A)[\udc00-\udfff]([^\udc00-\udfff]|\Z)').search
-
-
-# Helper functions
-def _sanitize_header(name, value):
-    # If the header value contains surrogates, return a Header using
-    # the unknown-8bit charset to encode the bytes as encoded words.
-    if not isinstance(value, str):
-        # Assume it is already a header object
-        return value
-    if _has_surrogates(value):
-        return header.Header(value, charset=_charset.UNKNOWN8BIT,
-                             header_name=name)
-    else:
-        return value
 
 def _splitparam(param):
     # Split header parameters.  BAW: this may be too simple.  It isn't
@@ -136,7 +118,8 @@ class Message:
     you must use the explicit API to set or get all the headers.  Not all of
     the mapping methods are implemented.
     """
-    def __init__(self):
+    def __init__(self, policy=compat32):
+        self.policy = policy
         self._headers = []
         self._unixfrom = None
         self._payload = None
@@ -246,7 +229,7 @@ class Message:
         cte = str(self.get('content-transfer-encoding', '')).lower()
         # payload may be bytes here.
         if isinstance(payload, str):
-            if _has_surrogates(payload):
+            if utils._has_surrogates(payload):
                 bpayload = payload.encode('ascii', 'surrogateescape')
                 if not decode:
                     try:
@@ -267,11 +250,12 @@ class Message:
         if cte == 'quoted-printable':
             return utils._qdecode(bpayload)
         elif cte == 'base64':
-            try:
-                return base64.b64decode(bpayload)
-            except binascii.Error:
-                # Incorrect padding
-                return bpayload
+            # XXX: this is a bit of a hack; decode_b should probably be factored
+            # out somewhere, but I haven't figured out where yet.
+            value, defects = decode_b(b''.join(bpayload.splitlines()))
+            for defect in defects:
+                self.policy.handle_defect(self, defect)
+            return value
         elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'):
             in_file = BytesIO(bpayload)
             out_file = BytesIO()
@@ -362,7 +346,17 @@ class Message:
         Note: this does not overwrite an existing header with the same field
         name.  Use __delitem__() first to delete any existing headers.
         """
-        self._headers.append((name, val))
+        max_count = self.policy.header_max_count(name)
+        if max_count:
+            lname = name.lower()
+            found = 0
+            for k, v in self._headers:
+                if k.lower() == lname:
+                    found += 1
+                    if found >= max_count:
+                        raise ValueError("There may be at most {} {} headers "
+                                         "in a message".format(max_count, name))
+        self._headers.append(self.policy.header_store_parse(name, val))
 
     def __delitem__(self, name):
         """Delete all occurrences of a header, if present.
@@ -401,7 +395,8 @@ class Message:
         Any fields deleted and re-inserted are always appended to the header
         list.
         """
-        return [_sanitize_header(k, v) for k, v in self._headers]
+        return [self.policy.header_fetch_parse(k, v)
+                for k, v in self._headers]
 
     def items(self):
         """Get all the message's header fields and values.
@@ -411,7 +406,8 @@ class Message:
         Any fields deleted and re-inserted are always appended to the header
         list.
         """
-        return [(k, _sanitize_header(k, v)) for k, v in self._headers]
+        return [(k, self.policy.header_fetch_parse(k, v))
+                for k, v in self._headers]
 
     def get(self, name, failobj=None):
         """Get a header value.
@@ -422,10 +418,29 @@ class Message:
         name = name.lower()
         for k, v in self._headers:
             if k.lower() == name:
-                return _sanitize_header(k, v)
+                return self.policy.header_fetch_parse(k, v)
         return failobj
 
     #
+    # "Internal" methods (public API, but only intended for use by a parser
+    # or generator, not normal application code.
+    #
+
+    def set_raw(self, name, value):
+        """Store name and value in the model without modification.
+
+        This is an "internal" API, intended only for use by a parser.
+        """
+        self._headers.append((name, value))
+
+    def raw_items(self):
+        """Return the (name, value) header pairs without modification.
+
+        This is an "internal" API, intended only for use by a generator.
+        """
+        return iter(self._headers.copy())
+
+    #
     # Additional useful stuff
     #
 
@@ -442,7 +457,7 @@ class Message:
         name = name.lower()
         for k, v in self._headers:
             if k.lower() == name:
-                values.append(_sanitize_header(k, v))
+                values.append(self.policy.header_fetch_parse(k, v))
         if not values:
             return failobj
         return values
@@ -475,7 +490,7 @@ class Message:
                 parts.append(_formatparam(k.replace('_', '-'), v))
         if _value is not None:
             parts.insert(0, _value)
-        self._headers.append((_name, SEMISPACE.join(parts)))
+        self[_name] = SEMISPACE.join(parts)
 
     def replace_header(self, _name, _value):
         """Replace a header.
@@ -487,7 +502,7 @@ class Message:
         _name = _name.lower()
         for i, (k, v) in zip(range(len(self._headers)), self._headers):
             if k.lower() == _name:
-                self._headers[i] = (k, _value)
+                self._headers[i] = self.policy.header_store_parse(k, _value)
                 break
         else:
             raise KeyError(_name)
@@ -803,7 +818,8 @@ class Message:
                         parts.append(k)
                     else:
                         parts.append('%s=%s' % (k, v))
-                newheaders.append((h, SEMISPACE.join(parts)))
+                val = SEMISPACE.join(parts)
+                newheaders.append(self.policy.header_store_parse(h, val))
 
             else:
                 newheaders.append((h, v))
diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py
index 5747db5d67..80ff95010f 100644
--- a/Lib/email/mime/text.py
+++ b/Lib/email/mime/text.py
@@ -14,7 +14,7 @@ from email.mime.nonmultipart import MIMENonMultipart
 class MIMEText(MIMENonMultipart):
     """Class for generating text/* type MIME documents."""
 
-    def __init__(self, _text, _subtype='plain', _charset='us-ascii'):
+    def __init__(self, _text, _subtype='plain', _charset=None):
         """Create a text/* type MIME document.
 
         _text is the string for this message object.
@@ -25,6 +25,18 @@ class MIMEText(MIMENonMultipart):
         header.  This defaults to "us-ascii".  Note that as a side-effect, the
         Content-Transfer-Encoding header will also be set.
         """
+
+        # If no _charset was specified, check to see see if there are non-ascii
+        # characters present. If not, use 'us-ascii', otherwise use utf-8.
+        # XXX: This can be removed once #7304 is fixed.
+        if _charset is None:
+            try:
+                _text.encode('us-ascii')
+                _charset = 'us-ascii'
+            except UnicodeEncodeError:
+                _charset = 'utf-8'
+
         MIMENonMultipart.__init__(self, 'text', _subtype,
                                   **{'charset': _charset})
+
         self.set_payload(_text, _charset)
diff --git a/Lib/email/parser.py b/Lib/email/parser.py
index 1c931ea9de..1aab012115 100644
--- a/Lib/email/parser.py
+++ b/Lib/email/parser.py
@@ -4,18 +4,19 @@
 
 """A parser of RFC 2822 and MIME email messages."""
 
-__all__ = ['Parser', 'HeaderParser', 'BytesParser']
+__all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser']
 
 import warnings
 from io import StringIO, TextIOWrapper
 
 from email.feedparser import FeedParser
 from email.message import Message
+from email._policybase import compat32
 
 
 
 class Parser:
-    def __init__(self, *args, **kws):
+    def __init__(self, _class=Message, *, policy=compat32):
         """Parser of RFC 2822 and MIME email messages.
 
         Creates an in-memory object tree representing the email message, which
@@ -30,28 +31,14 @@ class Parser:
         _class is the class to instantiate for new message objects when they
         must be created.  This class must have a constructor that can take
         zero arguments.  Default is Message.Message.
+
+        The policy keyword specifies a policy object that controls a number of
+        aspects of the parser's operation.  The default policy maintains
+        backward compatibility.
+
         """
-        if len(args) >= 1:
-            if '_class' in kws:
-                raise TypeError("Multiple values for keyword arg '_class'")
-            kws['_class'] = args[0]
-        if len(args) == 2:
-            if 'strict' in kws:
-                raise TypeError("Multiple values for keyword arg 'strict'")
-            kws['strict'] = args[1]
-        if len(args) > 2:
-            raise TypeError('Too many arguments')
-        if '_class' in kws:
-            self._class = kws['_class']
-            del kws['_class']
-        else:
-            self._class = Message
-        if 'strict' in kws:
-            warnings.warn("'strict' argument is deprecated (and ignored)",
-                          DeprecationWarning, 2)
-            del kws['strict']
-        if kws:
-            raise TypeError('Unexpected keyword arguments')
+        self._class = _class
+        self.policy = policy
 
     def parse(self, fp, headersonly=False):
         """Create a message structure from the data in a file.
@@ -61,7 +48,7 @@ class Parser:
         parsing after reading the headers or not.  The default is False,
         meaning it parses the entire contents of the file.
         """
-        feedparser = FeedParser(self._class)
+        feedparser = FeedParser(self._class, policy=self.policy)
         if headersonly:
             feedparser._set_headersonly()
         while True:
@@ -134,3 +121,11 @@ class BytesParser:
         """
         text = text.decode('ASCII', errors='surrogateescape')
         return self.parser.parsestr(text, headersonly)
+
+
+class BytesHeaderParser(BytesParser):
+    def parse(self, fp, headersonly=True):
+        return BytesParser.parse(self, fp, headersonly=True)
+
+    def parsebytes(self, text, headersonly=True):
+        return BytesParser.parsebytes(self, text, headersonly=True)
diff --git a/Lib/email/policy.py b/Lib/email/policy.py
new file mode 100644
index 0000000000..a17f598b50
--- /dev/null
+++ b/Lib/email/policy.py
@@ -0,0 +1,188 @@
+"""This will be the home for the policy that hooks in the new
+code that adds all the email6 features.
+"""
+
+from email._policybase import Policy, Compat32, compat32, _extend_docstrings
+from email.utils import _has_surrogates
+from email.headerregistry import HeaderRegistry as HeaderRegistry
+
+__all__ = [
+    'Compat32',
+    'compat32',
+    'Policy',
+    'EmailPolicy',
+    'default',
+    'strict',
+    'SMTP',
+    'HTTP',
+    ]
+
+@_extend_docstrings
+class EmailPolicy(Policy):
+
+    """+
+    PROVISIONAL
+
+    The API extensions enabled by this this policy are currently provisional.
+    Refer to the documentation for details.
+
+    This policy adds new header parsing and folding algorithms.  Instead of
+    simple strings, headers are custom objects with custom attributes
+    depending on the type of the field.  The folding algorithm fully
+    implements RFCs 2047 and 5322.
+
+    In addition to the settable attributes listed above that apply to
+    all Policies, this policy adds the following additional attributes:
+
+    refold_source       -- if the value for a header in the Message object
+                           came from the parsing of some source, this attribute
+                           indicates whether or not a generator should refold
+                           that value when transforming the message back into
+                           stream form.  The possible values are:
+
+                           none  -- all source values use original folding
+                           long  -- source values that have any line that is
+                                    longer than max_line_length will be
+                                    refolded
+                           all  -- all values are refolded.
+
+                           The default is 'long'.
+
+    header_factory      -- a callable that takes two arguments, 'name' and
+                           'value', where 'name' is a header field name and
+                           'value' is an unfolded header field value, and
+                           returns a string-like object that represents that
+                           header.  A default header_factory is provided that
+                           understands some of the RFC5322 header field types.
+                           (Currently address fields and date fields have
+                           special treatment, while all other fields are
+                           treated as unstructured.  This list will be
+                           completed before the extension is marked stable.)
+    """
+
+    refold_source = 'long'
+    header_factory = HeaderRegistry()
+
+    def __init__(self, **kw):
+        # Ensure that each new instance gets a unique header factory
+        # (as opposed to clones, which share the factory).
+        if 'header_factory' not in kw:
+            object.__setattr__(self, 'header_factory', HeaderRegistry())
+        super().__init__(**kw)
+
+    def header_max_count(self, name):
+        """+
+        The implementation for this class returns the max_count attribute from
+        the specialized header class that would be used to construct a header
+        of type 'name'.
+        """
+        return self.header_factory[name].max_count
+
+    # The logic of the next three methods is chosen such that it is possible to
+    # switch a Message object between a Compat32 policy and a policy derived
+    # from this class and have the results stay consistent.  This allows a
+    # Message object constructed with this policy to be passed to a library
+    # that only handles Compat32 objects, or to receive such an object and
+    # convert it to use the newer style by just changing its policy.  It is
+    # also chosen because it postpones the relatively expensive full rfc5322
+    # parse until as late as possible when parsing from source, since in many
+    # applications only a few headers will actually be inspected.
+
+    def header_source_parse(self, sourcelines):
+        """+
+        The name is parsed as everything up to the ':' and returned unmodified.
+        The value is determined by stripping leading whitespace off the
+        remainder of the first line, joining all subsequent lines together, and
+        stripping any trailing carriage return or linefeed characters.  (This
+        is the same as Compat32).
+
+        """
+        name, value = sourcelines[0].split(':', 1)
+        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        return (name, value.rstrip('\r\n'))
+
+    def header_store_parse(self, name, value):
+        """+
+        The name is returned unchanged.  If the input value has a 'name'
+        attribute and it matches the name ignoring case, the value is returned
+        unchanged.  Otherwise the name and value are passed to header_factory
+        method, and the resulting custom header object is returned as the
+        value.  In this case a ValueError is raised if the input value contains
+        CR or LF characters.
+
+        """
+        if hasattr(value, 'name') and value.name.lower() == name.lower():
+            return (name, value)
+        if isinstance(value, str) and len(value.splitlines())>1:
+            raise ValueError("Header values may not contain linefeed "
+                             "or carriage return characters")
+        return (name, self.header_factory(name, value))
+
+    def header_fetch_parse(self, name, value):
+        """+
+        If the value has a 'name' attribute, it is returned to unmodified.
+        Otherwise the name and the value with any linesep characters removed
+        are passed to the header_factory method, and the resulting custom
+        header object is returned.  Any surrogateescaped bytes get turned
+        into the unicode unknown-character glyph.
+
+        """
+        if hasattr(value, 'name'):
+            return value
+        return self.header_factory(name, ''.join(value.splitlines()))
+
+    def fold(self, name, value):
+        """+
+        Header folding is controlled by the refold_source policy setting.  A
+        value is considered to be a 'source value' if and only if it does not
+        have a 'name' attribute (having a 'name' attribute means it is a header
+        object of some sort).  If a source value needs to be refolded according
+        to the policy, it is converted into a custom header object by passing
+        the name and the value with any linesep characters removed to the
+        header_factory method.  Folding of a custom header object is done by
+        calling its fold method with the current policy.
+
+        Source values are split into lines using splitlines.  If the value is
+        not to be refolded, the lines are rejoined using the linesep from the
+        policy and returned.  The exception is lines containing non-ascii
+        binary data.  In that case the value is refolded regardless of the
+        refold_source setting, which causes the binary data to be CTE encoded
+        using the unknown-8bit charset.
+
+        """
+        return self._fold(name, value, refold_binary=True)
+
+    def fold_binary(self, name, value):
+        """+
+        The same as fold if cte_type is 7bit, except that the returned value is
+        bytes.
+
+        If cte_type is 8bit, non-ASCII binary data is converted back into
+        bytes.  Headers with binary data are not refolded, regardless of the
+        refold_header setting, since there is no way to know whether the binary
+        data consists of single byte characters or multibyte characters.
+
+        """
+        folded = self._fold(name, value, refold_binary=self.cte_type=='7bit')
+        return folded.encode('ascii', 'surrogateescape')
+
+    def _fold(self, name, value, refold_binary=False):
+        if hasattr(value, 'name'):
+            return value.fold(policy=self)
+        maxlen = self.max_line_length if self.max_line_length else float('inf')
+        lines = value.splitlines()
+        refold = (self.refold_source == 'all' or
+                  self.refold_source == 'long' and
+                    (lines and len(lines[0])+len(name)+2 > maxlen or
+                     any(len(x) > maxlen for x in lines[1:])))
+        if refold or refold_binary and _has_surrogates(value):
+            return self.header_factory(name, ''.join(lines)).fold(policy=self)
+        return name + ': ' + self.linesep.join(lines) + self.linesep
+
+
+default = EmailPolicy()
+# Make the default policy use the class default header_factory
+del default.header_factory
+strict = default.clone(raise_on_defect=True)
+SMTP = default.clone(linesep='\r\n')
+HTTP = default.clone(linesep='\r\n', max_line_length=None)
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index ac4da3705f..93a625c8b4 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -11,12 +11,14 @@ __all__ = [
     'encode_rfc2231',
     'formataddr',
     'formatdate',
+    'format_datetime',
     'getaddresses',
     'make_msgid',
     'mktime_tz',
     'parseaddr',
     'parsedate',
     'parsedate_tz',
+    'parsedate_to_datetime',
     'unquote',
     ]
 
@@ -26,6 +28,7 @@ import time
 import base64
 import random
 import socket
+import datetime
 import urllib.parse
 import warnings
 from io import StringIO
@@ -34,14 +37,13 @@ from email._parseaddr import quote
 from email._parseaddr import AddressList as _AddressList
 from email._parseaddr import mktime_tz
 
-# We need wormarounds for bugs in these methods in older Pythons (see below)
-from email._parseaddr import parsedate as _parsedate
-from email._parseaddr import parsedate_tz as _parsedate_tz
+from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz
 
 from quopri import decodestring as _qdecode
 
 # Intrapackage imports
 from email.encoders import _bencode, _qencode
+from email.charset import Charset
 
 COMMASPACE = ', '
 EMPTYSTRING = ''
@@ -50,27 +52,53 @@ CRLF = '\r\n'
 TICK = "'"
 
 specialsre = re.compile(r'[][\\()<>@,:;".]')
-escapesre = re.compile(r'[][\\()"]')
+escapesre = re.compile(r'[\\"]')
 
+# How to figure out if we are processing strings that come from a byte
+# source with undecodable characters.
+_has_surrogates = re.compile(
+    '([^\ud800-\udbff]|\A)[\udc00-\udfff]([^\udc00-\udfff]|\Z)').search
+
+# How to deal with a string containing bytes before handing it to the
+# application through the 'normal' interface.
+def _sanitize(string):
+    # Turn any escaped bytes into unicode 'unknown' char.
+    original_bytes = string.encode('ascii', 'surrogateescape')
+    return original_bytes.decode('ascii', 'replace')
 
 
 # Helpers
 
-def formataddr(pair):
+def formataddr(pair, charset='utf-8'):
     """The inverse of parseaddr(), this takes a 2-tuple of the form
     (realname, email_address) and returns the string value suitable
     for an RFC 2822 From, To or Cc header.
 
     If the first element of pair is false, then the second element is
     returned unmodified.
+
+    Optional charset if given is the character set that is used to encode
+    realname in case realname is not ASCII safe.  Can be an instance of str or
+    a Charset-like object which has a header_encode method.  Default is
+    'utf-8'.
     """
     name, address = pair
+    # The address MUST (per RFC) be ascii, so raise an UnicodeError if it isn't.
+    address.encode('ascii')
     if name:
-        quotes = ''
-        if specialsre.search(name):
-            quotes = '"'
-        name = escapesre.sub(r'\\\g<0>', name)
-        return '%s%s%s <%s>' % (quotes, name, quotes, address)
+        try:
+            name.encode('ascii')
+        except UnicodeEncodeError:
+            if isinstance(charset, str):
+                charset = Charset(charset)
+            encoded_name = charset.header_encode(name)
+            return "%s <%s>" % (encoded_name, address)
+        else:
+            quotes = ''
+            if specialsre.search(name):
+                quotes = '"'
+            name = escapesre.sub(r'\\\g<0>', name)
+            return '%s%s%s <%s>' % (quotes, name, quotes, address)
     return address
 
 
@@ -94,6 +122,14 @@ ecre = re.compile(r'''
   ''', re.VERBOSE | re.IGNORECASE)
 
 
+def _format_timetuple_and_zone(timetuple, zone):
+    return '%s, %02d %s %04d %02d:%02d:%02d %s' % (
+        ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][timetuple[6]],
+        timetuple[2],
+        ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+         'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][timetuple[1] - 1],
+        timetuple[0], timetuple[3], timetuple[4], timetuple[5],
+        zone)
 
 def formatdate(timeval=None, localtime=False, usegmt=False):
     """Returns a date string as specified by RFC 2822, e.g.:
@@ -138,14 +174,25 @@ def formatdate(timeval=None, localtime=False, usegmt=False):
             zone = 'GMT'
         else:
             zone = '-0000'
-    return '%s, %02d %s %04d %02d:%02d:%02d %s' % (
-        ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][now[6]],
-        now[2],
-        ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
-         'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][now[1] - 1],
-        now[0], now[3], now[4], now[5],
-        zone)
+    return _format_timetuple_and_zone(now, zone)
 
+def format_datetime(dt, usegmt=False):
+    """Turn a datetime into a date string as specified in RFC 2822.
+
+    If usegmt is True, dt must be an aware datetime with an offset of zero.  In
+    this case 'GMT' will be rendered instead of the normal +0000 required by
+    RFC2822.  This is to support HTTP headers involving date stamps.
+    """
+    now = dt.timetuple()
+    if usegmt:
+        if dt.tzinfo is None or dt.tzinfo != datetime.timezone.utc:
+            raise ValueError("usegmt option requires a UTC datetime")
+        zone = 'GMT'
+    elif dt.tzinfo is None:
+        zone = '-0000'
+    else:
+        zone = dt.strftime("%z")
+    return _format_timetuple_and_zone(now, zone)
 
 
 def make_msgid(idstring=None, domain=None):
@@ -172,20 +219,12 @@ def make_msgid(idstring=None, domain=None):
     return msgid
 
 
-
-# These functions are in the standalone mimelib version only because they've
-# subsequently been fixed in the latest Python versions.  We use this to worm
-# around broken older Pythons.
-def parsedate(data):
-    if not data:
-        return None
-    return _parsedate(data)
-
-
-def parsedate_tz(data):
-    if not data:
-        return None
-    return _parsedate_tz(data)
+def parsedate_to_datetime(data):
+    *dtuple, tz = _parsedate_tz(data)
+    if tz is None:
+        return datetime.datetime(*dtuple[:6])
+    return datetime.datetime(*dtuple[:6],
+            tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
 
 
 def parseaddr(addr):
@@ -304,3 +343,49 @@ def collapse_rfc2231_value(value, errors='replace',
     except LookupError:
         # charset is not a known codec.
         return unquote(text)
+
+
+#
+# datetime doesn't provide a localtime function yet, so provide one.  Code
+# adapted from the patch in issue 9527.  This may not be perfect, but it is
+# better than not having it.
+#
+
+def localtime(dt=None, isdst=-1):
+    """Return local time as an aware datetime object.
+
+    If called without arguments, return current time.  Otherwise *dt*
+    argument should be a datetime instance, and it is converted to the
+    local time zone according to the system time zone database.  If *dt* is
+    naive (that is, dt.tzinfo is None), it is assumed to be in local time.
+    In this case, a positive or zero value for *isdst* causes localtime to
+    presume initially that summer time (for example, Daylight Saving Time)
+    is or is not (respectively) in effect for the specified time.  A
+    negative value for *isdst* causes the localtime() function to attempt
+    to divine whether summer time is in effect for the specified time.
+
+    """
+    if dt is None:
+        return datetime.datetime.now(datetime.timezone.utc).astimezone()
+    if dt.tzinfo is not None:
+        return dt.astimezone()
+    # We have a naive datetime.  Convert to a (localtime) timetuple and pass to
+    # system mktime together with the isdst hint.  System mktime will return
+    # seconds since epoch.
+    tm = dt.timetuple()[:-1] + (isdst,)
+    seconds = time.mktime(tm)
+    localtm = time.localtime(seconds)
+    try:
+        delta = datetime.timedelta(seconds=localtm.tm_gmtoff)
+        tz = datetime.timezone(delta, localtm.tm_zone)
+    except AttributeError:
+        # Compute UTC offset and compare with the value implied by tm_isdst.
+        # If the values match, use the zone name implied by tm_isdst.
+        delta = dt - datetime.datetime(*time.gmtime(seconds)[:6])
+        dst = time.daylight and localtm.tm_isdst > 0
+        gmtoff = -(time.altzone if dst else time.timezone)
+        if delta == datetime.timedelta(seconds=gmtoff):
+            tz = datetime.timezone(delta, time.tzname[dst])
+        else:
+            tz = datetime.timezone(delta)
+    return dt.replace(tzinfo=tz)
diff --git a/Lib/encodings/cp037.py b/Lib/encodings/cp037.py
index 4edd708f3d..bfe2c1ed17 100644
--- a/Lib/encodings/cp037.py
+++ b/Lib/encodings/cp037.py
@@ -301,6 +301,7 @@ decoding_table = (
     '\xd9'     #  0xFD -> LATIN CAPITAL LETTER U WITH GRAVE
     '\xda'     #  0xFE -> LATIN CAPITAL LETTER U WITH ACUTE
     '\x9f'     #  0xFF -> CONTROL
+    '\ufffe'   ## Widen to UCS2 for optimization
 )
 
 ### Encoding table
diff --git a/Lib/encodings/cp500.py b/Lib/encodings/cp500.py
index 5f61535f82..a975be7b8d 100644
--- a/Lib/encodings/cp500.py
+++ b/Lib/encodings/cp500.py
@@ -301,6 +301,7 @@ decoding_table = (
     '\xd9'     #  0xFD -> LATIN CAPITAL LETTER U WITH GRAVE
     '\xda'     #  0xFE -> LATIN CAPITAL LETTER U WITH ACUTE
     '\x9f'     #  0xFF -> CONTROL
+    '\ufffe'   ## Widen to UCS2 for optimization
 )
 
 ### Encoding table
diff --git a/Lib/encodings/cp65001.py b/Lib/encodings/cp65001.py
new file mode 100644
index 0000000000..287eb877fe
--- /dev/null
+++ b/Lib/encodings/cp65001.py
@@ -0,0 +1,40 @@
+"""
+Code page 65001: Windows UTF-8 (CP_UTF8).
+"""
+
+import codecs
+import functools
+
+if not hasattr(codecs, 'code_page_encode'):
+    raise LookupError("cp65001 encoding is only available on Windows")
+
+### Codec APIs
+
+encode = functools.partial(codecs.code_page_encode, 65001)
+decode = functools.partial(codecs.code_page_decode, 65001)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    def encode(self, input, final=False):
+        return encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+    _buffer_decode = decode
+
+class StreamWriter(codecs.StreamWriter):
+    encode = encode
+
+class StreamReader(codecs.StreamReader):
+    decode = decode
+
+### encodings module API
+
+def getregentry():
+    return codecs.CodecInfo(
+        name='cp65001',
+        encode=encode,
+        decode=decode,
+        incrementalencoder=IncrementalEncoder,
+        incrementaldecoder=IncrementalDecoder,
+        streamreader=StreamReader,
+        streamwriter=StreamWriter,
+    )
diff --git a/Lib/encodings/hp_roman8.py b/Lib/encodings/hp_roman8.py
index dbaaa72d76..2334208812 100644
--- a/Lib/encodings/hp_roman8.py
+++ b/Lib/encodings/hp_roman8.py
@@ -14,18 +14,18 @@ import codecs
 class Codec(codecs.Codec):
 
     def encode(self,input,errors='strict'):
-        return codecs.charmap_encode(input,errors,encoding_map)
+        return codecs.charmap_encode(input,errors,encoding_table)
 
     def decode(self,input,errors='strict'):
-        return codecs.charmap_decode(input,errors,decoding_map)
+        return codecs.charmap_decode(input,errors,decoding_table)
 
 class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
-        return codecs.charmap_encode(input,self.errors,encoding_map)[0]
+        return codecs.charmap_encode(input,self.errors,encoding_table)[0]
 
 class IncrementalDecoder(codecs.IncrementalDecoder):
     def decode(self, input, final=False):
-        return codecs.charmap_decode(input,self.errors,decoding_map)[0]
+        return codecs.charmap_decode(input,self.errors,decoding_table)[0]
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
@@ -46,107 +46,267 @@ def getregentry():
         streamreader=StreamReader,
     )
 
-### Decoding Map
-
-decoding_map = codecs.make_identity_dict(range(256))
-decoding_map.update({
-        0x00a1: 0x00c0, #       LATIN CAPITAL LETTER A WITH GRAVE
-        0x00a2: 0x00c2, #       LATIN CAPITAL LETTER A WITH CIRCUMFLEX
-        0x00a3: 0x00c8, #       LATIN CAPITAL LETTER E WITH GRAVE
-        0x00a4: 0x00ca, #       LATIN CAPITAL LETTER E WITH CIRCUMFLEX
-        0x00a5: 0x00cb, #       LATIN CAPITAL LETTER E WITH DIAERESIS
-        0x00a6: 0x00ce, #       LATIN CAPITAL LETTER I WITH CIRCUMFLEX
-        0x00a7: 0x00cf, #       LATIN CAPITAL LETTER I WITH DIAERESIS
-        0x00a8: 0x00b4, #       ACUTE ACCENT
-        0x00a9: 0x02cb, #       MODIFIER LETTER GRAVE ACCENT (Mandarin Chinese fourth tone)
-        0x00aa: 0x02c6, #       MODIFIER LETTER CIRCUMFLEX ACCENT
-        0x00ab: 0x00a8, #       DIAERESIS
-        0x00ac: 0x02dc, #       SMALL TILDE
-        0x00ad: 0x00d9, #       LATIN CAPITAL LETTER U WITH GRAVE
-        0x00ae: 0x00db, #       LATIN CAPITAL LETTER U WITH CIRCUMFLEX
-        0x00af: 0x20a4, #       LIRA SIGN
-        0x00b0: 0x00af, #       MACRON
-        0x00b1: 0x00dd, #       LATIN CAPITAL LETTER Y WITH ACUTE
-        0x00b2: 0x00fd, #       LATIN SMALL LETTER Y WITH ACUTE
-        0x00b3: 0x00b0, #       DEGREE SIGN
-        0x00b4: 0x00c7, #       LATIN CAPITAL LETTER C WITH CEDILLA
-        0x00b5: 0x00e7, #       LATIN SMALL LETTER C WITH CEDILLA
-        0x00b6: 0x00d1, #       LATIN CAPITAL LETTER N WITH TILDE
-        0x00b7: 0x00f1, #       LATIN SMALL LETTER N WITH TILDE
-        0x00b8: 0x00a1, #       INVERTED EXCLAMATION MARK
-        0x00b9: 0x00bf, #       INVERTED QUESTION MARK
-        0x00ba: 0x00a4, #       CURRENCY SIGN
-        0x00bb: 0x00a3, #       POUND SIGN
-        0x00bc: 0x00a5, #       YEN SIGN
-        0x00bd: 0x00a7, #       SECTION SIGN
-        0x00be: 0x0192, #       LATIN SMALL LETTER F WITH HOOK
-        0x00bf: 0x00a2, #       CENT SIGN
-        0x00c0: 0x00e2, #       LATIN SMALL LETTER A WITH CIRCUMFLEX
-        0x00c1: 0x00ea, #       LATIN SMALL LETTER E WITH CIRCUMFLEX
-        0x00c2: 0x00f4, #       LATIN SMALL LETTER O WITH CIRCUMFLEX
-        0x00c3: 0x00fb, #       LATIN SMALL LETTER U WITH CIRCUMFLEX
-        0x00c4: 0x00e1, #       LATIN SMALL LETTER A WITH ACUTE
-        0x00c5: 0x00e9, #       LATIN SMALL LETTER E WITH ACUTE
-        0x00c6: 0x00f3, #       LATIN SMALL LETTER O WITH ACUTE
-        0x00c7: 0x00fa, #       LATIN SMALL LETTER U WITH ACUTE
-        0x00c8: 0x00e0, #       LATIN SMALL LETTER A WITH GRAVE
-        0x00c9: 0x00e8, #       LATIN SMALL LETTER E WITH GRAVE
-        0x00ca: 0x00f2, #       LATIN SMALL LETTER O WITH GRAVE
-        0x00cb: 0x00f9, #       LATIN SMALL LETTER U WITH GRAVE
-        0x00cc: 0x00e4, #       LATIN SMALL LETTER A WITH DIAERESIS
-        0x00cd: 0x00eb, #       LATIN SMALL LETTER E WITH DIAERESIS
-        0x00ce: 0x00f6, #       LATIN SMALL LETTER O WITH DIAERESIS
-        0x00cf: 0x00fc, #       LATIN SMALL LETTER U WITH DIAERESIS
-        0x00d0: 0x00c5, #       LATIN CAPITAL LETTER A WITH RING ABOVE
-        0x00d1: 0x00ee, #       LATIN SMALL LETTER I WITH CIRCUMFLEX
-        0x00d2: 0x00d8, #       LATIN CAPITAL LETTER O WITH STROKE
-        0x00d3: 0x00c6, #       LATIN CAPITAL LETTER AE
-        0x00d4: 0x00e5, #       LATIN SMALL LETTER A WITH RING ABOVE
-        0x00d5: 0x00ed, #       LATIN SMALL LETTER I WITH ACUTE
-        0x00d6: 0x00f8, #       LATIN SMALL LETTER O WITH STROKE
-        0x00d7: 0x00e6, #       LATIN SMALL LETTER AE
-        0x00d8: 0x00c4, #       LATIN CAPITAL LETTER A WITH DIAERESIS
-        0x00d9: 0x00ec, #       LATIN SMALL LETTER I WITH GRAVE
-        0x00da: 0x00d6, #       LATIN CAPITAL LETTER O WITH DIAERESIS
-        0x00db: 0x00dc, #       LATIN CAPITAL LETTER U WITH DIAERESIS
-        0x00dc: 0x00c9, #       LATIN CAPITAL LETTER E WITH ACUTE
-        0x00dd: 0x00ef, #       LATIN SMALL LETTER I WITH DIAERESIS
-        0x00de: 0x00df, #       LATIN SMALL LETTER SHARP S (German)
-        0x00df: 0x00d4, #       LATIN CAPITAL LETTER O WITH CIRCUMFLEX
-        0x00e0: 0x00c1, #       LATIN CAPITAL LETTER A WITH ACUTE
-        0x00e1: 0x00c3, #       LATIN CAPITAL LETTER A WITH TILDE
-        0x00e2: 0x00e3, #       LATIN SMALL LETTER A WITH TILDE
-        0x00e3: 0x00d0, #       LATIN CAPITAL LETTER ETH (Icelandic)
-        0x00e4: 0x00f0, #       LATIN SMALL LETTER ETH (Icelandic)
-        0x00e5: 0x00cd, #       LATIN CAPITAL LETTER I WITH ACUTE
-        0x00e6: 0x00cc, #       LATIN CAPITAL LETTER I WITH GRAVE
-        0x00e7: 0x00d3, #       LATIN CAPITAL LETTER O WITH ACUTE
-        0x00e8: 0x00d2, #       LATIN CAPITAL LETTER O WITH GRAVE
-        0x00e9: 0x00d5, #       LATIN CAPITAL LETTER O WITH TILDE
-        0x00ea: 0x00f5, #       LATIN SMALL LETTER O WITH TILDE
-        0x00eb: 0x0160, #       LATIN CAPITAL LETTER S WITH CARON
-        0x00ec: 0x0161, #       LATIN SMALL LETTER S WITH CARON
-        0x00ed: 0x00da, #       LATIN CAPITAL LETTER U WITH ACUTE
-        0x00ee: 0x0178, #       LATIN CAPITAL LETTER Y WITH DIAERESIS
-        0x00ef: 0x00ff, #       LATIN SMALL LETTER Y WITH DIAERESIS
-        0x00f0: 0x00de, #       LATIN CAPITAL LETTER THORN (Icelandic)
-        0x00f1: 0x00fe, #       LATIN SMALL LETTER THORN (Icelandic)
-        0x00f2: 0x00b7, #       MIDDLE DOT
-        0x00f3: 0x00b5, #       MICRO SIGN
-        0x00f4: 0x00b6, #       PILCROW SIGN
-        0x00f5: 0x00be, #       VULGAR FRACTION THREE QUARTERS
-        0x00f6: 0x2014, #       EM DASH
-        0x00f7: 0x00bc, #       VULGAR FRACTION ONE QUARTER
-        0x00f8: 0x00bd, #       VULGAR FRACTION ONE HALF
-        0x00f9: 0x00aa, #       FEMININE ORDINAL INDICATOR
-        0x00fa: 0x00ba, #       MASCULINE ORDINAL INDICATOR
-        0x00fb: 0x00ab, #       LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
-        0x00fc: 0x25a0, #       BLACK SQUARE
-        0x00fd: 0x00bb, #       RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
-        0x00fe: 0x00b1, #       PLUS-MINUS SIGN
-        0x00ff: None,
-})
-
-### Encoding Map
-
-encoding_map = codecs.make_encoding_map(decoding_map)
+
+### Decoding Table
+
+decoding_table = (
+    '\x00'     #  0x00 -> NULL
+    '\x01'     #  0x01 -> START OF HEADING
+    '\x02'     #  0x02 -> START OF TEXT
+    '\x03'     #  0x03 -> END OF TEXT
+    '\x04'     #  0x04 -> END OF TRANSMISSION
+    '\x05'     #  0x05 -> ENQUIRY
+    '\x06'     #  0x06 -> ACKNOWLEDGE
+    '\x07'     #  0x07 -> BELL
+    '\x08'     #  0x08 -> BACKSPACE
+    '\t'       #  0x09 -> HORIZONTAL TABULATION
+    '\n'       #  0x0A -> LINE FEED
+    '\x0b'     #  0x0B -> VERTICAL TABULATION
+    '\x0c'     #  0x0C -> FORM FEED
+    '\r'       #  0x0D -> CARRIAGE RETURN
+    '\x0e'     #  0x0E -> SHIFT OUT
+    '\x0f'     #  0x0F -> SHIFT IN
+    '\x10'     #  0x10 -> DATA LINK ESCAPE
+    '\x11'     #  0x11 -> DEVICE CONTROL ONE
+    '\x12'     #  0x12 -> DEVICE CONTROL TWO
+    '\x13'     #  0x13 -> DEVICE CONTROL THREE
+    '\x14'     #  0x14 -> DEVICE CONTROL FOUR
+    '\x15'     #  0x15 -> NEGATIVE ACKNOWLEDGE
+    '\x16'     #  0x16 -> SYNCHRONOUS IDLE
+    '\x17'     #  0x17 -> END OF TRANSMISSION BLOCK
+    '\x18'     #  0x18 -> CANCEL
+    '\x19'     #  0x19 -> END OF MEDIUM
+    '\x1a'     #  0x1A -> SUBSTITUTE
+    '\x1b'     #  0x1B -> ESCAPE
+    '\x1c'     #  0x1C -> FILE SEPARATOR
+    '\x1d'     #  0x1D -> GROUP SEPARATOR
+    '\x1e'     #  0x1E -> RECORD SEPARATOR
+    '\x1f'     #  0x1F -> UNIT SEPARATOR
+    ' '        #  0x20 -> SPACE
+    '!'        #  0x21 -> EXCLAMATION MARK
+    '"'        #  0x22 -> QUOTATION MARK
+    '#'        #  0x23 -> NUMBER SIGN
+    '$'        #  0x24 -> DOLLAR SIGN
+    '%'        #  0x25 -> PERCENT SIGN
+    '&'        #  0x26 -> AMPERSAND
+    "'"        #  0x27 -> APOSTROPHE
+    '('        #  0x28 -> LEFT PARENTHESIS
+    ')'        #  0x29 -> RIGHT PARENTHESIS
+    '*'        #  0x2A -> ASTERISK
+    '+'        #  0x2B -> PLUS SIGN
+    ','        #  0x2C -> COMMA
+    '-'        #  0x2D -> HYPHEN-MINUS
+    '.'        #  0x2E -> FULL STOP
+    '/'        #  0x2F -> SOLIDUS
+    '0'        #  0x30 -> DIGIT ZERO
+    '1'        #  0x31 -> DIGIT ONE
+    '2'        #  0x32 -> DIGIT TWO
+    '3'        #  0x33 -> DIGIT THREE
+    '4'        #  0x34 -> DIGIT FOUR
+    '5'        #  0x35 -> DIGIT FIVE
+    '6'        #  0x36 -> DIGIT SIX
+    '7'        #  0x37 -> DIGIT SEVEN
+    '8'        #  0x38 -> DIGIT EIGHT
+    '9'        #  0x39 -> DIGIT NINE
+    ':'        #  0x3A -> COLON
+    ';'        #  0x3B -> SEMICOLON
+    '<'        #  0x3C -> LESS-THAN SIGN
+    '='        #  0x3D -> EQUALS SIGN
+    '>'        #  0x3E -> GREATER-THAN SIGN
+    '?'        #  0x3F -> QUESTION MARK
+    '@'        #  0x40 -> COMMERCIAL AT
+    'A'        #  0x41 -> LATIN CAPITAL LETTER A
+    'B'        #  0x42 -> LATIN CAPITAL LETTER B
+    'C'        #  0x43 -> LATIN CAPITAL LETTER C
+    'D'        #  0x44 -> LATIN CAPITAL LETTER D
+    'E'        #  0x45 -> LATIN CAPITAL LETTER E
+    'F'        #  0x46 -> LATIN CAPITAL LETTER F
+    'G'        #  0x47 -> LATIN CAPITAL LETTER G
+    'H'        #  0x48 -> LATIN CAPITAL LETTER H
+    'I'        #  0x49 -> LATIN CAPITAL LETTER I
+    'J'        #  0x4A -> LATIN CAPITAL LETTER J
+    'K'        #  0x4B -> LATIN CAPITAL LETTER K
+    'L'        #  0x4C -> LATIN CAPITAL LETTER L
+    'M'        #  0x4D -> LATIN CAPITAL LETTER M
+    'N'        #  0x4E -> LATIN CAPITAL LETTER N
+    'O'        #  0x4F -> LATIN CAPITAL LETTER O
+    'P'        #  0x50 -> LATIN CAPITAL LETTER P
+    'Q'        #  0x51 -> LATIN CAPITAL LETTER Q
+    'R'        #  0x52 -> LATIN CAPITAL LETTER R
+    'S'        #  0x53 -> LATIN CAPITAL LETTER S
+    'T'        #  0x54 -> LATIN CAPITAL LETTER T
+    'U'        #  0x55 -> LATIN CAPITAL LETTER U
+    'V'        #  0x56 -> LATIN CAPITAL LETTER V
+    'W'        #  0x57 -> LATIN CAPITAL LETTER W
+    'X'        #  0x58 -> LATIN CAPITAL LETTER X
+    'Y'        #  0x59 -> LATIN CAPITAL LETTER Y
+    'Z'        #  0x5A -> LATIN CAPITAL LETTER Z
+    '['        #  0x5B -> LEFT SQUARE BRACKET
+    '\\'       #  0x5C -> REVERSE SOLIDUS
+    ']'        #  0x5D -> RIGHT SQUARE BRACKET
+    '^'        #  0x5E -> CIRCUMFLEX ACCENT
+    '_'        #  0x5F -> LOW LINE
+    '`'        #  0x60 -> GRAVE ACCENT
+    'a'        #  0x61 -> LATIN SMALL LETTER A
+    'b'        #  0x62 -> LATIN SMALL LETTER B
+    'c'        #  0x63 -> LATIN SMALL LETTER C
+    'd'        #  0x64 -> LATIN SMALL LETTER D
+    'e'        #  0x65 -> LATIN SMALL LETTER E
+    'f'        #  0x66 -> LATIN SMALL LETTER F
+    'g'        #  0x67 -> LATIN SMALL LETTER G
+    'h'        #  0x68 -> LATIN SMALL LETTER H
+    'i'        #  0x69 -> LATIN SMALL LETTER I
+    'j'        #  0x6A -> LATIN SMALL LETTER J
+    'k'        #  0x6B -> LATIN SMALL LETTER K
+    'l'        #  0x6C -> LATIN SMALL LETTER L
+    'm'        #  0x6D -> LATIN SMALL LETTER M
+    'n'        #  0x6E -> LATIN SMALL LETTER N
+    'o'        #  0x6F -> LATIN SMALL LETTER O
+    'p'        #  0x70 -> LATIN SMALL LETTER P
+    'q'        #  0x71 -> LATIN SMALL LETTER Q
+    'r'        #  0x72 -> LATIN SMALL LETTER R
+    's'        #  0x73 -> LATIN SMALL LETTER S
+    't'        #  0x74 -> LATIN SMALL LETTER T
+    'u'        #  0x75 -> LATIN SMALL LETTER U
+    'v'        #  0x76 -> LATIN SMALL LETTER V
+    'w'        #  0x77 -> LATIN SMALL LETTER W
+    'x'        #  0x78 -> LATIN SMALL LETTER X
+    'y'        #  0x79 -> LATIN SMALL LETTER Y
+    'z'        #  0x7A -> LATIN SMALL LETTER Z
+    '{'        #  0x7B -> LEFT CURLY BRACKET
+    '|'        #  0x7C -> VERTICAL LINE
+    '}'        #  0x7D -> RIGHT CURLY BRACKET
+    '~'        #  0x7E -> TILDE
+    '\x7f'     #  0x7F -> DELETE
+    '\x80'     #  0x80 -> <control>
+    '\x81'     #  0x81 -> <control>
+    '\x82'     #  0x82 -> <control>
+    '\x83'     #  0x83 -> <control>
+    '\x84'     #  0x84 -> <control>
+    '\x85'     #  0x85 -> <control>
+    '\x86'     #  0x86 -> <control>
+    '\x87'     #  0x87 -> <control>
+    '\x88'     #  0x88 -> <control>
+    '\x89'     #  0x89 -> <control>
+    '\x8a'     #  0x8A -> <control>
+    '\x8b'     #  0x8B -> <control>
+    '\x8c'     #  0x8C -> <control>
+    '\x8d'     #  0x8D -> <control>
+    '\x8e'     #  0x8E -> <control>
+    '\x8f'     #  0x8F -> <control>
+    '\x90'     #  0x90 -> <control>
+    '\x91'     #  0x91 -> <control>
+    '\x92'     #  0x92 -> <control>
+    '\x93'     #  0x93 -> <control>
+    '\x94'     #  0x94 -> <control>
+    '\x95'     #  0x95 -> <control>
+    '\x96'     #  0x96 -> <control>
+    '\x97'     #  0x97 -> <control>
+    '\x98'     #  0x98 -> <control>
+    '\x99'     #  0x99 -> <control>
+    '\x9a'     #  0x9A -> <control>
+    '\x9b'     #  0x9B -> <control>
+    '\x9c'     #  0x9C -> <control>
+    '\x9d'     #  0x9D -> <control>
+    '\x9e'     #  0x9E -> <control>
+    '\x9f'     #  0x9F -> <control>
+    '\xa0'     #  0xA0 -> NO-BREAK SPACE
+    '\xc0'     #  0xA1 -> LATIN CAPITAL LETTER A WITH GRAVE
+    '\xc2'     #  0xA2 -> LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+    '\xc8'     #  0xA3 -> LATIN CAPITAL LETTER E WITH GRAVE
+    '\xca'     #  0xA4 -> LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+    '\xcb'     #  0xA5 -> LATIN CAPITAL LETTER E WITH DIAERESIS
+    '\xce'     #  0xA6 -> LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+    '\xcf'     #  0xA7 -> LATIN CAPITAL LETTER I WITH DIAERESIS
+    '\xb4'     #  0xA8 -> ACUTE ACCENT
+    '\u02cb'   #  0xA9 -> MODIFIER LETTER GRAVE ACCENT (MANDARIN CHINESE FOURTH TONE)
+    '\u02c6'   #  0xAA -> MODIFIER LETTER CIRCUMFLEX ACCENT
+    '\xa8'     #  0xAB -> DIAERESIS
+    '\u02dc'   #  0xAC -> SMALL TILDE
+    '\xd9'     #  0xAD -> LATIN CAPITAL LETTER U WITH GRAVE
+    '\xdb'     #  0xAE -> LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+    '\u20a4'   #  0xAF -> LIRA SIGN
+    '\xaf'     #  0xB0 -> MACRON
+    '\xdd'     #  0xB1 -> LATIN CAPITAL LETTER Y WITH ACUTE
+    '\xfd'     #  0xB2 -> LATIN SMALL LETTER Y WITH ACUTE
+    '\xb0'     #  0xB3 -> DEGREE SIGN
+    '\xc7'     #  0xB4 -> LATIN CAPITAL LETTER C WITH CEDILLA
+    '\xe7'     #  0xB5 -> LATIN SMALL LETTER C WITH CEDILLA
+    '\xd1'     #  0xB6 -> LATIN CAPITAL LETTER N WITH TILDE
+    '\xf1'     #  0xB7 -> LATIN SMALL LETTER N WITH TILDE
+    '\xa1'     #  0xB8 -> INVERTED EXCLAMATION MARK
+    '\xbf'     #  0xB9 -> INVERTED QUESTION MARK
+    '\xa4'     #  0xBA -> CURRENCY SIGN
+    '\xa3'     #  0xBB -> POUND SIGN
+    '\xa5'     #  0xBC -> YEN SIGN
+    '\xa7'     #  0xBD -> SECTION SIGN
+    '\u0192'   #  0xBE -> LATIN SMALL LETTER F WITH HOOK
+    '\xa2'     #  0xBF -> CENT SIGN
+    '\xe2'     #  0xC0 -> LATIN SMALL LETTER A WITH CIRCUMFLEX
+    '\xea'     #  0xC1 -> LATIN SMALL LETTER E WITH CIRCUMFLEX
+    '\xf4'     #  0xC2 -> LATIN SMALL LETTER O WITH CIRCUMFLEX
+    '\xfb'     #  0xC3 -> LATIN SMALL LETTER U WITH CIRCUMFLEX
+    '\xe1'     #  0xC4 -> LATIN SMALL LETTER A WITH ACUTE
+    '\xe9'     #  0xC5 -> LATIN SMALL LETTER E WITH ACUTE
+    '\xf3'     #  0xC6 -> LATIN SMALL LETTER O WITH ACUTE
+    '\xfa'     #  0xC7 -> LATIN SMALL LETTER U WITH ACUTE
+    '\xe0'     #  0xC8 -> LATIN SMALL LETTER A WITH GRAVE
+    '\xe8'     #  0xC9 -> LATIN SMALL LETTER E WITH GRAVE
+    '\xf2'     #  0xCA -> LATIN SMALL LETTER O WITH GRAVE
+    '\xf9'     #  0xCB -> LATIN SMALL LETTER U WITH GRAVE
+    '\xe4'     #  0xCC -> LATIN SMALL LETTER A WITH DIAERESIS
+    '\xeb'     #  0xCD -> LATIN SMALL LETTER E WITH DIAERESIS
+    '\xf6'     #  0xCE -> LATIN SMALL LETTER O WITH DIAERESIS
+    '\xfc'     #  0xCF -> LATIN SMALL LETTER U WITH DIAERESIS
+    '\xc5'     #  0xD0 -> LATIN CAPITAL LETTER A WITH RING ABOVE
+    '\xee'     #  0xD1 -> LATIN SMALL LETTER I WITH CIRCUMFLEX
+    '\xd8'     #  0xD2 -> LATIN CAPITAL LETTER O WITH STROKE
+    '\xc6'     #  0xD3 -> LATIN CAPITAL LETTER AE
+    '\xe5'     #  0xD4 -> LATIN SMALL LETTER A WITH RING ABOVE
+    '\xed'     #  0xD5 -> LATIN SMALL LETTER I WITH ACUTE
+    '\xf8'     #  0xD6 -> LATIN SMALL LETTER O WITH STROKE
+    '\xe6'     #  0xD7 -> LATIN SMALL LETTER AE
+    '\xc4'     #  0xD8 -> LATIN CAPITAL LETTER A WITH DIAERESIS
+    '\xec'     #  0xD9 -> LATIN SMALL LETTER I WITH GRAVE
+    '\xd6'     #  0xDA -> LATIN CAPITAL LETTER O WITH DIAERESIS
+    '\xdc'     #  0xDB -> LATIN CAPITAL LETTER U WITH DIAERESIS
+    '\xc9'     #  0xDC -> LATIN CAPITAL LETTER E WITH ACUTE
+    '\xef'     #  0xDD -> LATIN SMALL LETTER I WITH DIAERESIS
+    '\xdf'     #  0xDE -> LATIN SMALL LETTER SHARP S (GERMAN)
+    '\xd4'     #  0xDF -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+    '\xc1'     #  0xE0 -> LATIN CAPITAL LETTER A WITH ACUTE
+    '\xc3'     #  0xE1 -> LATIN CAPITAL LETTER A WITH TILDE
+    '\xe3'     #  0xE2 -> LATIN SMALL LETTER A WITH TILDE
+    '\xd0'     #  0xE3 -> LATIN CAPITAL LETTER ETH (ICELANDIC)
+    '\xf0'     #  0xE4 -> LATIN SMALL LETTER ETH (ICELANDIC)
+    '\xcd'     #  0xE5 -> LATIN CAPITAL LETTER I WITH ACUTE
+    '\xcc'     #  0xE6 -> LATIN CAPITAL LETTER I WITH GRAVE
+    '\xd3'     #  0xE7 -> LATIN CAPITAL LETTER O WITH ACUTE
+    '\xd2'     #  0xE8 -> LATIN CAPITAL LETTER O WITH GRAVE
+    '\xd5'     #  0xE9 -> LATIN CAPITAL LETTER O WITH TILDE
+    '\xf5'     #  0xEA -> LATIN SMALL LETTER O WITH TILDE
+    '\u0160'   #  0xEB -> LATIN CAPITAL LETTER S WITH CARON
+    '\u0161'   #  0xEC -> LATIN SMALL LETTER S WITH CARON
+    '\xda'     #  0xED -> LATIN CAPITAL LETTER U WITH ACUTE
+    '\u0178'   #  0xEE -> LATIN CAPITAL LETTER Y WITH DIAERESIS
+    '\xff'     #  0xEF -> LATIN SMALL LETTER Y WITH DIAERESIS
+    '\xde'     #  0xF0 -> LATIN CAPITAL LETTER THORN (ICELANDIC)
+    '\xfe'     #  0xF1 -> LATIN SMALL LETTER THORN (ICELANDIC)
+    '\xb7'     #  0xF2 -> MIDDLE DOT
+    '\xb5'     #  0xF3 -> MICRO SIGN
+    '\xb6'     #  0xF4 -> PILCROW SIGN
+    '\xbe'     #  0xF5 -> VULGAR FRACTION THREE QUARTERS
+    '\u2014'   #  0xF6 -> EM DASH
+    '\xbc'     #  0xF7 -> VULGAR FRACTION ONE QUARTER
+    '\xbd'     #  0xF8 -> VULGAR FRACTION ONE HALF
+    '\xaa'     #  0xF9 -> FEMININE ORDINAL INDICATOR
+    '\xba'     #  0xFA -> MASCULINE ORDINAL INDICATOR
+    '\xab'     #  0xFB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\u25a0'   #  0xFC -> BLACK SQUARE
+    '\xbb'     #  0xFD -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\xb1'     #  0xFE -> PLUS-MINUS SIGN
+    '\ufffe'
+)
+
+### Encoding table
+encoding_table=codecs.charmap_build(decoding_table)
diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py
index 583bdf130a..ea4058512f 100644
--- a/Lib/encodings/idna.py
+++ b/Lib/encodings/idna.py
@@ -153,6 +153,20 @@ class Codec(codecs.Codec):
         if not input:
             return b'', 0
 
+        try:
+            result = input.encode('ascii')
+        except UnicodeEncodeError:
+            pass
+        else:
+            # ASCII name: fast path
+            labels = result.split(b'.')
+            for label in labels[:-1]:
+                if not (0 < len(label) < 64):
+                    raise UnicodeError("label empty or too long")
+            if len(labels[-1]) >= 64:
+                raise UnicodeError("label too long")
+            return result, len(input)
+
         result = bytearray()
         labels = dots.split(input)
         if labels and not labels[-1]:
@@ -179,6 +193,14 @@ class Codec(codecs.Codec):
         if not isinstance(input, bytes):
             # XXX obviously wrong, see #3232
             input = bytes(input)
+
+        if ace_prefix not in input:
+            # Fast path
+            try:
+                return input.decode('ascii'), len(input)
+            except UnicodeDecodeError:
+                pass
+
         labels = input.split(b".")
 
         if labels and len(labels[-1]) == 0:
diff --git a/Lib/encodings/iso8859_1.py b/Lib/encodings/iso8859_1.py
index 8cfc01fe14..d9cc516718 100644
--- a/Lib/encodings/iso8859_1.py
+++ b/Lib/encodings/iso8859_1.py
@@ -301,6 +301,7 @@ decoding_table = (
     '\xfd'     #  0xFD -> LATIN SMALL LETTER Y WITH ACUTE
     '\xfe'     #  0xFE -> LATIN SMALL LETTER THORN (Icelandic)
     '\xff'     #  0xFF -> LATIN SMALL LETTER Y WITH DIAERESIS
+    '\ufffe'   ## Widen to UCS2 for optimization
 )
 
 ### Encoding table
diff --git a/Lib/encodings/mac_latin2.py b/Lib/encodings/mac_latin2.py
index e322be236c..da9d4b134a 100644
--- a/Lib/encodings/mac_latin2.py
+++ b/Lib/encodings/mac_latin2.py
@@ -1,4 +1,4 @@
-""" Python Character Mapping Codec generated from 'LATIN2.TXT' with gencodec.py.
+""" Python Character Mapping Codec mac_latin2 generated from 'MAPPINGS/VENDORS/MICSFT/MAC/LATIN2.TXT' with gencodec.py.
 
 Written by Marc-Andre Lemburg (mal@lemburg.com).
 
@@ -14,18 +14,18 @@ import codecs
 class Codec(codecs.Codec):
 
     def encode(self,input,errors='strict'):
-        return codecs.charmap_encode(input,errors,encoding_map)
+        return codecs.charmap_encode(input,errors,encoding_table)
 
     def decode(self,input,errors='strict'):
-        return codecs.charmap_decode(input,errors,decoding_map)
+        return codecs.charmap_decode(input,errors,decoding_table)
 
 class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
-        return codecs.charmap_encode(input,self.errors,encoding_map)[0]
+        return codecs.charmap_encode(input,self.errors,encoding_table)[0]
 
 class IncrementalDecoder(codecs.IncrementalDecoder):
     def decode(self, input, final=False):
-        return codecs.charmap_decode(input,self.errors,decoding_map)[0]
+        return codecs.charmap_decode(input,self.errors,decoding_table)[0]
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
@@ -46,138 +46,267 @@ def getregentry():
         streamwriter=StreamWriter,
     )
 
-### Decoding Map
-
-decoding_map = codecs.make_identity_dict(range(256))
-decoding_map.update({
-        0x0080: 0x00c4, # LATIN CAPITAL LETTER A WITH DIAERESIS
-        0x0081: 0x0100, # LATIN CAPITAL LETTER A WITH MACRON
-        0x0082: 0x0101, # LATIN SMALL LETTER A WITH MACRON
-        0x0083: 0x00c9, # LATIN CAPITAL LETTER E WITH ACUTE
-        0x0084: 0x0104, # LATIN CAPITAL LETTER A WITH OGONEK
-        0x0085: 0x00d6, # LATIN CAPITAL LETTER O WITH DIAERESIS
-        0x0086: 0x00dc, # LATIN CAPITAL LETTER U WITH DIAERESIS
-        0x0087: 0x00e1, # LATIN SMALL LETTER A WITH ACUTE
-        0x0088: 0x0105, # LATIN SMALL LETTER A WITH OGONEK
-        0x0089: 0x010c, # LATIN CAPITAL LETTER C WITH CARON
-        0x008a: 0x00e4, # LATIN SMALL LETTER A WITH DIAERESIS
-        0x008b: 0x010d, # LATIN SMALL LETTER C WITH CARON
-        0x008c: 0x0106, # LATIN CAPITAL LETTER C WITH ACUTE
-        0x008d: 0x0107, # LATIN SMALL LETTER C WITH ACUTE
-        0x008e: 0x00e9, # LATIN SMALL LETTER E WITH ACUTE
-        0x008f: 0x0179, # LATIN CAPITAL LETTER Z WITH ACUTE
-        0x0090: 0x017a, # LATIN SMALL LETTER Z WITH ACUTE
-        0x0091: 0x010e, # LATIN CAPITAL LETTER D WITH CARON
-        0x0092: 0x00ed, # LATIN SMALL LETTER I WITH ACUTE
-        0x0093: 0x010f, # LATIN SMALL LETTER D WITH CARON
-        0x0094: 0x0112, # LATIN CAPITAL LETTER E WITH MACRON
-        0x0095: 0x0113, # LATIN SMALL LETTER E WITH MACRON
-        0x0096: 0x0116, # LATIN CAPITAL LETTER E WITH DOT ABOVE
-        0x0097: 0x00f3, # LATIN SMALL LETTER O WITH ACUTE
-        0x0098: 0x0117, # LATIN SMALL LETTER E WITH DOT ABOVE
-        0x0099: 0x00f4, # LATIN SMALL LETTER O WITH CIRCUMFLEX
-        0x009a: 0x00f6, # LATIN SMALL LETTER O WITH DIAERESIS
-        0x009b: 0x00f5, # LATIN SMALL LETTER O WITH TILDE
-        0x009c: 0x00fa, # LATIN SMALL LETTER U WITH ACUTE
-        0x009d: 0x011a, # LATIN CAPITAL LETTER E WITH CARON
-        0x009e: 0x011b, # LATIN SMALL LETTER E WITH CARON
-        0x009f: 0x00fc, # LATIN SMALL LETTER U WITH DIAERESIS
-        0x00a0: 0x2020, # DAGGER
-        0x00a1: 0x00b0, # DEGREE SIGN
-        0x00a2: 0x0118, # LATIN CAPITAL LETTER E WITH OGONEK
-        0x00a4: 0x00a7, # SECTION SIGN
-        0x00a5: 0x2022, # BULLET
-        0x00a6: 0x00b6, # PILCROW SIGN
-        0x00a7: 0x00df, # LATIN SMALL LETTER SHARP S
-        0x00a8: 0x00ae, # REGISTERED SIGN
-        0x00aa: 0x2122, # TRADE MARK SIGN
-        0x00ab: 0x0119, # LATIN SMALL LETTER E WITH OGONEK
-        0x00ac: 0x00a8, # DIAERESIS
-        0x00ad: 0x2260, # NOT EQUAL TO
-        0x00ae: 0x0123, # LATIN SMALL LETTER G WITH CEDILLA
-        0x00af: 0x012e, # LATIN CAPITAL LETTER I WITH OGONEK
-        0x00b0: 0x012f, # LATIN SMALL LETTER I WITH OGONEK
-        0x00b1: 0x012a, # LATIN CAPITAL LETTER I WITH MACRON
-        0x00b2: 0x2264, # LESS-THAN OR EQUAL TO
-        0x00b3: 0x2265, # GREATER-THAN OR EQUAL TO
-        0x00b4: 0x012b, # LATIN SMALL LETTER I WITH MACRON
-        0x00b5: 0x0136, # LATIN CAPITAL LETTER K WITH CEDILLA
-        0x00b6: 0x2202, # PARTIAL DIFFERENTIAL
-        0x00b7: 0x2211, # N-ARY SUMMATION
-        0x00b8: 0x0142, # LATIN SMALL LETTER L WITH STROKE
-        0x00b9: 0x013b, # LATIN CAPITAL LETTER L WITH CEDILLA
-        0x00ba: 0x013c, # LATIN SMALL LETTER L WITH CEDILLA
-        0x00bb: 0x013d, # LATIN CAPITAL LETTER L WITH CARON
-        0x00bc: 0x013e, # LATIN SMALL LETTER L WITH CARON
-        0x00bd: 0x0139, # LATIN CAPITAL LETTER L WITH ACUTE
-        0x00be: 0x013a, # LATIN SMALL LETTER L WITH ACUTE
-        0x00bf: 0x0145, # LATIN CAPITAL LETTER N WITH CEDILLA
-        0x00c0: 0x0146, # LATIN SMALL LETTER N WITH CEDILLA
-        0x00c1: 0x0143, # LATIN CAPITAL LETTER N WITH ACUTE
-        0x00c2: 0x00ac, # NOT SIGN
-        0x00c3: 0x221a, # SQUARE ROOT
-        0x00c4: 0x0144, # LATIN SMALL LETTER N WITH ACUTE
-        0x00c5: 0x0147, # LATIN CAPITAL LETTER N WITH CARON
-        0x00c6: 0x2206, # INCREMENT
-        0x00c7: 0x00ab, # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
-        0x00c8: 0x00bb, # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
-        0x00c9: 0x2026, # HORIZONTAL ELLIPSIS
-        0x00ca: 0x00a0, # NO-BREAK SPACE
-        0x00cb: 0x0148, # LATIN SMALL LETTER N WITH CARON
-        0x00cc: 0x0150, # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
-        0x00cd: 0x00d5, # LATIN CAPITAL LETTER O WITH TILDE
-        0x00ce: 0x0151, # LATIN SMALL LETTER O WITH DOUBLE ACUTE
-        0x00cf: 0x014c, # LATIN CAPITAL LETTER O WITH MACRON
-        0x00d0: 0x2013, # EN DASH
-        0x00d1: 0x2014, # EM DASH
-        0x00d2: 0x201c, # LEFT DOUBLE QUOTATION MARK
-        0x00d3: 0x201d, # RIGHT DOUBLE QUOTATION MARK
-        0x00d4: 0x2018, # LEFT SINGLE QUOTATION MARK
-        0x00d5: 0x2019, # RIGHT SINGLE QUOTATION MARK
-        0x00d6: 0x00f7, # DIVISION SIGN
-        0x00d7: 0x25ca, # LOZENGE
-        0x00d8: 0x014d, # LATIN SMALL LETTER O WITH MACRON
-        0x00d9: 0x0154, # LATIN CAPITAL LETTER R WITH ACUTE
-        0x00da: 0x0155, # LATIN SMALL LETTER R WITH ACUTE
-        0x00db: 0x0158, # LATIN CAPITAL LETTER R WITH CARON
-        0x00dc: 0x2039, # SINGLE LEFT-POINTING ANGLE QUOTATION MARK
-        0x00dd: 0x203a, # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
-        0x00de: 0x0159, # LATIN SMALL LETTER R WITH CARON
-        0x00df: 0x0156, # LATIN CAPITAL LETTER R WITH CEDILLA
-        0x00e0: 0x0157, # LATIN SMALL LETTER R WITH CEDILLA
-        0x00e1: 0x0160, # LATIN CAPITAL LETTER S WITH CARON
-        0x00e2: 0x201a, # SINGLE LOW-9 QUOTATION MARK
-        0x00e3: 0x201e, # DOUBLE LOW-9 QUOTATION MARK
-        0x00e4: 0x0161, # LATIN SMALL LETTER S WITH CARON
-        0x00e5: 0x015a, # LATIN CAPITAL LETTER S WITH ACUTE
-        0x00e6: 0x015b, # LATIN SMALL LETTER S WITH ACUTE
-        0x00e7: 0x00c1, # LATIN CAPITAL LETTER A WITH ACUTE
-        0x00e8: 0x0164, # LATIN CAPITAL LETTER T WITH CARON
-        0x00e9: 0x0165, # LATIN SMALL LETTER T WITH CARON
-        0x00ea: 0x00cd, # LATIN CAPITAL LETTER I WITH ACUTE
-        0x00eb: 0x017d, # LATIN CAPITAL LETTER Z WITH CARON
-        0x00ec: 0x017e, # LATIN SMALL LETTER Z WITH CARON
-        0x00ed: 0x016a, # LATIN CAPITAL LETTER U WITH MACRON
-        0x00ee: 0x00d3, # LATIN CAPITAL LETTER O WITH ACUTE
-        0x00ef: 0x00d4, # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
-        0x00f0: 0x016b, # LATIN SMALL LETTER U WITH MACRON
-        0x00f1: 0x016e, # LATIN CAPITAL LETTER U WITH RING ABOVE
-        0x00f2: 0x00da, # LATIN CAPITAL LETTER U WITH ACUTE
-        0x00f3: 0x016f, # LATIN SMALL LETTER U WITH RING ABOVE
-        0x00f4: 0x0170, # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
-        0x00f5: 0x0171, # LATIN SMALL LETTER U WITH DOUBLE ACUTE
-        0x00f6: 0x0172, # LATIN CAPITAL LETTER U WITH OGONEK
-        0x00f7: 0x0173, # LATIN SMALL LETTER U WITH OGONEK
-        0x00f8: 0x00dd, # LATIN CAPITAL LETTER Y WITH ACUTE
-        0x00f9: 0x00fd, # LATIN SMALL LETTER Y WITH ACUTE
-        0x00fa: 0x0137, # LATIN SMALL LETTER K WITH CEDILLA
-        0x00fb: 0x017b, # LATIN CAPITAL LETTER Z WITH DOT ABOVE
-        0x00fc: 0x0141, # LATIN CAPITAL LETTER L WITH STROKE
-        0x00fd: 0x017c, # LATIN SMALL LETTER Z WITH DOT ABOVE
-        0x00fe: 0x0122, # LATIN CAPITAL LETTER G WITH CEDILLA
-        0x00ff: 0x02c7, # CARON
-})
-
-### Encoding Map
-
-encoding_map = codecs.make_encoding_map(decoding_map)
+
+### Decoding Table
+
+decoding_table = (
+    '\x00'     #  0x00 -> NULL
+    '\x01'     #  0x01 -> START OF HEADING
+    '\x02'     #  0x02 -> START OF TEXT
+    '\x03'     #  0x03 -> END OF TEXT
+    '\x04'     #  0x04 -> END OF TRANSMISSION
+    '\x05'     #  0x05 -> ENQUIRY
+    '\x06'     #  0x06 -> ACKNOWLEDGE
+    '\x07'     #  0x07 -> BELL
+    '\x08'     #  0x08 -> BACKSPACE
+    '\t'       #  0x09 -> HORIZONTAL TABULATION
+    '\n'       #  0x0A -> LINE FEED
+    '\x0b'     #  0x0B -> VERTICAL TABULATION
+    '\x0c'     #  0x0C -> FORM FEED
+    '\r'       #  0x0D -> CARRIAGE RETURN
+    '\x0e'     #  0x0E -> SHIFT OUT
+    '\x0f'     #  0x0F -> SHIFT IN
+    '\x10'     #  0x10 -> DATA LINK ESCAPE
+    '\x11'     #  0x11 -> DEVICE CONTROL ONE
+    '\x12'     #  0x12 -> DEVICE CONTROL TWO
+    '\x13'     #  0x13 -> DEVICE CONTROL THREE
+    '\x14'     #  0x14 -> DEVICE CONTROL FOUR
+    '\x15'     #  0x15 -> NEGATIVE ACKNOWLEDGE
+    '\x16'     #  0x16 -> SYNCHRONOUS IDLE
+    '\x17'     #  0x17 -> END OF TRANSMISSION BLOCK
+    '\x18'     #  0x18 -> CANCEL
+    '\x19'     #  0x19 -> END OF MEDIUM
+    '\x1a'     #  0x1A -> SUBSTITUTE
+    '\x1b'     #  0x1B -> ESCAPE
+    '\x1c'     #  0x1C -> FILE SEPARATOR
+    '\x1d'     #  0x1D -> GROUP SEPARATOR
+    '\x1e'     #  0x1E -> RECORD SEPARATOR
+    '\x1f'     #  0x1F -> UNIT SEPARATOR
+    ' '        #  0x20 -> SPACE
+    '!'        #  0x21 -> EXCLAMATION MARK
+    '"'        #  0x22 -> QUOTATION MARK
+    '#'        #  0x23 -> NUMBER SIGN
+    '$'        #  0x24 -> DOLLAR SIGN
+    '%'        #  0x25 -> PERCENT SIGN
+    '&'        #  0x26 -> AMPERSAND
+    "'"        #  0x27 -> APOSTROPHE
+    '('        #  0x28 -> LEFT PARENTHESIS
+    ')'        #  0x29 -> RIGHT PARENTHESIS
+    '*'        #  0x2A -> ASTERISK
+    '+'        #  0x2B -> PLUS SIGN
+    ','        #  0x2C -> COMMA
+    '-'        #  0x2D -> HYPHEN-MINUS
+    '.'        #  0x2E -> FULL STOP
+    '/'        #  0x2F -> SOLIDUS
+    '0'        #  0x30 -> DIGIT ZERO
+    '1'        #  0x31 -> DIGIT ONE
+    '2'        #  0x32 -> DIGIT TWO
+    '3'        #  0x33 -> DIGIT THREE
+    '4'        #  0x34 -> DIGIT FOUR
+    '5'        #  0x35 -> DIGIT FIVE
+    '6'        #  0x36 -> DIGIT SIX
+    '7'        #  0x37 -> DIGIT SEVEN
+    '8'        #  0x38 -> DIGIT EIGHT
+    '9'        #  0x39 -> DIGIT NINE
+    ':'        #  0x3A -> COLON
+    ';'        #  0x3B -> SEMICOLON
+    '<'        #  0x3C -> LESS-THAN SIGN
+    '='        #  0x3D -> EQUALS SIGN
+    '>'        #  0x3E -> GREATER-THAN SIGN
+    '?'        #  0x3F -> QUESTION MARK
+    '@'        #  0x40 -> COMMERCIAL AT
+    'A'        #  0x41 -> LATIN CAPITAL LETTER A
+    'B'        #  0x42 -> LATIN CAPITAL LETTER B
+    'C'        #  0x43 -> LATIN CAPITAL LETTER C
+    'D'        #  0x44 -> LATIN CAPITAL LETTER D
+    'E'        #  0x45 -> LATIN CAPITAL LETTER E
+    'F'        #  0x46 -> LATIN CAPITAL LETTER F
+    'G'        #  0x47 -> LATIN CAPITAL LETTER G
+    'H'        #  0x48 -> LATIN CAPITAL LETTER H
+    'I'        #  0x49 -> LATIN CAPITAL LETTER I
+    'J'        #  0x4A -> LATIN CAPITAL LETTER J
+    'K'        #  0x4B -> LATIN CAPITAL LETTER K
+    'L'        #  0x4C -> LATIN CAPITAL LETTER L
+    'M'        #  0x4D -> LATIN CAPITAL LETTER M
+    'N'        #  0x4E -> LATIN CAPITAL LETTER N
+    'O'        #  0x4F -> LATIN CAPITAL LETTER O
+    'P'        #  0x50 -> LATIN CAPITAL LETTER P
+    'Q'        #  0x51 -> LATIN CAPITAL LETTER Q
+    'R'        #  0x52 -> LATIN CAPITAL LETTER R
+    'S'        #  0x53 -> LATIN CAPITAL LETTER S
+    'T'        #  0x54 -> LATIN CAPITAL LETTER T
+    'U'        #  0x55 -> LATIN CAPITAL LETTER U
+    'V'        #  0x56 -> LATIN CAPITAL LETTER V
+    'W'        #  0x57 -> LATIN CAPITAL LETTER W
+    'X'        #  0x58 -> LATIN CAPITAL LETTER X
+    'Y'        #  0x59 -> LATIN CAPITAL LETTER Y
+    'Z'        #  0x5A -> LATIN CAPITAL LETTER Z
+    '['        #  0x5B -> LEFT SQUARE BRACKET
+    '\\'       #  0x5C -> REVERSE SOLIDUS
+    ']'        #  0x5D -> RIGHT SQUARE BRACKET
+    '^'        #  0x5E -> CIRCUMFLEX ACCENT
+    '_'        #  0x5F -> LOW LINE
+    '`'        #  0x60 -> GRAVE ACCENT
+    'a'        #  0x61 -> LATIN SMALL LETTER A
+    'b'        #  0x62 -> LATIN SMALL LETTER B
+    'c'        #  0x63 -> LATIN SMALL LETTER C
+    'd'        #  0x64 -> LATIN SMALL LETTER D
+    'e'        #  0x65 -> LATIN SMALL LETTER E
+    'f'        #  0x66 -> LATIN SMALL LETTER F
+    'g'        #  0x67 -> LATIN SMALL LETTER G
+    'h'        #  0x68 -> LATIN SMALL LETTER H
+    'i'        #  0x69 -> LATIN SMALL LETTER I
+    'j'        #  0x6A -> LATIN SMALL LETTER J
+    'k'        #  0x6B -> LATIN SMALL LETTER K
+    'l'        #  0x6C -> LATIN SMALL LETTER L
+    'm'        #  0x6D -> LATIN SMALL LETTER M
+    'n'        #  0x6E -> LATIN SMALL LETTER N
+    'o'        #  0x6F -> LATIN SMALL LETTER O
+    'p'        #  0x70 -> LATIN SMALL LETTER P
+    'q'        #  0x71 -> LATIN SMALL LETTER Q
+    'r'        #  0x72 -> LATIN SMALL LETTER R
+    's'        #  0x73 -> LATIN SMALL LETTER S
+    't'        #  0x74 -> LATIN SMALL LETTER T
+    'u'        #  0x75 -> LATIN SMALL LETTER U
+    'v'        #  0x76 -> LATIN SMALL LETTER V
+    'w'        #  0x77 -> LATIN SMALL LETTER W
+    'x'        #  0x78 -> LATIN SMALL LETTER X
+    'y'        #  0x79 -> LATIN SMALL LETTER Y
+    'z'        #  0x7A -> LATIN SMALL LETTER Z
+    '{'        #  0x7B -> LEFT CURLY BRACKET
+    '|'        #  0x7C -> VERTICAL LINE
+    '}'        #  0x7D -> RIGHT CURLY BRACKET
+    '~'        #  0x7E -> TILDE
+    '\x7f'     #  0x7F -> DELETE
+    '\xc4'     #  0x80 -> LATIN CAPITAL LETTER A WITH DIAERESIS
+    '\u0100'   #  0x81 -> LATIN CAPITAL LETTER A WITH MACRON
+    '\u0101'   #  0x82 -> LATIN SMALL LETTER A WITH MACRON
+    '\xc9'     #  0x83 -> LATIN CAPITAL LETTER E WITH ACUTE
+    '\u0104'   #  0x84 -> LATIN CAPITAL LETTER A WITH OGONEK
+    '\xd6'     #  0x85 -> LATIN CAPITAL LETTER O WITH DIAERESIS
+    '\xdc'     #  0x86 -> LATIN CAPITAL LETTER U WITH DIAERESIS
+    '\xe1'     #  0x87 -> LATIN SMALL LETTER A WITH ACUTE
+    '\u0105'   #  0x88 -> LATIN SMALL LETTER A WITH OGONEK
+    '\u010c'   #  0x89 -> LATIN CAPITAL LETTER C WITH CARON
+    '\xe4'     #  0x8A -> LATIN SMALL LETTER A WITH DIAERESIS
+    '\u010d'   #  0x8B -> LATIN SMALL LETTER C WITH CARON
+    '\u0106'   #  0x8C -> LATIN CAPITAL LETTER C WITH ACUTE
+    '\u0107'   #  0x8D -> LATIN SMALL LETTER C WITH ACUTE
+    '\xe9'     #  0x8E -> LATIN SMALL LETTER E WITH ACUTE
+    '\u0179'   #  0x8F -> LATIN CAPITAL LETTER Z WITH ACUTE
+    '\u017a'   #  0x90 -> LATIN SMALL LETTER Z WITH ACUTE
+    '\u010e'   #  0x91 -> LATIN CAPITAL LETTER D WITH CARON
+    '\xed'     #  0x92 -> LATIN SMALL LETTER I WITH ACUTE
+    '\u010f'   #  0x93 -> LATIN SMALL LETTER D WITH CARON
+    '\u0112'   #  0x94 -> LATIN CAPITAL LETTER E WITH MACRON
+    '\u0113'   #  0x95 -> LATIN SMALL LETTER E WITH MACRON
+    '\u0116'   #  0x96 -> LATIN CAPITAL LETTER E WITH DOT ABOVE
+    '\xf3'     #  0x97 -> LATIN SMALL LETTER O WITH ACUTE
+    '\u0117'   #  0x98 -> LATIN SMALL LETTER E WITH DOT ABOVE
+    '\xf4'     #  0x99 -> LATIN SMALL LETTER O WITH CIRCUMFLEX
+    '\xf6'     #  0x9A -> LATIN SMALL LETTER O WITH DIAERESIS
+    '\xf5'     #  0x9B -> LATIN SMALL LETTER O WITH TILDE
+    '\xfa'     #  0x9C -> LATIN SMALL LETTER U WITH ACUTE
+    '\u011a'   #  0x9D -> LATIN CAPITAL LETTER E WITH CARON
+    '\u011b'   #  0x9E -> LATIN SMALL LETTER E WITH CARON
+    '\xfc'     #  0x9F -> LATIN SMALL LETTER U WITH DIAERESIS
+    '\u2020'   #  0xA0 -> DAGGER
+    '\xb0'     #  0xA1 -> DEGREE SIGN
+    '\u0118'   #  0xA2 -> LATIN CAPITAL LETTER E WITH OGONEK
+    '\xa3'     #  0xA3 -> POUND SIGN
+    '\xa7'     #  0xA4 -> SECTION SIGN
+    '\u2022'   #  0xA5 -> BULLET
+    '\xb6'     #  0xA6 -> PILCROW SIGN
+    '\xdf'     #  0xA7 -> LATIN SMALL LETTER SHARP S
+    '\xae'     #  0xA8 -> REGISTERED SIGN
+    '\xa9'     #  0xA9 -> COPYRIGHT SIGN
+    '\u2122'   #  0xAA -> TRADE MARK SIGN
+    '\u0119'   #  0xAB -> LATIN SMALL LETTER E WITH OGONEK
+    '\xa8'     #  0xAC -> DIAERESIS
+    '\u2260'   #  0xAD -> NOT EQUAL TO
+    '\u0123'   #  0xAE -> LATIN SMALL LETTER G WITH CEDILLA
+    '\u012e'   #  0xAF -> LATIN CAPITAL LETTER I WITH OGONEK
+    '\u012f'   #  0xB0 -> LATIN SMALL LETTER I WITH OGONEK
+    '\u012a'   #  0xB1 -> LATIN CAPITAL LETTER I WITH MACRON
+    '\u2264'   #  0xB2 -> LESS-THAN OR EQUAL TO
+    '\u2265'   #  0xB3 -> GREATER-THAN OR EQUAL TO
+    '\u012b'   #  0xB4 -> LATIN SMALL LETTER I WITH MACRON
+    '\u0136'   #  0xB5 -> LATIN CAPITAL LETTER K WITH CEDILLA
+    '\u2202'   #  0xB6 -> PARTIAL DIFFERENTIAL
+    '\u2211'   #  0xB7 -> N-ARY SUMMATION
+    '\u0142'   #  0xB8 -> LATIN SMALL LETTER L WITH STROKE
+    '\u013b'   #  0xB9 -> LATIN CAPITAL LETTER L WITH CEDILLA
+    '\u013c'   #  0xBA -> LATIN SMALL LETTER L WITH CEDILLA
+    '\u013d'   #  0xBB -> LATIN CAPITAL LETTER L WITH CARON
+    '\u013e'   #  0xBC -> LATIN SMALL LETTER L WITH CARON
+    '\u0139'   #  0xBD -> LATIN CAPITAL LETTER L WITH ACUTE
+    '\u013a'   #  0xBE -> LATIN SMALL LETTER L WITH ACUTE
+    '\u0145'   #  0xBF -> LATIN CAPITAL LETTER N WITH CEDILLA
+    '\u0146'   #  0xC0 -> LATIN SMALL LETTER N WITH CEDILLA
+    '\u0143'   #  0xC1 -> LATIN CAPITAL LETTER N WITH ACUTE
+    '\xac'     #  0xC2 -> NOT SIGN
+    '\u221a'   #  0xC3 -> SQUARE ROOT
+    '\u0144'   #  0xC4 -> LATIN SMALL LETTER N WITH ACUTE
+    '\u0147'   #  0xC5 -> LATIN CAPITAL LETTER N WITH CARON
+    '\u2206'   #  0xC6 -> INCREMENT
+    '\xab'     #  0xC7 -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\xbb'     #  0xC8 -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\u2026'   #  0xC9 -> HORIZONTAL ELLIPSIS
+    '\xa0'     #  0xCA -> NO-BREAK SPACE
+    '\u0148'   #  0xCB -> LATIN SMALL LETTER N WITH CARON
+    '\u0150'   #  0xCC -> LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+    '\xd5'     #  0xCD -> LATIN CAPITAL LETTER O WITH TILDE
+    '\u0151'   #  0xCE -> LATIN SMALL LETTER O WITH DOUBLE ACUTE
+    '\u014c'   #  0xCF -> LATIN CAPITAL LETTER O WITH MACRON
+    '\u2013'   #  0xD0 -> EN DASH
+    '\u2014'   #  0xD1 -> EM DASH
+    '\u201c'   #  0xD2 -> LEFT DOUBLE QUOTATION MARK
+    '\u201d'   #  0xD3 -> RIGHT DOUBLE QUOTATION MARK
+    '\u2018'   #  0xD4 -> LEFT SINGLE QUOTATION MARK
+    '\u2019'   #  0xD5 -> RIGHT SINGLE QUOTATION MARK
+    '\xf7'     #  0xD6 -> DIVISION SIGN
+    '\u25ca'   #  0xD7 -> LOZENGE
+    '\u014d'   #  0xD8 -> LATIN SMALL LETTER O WITH MACRON
+    '\u0154'   #  0xD9 -> LATIN CAPITAL LETTER R WITH ACUTE
+    '\u0155'   #  0xDA -> LATIN SMALL LETTER R WITH ACUTE
+    '\u0158'   #  0xDB -> LATIN CAPITAL LETTER R WITH CARON
+    '\u2039'   #  0xDC -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+    '\u203a'   #  0xDD -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+    '\u0159'   #  0xDE -> LATIN SMALL LETTER R WITH CARON
+    '\u0156'   #  0xDF -> LATIN CAPITAL LETTER R WITH CEDILLA
+    '\u0157'   #  0xE0 -> LATIN SMALL LETTER R WITH CEDILLA
+    '\u0160'   #  0xE1 -> LATIN CAPITAL LETTER S WITH CARON
+    '\u201a'   #  0xE2 -> SINGLE LOW-9 QUOTATION MARK
+    '\u201e'   #  0xE3 -> DOUBLE LOW-9 QUOTATION MARK
+    '\u0161'   #  0xE4 -> LATIN SMALL LETTER S WITH CARON
+    '\u015a'   #  0xE5 -> LATIN CAPITAL LETTER S WITH ACUTE
+    '\u015b'   #  0xE6 -> LATIN SMALL LETTER S WITH ACUTE
+    '\xc1'     #  0xE7 -> LATIN CAPITAL LETTER A WITH ACUTE
+    '\u0164'   #  0xE8 -> LATIN CAPITAL LETTER T WITH CARON
+    '\u0165'   #  0xE9 -> LATIN SMALL LETTER T WITH CARON
+    '\xcd'     #  0xEA -> LATIN CAPITAL LETTER I WITH ACUTE
+    '\u017d'   #  0xEB -> LATIN CAPITAL LETTER Z WITH CARON
+    '\u017e'   #  0xEC -> LATIN SMALL LETTER Z WITH CARON
+    '\u016a'   #  0xED -> LATIN CAPITAL LETTER U WITH MACRON
+    '\xd3'     #  0xEE -> LATIN CAPITAL LETTER O WITH ACUTE
+    '\xd4'     #  0xEF -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+    '\u016b'   #  0xF0 -> LATIN SMALL LETTER U WITH MACRON
+    '\u016e'   #  0xF1 -> LATIN CAPITAL LETTER U WITH RING ABOVE
+    '\xda'     #  0xF2 -> LATIN CAPITAL LETTER U WITH ACUTE
+    '\u016f'   #  0xF3 -> LATIN SMALL LETTER U WITH RING ABOVE
+    '\u0170'   #  0xF4 -> LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+    '\u0171'   #  0xF5 -> LATIN SMALL LETTER U WITH DOUBLE ACUTE
+    '\u0172'   #  0xF6 -> LATIN CAPITAL LETTER U WITH OGONEK
+    '\u0173'   #  0xF7 -> LATIN SMALL LETTER U WITH OGONEK
+    '\xdd'     #  0xF8 -> LATIN CAPITAL LETTER Y WITH ACUTE
+    '\xfd'     #  0xF9 -> LATIN SMALL LETTER Y WITH ACUTE
+    '\u0137'   #  0xFA -> LATIN SMALL LETTER K WITH CEDILLA
+    '\u017b'   #  0xFB -> LATIN CAPITAL LETTER Z WITH DOT ABOVE
+    '\u0141'   #  0xFC -> LATIN CAPITAL LETTER L WITH STROKE
+    '\u017c'   #  0xFD -> LATIN SMALL LETTER Z WITH DOT ABOVE
+    '\u0122'   #  0xFE -> LATIN CAPITAL LETTER G WITH CEDILLA
+    '\u02c7'   #  0xFF -> CARON
+)
+
+### Encoding table
+encoding_table=codecs.charmap_build(decoding_table)
diff --git a/Lib/encodings/palmos.py b/Lib/encodings/palmos.py
index 4b77e2ba91..c506d65452 100644
--- a/Lib/encodings/palmos.py
+++ b/Lib/encodings/palmos.py
@@ -10,18 +10,18 @@ import codecs
 
 class Codec(codecs.Codec):
     def encode(self,input,errors='strict'):
-        return codecs.charmap_encode(input,errors,encoding_map)
+        return codecs.charmap_encode(input,errors,encoding_table)
 
     def decode(self,input,errors='strict'):
-        return codecs.charmap_decode(input,errors,decoding_map)
+        return codecs.charmap_decode(input,errors,decoding_table)
 
 class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
-        return codecs.charmap_encode(input,self.errors,encoding_map)[0]
+        return codecs.charmap_encode(input,self.errors,encoding_table)[0]
 
 class IncrementalDecoder(codecs.IncrementalDecoder):
     def decode(self, input, final=False):
-        return codecs.charmap_decode(input,self.errors,decoding_map)[0]
+        return codecs.charmap_decode(input,self.errors,decoding_table)[0]
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
@@ -42,42 +42,267 @@ def getregentry():
         streamwriter=StreamWriter,
     )
 
-### Decoding Map
-
-decoding_map = codecs.make_identity_dict(range(256))
-
-# The PalmOS character set is mostly iso-8859-1 with some differences.
-decoding_map.update({
-        0x0080: 0x20ac, #       EURO SIGN
-        0x0082: 0x201a, #       SINGLE LOW-9 QUOTATION MARK
-        0x0083: 0x0192, #       LATIN SMALL LETTER F WITH HOOK
-        0x0084: 0x201e, #       DOUBLE LOW-9 QUOTATION MARK
-        0x0085: 0x2026, #       HORIZONTAL ELLIPSIS
-        0x0086: 0x2020, #       DAGGER
-        0x0087: 0x2021, #       DOUBLE DAGGER
-        0x0088: 0x02c6, #       MODIFIER LETTER CIRCUMFLEX ACCENT
-        0x0089: 0x2030, #       PER MILLE SIGN
-        0x008a: 0x0160, #       LATIN CAPITAL LETTER S WITH CARON
-        0x008b: 0x2039, #       SINGLE LEFT-POINTING ANGLE QUOTATION MARK
-        0x008c: 0x0152, #       LATIN CAPITAL LIGATURE OE
-        0x008d: 0x2666, #       BLACK DIAMOND SUIT
-        0x008e: 0x2663, #       BLACK CLUB SUIT
-        0x008f: 0x2665, #       BLACK HEART SUIT
-        0x0090: 0x2660, #       BLACK SPADE SUIT
-        0x0091: 0x2018, #       LEFT SINGLE QUOTATION MARK
-        0x0092: 0x2019, #       RIGHT SINGLE QUOTATION MARK
-        0x0093: 0x201c, #       LEFT DOUBLE QUOTATION MARK
-        0x0094: 0x201d, #       RIGHT DOUBLE QUOTATION MARK
-        0x0095: 0x2022, #       BULLET
-        0x0096: 0x2013, #       EN DASH
-        0x0097: 0x2014, #       EM DASH
-        0x0098: 0x02dc, #       SMALL TILDE
-        0x0099: 0x2122, #       TRADE MARK SIGN
-        0x009a: 0x0161, #       LATIN SMALL LETTER S WITH CARON
-        0x009c: 0x0153, #       LATIN SMALL LIGATURE OE
-        0x009f: 0x0178, #       LATIN CAPITAL LETTER Y WITH DIAERESIS
-})
-
-### Encoding Map
-
-encoding_map = codecs.make_encoding_map(decoding_map)
+
+### Decoding Table
+
+decoding_table = (
+    '\x00'     #  0x00 -> NULL
+    '\x01'     #  0x01 -> START OF HEADING
+    '\x02'     #  0x02 -> START OF TEXT
+    '\x03'     #  0x03 -> END OF TEXT
+    '\x04'     #  0x04 -> END OF TRANSMISSION
+    '\x05'     #  0x05 -> ENQUIRY
+    '\x06'     #  0x06 -> ACKNOWLEDGE
+    '\x07'     #  0x07 -> BELL
+    '\x08'     #  0x08 -> BACKSPACE
+    '\t'       #  0x09 -> HORIZONTAL TABULATION
+    '\n'       #  0x0A -> LINE FEED
+    '\x0b'     #  0x0B -> VERTICAL TABULATION
+    '\x0c'     #  0x0C -> FORM FEED
+    '\r'       #  0x0D -> CARRIAGE RETURN
+    '\x0e'     #  0x0E -> SHIFT OUT
+    '\x0f'     #  0x0F -> SHIFT IN
+    '\x10'     #  0x10 -> DATA LINK ESCAPE
+    '\x11'     #  0x11 -> DEVICE CONTROL ONE
+    '\x12'     #  0x12 -> DEVICE CONTROL TWO
+    '\x13'     #  0x13 -> DEVICE CONTROL THREE
+    '\x14'     #  0x14 -> DEVICE CONTROL FOUR
+    '\x15'     #  0x15 -> NEGATIVE ACKNOWLEDGE
+    '\x16'     #  0x16 -> SYNCHRONOUS IDLE
+    '\x17'     #  0x17 -> END OF TRANSMISSION BLOCK
+    '\x18'     #  0x18 -> CANCEL
+    '\x19'     #  0x19 -> END OF MEDIUM
+    '\x1a'     #  0x1A -> SUBSTITUTE
+    '\x1b'     #  0x1B -> ESCAPE
+    '\x1c'     #  0x1C -> FILE SEPARATOR
+    '\x1d'     #  0x1D -> GROUP SEPARATOR
+    '\x1e'     #  0x1E -> RECORD SEPARATOR
+    '\x1f'     #  0x1F -> UNIT SEPARATOR
+    ' '        #  0x20 -> SPACE
+    '!'        #  0x21 -> EXCLAMATION MARK
+    '"'        #  0x22 -> QUOTATION MARK
+    '#'        #  0x23 -> NUMBER SIGN
+    '$'        #  0x24 -> DOLLAR SIGN
+    '%'        #  0x25 -> PERCENT SIGN
+    '&'        #  0x26 -> AMPERSAND
+    "'"        #  0x27 -> APOSTROPHE
+    '('        #  0x28 -> LEFT PARENTHESIS
+    ')'        #  0x29 -> RIGHT PARENTHESIS
+    '*'        #  0x2A -> ASTERISK
+    '+'        #  0x2B -> PLUS SIGN
+    ','        #  0x2C -> COMMA
+    '-'        #  0x2D -> HYPHEN-MINUS
+    '.'        #  0x2E -> FULL STOP
+    '/'        #  0x2F -> SOLIDUS
+    '0'        #  0x30 -> DIGIT ZERO
+    '1'        #  0x31 -> DIGIT ONE
+    '2'        #  0x32 -> DIGIT TWO
+    '3'        #  0x33 -> DIGIT THREE
+    '4'        #  0x34 -> DIGIT FOUR
+    '5'        #  0x35 -> DIGIT FIVE
+    '6'        #  0x36 -> DIGIT SIX
+    '7'        #  0x37 -> DIGIT SEVEN
+    '8'        #  0x38 -> DIGIT EIGHT
+    '9'        #  0x39 -> DIGIT NINE
+    ':'        #  0x3A -> COLON
+    ';'        #  0x3B -> SEMICOLON
+    '<'        #  0x3C -> LESS-THAN SIGN
+    '='        #  0x3D -> EQUALS SIGN
+    '>'        #  0x3E -> GREATER-THAN SIGN
+    '?'        #  0x3F -> QUESTION MARK
+    '@'        #  0x40 -> COMMERCIAL AT
+    'A'        #  0x41 -> LATIN CAPITAL LETTER A
+    'B'        #  0x42 -> LATIN CAPITAL LETTER B
+    'C'        #  0x43 -> LATIN CAPITAL LETTER C
+    'D'        #  0x44 -> LATIN CAPITAL LETTER D
+    'E'        #  0x45 -> LATIN CAPITAL LETTER E
+    'F'        #  0x46 -> LATIN CAPITAL LETTER F
+    'G'        #  0x47 -> LATIN CAPITAL LETTER G
+    'H'        #  0x48 -> LATIN CAPITAL LETTER H
+    'I'        #  0x49 -> LATIN CAPITAL LETTER I
+    'J'        #  0x4A -> LATIN CAPITAL LETTER J
+    'K'        #  0x4B -> LATIN CAPITAL LETTER K
+    'L'        #  0x4C -> LATIN CAPITAL LETTER L
+    'M'        #  0x4D -> LATIN CAPITAL LETTER M
+    'N'        #  0x4E -> LATIN CAPITAL LETTER N
+    'O'        #  0x4F -> LATIN CAPITAL LETTER O
+    'P'        #  0x50 -> LATIN CAPITAL LETTER P
+    'Q'        #  0x51 -> LATIN CAPITAL LETTER Q
+    'R'        #  0x52 -> LATIN CAPITAL LETTER R
+    'S'        #  0x53 -> LATIN CAPITAL LETTER S
+    'T'        #  0x54 -> LATIN CAPITAL LETTER T
+    'U'        #  0x55 -> LATIN CAPITAL LETTER U
+    'V'        #  0x56 -> LATIN CAPITAL LETTER V
+    'W'        #  0x57 -> LATIN CAPITAL LETTER W
+    'X'        #  0x58 -> LATIN CAPITAL LETTER X
+    'Y'        #  0x59 -> LATIN CAPITAL LETTER Y
+    'Z'        #  0x5A -> LATIN CAPITAL LETTER Z
+    '['        #  0x5B -> LEFT SQUARE BRACKET
+    '\\'       #  0x5C -> REVERSE SOLIDUS
+    ']'        #  0x5D -> RIGHT SQUARE BRACKET
+    '^'        #  0x5E -> CIRCUMFLEX ACCENT
+    '_'        #  0x5F -> LOW LINE
+    '`'        #  0x60 -> GRAVE ACCENT
+    'a'        #  0x61 -> LATIN SMALL LETTER A
+    'b'        #  0x62 -> LATIN SMALL LETTER B
+    'c'        #  0x63 -> LATIN SMALL LETTER C
+    'd'        #  0x64 -> LATIN SMALL LETTER D
+    'e'        #  0x65 -> LATIN SMALL LETTER E
+    'f'        #  0x66 -> LATIN SMALL LETTER F
+    'g'        #  0x67 -> LATIN SMALL LETTER G
+    'h'        #  0x68 -> LATIN SMALL LETTER H
+    'i'        #  0x69 -> LATIN SMALL LETTER I
+    'j'        #  0x6A -> LATIN SMALL LETTER J
+    'k'        #  0x6B -> LATIN SMALL LETTER K
+    'l'        #  0x6C -> LATIN SMALL LETTER L
+    'm'        #  0x6D -> LATIN SMALL LETTER M
+    'n'        #  0x6E -> LATIN SMALL LETTER N
+    'o'        #  0x6F -> LATIN SMALL LETTER O
+    'p'        #  0x70 -> LATIN SMALL LETTER P
+    'q'        #  0x71 -> LATIN SMALL LETTER Q
+    'r'        #  0x72 -> LATIN SMALL LETTER R
+    's'        #  0x73 -> LATIN SMALL LETTER S
+    't'        #  0x74 -> LATIN SMALL LETTER T
+    'u'        #  0x75 -> LATIN SMALL LETTER U
+    'v'        #  0x76 -> LATIN SMALL LETTER V
+    'w'        #  0x77 -> LATIN SMALL LETTER W
+    'x'        #  0x78 -> LATIN SMALL LETTER X
+    'y'        #  0x79 -> LATIN SMALL LETTER Y
+    'z'        #  0x7A -> LATIN SMALL LETTER Z
+    '{'        #  0x7B -> LEFT CURLY BRACKET
+    '|'        #  0x7C -> VERTICAL LINE
+    '}'        #  0x7D -> RIGHT CURLY BRACKET
+    '~'        #  0x7E -> TILDE
+    '\x7f'     #  0x7F -> DELETE
+    '\u20ac'   #  0x80 -> EURO SIGN
+    '\x81'     #  0x81 -> <control>
+    '\u201a'   #  0x82 -> SINGLE LOW-9 QUOTATION MARK
+    '\u0192'   #  0x83 -> LATIN SMALL LETTER F WITH HOOK
+    '\u201e'   #  0x84 -> DOUBLE LOW-9 QUOTATION MARK
+    '\u2026'   #  0x85 -> HORIZONTAL ELLIPSIS
+    '\u2020'   #  0x86 -> DAGGER
+    '\u2021'   #  0x87 -> DOUBLE DAGGER
+    '\u02c6'   #  0x88 -> MODIFIER LETTER CIRCUMFLEX ACCENT
+    '\u2030'   #  0x89 -> PER MILLE SIGN
+    '\u0160'   #  0x8A -> LATIN CAPITAL LETTER S WITH CARON
+    '\u2039'   #  0x8B -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+    '\u0152'   #  0x8C -> LATIN CAPITAL LIGATURE OE
+    '\u2666'   #  0x8D -> BLACK DIAMOND SUIT
+    '\u2663'   #  0x8E -> BLACK CLUB SUIT
+    '\u2665'   #  0x8F -> BLACK HEART SUIT
+    '\u2660'   #  0x90 -> BLACK SPADE SUIT
+    '\u2018'   #  0x91 -> LEFT SINGLE QUOTATION MARK
+    '\u2019'   #  0x92 -> RIGHT SINGLE QUOTATION MARK
+    '\u201c'   #  0x93 -> LEFT DOUBLE QUOTATION MARK
+    '\u201d'   #  0x94 -> RIGHT DOUBLE QUOTATION MARK
+    '\u2022'   #  0x95 -> BULLET
+    '\u2013'   #  0x96 -> EN DASH
+    '\u2014'   #  0x97 -> EM DASH
+    '\u02dc'   #  0x98 -> SMALL TILDE
+    '\u2122'   #  0x99 -> TRADE MARK SIGN
+    '\u0161'   #  0x9A -> LATIN SMALL LETTER S WITH CARON
+    '\x9b'     #  0x9B -> <control>
+    '\u0153'   #  0x9C -> LATIN SMALL LIGATURE OE
+    '\x9d'     #  0x9D -> <control>
+    '\x9e'     #  0x9E -> <control>
+    '\u0178'   #  0x9F -> LATIN CAPITAL LETTER Y WITH DIAERESIS
+    '\xa0'     #  0xA0 -> NO-BREAK SPACE
+    '\xa1'     #  0xA1 -> INVERTED EXCLAMATION MARK
+    '\xa2'     #  0xA2 -> CENT SIGN
+    '\xa3'     #  0xA3 -> POUND SIGN
+    '\xa4'     #  0xA4 -> CURRENCY SIGN
+    '\xa5'     #  0xA5 -> YEN SIGN
+    '\xa6'     #  0xA6 -> BROKEN BAR
+    '\xa7'     #  0xA7 -> SECTION SIGN
+    '\xa8'     #  0xA8 -> DIAERESIS
+    '\xa9'     #  0xA9 -> COPYRIGHT SIGN
+    '\xaa'     #  0xAA -> FEMININE ORDINAL INDICATOR
+    '\xab'     #  0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\xac'     #  0xAC -> NOT SIGN
+    '\xad'     #  0xAD -> SOFT HYPHEN
+    '\xae'     #  0xAE -> REGISTERED SIGN
+    '\xaf'     #  0xAF -> MACRON
+    '\xb0'     #  0xB0 -> DEGREE SIGN
+    '\xb1'     #  0xB1 -> PLUS-MINUS SIGN
+    '\xb2'     #  0xB2 -> SUPERSCRIPT TWO
+    '\xb3'     #  0xB3 -> SUPERSCRIPT THREE
+    '\xb4'     #  0xB4 -> ACUTE ACCENT
+    '\xb5'     #  0xB5 -> MICRO SIGN
+    '\xb6'     #  0xB6 -> PILCROW SIGN
+    '\xb7'     #  0xB7 -> MIDDLE DOT
+    '\xb8'     #  0xB8 -> CEDILLA
+    '\xb9'     #  0xB9 -> SUPERSCRIPT ONE
+    '\xba'     #  0xBA -> MASCULINE ORDINAL INDICATOR
+    '\xbb'     #  0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\xbc'     #  0xBC -> VULGAR FRACTION ONE QUARTER
+    '\xbd'     #  0xBD -> VULGAR FRACTION ONE HALF
+    '\xbe'     #  0xBE -> VULGAR FRACTION THREE QUARTERS
+    '\xbf'     #  0xBF -> INVERTED QUESTION MARK
+    '\xc0'     #  0xC0 -> LATIN CAPITAL LETTER A WITH GRAVE
+    '\xc1'     #  0xC1 -> LATIN CAPITAL LETTER A WITH ACUTE
+    '\xc2'     #  0xC2 -> LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+    '\xc3'     #  0xC3 -> LATIN CAPITAL LETTER A WITH TILDE
+    '\xc4'     #  0xC4 -> LATIN CAPITAL LETTER A WITH DIAERESIS
+    '\xc5'     #  0xC5 -> LATIN CAPITAL LETTER A WITH RING ABOVE
+    '\xc6'     #  0xC6 -> LATIN CAPITAL LETTER AE
+    '\xc7'     #  0xC7 -> LATIN CAPITAL LETTER C WITH CEDILLA
+    '\xc8'     #  0xC8 -> LATIN CAPITAL LETTER E WITH GRAVE
+    '\xc9'     #  0xC9 -> LATIN CAPITAL LETTER E WITH ACUTE
+    '\xca'     #  0xCA -> LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+    '\xcb'     #  0xCB -> LATIN CAPITAL LETTER E WITH DIAERESIS
+    '\xcc'     #  0xCC -> LATIN CAPITAL LETTER I WITH GRAVE
+    '\xcd'     #  0xCD -> LATIN CAPITAL LETTER I WITH ACUTE
+    '\xce'     #  0xCE -> LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+    '\xcf'     #  0xCF -> LATIN CAPITAL LETTER I WITH DIAERESIS
+    '\xd0'     #  0xD0 -> LATIN CAPITAL LETTER ETH (Icelandic)
+    '\xd1'     #  0xD1 -> LATIN CAPITAL LETTER N WITH TILDE
+    '\xd2'     #  0xD2 -> LATIN CAPITAL LETTER O WITH GRAVE
+    '\xd3'     #  0xD3 -> LATIN CAPITAL LETTER O WITH ACUTE
+    '\xd4'     #  0xD4 -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+    '\xd5'     #  0xD5 -> LATIN CAPITAL LETTER O WITH TILDE
+    '\xd6'     #  0xD6 -> LATIN CAPITAL LETTER O WITH DIAERESIS
+    '\xd7'     #  0xD7 -> MULTIPLICATION SIGN
+    '\xd8'     #  0xD8 -> LATIN CAPITAL LETTER O WITH STROKE
+    '\xd9'     #  0xD9 -> LATIN CAPITAL LETTER U WITH GRAVE
+    '\xda'     #  0xDA -> LATIN CAPITAL LETTER U WITH ACUTE
+    '\xdb'     #  0xDB -> LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+    '\xdc'     #  0xDC -> LATIN CAPITAL LETTER U WITH DIAERESIS
+    '\xdd'     #  0xDD -> LATIN CAPITAL LETTER Y WITH ACUTE
+    '\xde'     #  0xDE -> LATIN CAPITAL LETTER THORN (Icelandic)
+    '\xdf'     #  0xDF -> LATIN SMALL LETTER SHARP S (German)
+    '\xe0'     #  0xE0 -> LATIN SMALL LETTER A WITH GRAVE
+    '\xe1'     #  0xE1 -> LATIN SMALL LETTER A WITH ACUTE
+    '\xe2'     #  0xE2 -> LATIN SMALL LETTER A WITH CIRCUMFLEX
+    '\xe3'     #  0xE3 -> LATIN SMALL LETTER A WITH TILDE
+    '\xe4'     #  0xE4 -> LATIN SMALL LETTER A WITH DIAERESIS
+    '\xe5'     #  0xE5 -> LATIN SMALL LETTER A WITH RING ABOVE
+    '\xe6'     #  0xE6 -> LATIN SMALL LETTER AE
+    '\xe7'     #  0xE7 -> LATIN SMALL LETTER C WITH CEDILLA
+    '\xe8'     #  0xE8 -> LATIN SMALL LETTER E WITH GRAVE
+    '\xe9'     #  0xE9 -> LATIN SMALL LETTER E WITH ACUTE
+    '\xea'     #  0xEA -> LATIN SMALL LETTER E WITH CIRCUMFLEX
+    '\xeb'     #  0xEB -> LATIN SMALL LETTER E WITH DIAERESIS
+    '\xec'     #  0xEC -> LATIN SMALL LETTER I WITH GRAVE
+    '\xed'     #  0xED -> LATIN SMALL LETTER I WITH ACUTE
+    '\xee'     #  0xEE -> LATIN SMALL LETTER I WITH CIRCUMFLEX
+    '\xef'     #  0xEF -> LATIN SMALL LETTER I WITH DIAERESIS
+    '\xf0'     #  0xF0 -> LATIN SMALL LETTER ETH (Icelandic)
+    '\xf1'     #  0xF1 -> LATIN SMALL LETTER N WITH TILDE
+    '\xf2'     #  0xF2 -> LATIN SMALL LETTER O WITH GRAVE
+    '\xf3'     #  0xF3 -> LATIN SMALL LETTER O WITH ACUTE
+    '\xf4'     #  0xF4 -> LATIN SMALL LETTER O WITH CIRCUMFLEX
+    '\xf5'     #  0xF5 -> LATIN SMALL LETTER O WITH TILDE
+    '\xf6'     #  0xF6 -> LATIN SMALL LETTER O WITH DIAERESIS
+    '\xf7'     #  0xF7 -> DIVISION SIGN
+    '\xf8'     #  0xF8 -> LATIN SMALL LETTER O WITH STROKE
+    '\xf9'     #  0xF9 -> LATIN SMALL LETTER U WITH GRAVE
+    '\xfa'     #  0xFA -> LATIN SMALL LETTER U WITH ACUTE
+    '\xfb'     #  0xFB -> LATIN SMALL LETTER U WITH CIRCUMFLEX
+    '\xfc'     #  0xFC -> LATIN SMALL LETTER U WITH DIAERESIS
+    '\xfd'     #  0xFD -> LATIN SMALL LETTER Y WITH ACUTE
+    '\xfe'     #  0xFE -> LATIN SMALL LETTER THORN (Icelandic)
+    '\xff'     #  0xFF -> LATIN SMALL LETTER Y WITH DIAERESIS
+)
+
+### Encoding table
+encoding_table=codecs.charmap_build(decoding_table)
diff --git a/Lib/encodings/ptcp154.py b/Lib/encodings/ptcp154.py
index aef897538f..656b79d6ac 100644
--- a/Lib/encodings/ptcp154.py
+++ b/Lib/encodings/ptcp154.py
@@ -14,18 +14,18 @@ import codecs
 class Codec(codecs.Codec):
 
     def encode(self,input,errors='strict'):
-        return codecs.charmap_encode(input,errors,encoding_map)
+        return codecs.charmap_encode(input,errors,encoding_table)
 
     def decode(self,input,errors='strict'):
-        return codecs.charmap_decode(input,errors,decoding_map)
+        return codecs.charmap_decode(input,errors,decoding_table)
 
 class IncrementalEncoder(codecs.IncrementalEncoder):
     def encode(self, input, final=False):
-        return codecs.charmap_encode(input,self.errors,encoding_map)[0]
+        return codecs.charmap_encode(input,self.errors,encoding_table)[0]
 
 class IncrementalDecoder(codecs.IncrementalDecoder):
     def decode(self, input, final=False):
-        return codecs.charmap_decode(input,self.errors,decoding_map)[0]
+        return codecs.charmap_decode(input,self.errors,decoding_table)[0]
 
 class StreamWriter(Codec,codecs.StreamWriter):
     pass
@@ -46,130 +46,267 @@ def getregentry():
         streamwriter=StreamWriter,
     )
 
-### Decoding Map
-
-decoding_map = codecs.make_identity_dict(range(256))
-decoding_map.update({
-        0x0080: 0x0496, #        CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
-        0x0081: 0x0492, #        CYRILLIC CAPITAL LETTER GHE WITH STROKE
-        0x0082: 0x04ee, #        CYRILLIC CAPITAL LETTER U WITH MACRON
-        0x0083: 0x0493, #        CYRILLIC SMALL LETTER GHE WITH STROKE
-        0x0084: 0x201e, #        DOUBLE LOW-9 QUOTATION MARK
-        0x0085: 0x2026, #        HORIZONTAL ELLIPSIS
-        0x0086: 0x04b6, #        CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
-        0x0087: 0x04ae, #        CYRILLIC CAPITAL LETTER STRAIGHT U
-        0x0088: 0x04b2, #        CYRILLIC CAPITAL LETTER HA WITH DESCENDER
-        0x0089: 0x04af, #        CYRILLIC SMALL LETTER STRAIGHT U
-        0x008a: 0x04a0, #        CYRILLIC CAPITAL LETTER BASHKIR KA
-        0x008b: 0x04e2, #        CYRILLIC CAPITAL LETTER I WITH MACRON
-        0x008c: 0x04a2, #        CYRILLIC CAPITAL LETTER EN WITH DESCENDER
-        0x008d: 0x049a, #        CYRILLIC CAPITAL LETTER KA WITH DESCENDER
-        0x008e: 0x04ba, #        CYRILLIC CAPITAL LETTER SHHA
-        0x008f: 0x04b8, #        CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
-        0x0090: 0x0497, #        CYRILLIC SMALL LETTER ZHE WITH DESCENDER
-        0x0091: 0x2018, #        LEFT SINGLE QUOTATION MARK
-        0x0092: 0x2019, #        RIGHT SINGLE QUOTATION MARK
-        0x0093: 0x201c, #        LEFT DOUBLE QUOTATION MARK
-        0x0094: 0x201d, #        RIGHT DOUBLE QUOTATION MARK
-        0x0095: 0x2022, #        BULLET
-        0x0096: 0x2013, #        EN DASH
-        0x0097: 0x2014, #        EM DASH
-        0x0098: 0x04b3, #        CYRILLIC SMALL LETTER HA WITH DESCENDER
-        0x0099: 0x04b7, #        CYRILLIC SMALL LETTER CHE WITH DESCENDER
-        0x009a: 0x04a1, #        CYRILLIC SMALL LETTER BASHKIR KA
-        0x009b: 0x04e3, #        CYRILLIC SMALL LETTER I WITH MACRON
-        0x009c: 0x04a3, #        CYRILLIC SMALL LETTER EN WITH DESCENDER
-        0x009d: 0x049b, #        CYRILLIC SMALL LETTER KA WITH DESCENDER
-        0x009e: 0x04bb, #        CYRILLIC SMALL LETTER SHHA
-        0x009f: 0x04b9, #        CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE
-        0x00a1: 0x040e, #        CYRILLIC CAPITAL LETTER SHORT U (Byelorussian)
-        0x00a2: 0x045e, #        CYRILLIC SMALL LETTER SHORT U (Byelorussian)
-        0x00a3: 0x0408, #        CYRILLIC CAPITAL LETTER JE
-        0x00a4: 0x04e8, #        CYRILLIC CAPITAL LETTER BARRED O
-        0x00a5: 0x0498, #        CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
-        0x00a6: 0x04b0, #        CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
-        0x00a8: 0x0401, #        CYRILLIC CAPITAL LETTER IO
-        0x00aa: 0x04d8, #        CYRILLIC CAPITAL LETTER SCHWA
-        0x00ad: 0x04ef, #        CYRILLIC SMALL LETTER U WITH MACRON
-        0x00af: 0x049c, #        CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
-        0x00b1: 0x04b1, #        CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE
-        0x00b2: 0x0406, #        CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
-        0x00b3: 0x0456, #        CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I
-        0x00b4: 0x0499, #        CYRILLIC SMALL LETTER ZE WITH DESCENDER
-        0x00b5: 0x04e9, #        CYRILLIC SMALL LETTER BARRED O
-        0x00b8: 0x0451, #        CYRILLIC SMALL LETTER IO
-        0x00b9: 0x2116, #        NUMERO SIGN
-        0x00ba: 0x04d9, #        CYRILLIC SMALL LETTER SCHWA
-        0x00bc: 0x0458, #        CYRILLIC SMALL LETTER JE
-        0x00bd: 0x04aa, #        CYRILLIC CAPITAL LETTER ES WITH DESCENDER
-        0x00be: 0x04ab, #        CYRILLIC SMALL LETTER ES WITH DESCENDER
-        0x00bf: 0x049d, #        CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE
-        0x00c0: 0x0410, #        CYRILLIC CAPITAL LETTER A
-        0x00c1: 0x0411, #        CYRILLIC CAPITAL LETTER BE
-        0x00c2: 0x0412, #        CYRILLIC CAPITAL LETTER VE
-        0x00c3: 0x0413, #        CYRILLIC CAPITAL LETTER GHE
-        0x00c4: 0x0414, #        CYRILLIC CAPITAL LETTER DE
-        0x00c5: 0x0415, #        CYRILLIC CAPITAL LETTER IE
-        0x00c6: 0x0416, #        CYRILLIC CAPITAL LETTER ZHE
-        0x00c7: 0x0417, #        CYRILLIC CAPITAL LETTER ZE
-        0x00c8: 0x0418, #        CYRILLIC CAPITAL LETTER I
-        0x00c9: 0x0419, #        CYRILLIC CAPITAL LETTER SHORT I
-        0x00ca: 0x041a, #        CYRILLIC CAPITAL LETTER KA
-        0x00cb: 0x041b, #        CYRILLIC CAPITAL LETTER EL
-        0x00cc: 0x041c, #        CYRILLIC CAPITAL LETTER EM
-        0x00cd: 0x041d, #        CYRILLIC CAPITAL LETTER EN
-        0x00ce: 0x041e, #        CYRILLIC CAPITAL LETTER O
-        0x00cf: 0x041f, #        CYRILLIC CAPITAL LETTER PE
-        0x00d0: 0x0420, #        CYRILLIC CAPITAL LETTER ER
-        0x00d1: 0x0421, #        CYRILLIC CAPITAL LETTER ES
-        0x00d2: 0x0422, #        CYRILLIC CAPITAL LETTER TE
-        0x00d3: 0x0423, #        CYRILLIC CAPITAL LETTER U
-        0x00d4: 0x0424, #        CYRILLIC CAPITAL LETTER EF
-        0x00d5: 0x0425, #        CYRILLIC CAPITAL LETTER HA
-        0x00d6: 0x0426, #        CYRILLIC CAPITAL LETTER TSE
-        0x00d7: 0x0427, #        CYRILLIC CAPITAL LETTER CHE
-        0x00d8: 0x0428, #        CYRILLIC CAPITAL LETTER SHA
-        0x00d9: 0x0429, #        CYRILLIC CAPITAL LETTER SHCHA
-        0x00da: 0x042a, #        CYRILLIC CAPITAL LETTER HARD SIGN
-        0x00db: 0x042b, #        CYRILLIC CAPITAL LETTER YERU
-        0x00dc: 0x042c, #        CYRILLIC CAPITAL LETTER SOFT SIGN
-        0x00dd: 0x042d, #        CYRILLIC CAPITAL LETTER E
-        0x00de: 0x042e, #        CYRILLIC CAPITAL LETTER YU
-        0x00df: 0x042f, #        CYRILLIC CAPITAL LETTER YA
-        0x00e0: 0x0430, #        CYRILLIC SMALL LETTER A
-        0x00e1: 0x0431, #        CYRILLIC SMALL LETTER BE
-        0x00e2: 0x0432, #        CYRILLIC SMALL LETTER VE
-        0x00e3: 0x0433, #        CYRILLIC SMALL LETTER GHE
-        0x00e4: 0x0434, #        CYRILLIC SMALL LETTER DE
-        0x00e5: 0x0435, #        CYRILLIC SMALL LETTER IE
-        0x00e6: 0x0436, #        CYRILLIC SMALL LETTER ZHE
-        0x00e7: 0x0437, #        CYRILLIC SMALL LETTER ZE
-        0x00e8: 0x0438, #        CYRILLIC SMALL LETTER I
-        0x00e9: 0x0439, #        CYRILLIC SMALL LETTER SHORT I
-        0x00ea: 0x043a, #        CYRILLIC SMALL LETTER KA
-        0x00eb: 0x043b, #        CYRILLIC SMALL LETTER EL
-        0x00ec: 0x043c, #        CYRILLIC SMALL LETTER EM
-        0x00ed: 0x043d, #        CYRILLIC SMALL LETTER EN
-        0x00ee: 0x043e, #        CYRILLIC SMALL LETTER O
-        0x00ef: 0x043f, #        CYRILLIC SMALL LETTER PE
-        0x00f0: 0x0440, #        CYRILLIC SMALL LETTER ER
-        0x00f1: 0x0441, #        CYRILLIC SMALL LETTER ES
-        0x00f2: 0x0442, #        CYRILLIC SMALL LETTER TE
-        0x00f3: 0x0443, #        CYRILLIC SMALL LETTER U
-        0x00f4: 0x0444, #        CYRILLIC SMALL LETTER EF
-        0x00f5: 0x0445, #        CYRILLIC SMALL LETTER HA
-        0x00f6: 0x0446, #        CYRILLIC SMALL LETTER TSE
-        0x00f7: 0x0447, #        CYRILLIC SMALL LETTER CHE
-        0x00f8: 0x0448, #        CYRILLIC SMALL LETTER SHA
-        0x00f9: 0x0449, #        CYRILLIC SMALL LETTER SHCHA
-        0x00fa: 0x044a, #        CYRILLIC SMALL LETTER HARD SIGN
-        0x00fb: 0x044b, #        CYRILLIC SMALL LETTER YERU
-        0x00fc: 0x044c, #        CYRILLIC SMALL LETTER SOFT SIGN
-        0x00fd: 0x044d, #        CYRILLIC SMALL LETTER E
-        0x00fe: 0x044e, #        CYRILLIC SMALL LETTER YU
-        0x00ff: 0x044f, #        CYRILLIC SMALL LETTER YA
-})
-
-### Encoding Map
-
-encoding_map = codecs.make_encoding_map(decoding_map)
+
+### Decoding Table
+
+decoding_table = (
+    '\x00'     #  0x00 -> NULL
+    '\x01'     #  0x01 -> START OF HEADING
+    '\x02'     #  0x02 -> START OF TEXT
+    '\x03'     #  0x03 -> END OF TEXT
+    '\x04'     #  0x04 -> END OF TRANSMISSION
+    '\x05'     #  0x05 -> ENQUIRY
+    '\x06'     #  0x06 -> ACKNOWLEDGE
+    '\x07'     #  0x07 -> BELL
+    '\x08'     #  0x08 -> BACKSPACE
+    '\t'       #  0x09 -> HORIZONTAL TABULATION
+    '\n'       #  0x0A -> LINE FEED
+    '\x0b'     #  0x0B -> VERTICAL TABULATION
+    '\x0c'     #  0x0C -> FORM FEED
+    '\r'       #  0x0D -> CARRIAGE RETURN
+    '\x0e'     #  0x0E -> SHIFT OUT
+    '\x0f'     #  0x0F -> SHIFT IN
+    '\x10'     #  0x10 -> DATA LINK ESCAPE
+    '\x11'     #  0x11 -> DEVICE CONTROL ONE
+    '\x12'     #  0x12 -> DEVICE CONTROL TWO
+    '\x13'     #  0x13 -> DEVICE CONTROL THREE
+    '\x14'     #  0x14 -> DEVICE CONTROL FOUR
+    '\x15'     #  0x15 -> NEGATIVE ACKNOWLEDGE
+    '\x16'     #  0x16 -> SYNCHRONOUS IDLE
+    '\x17'     #  0x17 -> END OF TRANSMISSION BLOCK
+    '\x18'     #  0x18 -> CANCEL
+    '\x19'     #  0x19 -> END OF MEDIUM
+    '\x1a'     #  0x1A -> SUBSTITUTE
+    '\x1b'     #  0x1B -> ESCAPE
+    '\x1c'     #  0x1C -> FILE SEPARATOR
+    '\x1d'     #  0x1D -> GROUP SEPARATOR
+    '\x1e'     #  0x1E -> RECORD SEPARATOR
+    '\x1f'     #  0x1F -> UNIT SEPARATOR
+    ' '        #  0x20 -> SPACE
+    '!'        #  0x21 -> EXCLAMATION MARK
+    '"'        #  0x22 -> QUOTATION MARK
+    '#'        #  0x23 -> NUMBER SIGN
+    '$'        #  0x24 -> DOLLAR SIGN
+    '%'        #  0x25 -> PERCENT SIGN
+    '&'        #  0x26 -> AMPERSAND
+    "'"        #  0x27 -> APOSTROPHE
+    '('        #  0x28 -> LEFT PARENTHESIS
+    ')'        #  0x29 -> RIGHT PARENTHESIS
+    '*'        #  0x2A -> ASTERISK
+    '+'        #  0x2B -> PLUS SIGN
+    ','        #  0x2C -> COMMA
+    '-'        #  0x2D -> HYPHEN-MINUS
+    '.'        #  0x2E -> FULL STOP
+    '/'        #  0x2F -> SOLIDUS
+    '0'        #  0x30 -> DIGIT ZERO
+    '1'        #  0x31 -> DIGIT ONE
+    '2'        #  0x32 -> DIGIT TWO
+    '3'        #  0x33 -> DIGIT THREE
+    '4'        #  0x34 -> DIGIT FOUR
+    '5'        #  0x35 -> DIGIT FIVE
+    '6'        #  0x36 -> DIGIT SIX
+    '7'        #  0x37 -> DIGIT SEVEN
+    '8'        #  0x38 -> DIGIT EIGHT
+    '9'        #  0x39 -> DIGIT NINE
+    ':'        #  0x3A -> COLON
+    ';'        #  0x3B -> SEMICOLON
+    '<'        #  0x3C -> LESS-THAN SIGN
+    '='        #  0x3D -> EQUALS SIGN
+    '>'        #  0x3E -> GREATER-THAN SIGN
+    '?'        #  0x3F -> QUESTION MARK
+    '@'        #  0x40 -> COMMERCIAL AT
+    'A'        #  0x41 -> LATIN CAPITAL LETTER A
+    'B'        #  0x42 -> LATIN CAPITAL LETTER B
+    'C'        #  0x43 -> LATIN CAPITAL LETTER C
+    'D'        #  0x44 -> LATIN CAPITAL LETTER D
+    'E'        #  0x45 -> LATIN CAPITAL LETTER E
+    'F'        #  0x46 -> LATIN CAPITAL LETTER F
+    'G'        #  0x47 -> LATIN CAPITAL LETTER G
+    'H'        #  0x48 -> LATIN CAPITAL LETTER H
+    'I'        #  0x49 -> LATIN CAPITAL LETTER I
+    'J'        #  0x4A -> LATIN CAPITAL LETTER J
+    'K'        #  0x4B -> LATIN CAPITAL LETTER K
+    'L'        #  0x4C -> LATIN CAPITAL LETTER L
+    'M'        #  0x4D -> LATIN CAPITAL LETTER M
+    'N'        #  0x4E -> LATIN CAPITAL LETTER N
+    'O'        #  0x4F -> LATIN CAPITAL LETTER O
+    'P'        #  0x50 -> LATIN CAPITAL LETTER P
+    'Q'        #  0x51 -> LATIN CAPITAL LETTER Q
+    'R'        #  0x52 -> LATIN CAPITAL LETTER R
+    'S'        #  0x53 -> LATIN CAPITAL LETTER S
+    'T'        #  0x54 -> LATIN CAPITAL LETTER T
+    'U'        #  0x55 -> LATIN CAPITAL LETTER U
+    'V'        #  0x56 -> LATIN CAPITAL LETTER V
+    'W'        #  0x57 -> LATIN CAPITAL LETTER W
+    'X'        #  0x58 -> LATIN CAPITAL LETTER X
+    'Y'        #  0x59 -> LATIN CAPITAL LETTER Y
+    'Z'        #  0x5A -> LATIN CAPITAL LETTER Z
+    '['        #  0x5B -> LEFT SQUARE BRACKET
+    '\\'       #  0x5C -> REVERSE SOLIDUS
+    ']'        #  0x5D -> RIGHT SQUARE BRACKET
+    '^'        #  0x5E -> CIRCUMFLEX ACCENT
+    '_'        #  0x5F -> LOW LINE
+    '`'        #  0x60 -> GRAVE ACCENT
+    'a'        #  0x61 -> LATIN SMALL LETTER A
+    'b'        #  0x62 -> LATIN SMALL LETTER B
+    'c'        #  0x63 -> LATIN SMALL LETTER C
+    'd'        #  0x64 -> LATIN SMALL LETTER D
+    'e'        #  0x65 -> LATIN SMALL LETTER E
+    'f'        #  0x66 -> LATIN SMALL LETTER F
+    'g'        #  0x67 -> LATIN SMALL LETTER G
+    'h'        #  0x68 -> LATIN SMALL LETTER H
+    'i'        #  0x69 -> LATIN SMALL LETTER I
+    'j'        #  0x6A -> LATIN SMALL LETTER J
+    'k'        #  0x6B -> LATIN SMALL LETTER K
+    'l'        #  0x6C -> LATIN SMALL LETTER L
+    'm'        #  0x6D -> LATIN SMALL LETTER M
+    'n'        #  0x6E -> LATIN SMALL LETTER N
+    'o'        #  0x6F -> LATIN SMALL LETTER O
+    'p'        #  0x70 -> LATIN SMALL LETTER P
+    'q'        #  0x71 -> LATIN SMALL LETTER Q
+    'r'        #  0x72 -> LATIN SMALL LETTER R
+    's'        #  0x73 -> LATIN SMALL LETTER S
+    't'        #  0x74 -> LATIN SMALL LETTER T
+    'u'        #  0x75 -> LATIN SMALL LETTER U
+    'v'        #  0x76 -> LATIN SMALL LETTER V
+    'w'        #  0x77 -> LATIN SMALL LETTER W
+    'x'        #  0x78 -> LATIN SMALL LETTER X
+    'y'        #  0x79 -> LATIN SMALL LETTER Y
+    'z'        #  0x7A -> LATIN SMALL LETTER Z
+    '{'        #  0x7B -> LEFT CURLY BRACKET
+    '|'        #  0x7C -> VERTICAL LINE
+    '}'        #  0x7D -> RIGHT CURLY BRACKET
+    '~'        #  0x7E -> TILDE
+    '\x7f'     #  0x7F -> DELETE (DEL)
+    '\u0496'   #  0x80 -> CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+    '\u0492'   #  0x81 -> CYRILLIC CAPITAL LETTER GHE WITH STROKE
+    '\u04ee'   #  0x82 -> CYRILLIC CAPITAL LETTER U WITH MACRON
+    '\u0493'   #  0x83 -> CYRILLIC SMALL LETTER GHE WITH STROKE
+    '\u201e'   #  0x84 -> DOUBLE LOW-9 QUOTATION MARK
+    '\u2026'   #  0x85 -> HORIZONTAL ELLIPSIS
+    '\u04b6'   #  0x86 -> CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+    '\u04ae'   #  0x87 -> CYRILLIC CAPITAL LETTER STRAIGHT U
+    '\u04b2'   #  0x88 -> CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+    '\u04af'   #  0x89 -> CYRILLIC SMALL LETTER STRAIGHT U
+    '\u04a0'   #  0x8A -> CYRILLIC CAPITAL LETTER BASHKIR KA
+    '\u04e2'   #  0x8B -> CYRILLIC CAPITAL LETTER I WITH MACRON
+    '\u04a2'   #  0x8C -> CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+    '\u049a'   #  0x8D -> CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+    '\u04ba'   #  0x8E -> CYRILLIC CAPITAL LETTER SHHA
+    '\u04b8'   #  0x8F -> CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+    '\u0497'   #  0x90 -> CYRILLIC SMALL LETTER ZHE WITH DESCENDER
+    '\u2018'   #  0x91 -> LEFT SINGLE QUOTATION MARK
+    '\u2019'   #  0x92 -> RIGHT SINGLE QUOTATION MARK
+    '\u201c'   #  0x93 -> LEFT DOUBLE QUOTATION MARK
+    '\u201d'   #  0x94 -> RIGHT DOUBLE QUOTATION MARK
+    '\u2022'   #  0x95 -> BULLET
+    '\u2013'   #  0x96 -> EN DASH
+    '\u2014'   #  0x97 -> EM DASH
+    '\u04b3'   #  0x98 -> CYRILLIC SMALL LETTER HA WITH DESCENDER
+    '\u04b7'   #  0x99 -> CYRILLIC SMALL LETTER CHE WITH DESCENDER
+    '\u04a1'   #  0x9A -> CYRILLIC SMALL LETTER BASHKIR KA
+    '\u04e3'   #  0x9B -> CYRILLIC SMALL LETTER I WITH MACRON
+    '\u04a3'   #  0x9C -> CYRILLIC SMALL LETTER EN WITH DESCENDER
+    '\u049b'   #  0x9D -> CYRILLIC SMALL LETTER KA WITH DESCENDER
+    '\u04bb'   #  0x9E -> CYRILLIC SMALL LETTER SHHA
+    '\u04b9'   #  0x9F -> CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE
+    '\xa0'     #  0xA0 -> NO-BREAK SPACE
+    '\u040e'   #  0xA1 -> CYRILLIC CAPITAL LETTER SHORT U (Byelorussian)
+    '\u045e'   #  0xA2 -> CYRILLIC SMALL LETTER SHORT U (Byelorussian)
+    '\u0408'   #  0xA3 -> CYRILLIC CAPITAL LETTER JE
+    '\u04e8'   #  0xA4 -> CYRILLIC CAPITAL LETTER BARRED O
+    '\u0498'   #  0xA5 -> CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+    '\u04b0'   #  0xA6 -> CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+    '\xa7'     #  0xA7 -> SECTION SIGN
+    '\u0401'   #  0xA8 -> CYRILLIC CAPITAL LETTER IO
+    '\xa9'     #  0xA9 -> COPYRIGHT SIGN
+    '\u04d8'   #  0xAA -> CYRILLIC CAPITAL LETTER SCHWA
+    '\xab'     #  0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\xac'     #  0xAC -> NOT SIGN
+    '\u04ef'   #  0xAD -> CYRILLIC SMALL LETTER U WITH MACRON
+    '\xae'     #  0xAE -> REGISTERED SIGN
+    '\u049c'   #  0xAF -> CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+    '\xb0'     #  0xB0 -> DEGREE SIGN
+    '\u04b1'   #  0xB1 -> CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE
+    '\u0406'   #  0xB2 -> CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+    '\u0456'   #  0xB3 -> CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I
+    '\u0499'   #  0xB4 -> CYRILLIC SMALL LETTER ZE WITH DESCENDER
+    '\u04e9'   #  0xB5 -> CYRILLIC SMALL LETTER BARRED O
+    '\xb6'     #  0xB6 -> PILCROW SIGN
+    '\xb7'     #  0xB7 -> MIDDLE DOT
+    '\u0451'   #  0xB8 -> CYRILLIC SMALL LETTER IO
+    '\u2116'   #  0xB9 -> NUMERO SIGN
+    '\u04d9'   #  0xBA -> CYRILLIC SMALL LETTER SCHWA
+    '\xbb'     #  0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+    '\u0458'   #  0xBC -> CYRILLIC SMALL LETTER JE
+    '\u04aa'   #  0xBD -> CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+    '\u04ab'   #  0xBE -> CYRILLIC SMALL LETTER ES WITH DESCENDER
+    '\u049d'   #  0xBF -> CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE
+    '\u0410'   #  0xC0 -> CYRILLIC CAPITAL LETTER A
+    '\u0411'   #  0xC1 -> CYRILLIC CAPITAL LETTER BE
+    '\u0412'   #  0xC2 -> CYRILLIC CAPITAL LETTER VE
+    '\u0413'   #  0xC3 -> CYRILLIC CAPITAL LETTER GHE
+    '\u0414'   #  0xC4 -> CYRILLIC CAPITAL LETTER DE
+    '\u0415'   #  0xC5 -> CYRILLIC CAPITAL LETTER IE
+    '\u0416'   #  0xC6 -> CYRILLIC CAPITAL LETTER ZHE
+    '\u0417'   #  0xC7 -> CYRILLIC CAPITAL LETTER ZE
+    '\u0418'   #  0xC8 -> CYRILLIC CAPITAL LETTER I
+    '\u0419'   #  0xC9 -> CYRILLIC CAPITAL LETTER SHORT I
+    '\u041a'   #  0xCA -> CYRILLIC CAPITAL LETTER KA
+    '\u041b'   #  0xCB -> CYRILLIC CAPITAL LETTER EL
+    '\u041c'   #  0xCC -> CYRILLIC CAPITAL LETTER EM
+    '\u041d'   #  0xCD -> CYRILLIC CAPITAL LETTER EN
+    '\u041e'   #  0xCE -> CYRILLIC CAPITAL LETTER O
+    '\u041f'   #  0xCF -> CYRILLIC CAPITAL LETTER PE
+    '\u0420'   #  0xD0 -> CYRILLIC CAPITAL LETTER ER
+    '\u0421'   #  0xD1 -> CYRILLIC CAPITAL LETTER ES
+    '\u0422'   #  0xD2 -> CYRILLIC CAPITAL LETTER TE
+    '\u0423'   #  0xD3 -> CYRILLIC CAPITAL LETTER U
+    '\u0424'   #  0xD4 -> CYRILLIC CAPITAL LETTER EF
+    '\u0425'   #  0xD5 -> CYRILLIC CAPITAL LETTER HA
+    '\u0426'   #  0xD6 -> CYRILLIC CAPITAL LETTER TSE
+    '\u0427'   #  0xD7 -> CYRILLIC CAPITAL LETTER CHE
+    '\u0428'   #  0xD8 -> CYRILLIC CAPITAL LETTER SHA
+    '\u0429'   #  0xD9 -> CYRILLIC CAPITAL LETTER SHCHA
+    '\u042a'   #  0xDA -> CYRILLIC CAPITAL LETTER HARD SIGN
+    '\u042b'   #  0xDB -> CYRILLIC CAPITAL LETTER YERU
+    '\u042c'   #  0xDC -> CYRILLIC CAPITAL LETTER SOFT SIGN
+    '\u042d'   #  0xDD -> CYRILLIC CAPITAL LETTER E
+    '\u042e'   #  0xDE -> CYRILLIC CAPITAL LETTER YU
+    '\u042f'   #  0xDF -> CYRILLIC CAPITAL LETTER YA
+    '\u0430'   #  0xE0 -> CYRILLIC SMALL LETTER A
+    '\u0431'   #  0xE1 -> CYRILLIC SMALL LETTER BE
+    '\u0432'   #  0xE2 -> CYRILLIC SMALL LETTER VE
+    '\u0433'   #  0xE3 -> CYRILLIC SMALL LETTER GHE
+    '\u0434'   #  0xE4 -> CYRILLIC SMALL LETTER DE
+    '\u0435'   #  0xE5 -> CYRILLIC SMALL LETTER IE
+    '\u0436'   #  0xE6 -> CYRILLIC SMALL LETTER ZHE
+    '\u0437'   #  0xE7 -> CYRILLIC SMALL LETTER ZE
+    '\u0438'   #  0xE8 -> CYRILLIC SMALL LETTER I
+    '\u0439'   #  0xE9 -> CYRILLIC SMALL LETTER SHORT I
+    '\u043a'   #  0xEA -> CYRILLIC SMALL LETTER KA
+    '\u043b'   #  0xEB -> CYRILLIC SMALL LETTER EL
+    '\u043c'   #  0xEC -> CYRILLIC SMALL LETTER EM
+    '\u043d'   #  0xED -> CYRILLIC SMALL LETTER EN
+    '\u043e'   #  0xEE -> CYRILLIC SMALL LETTER O
+    '\u043f'   #  0xEF -> CYRILLIC SMALL LETTER PE
+    '\u0440'   #  0xF0 -> CYRILLIC SMALL LETTER ER
+    '\u0441'   #  0xF1 -> CYRILLIC SMALL LETTER ES
+    '\u0442'   #  0xF2 -> CYRILLIC SMALL LETTER TE
+    '\u0443'   #  0xF3 -> CYRILLIC SMALL LETTER U
+    '\u0444'   #  0xF4 -> CYRILLIC SMALL LETTER EF
+    '\u0445'   #  0xF5 -> CYRILLIC SMALL LETTER HA
+    '\u0446'   #  0xF6 -> CYRILLIC SMALL LETTER TSE
+    '\u0447'   #  0xF7 -> CYRILLIC SMALL LETTER CHE
+    '\u0448'   #  0xF8 -> CYRILLIC SMALL LETTER SHA
+    '\u0449'   #  0xF9 -> CYRILLIC SMALL LETTER SHCHA
+    '\u044a'   #  0xFA -> CYRILLIC SMALL LETTER HARD SIGN
+    '\u044b'   #  0xFB -> CYRILLIC SMALL LETTER YERU
+    '\u044c'   #  0xFC -> CYRILLIC SMALL LETTER SOFT SIGN
+    '\u044d'   #  0xFD -> CYRILLIC SMALL LETTER E
+    '\u044e'   #  0xFE -> CYRILLIC SMALL LETTER YU
+    '\u044f'   #  0xFF -> CYRILLIC SMALL LETTER YA
+)
+
+### Encoding table
+encoding_table=codecs.charmap_build(decoding_table)
diff --git a/Lib/fileinput.py b/Lib/fileinput.py
index 554beb2448..dbbbb2192e 100644
--- a/Lib/fileinput.py
+++ b/Lib/fileinput.py
@@ -398,9 +398,8 @@ def hook_compressed(filename, mode):
 
 
 def hook_encoded(encoding):
-    import codecs
     def openhook(filename, mode):
-        return codecs.open(filename, mode, encoding)
+        return open(filename, mode, encoding=encoding)
     return openhook
 
 
diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py
index 726fbe556e..6330b0cfda 100644
--- a/Lib/fnmatch.py
+++ b/Lib/fnmatch.py
@@ -35,9 +35,9 @@ def fnmatch(name, pat):
     pat = os.path.normcase(pat)
     return fnmatchcase(name, pat)
 
-@functools.lru_cache(maxsize=250)
-def _compile_pattern(pat, is_bytes=False):
-    if is_bytes:
+@functools.lru_cache(maxsize=256, typed=True)
+def _compile_pattern(pat):
+    if isinstance(pat, bytes):
         pat_str = str(pat, 'ISO-8859-1')
         res_str = translate(pat_str)
         res = bytes(res_str, 'ISO-8859-1')
@@ -49,7 +49,7 @@ def filter(names, pat):
     """Return the subset of the list NAMES that match PAT."""
     result = []
     pat = os.path.normcase(pat)
-    match = _compile_pattern(pat, isinstance(pat, bytes))
+    match = _compile_pattern(pat)
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -67,7 +67,7 @@ def fnmatchcase(name, pat):
     This is a version of fnmatch() which doesn't case-normalize
     its arguments.
     """
-    match = _compile_pattern(pat, isinstance(pat, bytes))
+    match = _compile_pattern(pat)
     return match(name) is not None
 
 
diff --git a/Lib/ftplib.py b/Lib/ftplib.py
index a0f0f90fd0..5efae95a60 100644
--- a/Lib/ftplib.py
+++ b/Lib/ftplib.py
@@ -100,14 +100,15 @@ class FTP:
     file = None
     welcome = None
     passiveserver = 1
-    encoding = "latin1"
+    encoding = "latin-1"
 
     # Initialization method (called by class instantiation).
     # Initialize host to localhost, port to standard ftp port
     # Optional arguments are host (for connect()),
     # and user, passwd, acct (for login())
     def __init__(self, host='', user='', passwd='', acct='',
-                 timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                 timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.source_address = source_address
         self.timeout = timeout
         if host:
             self.connect(host)
@@ -128,10 +129,12 @@ class FTP:
                 if self.sock is not None:
                     self.close()
 
-    def connect(self, host='', port=0, timeout=-999):
+    def connect(self, host='', port=0, timeout=-999, source_address=None):
         '''Connect to host.  Arguments are:
          - host: hostname to connect to (string, default previous host)
          - port: port to connect to (integer, default previous port)
+         - source_address: a 2-tuple (host, port) for the socket to bind
+           to as its source address before connecting.
         '''
         if host != '':
             self.host = host
@@ -139,7 +142,10 @@ class FTP:
             self.port = port
         if timeout != -999:
             self.timeout = timeout
-        self.sock = socket.create_connection((self.host, self.port), self.timeout)
+        if source_address is not None:
+            self.source_address = source_address
+        self.sock = socket.create_connection((self.host, self.port), self.timeout,
+                                             source_address=self.source_address)
         self.af = self.sock.family
         self.file = self.sock.makefile('r', encoding=self.encoding)
         self.welcome = self.getresp()
@@ -169,10 +175,8 @@ class FTP:
 
     # Internal: "sanitize" a string for printing
     def sanitize(self, s):
-        if s[:5] == 'pass ' or s[:5] == 'PASS ':
-            i = len(s)
-            while i > 5 and s[i-1] in {'\r', '\n'}:
-                i = i-1
+        if s[:5] in {'pass ', 'PASS '}:
+            i = len(s.rstrip('\r\n'))
             s = s[:5] + '*'*(i-5) + s[i:]
         return repr(s)
 
@@ -340,7 +344,8 @@ class FTP:
         size = None
         if self.passiveserver:
             host, port = self.makepasv()
-            conn = socket.create_connection((host, port), self.timeout)
+            conn = socket.create_connection((host, port), self.timeout,
+                                            source_address=self.source_address)
             try:
                 if rest is not None:
                     self.sendcmd("REST %s" % rest)
@@ -359,8 +364,7 @@ class FTP:
                 conn.close()
                 raise
         else:
-            sock = self.makeport()
-            try:
+            with self.makeport() as sock:
                 if rest is not None:
                     self.sendcmd("REST %s" % rest)
                 resp = self.sendcmd(cmd)
@@ -372,8 +376,6 @@ class FTP:
                 conn, sockaddr = sock.accept()
                 if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT:
                     conn.settimeout(self.timeout)
-            finally:
-                sock.close()
         if resp[:3] == '150':
             # this is conditional in case we received a 125
             size = parse150(resp)
@@ -431,7 +433,7 @@ class FTP:
         """Retrieve data in line mode.  A new port is created for you.
 
         Args:
-          cmd: A RETR, LIST, NLST, or MLSD command.
+          cmd: A RETR, LIST, or NLST command.
           callback: An optional single parameter callable that is called
                     for each line with the trailing CRLF stripped.
                     [default: print_line()]
@@ -532,6 +534,34 @@ class FTP:
                 cmd = cmd + (' ' + arg)
         self.retrlines(cmd, func)
 
+    def mlsd(self, path="", facts=[]):
+        '''List a directory in a standardized format by using MLSD
+        command (RFC-3659). If path is omitted the current directory
+        is assumed. "facts" is a list of strings representing the type
+        of information desired (e.g. ["type", "size", "perm"]).
+
+        Return a generator object yielding a tuple of two elements
+        for every file found in path.
+        First element is the file name, the second one is a dictionary
+        including a variable number of "facts" depending on the server
+        and whether "facts" argument has been provided.
+        '''
+        if facts:
+            self.sendcmd("OPTS MLST " + ";".join(facts) + ";")
+        if path:
+            cmd = "MLSD %s" % path
+        else:
+            cmd = "MLSD"
+        lines = []
+        self.retrlines(cmd, lines.append)
+        for line in lines:
+            facts_found, _, name = line.rstrip(CRLF).partition(' ')
+            entry = {}
+            for fact in facts_found[:-1].split(";"):
+                key, _, value = fact.partition("=")
+                entry[key.lower()] = value
+            yield (name, entry)
+
     def rename(self, fromname, toname):
         '''Rename a file.'''
         resp = self.sendcmd('RNFR ' + fromname)
@@ -566,10 +596,7 @@ class FTP:
         resp = self.sendcmd('SIZE ' + filename)
         if resp[:3] == '213':
             s = resp[3:].strip()
-            try:
-                return int(s)
-            except (OverflowError, ValueError):
-                return int(s)
+            return int(s)
 
     def mkd(self, dirname):
         '''Make a directory, return its full pathname.'''
@@ -601,11 +628,11 @@ class FTP:
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file:
+        if self.file is not None:
             self.file.close()
+        if self.sock is not None:
             self.sock.close()
-            self.file = self.sock = None
-
+        self.file = self.sock = None
 
 try:
     import ssl
@@ -649,7 +676,7 @@ else:
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
                      certfile=None, context=None,
-                     timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
             if context is not None and keyfile is not None:
                 raise ValueError("context and keyfile arguments are mutually "
                                  "exclusive")
@@ -660,7 +687,7 @@ else:
             self.certfile = certfile
             self.context = context
             self._prot_p = False
-            FTP.__init__(self, host, user, passwd, acct, timeout)
+            FTP.__init__(self, host, user, passwd, acct, timeout, source_address)
 
         def login(self, user='', passwd='', acct='', secure=True):
             if secure and not isinstance(self.sock, ssl.SSLSocket):
@@ -684,6 +711,14 @@ else:
             self.file = self.sock.makefile(mode='r', encoding=self.encoding)
             return resp
 
+        def ccc(self):
+            '''Switch back to a clear-text control connection.'''
+            if not isinstance(self.sock, ssl.SSLSocket):
+                raise ValueError("not using TLS")
+            resp = self.voidcmd('CCC')
+            self.sock = self.sock.unwrap()
+            return resp
+
         def prot_p(self):
             '''Set up secure data connection.'''
             # PROT defines whether or not the data channel is to be protected.
@@ -720,8 +755,7 @@ else:
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
             self.voidcmd('TYPE I')
-            conn = self.transfercmd(cmd, rest)
-            try:
+            with self.transfercmd(cmd, rest) as conn:
                 while 1:
                     data = conn.recv(blocksize)
                     if not data:
@@ -730,8 +764,6 @@ else:
                 # shutdown ssl layer
                 if isinstance(conn, ssl.SSLSocket):
                     conn.unwrap()
-            finally:
-                conn.close()
             return self.voidresp()
 
         def retrlines(self, cmd, callback = None):
@@ -739,7 +771,7 @@ else:
             resp = self.sendcmd('TYPE A')
             conn = self.transfercmd(cmd)
             fp = conn.makefile('r', encoding=self.encoding)
-            try:
+            with fp, conn:
                 while 1:
                     line = fp.readline()
                     if self.debugging > 2: print('*retr*', repr(line))
@@ -753,15 +785,11 @@ else:
                 # shutdown ssl layer
                 if isinstance(conn, ssl.SSLSocket):
                     conn.unwrap()
-            finally:
-                fp.close()
-                conn.close()
             return self.voidresp()
 
         def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None):
             self.voidcmd('TYPE I')
-            conn = self.transfercmd(cmd, rest)
-            try:
+            with self.transfercmd(cmd, rest) as conn:
                 while 1:
                     buf = fp.read(blocksize)
                     if not buf: break
@@ -770,14 +798,11 @@ else:
                 # shutdown ssl layer
                 if isinstance(conn, ssl.SSLSocket):
                     conn.unwrap()
-            finally:
-                conn.close()
             return self.voidresp()
 
         def storlines(self, cmd, fp, callback=None):
             self.voidcmd('TYPE A')
-            conn = self.transfercmd(cmd)
-            try:
+            with self.transfercmd(cmd) as conn:
                 while 1:
                     buf = fp.readline()
                     if not buf: break
@@ -789,8 +814,6 @@ else:
                 # shutdown ssl layer
                 if isinstance(conn, ssl.SSLSocket):
                     conn.unwrap()
-            finally:
-                conn.close()
             return self.voidresp()
 
         def abort(self):
@@ -823,11 +846,7 @@ def parse150(resp):
     m = _150_re.match(resp)
     if not m:
         return None
-    s = m.group(1)
-    try:
-        return int(s)
-    except (OverflowError, ValueError):
-        return int(s)
+    return int(m.group(1))
 
 
 _227_re = None
diff --git a/Lib/functools.py b/Lib/functools.py
index 85ea257abd..ba6f9cd5d4 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -12,16 +12,25 @@ __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
            'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial']
 
 from _functools import partial, reduce
-from collections import OrderedDict, namedtuple
+from collections import namedtuple
 try:
-    from _thread import allocate_lock as Lock
+    from _thread import RLock
 except:
-    from _dummy_thread import allocate_lock as Lock
+    class RLock:
+        'Dummy reentrant lock'
+        def __enter__(self): pass
+        def __exit__(self, exctype, excinst, exctb): pass
+
+
+################################################################################
+### update_wrapper() and wraps() decorator
+################################################################################
 
 # update_wrapper() and wraps() are tools to help write
 # wrapper functions that can handle naive introspection
 
-WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__', '__annotations__')
+WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
+                       '__annotations__')
 WRAPPER_UPDATES = ('__dict__',)
 def update_wrapper(wrapper,
                    wrapped,
@@ -65,6 +74,11 @@ def wraps(wrapped,
     return partial(update_wrapper, wrapped=wrapped,
                    assigned=assigned, updated=updated)
 
+
+################################################################################
+### total_ordering class decorator
+################################################################################
+
 def total_ordering(cls):
     """Class decorator that fills in missing ordering methods"""
     convert = {
@@ -93,6 +107,11 @@ def total_ordering(cls):
             setattr(cls, opname, opfunc)
     return cls
 
+
+################################################################################
+### cmp_to_key() function converter
+################################################################################
+
 def cmp_to_key(mycmp):
     """Convert a cmp= function into a key= function"""
     class K(object):
@@ -114,95 +133,178 @@ def cmp_to_key(mycmp):
         __hash__ = None
     return K
 
-_CacheInfo = namedtuple("CacheInfo", "hits misses maxsize currsize")
+try:
+    from _functools import cmp_to_key
+except ImportError:
+    pass
+
+
+################################################################################
+### LRU Cache function decorator
+################################################################################
+
+_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
+
+class _HashedSeq(list):
+    __slots__ = 'hashvalue'
+
+    def __init__(self, tup, hash=hash):
+        self[:] = tup
+        self.hashvalue = hash(tup)
+
+    def __hash__(self):
+        return self.hashvalue
 
-def lru_cache(maxsize=100):
+def _make_key(args, kwds, typed,
+             kwd_mark = (object(),),
+             fasttypes = {int, str, frozenset, type(None)},
+             sorted=sorted, tuple=tuple, type=type, len=len):
+    'Make a cache key from optionally typed positional and keyword arguments'
+    key = args
+    if kwds:
+        sorted_items = sorted(kwds.items())
+        key += kwd_mark
+        for item in sorted_items:
+            key += item
+    if typed:
+        key += tuple(type(v) for v in args)
+        if kwds:
+            key += tuple(type(v) for k, v in sorted_items)
+    elif len(key) == 1 and type(key[0]) in fasttypes:
+        return key[0]
+    return _HashedSeq(key)
+
+def lru_cache(maxsize=128, typed=False):
     """Least-recently-used cache decorator.
 
     If *maxsize* is set to None, the LRU features are disabled and the cache
     can grow without bound.
 
+    If *typed* is True, arguments of different types will be cached separately.
+    For example, f(3.0) and f(3) will be treated as distinct calls with
+    distinct results.
+
     Arguments to the cached function must be hashable.
 
-    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
-    f.cache_info().  Clear the cache and statistics with f.cache_clear().
+    View the cache statistics named tuple (hits, misses, maxsize, currsize)
+    with f.cache_info().  Clear the cache and statistics with f.cache_clear().
     Access the underlying function with f.__wrapped__.
 
     See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
 
     """
+
     # Users should only access the lru_cache through its public API:
     #       cache_info, cache_clear, and f.__wrapped__
     # The internals of the lru_cache are encapsulated for thread safety and
     # to allow the implementation to change (including a possible C version).
 
-    def decorating_function(user_function,
-                tuple=tuple, sorted=sorted, len=len, KeyError=KeyError):
+    # Constants shared by all lru cache instances:
+    sentinel = object()          # unique object used to signal cache misses
+    make_key = _make_key         # build a key from the function arguments
+    PREV, NEXT, KEY, RESULT = 0, 1, 2, 3   # names for the link fields
+
+    def decorating_function(user_function):
 
-        hits = misses = 0
-        kwd_mark = (object(),)          # separates positional and keyword args
-        lock = Lock()                   # needed because OrderedDict isn't threadsafe
+        cache = {}
+        hits = misses = currsize = 0
+        full = False
+        cache_get = cache.get    # bound method to lookup a key or return None
+        lock = RLock()           # because linkedlist updates aren't threadsafe
+        root = []                # root of the circular doubly linked list
+        root[:] = [root, root, None, None]     # initialize by pointing to self
 
-        if maxsize is None:
-            cache = dict()              # simple cache without ordering or size limit
+        if maxsize == 0:
 
-            @wraps(user_function)
             def wrapper(*args, **kwds):
-                nonlocal hits, misses
-                key = args
-                if kwds:
-                    key += kwd_mark + tuple(sorted(kwds.items()))
-                try:
-                    result = cache[key]
+                # no caching, just a statistics update after a successful call
+                nonlocal misses
+                result = user_function(*args, **kwds)
+                misses += 1
+                return result
+
+        elif maxsize is None:
+
+            def wrapper(*args, **kwds):
+                # simple caching without ordering or size limit
+                nonlocal hits, misses, currsize
+                key = make_key(args, kwds, typed)
+                result = cache_get(key, sentinel)
+                if result is not sentinel:
                     hits += 1
                     return result
-                except KeyError:
-                    pass
                 result = user_function(*args, **kwds)
                 cache[key] = result
                 misses += 1
+                currsize += 1
                 return result
+
         else:
-            cache = OrderedDict()           # ordered least recent to most recent
-            cache_popitem = cache.popitem
-            cache_renew = cache.move_to_end
 
-            @wraps(user_function)
             def wrapper(*args, **kwds):
-                nonlocal hits, misses
-                key = args
-                if kwds:
-                    key += kwd_mark + tuple(sorted(kwds.items()))
+                # size limited caching that tracks accesses by recency
+                nonlocal root, hits, misses, currsize, full
+                key = make_key(args, kwds, typed)
                 with lock:
-                    try:
-                        result = cache[key]
-                        cache_renew(key)    # record recent use of this key
+                    link = cache_get(key)
+                    if link is not None:
+                        # move the link to the front of the circular queue
+                        link_prev, link_next, key, result = link
+                        link_prev[NEXT] = link_next
+                        link_next[PREV] = link_prev
+                        last = root[PREV]
+                        last[NEXT] = root[PREV] = link
+                        link[PREV] = last
+                        link[NEXT] = root
                         hits += 1
                         return result
-                    except KeyError:
-                        pass
                 result = user_function(*args, **kwds)
                 with lock:
-                    cache[key] = result     # record recent use of this key
+                    if key in cache:
+                        # getting here means that this same key was added to the
+                        # cache while the lock was released.  since the link
+                        # update is already done, we need only return the
+                        # computed result and update the count of misses.
+                        pass
+                    elif full:
+                        # use the old root to store the new key and result
+                        oldroot = root
+                        oldroot[KEY] = key
+                        oldroot[RESULT] = result
+                        # empty the oldest link and make it the new root
+                        root = oldroot[NEXT]
+                        oldkey = root[KEY]
+                        oldvalue = root[RESULT]
+                        root[KEY] = root[RESULT] = None
+                        # now update the cache dictionary for the new links
+                        del cache[oldkey]
+                        cache[key] = oldroot
+                    else:
+                        # put result in a new link at the front of the queue
+                        last = root[PREV]
+                        link = [last, root, key, result]
+                        last[NEXT] = root[PREV] = cache[key] = link
+                        currsize += 1
+                        full = (currsize >= maxsize)
                     misses += 1
-                    if len(cache) > maxsize:
-                        cache_popitem(0)    # purge least recently used cache entry
                 return result
 
         def cache_info():
             """Report cache statistics"""
             with lock:
-                return _CacheInfo(hits, misses, maxsize, len(cache))
+                return _CacheInfo(hits, misses, maxsize, currsize)
 
         def cache_clear():
             """Clear the cache and cache statistics"""
-            nonlocal hits, misses
+            nonlocal hits, misses, currsize, full
             with lock:
                 cache.clear()
-                hits = misses = 0
+                root[:] = [root, root, None, None]
+                hits = misses = currsize = 0
+                full = False
 
         wrapper.cache_info = cache_info
         wrapper.cache_clear = cache_clear
-        return wrapper
+        return update_wrapper(wrapper, user_function)
 
     return decorating_function
diff --git a/Lib/getopt.py b/Lib/getopt.py
index 980861d251..3d6ecbddb9 100644
--- a/Lib/getopt.py
+++ b/Lib/getopt.py
@@ -19,7 +19,7 @@ option involved with the exception.
 # Gerrit Holl <gerrit@nl.linux.org> moved the string-based exceptions
 # to class-based exceptions.
 #
-# Peter Åstrand <astrand@lysator.liu.se> added gnu_getopt().
+# Peter Ã…strand <astrand@lysator.liu.se> added gnu_getopt().
 #
 # TODO for gnu_getopt():
 #
@@ -34,6 +34,11 @@ option involved with the exception.
 __all__ = ["GetoptError","error","getopt","gnu_getopt"]
 
 import os
+try:
+    from gettext import gettext as _
+except ImportError:
+    # Bootstrapping Python: gettext's dependencies not built yet
+    def _(s): return s
 
 class GetoptError(Exception):
     opt = ''
@@ -153,10 +158,10 @@ def do_longs(opts, opt, longopts, args):
     if has_arg:
         if optarg is None:
             if not args:
-                raise GetoptError('option --%s requires argument' % opt, opt)
+                raise GetoptError(_('option --%s requires argument') % opt, opt)
             optarg, args = args[0], args[1:]
     elif optarg is not None:
-        raise GetoptError('option --%s must not have an argument' % opt, opt)
+        raise GetoptError(_('option --%s must not have an argument') % opt, opt)
     opts.append(('--' + opt, optarg or ''))
     return opts, args
 
@@ -166,7 +171,7 @@ def do_longs(opts, opt, longopts, args):
 def long_has_args(opt, longopts):
     possibilities = [o for o in longopts if o.startswith(opt)]
     if not possibilities:
-        raise GetoptError('option --%s not recognized' % opt, opt)
+        raise GetoptError(_('option --%s not recognized') % opt, opt)
     # Is there an exact match?
     if opt in possibilities:
         return False, opt
@@ -176,7 +181,7 @@ def long_has_args(opt, longopts):
     if len(possibilities) > 1:
         # XXX since possibilities contains all valid continuations, might be
         # nice to work them into the error msg
-        raise GetoptError('option --%s not a unique prefix' % opt, opt)
+        raise GetoptError(_('option --%s not a unique prefix') % opt, opt)
     assert len(possibilities) == 1
     unique_match = possibilities[0]
     has_arg = unique_match.endswith('=')
@@ -190,7 +195,7 @@ def do_shorts(opts, optstring, shortopts, args):
         if short_has_arg(opt, shortopts):
             if optstring == '':
                 if not args:
-                    raise GetoptError('option -%s requires argument' % opt,
+                    raise GetoptError(_('option -%s requires argument') % opt,
                                       opt)
                 optstring, args = args[0], args[1:]
             optarg, optstring = optstring, ''
@@ -203,7 +208,7 @@ def short_has_arg(opt, shortopts):
     for i in range(len(shortopts)):
         if opt == shortopts[i] != ':':
             return shortopts.startswith(':', i+1)
-    raise GetoptError('option -%s not recognized' % opt, opt)
+    raise GetoptError(_('option -%s not recognized') % opt, opt)
 
 if __name__ == '__main__':
     import sys
diff --git a/Lib/getpass.py b/Lib/getpass.py
index dc02bd1eae..0044742193 100644
--- a/Lib/getpass.py
+++ b/Lib/getpass.py
@@ -72,7 +72,7 @@ def unix_getpass(prompt='Password: ', stream=None):
             finally:
                 termios.tcsetattr(fd, tcsetattr_flags, old)
                 stream.flush()  # issue7208
-        except termios.error as e:
+        except termios.error:
             if passwd is not None:
                 # _raw_input succeeded.  The final tcsetattr failed.  Reraise
                 # instead of leaving the terminal in an unknown state.
@@ -145,8 +145,6 @@ def getuser():
 
     """
 
-    import os
-
     for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'):
         user = os.environ.get(name)
         if user:
diff --git a/Lib/gettext.py b/Lib/gettext.py
index 256e331eba..e43f044cc7 100644
--- a/Lib/gettext.py
+++ b/Lib/gettext.py
@@ -55,7 +55,7 @@ __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'dgettext', 'dngettext', 'gettext', 'ngettext',
            ]
 
-_default_localedir = os.path.join(sys.prefix, 'share', 'locale')
+_default_localedir = os.path.join(sys.base_prefix, 'share', 'locale')
 
 
 def c2py(plural):
diff --git a/Lib/gzip.py b/Lib/gzip.py
index 8fb1ed06c9..d7da02ca1b 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -16,31 +16,54 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
 
 READ, WRITE = 1, 2
 
-def U32(i):
-    """Return i as an unsigned integer, assuming it fits in 32 bits.
-    If it's >= 2GB when viewed as a 32-bit unsigned int, return a long.
-    """
-    if i < 0:
-        i += 1 << 32
-    return i
+def open(filename, mode="rb", compresslevel=9,
+         encoding=None, errors=None, newline=None):
+    """Open a gzip-compressed file in binary or text mode.
 
-def LOWU32(i):
-    """Return the low-order 32 bits, as a non-negative int"""
-    return i & 0xFFFFFFFF
+    The filename argument can be an actual filename (a str or bytes object), or
+    an existing file object to read from or write to.
 
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
+    The mode argument can be "r", "rb", "w", "wb", "a" or "ab" for binary mode,
+    or "rt", "wt" or "at" for text mode. The default mode is "rb", and the
+    default compresslevel is 9.
 
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
+    For binary mode, this function is equivalent to the GzipFile constructor:
+    GzipFile(filename, mode, compresslevel). In this case, the encoding, errors
+    and newline arguments must not be provided.
 
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
+    For text mode, a GzipFile object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
 
     """
-    return GzipFile(filename, mode, compresslevel)
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    gz_mode = mode.replace("t", "")
+    if isinstance(filename, (str, bytes)):
+        binary_file = GzipFile(filename, gz_mode, compresslevel)
+    elif hasattr(filename, "read") or hasattr(filename, "write"):
+        binary_file = GzipFile(None, gz_mode, compresslevel, filename)
+    else:
+        raise TypeError("filename must be a str or bytes object, or a file")
+
+    if "t" in mode:
+        return io.TextIOWrapper(binary_file, encoding, errors, newline)
+    else:
+        return binary_file
+
+def write32u(output, value):
+    # The L format writes the bit pattern correctly whether signed
+    # or unsigned.
+    output.write(struct.pack("<L", value))
 
 class _PaddedFile:
     """Minimal read-only file object that prepends a string to the contents
@@ -103,7 +126,7 @@ class GzipFile(io.BufferedIOBase):
     the exception of the readinto() and truncate() methods.
 
     This class only supports opening files in binary mode. If you need to open a
-    compressed file in text mode, wrap your GzipFile with an io.TextIOWrapper.
+    compressed file in text mode, use the gzip.open() function.
 
     """
 
@@ -151,7 +174,7 @@ class GzipFile(io.BufferedIOBase):
         """
 
         if mode and ('t' in mode or 'U' in mode):
-            raise IOError("Mode " + mode + " not supported")
+            raise ValueError("Invalid mode: {!r}".format(mode))
         if mode and 'b' not in mode:
             mode += 'b'
         if fileobj is None:
@@ -161,10 +184,9 @@ class GzipFile(io.BufferedIOBase):
             if not isinstance(filename, (str, bytes)):
                 filename = ''
         if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
+            mode = getattr(fileobj, 'mode', 'rb')
 
-        if mode[0:1] == 'r':
+        if mode.startswith('r'):
             self.mode = READ
             # Set flag indicating start of a new member
             self._new_member = True
@@ -179,7 +201,7 @@ class GzipFile(io.BufferedIOBase):
             self.min_readsize = 100
             fileobj = _PaddedFile(fileobj)
 
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
+        elif mode.startswith(('w', 'a')):
             self.mode = WRITE
             self._init_write(filename)
             self.compress = zlib.compressobj(compresslevel,
@@ -188,7 +210,7 @@ class GzipFile(io.BufferedIOBase):
                                              zlib.DEF_MEM_LEVEL,
                                              0)
         else:
-            raise IOError("Mode " + mode + " not supported")
+            raise ValueError("Invalid mode: {!r}".format(mode))
 
         self.fileobj = fileobj
         self.offset = 0
@@ -352,6 +374,28 @@ class GzipFile(io.BufferedIOBase):
         self.offset += size
         return chunk
 
+    def read1(self, size=-1):
+        self._check_closed()
+        if self.mode != READ:
+            import errno
+            raise IOError(errno.EBADF, "read1() on write-only GzipFile object")
+
+        if self.extrasize <= 0 and self.fileobj is None:
+            return b''
+
+        # For certain input data, a single call to _read() may not return
+        # any data. In this case, retry until we get some data or reach EOF.
+        while self.extrasize <= 0 and self._read():
+            pass
+        if size < 0 or size > self.extrasize:
+            size = self.extrasize
+
+        offset = self.offset - self.extrastart
+        chunk = self.extrabuf[offset: offset + size]
+        self.extrasize -= size
+        self.offset += size
+        return chunk
+
     def peek(self, n):
         if self.mode != READ:
             import errno
diff --git a/Lib/heapq.py b/Lib/heapq.py
index dec15aec9c..00b429c2d3 100644
--- a/Lib/heapq.py
+++ b/Lib/heapq.py
@@ -127,8 +127,7 @@ From all times, sorting has always been a Great Art! :-)
 __all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
            'nlargest', 'nsmallest', 'heappushpop']
 
-from itertools import islice, repeat, count, tee, chain
-import bisect
+from itertools import islice, count, tee, chain
 
 def heappush(heap, item):
     """Push item onto heap, maintaining the heap invariant."""
@@ -180,6 +179,19 @@ def heapify(x):
     for i in reversed(range(n//2)):
         _siftup(x, i)
 
+def _heappushpop_max(heap, item):
+    """Maxheap version of a heappush followed by a heappop."""
+    if heap and item < heap[0]:
+        item, heap[0] = heap[0], item
+        _siftup_max(heap, 0)
+    return item
+
+def _heapify_max(x):
+    """Transform list into a maxheap, in-place, in O(len(x)) time."""
+    n = len(x)
+    for i in reversed(range(n//2)):
+        _siftup_max(x, i)
+
 def nlargest(n, iterable):
     """Find the n largest elements in a dataset.
 
@@ -205,30 +217,16 @@ def nsmallest(n, iterable):
     """
     if n < 0:
         return []
-    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
-        # For smaller values of n, the bisect method is faster than a minheap.
-        # It is also memory efficient, consuming only n elements of space.
-        it = iter(iterable)
-        result = sorted(islice(it, 0, n))
-        if not result:
-            return result
-        insort = bisect.insort
-        pop = result.pop
-        los = result[-1]    # los --> Largest of the nsmallest
-        for elem in it:
-            if elem < los:
-                insort(result, elem)
-                pop()
-                los = result[-1]
+    it = iter(iterable)
+    result = list(islice(it, n))
+    if not result:
         return result
-    # An alternative approach manifests the whole iterable in memory but
-    # saves comparisons by heapifying all at once.  Also, saves time
-    # over bisect.insort() which has O(n) data movement time for every
-    # insertion.  Finding the n smallest of an m length iterable requires
-    #    O(m) + O(n log m) comparisons.
-    h = list(iterable)
-    heapify(h)
-    return list(map(heappop, repeat(h, min(n, len(h)))))
+    _heapify_max(result)
+    _heappushpop = _heappushpop_max
+    for elem in it:
+        _heappushpop(result, elem)
+    result.sort()
+    return result
 
 # 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
 # is the index of a leaf with a possibly out-of-order value.  Restore the
@@ -306,6 +304,42 @@ def _siftup(heap, pos):
     heap[pos] = newitem
     _siftdown(heap, startpos, pos)
 
+def _siftdown_max(heap, startpos, pos):
+    'Maxheap variant of _siftdown'
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if parent < newitem:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+def _siftup_max(heap, pos):
+    'Maxheap variant of _siftup'
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the larger child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of larger child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[rightpos] < heap[childpos]:
+            childpos = rightpos
+        # Move the larger child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown_max(heap, startpos, pos)
+
 # If available, use C implementation
 try:
     from _heapq import *
diff --git a/Lib/hmac.py b/Lib/hmac.py
index 956fc65d2c..4297a7171a 100644
--- a/Lib/hmac.py
+++ b/Lib/hmac.py
@@ -4,6 +4,7 @@ Implements the HMAC algorithm as described by RFC 2104.
 """
 
 import warnings as _warnings
+from operator import _compare_digest as compare_digest
 
 trans_5C = bytes((x ^ 0x5C) for x in range(256))
 trans_36 = bytes((x ^ 0x36) for x in range(256))
@@ -13,6 +14,7 @@ trans_36 = bytes((x ^ 0x36) for x in range(256))
 digest_size = None
 
 
+
 class HMAC:
     """RFC 2104 HMAC class.  Also complies with RFC 4231.
 
@@ -33,7 +35,7 @@ class HMAC:
         """
 
         if not isinstance(key, bytes):
-            raise TypeError("expected bytes, but got %r" % type(key).__name__)
+            raise TypeError("key: expected bytes, but got %r" % type(key).__name__)
 
         if digestmod is None:
             import hashlib
diff --git a/Lib/html/entities.py b/Lib/html/entities.py
index e2b7bf1fb8..e891ad6599 100644
--- a/Lib/html/entities.py
+++ b/Lib/html/entities.py
@@ -256,6 +256,2242 @@ name2codepoint = {
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
+
+# maps the HTML5 named character references to the equivalent Unicode character(s)
+html5 = {
+    'Aacute': '\xc1',
+    'aacute': '\xe1',
+    'Aacute;': '\xc1',
+    'aacute;': '\xe1',
+    'Abreve;': '\u0102',
+    'abreve;': '\u0103',
+    'ac;': '\u223e',
+    'acd;': '\u223f',
+    'acE;': '\u223e\u0333',
+    'Acirc': '\xc2',
+    'acirc': '\xe2',
+    'Acirc;': '\xc2',
+    'acirc;': '\xe2',
+    'acute': '\xb4',
+    'acute;': '\xb4',
+    'Acy;': '\u0410',
+    'acy;': '\u0430',
+    'AElig': '\xc6',
+    'aelig': '\xe6',
+    'AElig;': '\xc6',
+    'aelig;': '\xe6',
+    'af;': '\u2061',
+    'Afr;': '\U0001d504',
+    'afr;': '\U0001d51e',
+    'Agrave': '\xc0',
+    'agrave': '\xe0',
+    'Agrave;': '\xc0',
+    'agrave;': '\xe0',
+    'alefsym;': '\u2135',
+    'aleph;': '\u2135',
+    'Alpha;': '\u0391',
+    'alpha;': '\u03b1',
+    'Amacr;': '\u0100',
+    'amacr;': '\u0101',
+    'amalg;': '\u2a3f',
+    'AMP': '&',
+    'amp': '&',
+    'AMP;': '&',
+    'amp;': '&',
+    'And;': '\u2a53',
+    'and;': '\u2227',
+    'andand;': '\u2a55',
+    'andd;': '\u2a5c',
+    'andslope;': '\u2a58',
+    'andv;': '\u2a5a',
+    'ang;': '\u2220',
+    'ange;': '\u29a4',
+    'angle;': '\u2220',
+    'angmsd;': '\u2221',
+    'angmsdaa;': '\u29a8',
+    'angmsdab;': '\u29a9',
+    'angmsdac;': '\u29aa',
+    'angmsdad;': '\u29ab',
+    'angmsdae;': '\u29ac',
+    'angmsdaf;': '\u29ad',
+    'angmsdag;': '\u29ae',
+    'angmsdah;': '\u29af',
+    'angrt;': '\u221f',
+    'angrtvb;': '\u22be',
+    'angrtvbd;': '\u299d',
+    'angsph;': '\u2222',
+    'angst;': '\xc5',
+    'angzarr;': '\u237c',
+    'Aogon;': '\u0104',
+    'aogon;': '\u0105',
+    'Aopf;': '\U0001d538',
+    'aopf;': '\U0001d552',
+    'ap;': '\u2248',
+    'apacir;': '\u2a6f',
+    'apE;': '\u2a70',
+    'ape;': '\u224a',
+    'apid;': '\u224b',
+    'apos;': "'",
+    'ApplyFunction;': '\u2061',
+    'approx;': '\u2248',
+    'approxeq;': '\u224a',
+    'Aring': '\xc5',
+    'aring': '\xe5',
+    'Aring;': '\xc5',
+    'aring;': '\xe5',
+    'Ascr;': '\U0001d49c',
+    'ascr;': '\U0001d4b6',
+    'Assign;': '\u2254',
+    'ast;': '*',
+    'asymp;': '\u2248',
+    'asympeq;': '\u224d',
+    'Atilde': '\xc3',
+    'atilde': '\xe3',
+    'Atilde;': '\xc3',
+    'atilde;': '\xe3',
+    'Auml': '\xc4',
+    'auml': '\xe4',
+    'Auml;': '\xc4',
+    'auml;': '\xe4',
+    'awconint;': '\u2233',
+    'awint;': '\u2a11',
+    'backcong;': '\u224c',
+    'backepsilon;': '\u03f6',
+    'backprime;': '\u2035',
+    'backsim;': '\u223d',
+    'backsimeq;': '\u22cd',
+    'Backslash;': '\u2216',
+    'Barv;': '\u2ae7',
+    'barvee;': '\u22bd',
+    'Barwed;': '\u2306',
+    'barwed;': '\u2305',
+    'barwedge;': '\u2305',
+    'bbrk;': '\u23b5',
+    'bbrktbrk;': '\u23b6',
+    'bcong;': '\u224c',
+    'Bcy;': '\u0411',
+    'bcy;': '\u0431',
+    'bdquo;': '\u201e',
+    'becaus;': '\u2235',
+    'Because;': '\u2235',
+    'because;': '\u2235',
+    'bemptyv;': '\u29b0',
+    'bepsi;': '\u03f6',
+    'bernou;': '\u212c',
+    'Bernoullis;': '\u212c',
+    'Beta;': '\u0392',
+    'beta;': '\u03b2',
+    'beth;': '\u2136',
+    'between;': '\u226c',
+    'Bfr;': '\U0001d505',
+    'bfr;': '\U0001d51f',
+    'bigcap;': '\u22c2',
+    'bigcirc;': '\u25ef',
+    'bigcup;': '\u22c3',
+    'bigodot;': '\u2a00',
+    'bigoplus;': '\u2a01',
+    'bigotimes;': '\u2a02',
+    'bigsqcup;': '\u2a06',
+    'bigstar;': '\u2605',
+    'bigtriangledown;': '\u25bd',
+    'bigtriangleup;': '\u25b3',
+    'biguplus;': '\u2a04',
+    'bigvee;': '\u22c1',
+    'bigwedge;': '\u22c0',
+    'bkarow;': '\u290d',
+    'blacklozenge;': '\u29eb',
+    'blacksquare;': '\u25aa',
+    'blacktriangle;': '\u25b4',
+    'blacktriangledown;': '\u25be',
+    'blacktriangleleft;': '\u25c2',
+    'blacktriangleright;': '\u25b8',
+    'blank;': '\u2423',
+    'blk12;': '\u2592',
+    'blk14;': '\u2591',
+    'blk34;': '\u2593',
+    'block;': '\u2588',
+    'bne;': '=\u20e5',
+    'bnequiv;': '\u2261\u20e5',
+    'bNot;': '\u2aed',
+    'bnot;': '\u2310',
+    'Bopf;': '\U0001d539',
+    'bopf;': '\U0001d553',
+    'bot;': '\u22a5',
+    'bottom;': '\u22a5',
+    'bowtie;': '\u22c8',
+    'boxbox;': '\u29c9',
+    'boxDL;': '\u2557',
+    'boxDl;': '\u2556',
+    'boxdL;': '\u2555',
+    'boxdl;': '\u2510',
+    'boxDR;': '\u2554',
+    'boxDr;': '\u2553',
+    'boxdR;': '\u2552',
+    'boxdr;': '\u250c',
+    'boxH;': '\u2550',
+    'boxh;': '\u2500',
+    'boxHD;': '\u2566',
+    'boxHd;': '\u2564',
+    'boxhD;': '\u2565',
+    'boxhd;': '\u252c',
+    'boxHU;': '\u2569',
+    'boxHu;': '\u2567',
+    'boxhU;': '\u2568',
+    'boxhu;': '\u2534',
+    'boxminus;': '\u229f',
+    'boxplus;': '\u229e',
+    'boxtimes;': '\u22a0',
+    'boxUL;': '\u255d',
+    'boxUl;': '\u255c',
+    'boxuL;': '\u255b',
+    'boxul;': '\u2518',
+    'boxUR;': '\u255a',
+    'boxUr;': '\u2559',
+    'boxuR;': '\u2558',
+    'boxur;': '\u2514',
+    'boxV;': '\u2551',
+    'boxv;': '\u2502',
+    'boxVH;': '\u256c',
+    'boxVh;': '\u256b',
+    'boxvH;': '\u256a',
+    'boxvh;': '\u253c',
+    'boxVL;': '\u2563',
+    'boxVl;': '\u2562',
+    'boxvL;': '\u2561',
+    'boxvl;': '\u2524',
+    'boxVR;': '\u2560',
+    'boxVr;': '\u255f',
+    'boxvR;': '\u255e',
+    'boxvr;': '\u251c',
+    'bprime;': '\u2035',
+    'Breve;': '\u02d8',
+    'breve;': '\u02d8',
+    'brvbar': '\xa6',
+    'brvbar;': '\xa6',
+    'Bscr;': '\u212c',
+    'bscr;': '\U0001d4b7',
+    'bsemi;': '\u204f',
+    'bsim;': '\u223d',
+    'bsime;': '\u22cd',
+    'bsol;': '\\',
+    'bsolb;': '\u29c5',
+    'bsolhsub;': '\u27c8',
+    'bull;': '\u2022',
+    'bullet;': '\u2022',
+    'bump;': '\u224e',
+    'bumpE;': '\u2aae',
+    'bumpe;': '\u224f',
+    'Bumpeq;': '\u224e',
+    'bumpeq;': '\u224f',
+    'Cacute;': '\u0106',
+    'cacute;': '\u0107',
+    'Cap;': '\u22d2',
+    'cap;': '\u2229',
+    'capand;': '\u2a44',
+    'capbrcup;': '\u2a49',
+    'capcap;': '\u2a4b',
+    'capcup;': '\u2a47',
+    'capdot;': '\u2a40',
+    'CapitalDifferentialD;': '\u2145',
+    'caps;': '\u2229\ufe00',
+    'caret;': '\u2041',
+    'caron;': '\u02c7',
+    'Cayleys;': '\u212d',
+    'ccaps;': '\u2a4d',
+    'Ccaron;': '\u010c',
+    'ccaron;': '\u010d',
+    'Ccedil': '\xc7',
+    'ccedil': '\xe7',
+    'Ccedil;': '\xc7',
+    'ccedil;': '\xe7',
+    'Ccirc;': '\u0108',
+    'ccirc;': '\u0109',
+    'Cconint;': '\u2230',
+    'ccups;': '\u2a4c',
+    'ccupssm;': '\u2a50',
+    'Cdot;': '\u010a',
+    'cdot;': '\u010b',
+    'cedil': '\xb8',
+    'cedil;': '\xb8',
+    'Cedilla;': '\xb8',
+    'cemptyv;': '\u29b2',
+    'cent': '\xa2',
+    'cent;': '\xa2',
+    'CenterDot;': '\xb7',
+    'centerdot;': '\xb7',
+    'Cfr;': '\u212d',
+    'cfr;': '\U0001d520',
+    'CHcy;': '\u0427',
+    'chcy;': '\u0447',
+    'check;': '\u2713',
+    'checkmark;': '\u2713',
+    'Chi;': '\u03a7',
+    'chi;': '\u03c7',
+    'cir;': '\u25cb',
+    'circ;': '\u02c6',
+    'circeq;': '\u2257',
+    'circlearrowleft;': '\u21ba',
+    'circlearrowright;': '\u21bb',
+    'circledast;': '\u229b',
+    'circledcirc;': '\u229a',
+    'circleddash;': '\u229d',
+    'CircleDot;': '\u2299',
+    'circledR;': '\xae',
+    'circledS;': '\u24c8',
+    'CircleMinus;': '\u2296',
+    'CirclePlus;': '\u2295',
+    'CircleTimes;': '\u2297',
+    'cirE;': '\u29c3',
+    'cire;': '\u2257',
+    'cirfnint;': '\u2a10',
+    'cirmid;': '\u2aef',
+    'cirscir;': '\u29c2',
+    'ClockwiseContourIntegral;': '\u2232',
+    'CloseCurlyDoubleQuote;': '\u201d',
+    'CloseCurlyQuote;': '\u2019',
+    'clubs;': '\u2663',
+    'clubsuit;': '\u2663',
+    'Colon;': '\u2237',
+    'colon;': ':',
+    'Colone;': '\u2a74',
+    'colone;': '\u2254',
+    'coloneq;': '\u2254',
+    'comma;': ',',
+    'commat;': '@',
+    'comp;': '\u2201',
+    'compfn;': '\u2218',
+    'complement;': '\u2201',
+    'complexes;': '\u2102',
+    'cong;': '\u2245',
+    'congdot;': '\u2a6d',
+    'Congruent;': '\u2261',
+    'Conint;': '\u222f',
+    'conint;': '\u222e',
+    'ContourIntegral;': '\u222e',
+    'Copf;': '\u2102',
+    'copf;': '\U0001d554',
+    'coprod;': '\u2210',
+    'Coproduct;': '\u2210',
+    'COPY': '\xa9',
+    'copy': '\xa9',
+    'COPY;': '\xa9',
+    'copy;': '\xa9',
+    'copysr;': '\u2117',
+    'CounterClockwiseContourIntegral;': '\u2233',
+    'crarr;': '\u21b5',
+    'Cross;': '\u2a2f',
+    'cross;': '\u2717',
+    'Cscr;': '\U0001d49e',
+    'cscr;': '\U0001d4b8',
+    'csub;': '\u2acf',
+    'csube;': '\u2ad1',
+    'csup;': '\u2ad0',
+    'csupe;': '\u2ad2',
+    'ctdot;': '\u22ef',
+    'cudarrl;': '\u2938',
+    'cudarrr;': '\u2935',
+    'cuepr;': '\u22de',
+    'cuesc;': '\u22df',
+    'cularr;': '\u21b6',
+    'cularrp;': '\u293d',
+    'Cup;': '\u22d3',
+    'cup;': '\u222a',
+    'cupbrcap;': '\u2a48',
+    'CupCap;': '\u224d',
+    'cupcap;': '\u2a46',
+    'cupcup;': '\u2a4a',
+    'cupdot;': '\u228d',
+    'cupor;': '\u2a45',
+    'cups;': '\u222a\ufe00',
+    'curarr;': '\u21b7',
+    'curarrm;': '\u293c',
+    'curlyeqprec;': '\u22de',
+    'curlyeqsucc;': '\u22df',
+    'curlyvee;': '\u22ce',
+    'curlywedge;': '\u22cf',
+    'curren': '\xa4',
+    'curren;': '\xa4',
+    'curvearrowleft;': '\u21b6',
+    'curvearrowright;': '\u21b7',
+    'cuvee;': '\u22ce',
+    'cuwed;': '\u22cf',
+    'cwconint;': '\u2232',
+    'cwint;': '\u2231',
+    'cylcty;': '\u232d',
+    'Dagger;': '\u2021',
+    'dagger;': '\u2020',
+    'daleth;': '\u2138',
+    'Darr;': '\u21a1',
+    'dArr;': '\u21d3',
+    'darr;': '\u2193',
+    'dash;': '\u2010',
+    'Dashv;': '\u2ae4',
+    'dashv;': '\u22a3',
+    'dbkarow;': '\u290f',
+    'dblac;': '\u02dd',
+    'Dcaron;': '\u010e',
+    'dcaron;': '\u010f',
+    'Dcy;': '\u0414',
+    'dcy;': '\u0434',
+    'DD;': '\u2145',
+    'dd;': '\u2146',
+    'ddagger;': '\u2021',
+    'ddarr;': '\u21ca',
+    'DDotrahd;': '\u2911',
+    'ddotseq;': '\u2a77',
+    'deg': '\xb0',
+    'deg;': '\xb0',
+    'Del;': '\u2207',
+    'Delta;': '\u0394',
+    'delta;': '\u03b4',
+    'demptyv;': '\u29b1',
+    'dfisht;': '\u297f',
+    'Dfr;': '\U0001d507',
+    'dfr;': '\U0001d521',
+    'dHar;': '\u2965',
+    'dharl;': '\u21c3',
+    'dharr;': '\u21c2',
+    'DiacriticalAcute;': '\xb4',
+    'DiacriticalDot;': '\u02d9',
+    'DiacriticalDoubleAcute;': '\u02dd',
+    'DiacriticalGrave;': '`',
+    'DiacriticalTilde;': '\u02dc',
+    'diam;': '\u22c4',
+    'Diamond;': '\u22c4',
+    'diamond;': '\u22c4',
+    'diamondsuit;': '\u2666',
+    'diams;': '\u2666',
+    'die;': '\xa8',
+    'DifferentialD;': '\u2146',
+    'digamma;': '\u03dd',
+    'disin;': '\u22f2',
+    'div;': '\xf7',
+    'divide': '\xf7',
+    'divide;': '\xf7',
+    'divideontimes;': '\u22c7',
+    'divonx;': '\u22c7',
+    'DJcy;': '\u0402',
+    'djcy;': '\u0452',
+    'dlcorn;': '\u231e',
+    'dlcrop;': '\u230d',
+    'dollar;': '$',
+    'Dopf;': '\U0001d53b',
+    'dopf;': '\U0001d555',
+    'Dot;': '\xa8',
+    'dot;': '\u02d9',
+    'DotDot;': '\u20dc',
+    'doteq;': '\u2250',
+    'doteqdot;': '\u2251',
+    'DotEqual;': '\u2250',
+    'dotminus;': '\u2238',
+    'dotplus;': '\u2214',
+    'dotsquare;': '\u22a1',
+    'doublebarwedge;': '\u2306',
+    'DoubleContourIntegral;': '\u222f',
+    'DoubleDot;': '\xa8',
+    'DoubleDownArrow;': '\u21d3',
+    'DoubleLeftArrow;': '\u21d0',
+    'DoubleLeftRightArrow;': '\u21d4',
+    'DoubleLeftTee;': '\u2ae4',
+    'DoubleLongLeftArrow;': '\u27f8',
+    'DoubleLongLeftRightArrow;': '\u27fa',
+    'DoubleLongRightArrow;': '\u27f9',
+    'DoubleRightArrow;': '\u21d2',
+    'DoubleRightTee;': '\u22a8',
+    'DoubleUpArrow;': '\u21d1',
+    'DoubleUpDownArrow;': '\u21d5',
+    'DoubleVerticalBar;': '\u2225',
+    'DownArrow;': '\u2193',
+    'Downarrow;': '\u21d3',
+    'downarrow;': '\u2193',
+    'DownArrowBar;': '\u2913',
+    'DownArrowUpArrow;': '\u21f5',
+    'DownBreve;': '\u0311',
+    'downdownarrows;': '\u21ca',
+    'downharpoonleft;': '\u21c3',
+    'downharpoonright;': '\u21c2',
+    'DownLeftRightVector;': '\u2950',
+    'DownLeftTeeVector;': '\u295e',
+    'DownLeftVector;': '\u21bd',
+    'DownLeftVectorBar;': '\u2956',
+    'DownRightTeeVector;': '\u295f',
+    'DownRightVector;': '\u21c1',
+    'DownRightVectorBar;': '\u2957',
+    'DownTee;': '\u22a4',
+    'DownTeeArrow;': '\u21a7',
+    'drbkarow;': '\u2910',
+    'drcorn;': '\u231f',
+    'drcrop;': '\u230c',
+    'Dscr;': '\U0001d49f',
+    'dscr;': '\U0001d4b9',
+    'DScy;': '\u0405',
+    'dscy;': '\u0455',
+    'dsol;': '\u29f6',
+    'Dstrok;': '\u0110',
+    'dstrok;': '\u0111',
+    'dtdot;': '\u22f1',
+    'dtri;': '\u25bf',
+    'dtrif;': '\u25be',
+    'duarr;': '\u21f5',
+    'duhar;': '\u296f',
+    'dwangle;': '\u29a6',
+    'DZcy;': '\u040f',
+    'dzcy;': '\u045f',
+    'dzigrarr;': '\u27ff',
+    'Eacute': '\xc9',
+    'eacute': '\xe9',
+    'Eacute;': '\xc9',
+    'eacute;': '\xe9',
+    'easter;': '\u2a6e',
+    'Ecaron;': '\u011a',
+    'ecaron;': '\u011b',
+    'ecir;': '\u2256',
+    'Ecirc': '\xca',
+    'ecirc': '\xea',
+    'Ecirc;': '\xca',
+    'ecirc;': '\xea',
+    'ecolon;': '\u2255',
+    'Ecy;': '\u042d',
+    'ecy;': '\u044d',
+    'eDDot;': '\u2a77',
+    'Edot;': '\u0116',
+    'eDot;': '\u2251',
+    'edot;': '\u0117',
+    'ee;': '\u2147',
+    'efDot;': '\u2252',
+    'Efr;': '\U0001d508',
+    'efr;': '\U0001d522',
+    'eg;': '\u2a9a',
+    'Egrave': '\xc8',
+    'egrave': '\xe8',
+    'Egrave;': '\xc8',
+    'egrave;': '\xe8',
+    'egs;': '\u2a96',
+    'egsdot;': '\u2a98',
+    'el;': '\u2a99',
+    'Element;': '\u2208',
+    'elinters;': '\u23e7',
+    'ell;': '\u2113',
+    'els;': '\u2a95',
+    'elsdot;': '\u2a97',
+    'Emacr;': '\u0112',
+    'emacr;': '\u0113',
+    'empty;': '\u2205',
+    'emptyset;': '\u2205',
+    'EmptySmallSquare;': '\u25fb',
+    'emptyv;': '\u2205',
+    'EmptyVerySmallSquare;': '\u25ab',
+    'emsp13;': '\u2004',
+    'emsp14;': '\u2005',
+    'emsp;': '\u2003',
+    'ENG;': '\u014a',
+    'eng;': '\u014b',
+    'ensp;': '\u2002',
+    'Eogon;': '\u0118',
+    'eogon;': '\u0119',
+    'Eopf;': '\U0001d53c',
+    'eopf;': '\U0001d556',
+    'epar;': '\u22d5',
+    'eparsl;': '\u29e3',
+    'eplus;': '\u2a71',
+    'epsi;': '\u03b5',
+    'Epsilon;': '\u0395',
+    'epsilon;': '\u03b5',
+    'epsiv;': '\u03f5',
+    'eqcirc;': '\u2256',
+    'eqcolon;': '\u2255',
+    'eqsim;': '\u2242',
+    'eqslantgtr;': '\u2a96',
+    'eqslantless;': '\u2a95',
+    'Equal;': '\u2a75',
+    'equals;': '=',
+    'EqualTilde;': '\u2242',
+    'equest;': '\u225f',
+    'Equilibrium;': '\u21cc',
+    'equiv;': '\u2261',
+    'equivDD;': '\u2a78',
+    'eqvparsl;': '\u29e5',
+    'erarr;': '\u2971',
+    'erDot;': '\u2253',
+    'Escr;': '\u2130',
+    'escr;': '\u212f',
+    'esdot;': '\u2250',
+    'Esim;': '\u2a73',
+    'esim;': '\u2242',
+    'Eta;': '\u0397',
+    'eta;': '\u03b7',
+    'ETH': '\xd0',
+    'eth': '\xf0',
+    'ETH;': '\xd0',
+    'eth;': '\xf0',
+    'Euml': '\xcb',
+    'euml': '\xeb',
+    'Euml;': '\xcb',
+    'euml;': '\xeb',
+    'euro;': '\u20ac',
+    'excl;': '!',
+    'exist;': '\u2203',
+    'Exists;': '\u2203',
+    'expectation;': '\u2130',
+    'ExponentialE;': '\u2147',
+    'exponentiale;': '\u2147',
+    'fallingdotseq;': '\u2252',
+    'Fcy;': '\u0424',
+    'fcy;': '\u0444',
+    'female;': '\u2640',
+    'ffilig;': '\ufb03',
+    'fflig;': '\ufb00',
+    'ffllig;': '\ufb04',
+    'Ffr;': '\U0001d509',
+    'ffr;': '\U0001d523',
+    'filig;': '\ufb01',
+    'FilledSmallSquare;': '\u25fc',
+    'FilledVerySmallSquare;': '\u25aa',
+    'fjlig;': 'fj',
+    'flat;': '\u266d',
+    'fllig;': '\ufb02',
+    'fltns;': '\u25b1',
+    'fnof;': '\u0192',
+    'Fopf;': '\U0001d53d',
+    'fopf;': '\U0001d557',
+    'ForAll;': '\u2200',
+    'forall;': '\u2200',
+    'fork;': '\u22d4',
+    'forkv;': '\u2ad9',
+    'Fouriertrf;': '\u2131',
+    'fpartint;': '\u2a0d',
+    'frac12': '\xbd',
+    'frac12;': '\xbd',
+    'frac13;': '\u2153',
+    'frac14': '\xbc',
+    'frac14;': '\xbc',
+    'frac15;': '\u2155',
+    'frac16;': '\u2159',
+    'frac18;': '\u215b',
+    'frac23;': '\u2154',
+    'frac25;': '\u2156',
+    'frac34': '\xbe',
+    'frac34;': '\xbe',
+    'frac35;': '\u2157',
+    'frac38;': '\u215c',
+    'frac45;': '\u2158',
+    'frac56;': '\u215a',
+    'frac58;': '\u215d',
+    'frac78;': '\u215e',
+    'frasl;': '\u2044',
+    'frown;': '\u2322',
+    'Fscr;': '\u2131',
+    'fscr;': '\U0001d4bb',
+    'gacute;': '\u01f5',
+    'Gamma;': '\u0393',
+    'gamma;': '\u03b3',
+    'Gammad;': '\u03dc',
+    'gammad;': '\u03dd',
+    'gap;': '\u2a86',
+    'Gbreve;': '\u011e',
+    'gbreve;': '\u011f',
+    'Gcedil;': '\u0122',
+    'Gcirc;': '\u011c',
+    'gcirc;': '\u011d',
+    'Gcy;': '\u0413',
+    'gcy;': '\u0433',
+    'Gdot;': '\u0120',
+    'gdot;': '\u0121',
+    'gE;': '\u2267',
+    'ge;': '\u2265',
+    'gEl;': '\u2a8c',
+    'gel;': '\u22db',
+    'geq;': '\u2265',
+    'geqq;': '\u2267',
+    'geqslant;': '\u2a7e',
+    'ges;': '\u2a7e',
+    'gescc;': '\u2aa9',
+    'gesdot;': '\u2a80',
+    'gesdoto;': '\u2a82',
+    'gesdotol;': '\u2a84',
+    'gesl;': '\u22db\ufe00',
+    'gesles;': '\u2a94',
+    'Gfr;': '\U0001d50a',
+    'gfr;': '\U0001d524',
+    'Gg;': '\u22d9',
+    'gg;': '\u226b',
+    'ggg;': '\u22d9',
+    'gimel;': '\u2137',
+    'GJcy;': '\u0403',
+    'gjcy;': '\u0453',
+    'gl;': '\u2277',
+    'gla;': '\u2aa5',
+    'glE;': '\u2a92',
+    'glj;': '\u2aa4',
+    'gnap;': '\u2a8a',
+    'gnapprox;': '\u2a8a',
+    'gnE;': '\u2269',
+    'gne;': '\u2a88',
+    'gneq;': '\u2a88',
+    'gneqq;': '\u2269',
+    'gnsim;': '\u22e7',
+    'Gopf;': '\U0001d53e',
+    'gopf;': '\U0001d558',
+    'grave;': '`',
+    'GreaterEqual;': '\u2265',
+    'GreaterEqualLess;': '\u22db',
+    'GreaterFullEqual;': '\u2267',
+    'GreaterGreater;': '\u2aa2',
+    'GreaterLess;': '\u2277',
+    'GreaterSlantEqual;': '\u2a7e',
+    'GreaterTilde;': '\u2273',
+    'Gscr;': '\U0001d4a2',
+    'gscr;': '\u210a',
+    'gsim;': '\u2273',
+    'gsime;': '\u2a8e',
+    'gsiml;': '\u2a90',
+    'GT': '>',
+    'gt': '>',
+    'GT;': '>',
+    'Gt;': '\u226b',
+    'gt;': '>',
+    'gtcc;': '\u2aa7',
+    'gtcir;': '\u2a7a',
+    'gtdot;': '\u22d7',
+    'gtlPar;': '\u2995',
+    'gtquest;': '\u2a7c',
+    'gtrapprox;': '\u2a86',
+    'gtrarr;': '\u2978',
+    'gtrdot;': '\u22d7',
+    'gtreqless;': '\u22db',
+    'gtreqqless;': '\u2a8c',
+    'gtrless;': '\u2277',
+    'gtrsim;': '\u2273',
+    'gvertneqq;': '\u2269\ufe00',
+    'gvnE;': '\u2269\ufe00',
+    'Hacek;': '\u02c7',
+    'hairsp;': '\u200a',
+    'half;': '\xbd',
+    'hamilt;': '\u210b',
+    'HARDcy;': '\u042a',
+    'hardcy;': '\u044a',
+    'hArr;': '\u21d4',
+    'harr;': '\u2194',
+    'harrcir;': '\u2948',
+    'harrw;': '\u21ad',
+    'Hat;': '^',
+    'hbar;': '\u210f',
+    'Hcirc;': '\u0124',
+    'hcirc;': '\u0125',
+    'hearts;': '\u2665',
+    'heartsuit;': '\u2665',
+    'hellip;': '\u2026',
+    'hercon;': '\u22b9',
+    'Hfr;': '\u210c',
+    'hfr;': '\U0001d525',
+    'HilbertSpace;': '\u210b',
+    'hksearow;': '\u2925',
+    'hkswarow;': '\u2926',
+    'hoarr;': '\u21ff',
+    'homtht;': '\u223b',
+    'hookleftarrow;': '\u21a9',
+    'hookrightarrow;': '\u21aa',
+    'Hopf;': '\u210d',
+    'hopf;': '\U0001d559',
+    'horbar;': '\u2015',
+    'HorizontalLine;': '\u2500',
+    'Hscr;': '\u210b',
+    'hscr;': '\U0001d4bd',
+    'hslash;': '\u210f',
+    'Hstrok;': '\u0126',
+    'hstrok;': '\u0127',
+    'HumpDownHump;': '\u224e',
+    'HumpEqual;': '\u224f',
+    'hybull;': '\u2043',
+    'hyphen;': '\u2010',
+    'Iacute': '\xcd',
+    'iacute': '\xed',
+    'Iacute;': '\xcd',
+    'iacute;': '\xed',
+    'ic;': '\u2063',
+    'Icirc': '\xce',
+    'icirc': '\xee',
+    'Icirc;': '\xce',
+    'icirc;': '\xee',
+    'Icy;': '\u0418',
+    'icy;': '\u0438',
+    'Idot;': '\u0130',
+    'IEcy;': '\u0415',
+    'iecy;': '\u0435',
+    'iexcl': '\xa1',
+    'iexcl;': '\xa1',
+    'iff;': '\u21d4',
+    'Ifr;': '\u2111',
+    'ifr;': '\U0001d526',
+    'Igrave': '\xcc',
+    'igrave': '\xec',
+    'Igrave;': '\xcc',
+    'igrave;': '\xec',
+    'ii;': '\u2148',
+    'iiiint;': '\u2a0c',
+    'iiint;': '\u222d',
+    'iinfin;': '\u29dc',
+    'iiota;': '\u2129',
+    'IJlig;': '\u0132',
+    'ijlig;': '\u0133',
+    'Im;': '\u2111',
+    'Imacr;': '\u012a',
+    'imacr;': '\u012b',
+    'image;': '\u2111',
+    'ImaginaryI;': '\u2148',
+    'imagline;': '\u2110',
+    'imagpart;': '\u2111',
+    'imath;': '\u0131',
+    'imof;': '\u22b7',
+    'imped;': '\u01b5',
+    'Implies;': '\u21d2',
+    'in;': '\u2208',
+    'incare;': '\u2105',
+    'infin;': '\u221e',
+    'infintie;': '\u29dd',
+    'inodot;': '\u0131',
+    'Int;': '\u222c',
+    'int;': '\u222b',
+    'intcal;': '\u22ba',
+    'integers;': '\u2124',
+    'Integral;': '\u222b',
+    'intercal;': '\u22ba',
+    'Intersection;': '\u22c2',
+    'intlarhk;': '\u2a17',
+    'intprod;': '\u2a3c',
+    'InvisibleComma;': '\u2063',
+    'InvisibleTimes;': '\u2062',
+    'IOcy;': '\u0401',
+    'iocy;': '\u0451',
+    'Iogon;': '\u012e',
+    'iogon;': '\u012f',
+    'Iopf;': '\U0001d540',
+    'iopf;': '\U0001d55a',
+    'Iota;': '\u0399',
+    'iota;': '\u03b9',
+    'iprod;': '\u2a3c',
+    'iquest': '\xbf',
+    'iquest;': '\xbf',
+    'Iscr;': '\u2110',
+    'iscr;': '\U0001d4be',
+    'isin;': '\u2208',
+    'isindot;': '\u22f5',
+    'isinE;': '\u22f9',
+    'isins;': '\u22f4',
+    'isinsv;': '\u22f3',
+    'isinv;': '\u2208',
+    'it;': '\u2062',
+    'Itilde;': '\u0128',
+    'itilde;': '\u0129',
+    'Iukcy;': '\u0406',
+    'iukcy;': '\u0456',
+    'Iuml': '\xcf',
+    'iuml': '\xef',
+    'Iuml;': '\xcf',
+    'iuml;': '\xef',
+    'Jcirc;': '\u0134',
+    'jcirc;': '\u0135',
+    'Jcy;': '\u0419',
+    'jcy;': '\u0439',
+    'Jfr;': '\U0001d50d',
+    'jfr;': '\U0001d527',
+    'jmath;': '\u0237',
+    'Jopf;': '\U0001d541',
+    'jopf;': '\U0001d55b',
+    'Jscr;': '\U0001d4a5',
+    'jscr;': '\U0001d4bf',
+    'Jsercy;': '\u0408',
+    'jsercy;': '\u0458',
+    'Jukcy;': '\u0404',
+    'jukcy;': '\u0454',
+    'Kappa;': '\u039a',
+    'kappa;': '\u03ba',
+    'kappav;': '\u03f0',
+    'Kcedil;': '\u0136',
+    'kcedil;': '\u0137',
+    'Kcy;': '\u041a',
+    'kcy;': '\u043a',
+    'Kfr;': '\U0001d50e',
+    'kfr;': '\U0001d528',
+    'kgreen;': '\u0138',
+    'KHcy;': '\u0425',
+    'khcy;': '\u0445',
+    'KJcy;': '\u040c',
+    'kjcy;': '\u045c',
+    'Kopf;': '\U0001d542',
+    'kopf;': '\U0001d55c',
+    'Kscr;': '\U0001d4a6',
+    'kscr;': '\U0001d4c0',
+    'lAarr;': '\u21da',
+    'Lacute;': '\u0139',
+    'lacute;': '\u013a',
+    'laemptyv;': '\u29b4',
+    'lagran;': '\u2112',
+    'Lambda;': '\u039b',
+    'lambda;': '\u03bb',
+    'Lang;': '\u27ea',
+    'lang;': '\u27e8',
+    'langd;': '\u2991',
+    'langle;': '\u27e8',
+    'lap;': '\u2a85',
+    'Laplacetrf;': '\u2112',
+    'laquo': '\xab',
+    'laquo;': '\xab',
+    'Larr;': '\u219e',
+    'lArr;': '\u21d0',
+    'larr;': '\u2190',
+    'larrb;': '\u21e4',
+    'larrbfs;': '\u291f',
+    'larrfs;': '\u291d',
+    'larrhk;': '\u21a9',
+    'larrlp;': '\u21ab',
+    'larrpl;': '\u2939',
+    'larrsim;': '\u2973',
+    'larrtl;': '\u21a2',
+    'lat;': '\u2aab',
+    'lAtail;': '\u291b',
+    'latail;': '\u2919',
+    'late;': '\u2aad',
+    'lates;': '\u2aad\ufe00',
+    'lBarr;': '\u290e',
+    'lbarr;': '\u290c',
+    'lbbrk;': '\u2772',
+    'lbrace;': '{',
+    'lbrack;': '[',
+    'lbrke;': '\u298b',
+    'lbrksld;': '\u298f',
+    'lbrkslu;': '\u298d',
+    'Lcaron;': '\u013d',
+    'lcaron;': '\u013e',
+    'Lcedil;': '\u013b',
+    'lcedil;': '\u013c',
+    'lceil;': '\u2308',
+    'lcub;': '{',
+    'Lcy;': '\u041b',
+    'lcy;': '\u043b',
+    'ldca;': '\u2936',
+    'ldquo;': '\u201c',
+    'ldquor;': '\u201e',
+    'ldrdhar;': '\u2967',
+    'ldrushar;': '\u294b',
+    'ldsh;': '\u21b2',
+    'lE;': '\u2266',
+    'le;': '\u2264',
+    'LeftAngleBracket;': '\u27e8',
+    'LeftArrow;': '\u2190',
+    'Leftarrow;': '\u21d0',
+    'leftarrow;': '\u2190',
+    'LeftArrowBar;': '\u21e4',
+    'LeftArrowRightArrow;': '\u21c6',
+    'leftarrowtail;': '\u21a2',
+    'LeftCeiling;': '\u2308',
+    'LeftDoubleBracket;': '\u27e6',
+    'LeftDownTeeVector;': '\u2961',
+    'LeftDownVector;': '\u21c3',
+    'LeftDownVectorBar;': '\u2959',
+    'LeftFloor;': '\u230a',
+    'leftharpoondown;': '\u21bd',
+    'leftharpoonup;': '\u21bc',
+    'leftleftarrows;': '\u21c7',
+    'LeftRightArrow;': '\u2194',
+    'Leftrightarrow;': '\u21d4',
+    'leftrightarrow;': '\u2194',
+    'leftrightarrows;': '\u21c6',
+    'leftrightharpoons;': '\u21cb',
+    'leftrightsquigarrow;': '\u21ad',
+    'LeftRightVector;': '\u294e',
+    'LeftTee;': '\u22a3',
+    'LeftTeeArrow;': '\u21a4',
+    'LeftTeeVector;': '\u295a',
+    'leftthreetimes;': '\u22cb',
+    'LeftTriangle;': '\u22b2',
+    'LeftTriangleBar;': '\u29cf',
+    'LeftTriangleEqual;': '\u22b4',
+    'LeftUpDownVector;': '\u2951',
+    'LeftUpTeeVector;': '\u2960',
+    'LeftUpVector;': '\u21bf',
+    'LeftUpVectorBar;': '\u2958',
+    'LeftVector;': '\u21bc',
+    'LeftVectorBar;': '\u2952',
+    'lEg;': '\u2a8b',
+    'leg;': '\u22da',
+    'leq;': '\u2264',
+    'leqq;': '\u2266',
+    'leqslant;': '\u2a7d',
+    'les;': '\u2a7d',
+    'lescc;': '\u2aa8',
+    'lesdot;': '\u2a7f',
+    'lesdoto;': '\u2a81',
+    'lesdotor;': '\u2a83',
+    'lesg;': '\u22da\ufe00',
+    'lesges;': '\u2a93',
+    'lessapprox;': '\u2a85',
+    'lessdot;': '\u22d6',
+    'lesseqgtr;': '\u22da',
+    'lesseqqgtr;': '\u2a8b',
+    'LessEqualGreater;': '\u22da',
+    'LessFullEqual;': '\u2266',
+    'LessGreater;': '\u2276',
+    'lessgtr;': '\u2276',
+    'LessLess;': '\u2aa1',
+    'lesssim;': '\u2272',
+    'LessSlantEqual;': '\u2a7d',
+    'LessTilde;': '\u2272',
+    'lfisht;': '\u297c',
+    'lfloor;': '\u230a',
+    'Lfr;': '\U0001d50f',
+    'lfr;': '\U0001d529',
+    'lg;': '\u2276',
+    'lgE;': '\u2a91',
+    'lHar;': '\u2962',
+    'lhard;': '\u21bd',
+    'lharu;': '\u21bc',
+    'lharul;': '\u296a',
+    'lhblk;': '\u2584',
+    'LJcy;': '\u0409',
+    'ljcy;': '\u0459',
+    'Ll;': '\u22d8',
+    'll;': '\u226a',
+    'llarr;': '\u21c7',
+    'llcorner;': '\u231e',
+    'Lleftarrow;': '\u21da',
+    'llhard;': '\u296b',
+    'lltri;': '\u25fa',
+    'Lmidot;': '\u013f',
+    'lmidot;': '\u0140',
+    'lmoust;': '\u23b0',
+    'lmoustache;': '\u23b0',
+    'lnap;': '\u2a89',
+    'lnapprox;': '\u2a89',
+    'lnE;': '\u2268',
+    'lne;': '\u2a87',
+    'lneq;': '\u2a87',
+    'lneqq;': '\u2268',
+    'lnsim;': '\u22e6',
+    'loang;': '\u27ec',
+    'loarr;': '\u21fd',
+    'lobrk;': '\u27e6',
+    'LongLeftArrow;': '\u27f5',
+    'Longleftarrow;': '\u27f8',
+    'longleftarrow;': '\u27f5',
+    'LongLeftRightArrow;': '\u27f7',
+    'Longleftrightarrow;': '\u27fa',
+    'longleftrightarrow;': '\u27f7',
+    'longmapsto;': '\u27fc',
+    'LongRightArrow;': '\u27f6',
+    'Longrightarrow;': '\u27f9',
+    'longrightarrow;': '\u27f6',
+    'looparrowleft;': '\u21ab',
+    'looparrowright;': '\u21ac',
+    'lopar;': '\u2985',
+    'Lopf;': '\U0001d543',
+    'lopf;': '\U0001d55d',
+    'loplus;': '\u2a2d',
+    'lotimes;': '\u2a34',
+    'lowast;': '\u2217',
+    'lowbar;': '_',
+    'LowerLeftArrow;': '\u2199',
+    'LowerRightArrow;': '\u2198',
+    'loz;': '\u25ca',
+    'lozenge;': '\u25ca',
+    'lozf;': '\u29eb',
+    'lpar;': '(',
+    'lparlt;': '\u2993',
+    'lrarr;': '\u21c6',
+    'lrcorner;': '\u231f',
+    'lrhar;': '\u21cb',
+    'lrhard;': '\u296d',
+    'lrm;': '\u200e',
+    'lrtri;': '\u22bf',
+    'lsaquo;': '\u2039',
+    'Lscr;': '\u2112',
+    'lscr;': '\U0001d4c1',
+    'Lsh;': '\u21b0',
+    'lsh;': '\u21b0',
+    'lsim;': '\u2272',
+    'lsime;': '\u2a8d',
+    'lsimg;': '\u2a8f',
+    'lsqb;': '[',
+    'lsquo;': '\u2018',
+    'lsquor;': '\u201a',
+    'Lstrok;': '\u0141',
+    'lstrok;': '\u0142',
+    'LT': '<',
+    'lt': '<',
+    'LT;': '<',
+    'Lt;': '\u226a',
+    'lt;': '<',
+    'ltcc;': '\u2aa6',
+    'ltcir;': '\u2a79',
+    'ltdot;': '\u22d6',
+    'lthree;': '\u22cb',
+    'ltimes;': '\u22c9',
+    'ltlarr;': '\u2976',
+    'ltquest;': '\u2a7b',
+    'ltri;': '\u25c3',
+    'ltrie;': '\u22b4',
+    'ltrif;': '\u25c2',
+    'ltrPar;': '\u2996',
+    'lurdshar;': '\u294a',
+    'luruhar;': '\u2966',
+    'lvertneqq;': '\u2268\ufe00',
+    'lvnE;': '\u2268\ufe00',
+    'macr': '\xaf',
+    'macr;': '\xaf',
+    'male;': '\u2642',
+    'malt;': '\u2720',
+    'maltese;': '\u2720',
+    'Map;': '\u2905',
+    'map;': '\u21a6',
+    'mapsto;': '\u21a6',
+    'mapstodown;': '\u21a7',
+    'mapstoleft;': '\u21a4',
+    'mapstoup;': '\u21a5',
+    'marker;': '\u25ae',
+    'mcomma;': '\u2a29',
+    'Mcy;': '\u041c',
+    'mcy;': '\u043c',
+    'mdash;': '\u2014',
+    'mDDot;': '\u223a',
+    'measuredangle;': '\u2221',
+    'MediumSpace;': '\u205f',
+    'Mellintrf;': '\u2133',
+    'Mfr;': '\U0001d510',
+    'mfr;': '\U0001d52a',
+    'mho;': '\u2127',
+    'micro': '\xb5',
+    'micro;': '\xb5',
+    'mid;': '\u2223',
+    'midast;': '*',
+    'midcir;': '\u2af0',
+    'middot': '\xb7',
+    'middot;': '\xb7',
+    'minus;': '\u2212',
+    'minusb;': '\u229f',
+    'minusd;': '\u2238',
+    'minusdu;': '\u2a2a',
+    'MinusPlus;': '\u2213',
+    'mlcp;': '\u2adb',
+    'mldr;': '\u2026',
+    'mnplus;': '\u2213',
+    'models;': '\u22a7',
+    'Mopf;': '\U0001d544',
+    'mopf;': '\U0001d55e',
+    'mp;': '\u2213',
+    'Mscr;': '\u2133',
+    'mscr;': '\U0001d4c2',
+    'mstpos;': '\u223e',
+    'Mu;': '\u039c',
+    'mu;': '\u03bc',
+    'multimap;': '\u22b8',
+    'mumap;': '\u22b8',
+    'nabla;': '\u2207',
+    'Nacute;': '\u0143',
+    'nacute;': '\u0144',
+    'nang;': '\u2220\u20d2',
+    'nap;': '\u2249',
+    'napE;': '\u2a70\u0338',
+    'napid;': '\u224b\u0338',
+    'napos;': '\u0149',
+    'napprox;': '\u2249',
+    'natur;': '\u266e',
+    'natural;': '\u266e',
+    'naturals;': '\u2115',
+    'nbsp': '\xa0',
+    'nbsp;': '\xa0',
+    'nbump;': '\u224e\u0338',
+    'nbumpe;': '\u224f\u0338',
+    'ncap;': '\u2a43',
+    'Ncaron;': '\u0147',
+    'ncaron;': '\u0148',
+    'Ncedil;': '\u0145',
+    'ncedil;': '\u0146',
+    'ncong;': '\u2247',
+    'ncongdot;': '\u2a6d\u0338',
+    'ncup;': '\u2a42',
+    'Ncy;': '\u041d',
+    'ncy;': '\u043d',
+    'ndash;': '\u2013',
+    'ne;': '\u2260',
+    'nearhk;': '\u2924',
+    'neArr;': '\u21d7',
+    'nearr;': '\u2197',
+    'nearrow;': '\u2197',
+    'nedot;': '\u2250\u0338',
+    'NegativeMediumSpace;': '\u200b',
+    'NegativeThickSpace;': '\u200b',
+    'NegativeThinSpace;': '\u200b',
+    'NegativeVeryThinSpace;': '\u200b',
+    'nequiv;': '\u2262',
+    'nesear;': '\u2928',
+    'nesim;': '\u2242\u0338',
+    'NestedGreaterGreater;': '\u226b',
+    'NestedLessLess;': '\u226a',
+    'NewLine;': '\n',
+    'nexist;': '\u2204',
+    'nexists;': '\u2204',
+    'Nfr;': '\U0001d511',
+    'nfr;': '\U0001d52b',
+    'ngE;': '\u2267\u0338',
+    'nge;': '\u2271',
+    'ngeq;': '\u2271',
+    'ngeqq;': '\u2267\u0338',
+    'ngeqslant;': '\u2a7e\u0338',
+    'nges;': '\u2a7e\u0338',
+    'nGg;': '\u22d9\u0338',
+    'ngsim;': '\u2275',
+    'nGt;': '\u226b\u20d2',
+    'ngt;': '\u226f',
+    'ngtr;': '\u226f',
+    'nGtv;': '\u226b\u0338',
+    'nhArr;': '\u21ce',
+    'nharr;': '\u21ae',
+    'nhpar;': '\u2af2',
+    'ni;': '\u220b',
+    'nis;': '\u22fc',
+    'nisd;': '\u22fa',
+    'niv;': '\u220b',
+    'NJcy;': '\u040a',
+    'njcy;': '\u045a',
+    'nlArr;': '\u21cd',
+    'nlarr;': '\u219a',
+    'nldr;': '\u2025',
+    'nlE;': '\u2266\u0338',
+    'nle;': '\u2270',
+    'nLeftarrow;': '\u21cd',
+    'nleftarrow;': '\u219a',
+    'nLeftrightarrow;': '\u21ce',
+    'nleftrightarrow;': '\u21ae',
+    'nleq;': '\u2270',
+    'nleqq;': '\u2266\u0338',
+    'nleqslant;': '\u2a7d\u0338',
+    'nles;': '\u2a7d\u0338',
+    'nless;': '\u226e',
+    'nLl;': '\u22d8\u0338',
+    'nlsim;': '\u2274',
+    'nLt;': '\u226a\u20d2',
+    'nlt;': '\u226e',
+    'nltri;': '\u22ea',
+    'nltrie;': '\u22ec',
+    'nLtv;': '\u226a\u0338',
+    'nmid;': '\u2224',
+    'NoBreak;': '\u2060',
+    'NonBreakingSpace;': '\xa0',
+    'Nopf;': '\u2115',
+    'nopf;': '\U0001d55f',
+    'not': '\xac',
+    'Not;': '\u2aec',
+    'not;': '\xac',
+    'NotCongruent;': '\u2262',
+    'NotCupCap;': '\u226d',
+    'NotDoubleVerticalBar;': '\u2226',
+    'NotElement;': '\u2209',
+    'NotEqual;': '\u2260',
+    'NotEqualTilde;': '\u2242\u0338',
+    'NotExists;': '\u2204',
+    'NotGreater;': '\u226f',
+    'NotGreaterEqual;': '\u2271',
+    'NotGreaterFullEqual;': '\u2267\u0338',
+    'NotGreaterGreater;': '\u226b\u0338',
+    'NotGreaterLess;': '\u2279',
+    'NotGreaterSlantEqual;': '\u2a7e\u0338',
+    'NotGreaterTilde;': '\u2275',
+    'NotHumpDownHump;': '\u224e\u0338',
+    'NotHumpEqual;': '\u224f\u0338',
+    'notin;': '\u2209',
+    'notindot;': '\u22f5\u0338',
+    'notinE;': '\u22f9\u0338',
+    'notinva;': '\u2209',
+    'notinvb;': '\u22f7',
+    'notinvc;': '\u22f6',
+    'NotLeftTriangle;': '\u22ea',
+    'NotLeftTriangleBar;': '\u29cf\u0338',
+    'NotLeftTriangleEqual;': '\u22ec',
+    'NotLess;': '\u226e',
+    'NotLessEqual;': '\u2270',
+    'NotLessGreater;': '\u2278',
+    'NotLessLess;': '\u226a\u0338',
+    'NotLessSlantEqual;': '\u2a7d\u0338',
+    'NotLessTilde;': '\u2274',
+    'NotNestedGreaterGreater;': '\u2aa2\u0338',
+    'NotNestedLessLess;': '\u2aa1\u0338',
+    'notni;': '\u220c',
+    'notniva;': '\u220c',
+    'notnivb;': '\u22fe',
+    'notnivc;': '\u22fd',
+    'NotPrecedes;': '\u2280',
+    'NotPrecedesEqual;': '\u2aaf\u0338',
+    'NotPrecedesSlantEqual;': '\u22e0',
+    'NotReverseElement;': '\u220c',
+    'NotRightTriangle;': '\u22eb',
+    'NotRightTriangleBar;': '\u29d0\u0338',
+    'NotRightTriangleEqual;': '\u22ed',
+    'NotSquareSubset;': '\u228f\u0338',
+    'NotSquareSubsetEqual;': '\u22e2',
+    'NotSquareSuperset;': '\u2290\u0338',
+    'NotSquareSupersetEqual;': '\u22e3',
+    'NotSubset;': '\u2282\u20d2',
+    'NotSubsetEqual;': '\u2288',
+    'NotSucceeds;': '\u2281',
+    'NotSucceedsEqual;': '\u2ab0\u0338',
+    'NotSucceedsSlantEqual;': '\u22e1',
+    'NotSucceedsTilde;': '\u227f\u0338',
+    'NotSuperset;': '\u2283\u20d2',
+    'NotSupersetEqual;': '\u2289',
+    'NotTilde;': '\u2241',
+    'NotTildeEqual;': '\u2244',
+    'NotTildeFullEqual;': '\u2247',
+    'NotTildeTilde;': '\u2249',
+    'NotVerticalBar;': '\u2224',
+    'npar;': '\u2226',
+    'nparallel;': '\u2226',
+    'nparsl;': '\u2afd\u20e5',
+    'npart;': '\u2202\u0338',
+    'npolint;': '\u2a14',
+    'npr;': '\u2280',
+    'nprcue;': '\u22e0',
+    'npre;': '\u2aaf\u0338',
+    'nprec;': '\u2280',
+    'npreceq;': '\u2aaf\u0338',
+    'nrArr;': '\u21cf',
+    'nrarr;': '\u219b',
+    'nrarrc;': '\u2933\u0338',
+    'nrarrw;': '\u219d\u0338',
+    'nRightarrow;': '\u21cf',
+    'nrightarrow;': '\u219b',
+    'nrtri;': '\u22eb',
+    'nrtrie;': '\u22ed',
+    'nsc;': '\u2281',
+    'nsccue;': '\u22e1',
+    'nsce;': '\u2ab0\u0338',
+    'Nscr;': '\U0001d4a9',
+    'nscr;': '\U0001d4c3',
+    'nshortmid;': '\u2224',
+    'nshortparallel;': '\u2226',
+    'nsim;': '\u2241',
+    'nsime;': '\u2244',
+    'nsimeq;': '\u2244',
+    'nsmid;': '\u2224',
+    'nspar;': '\u2226',
+    'nsqsube;': '\u22e2',
+    'nsqsupe;': '\u22e3',
+    'nsub;': '\u2284',
+    'nsubE;': '\u2ac5\u0338',
+    'nsube;': '\u2288',
+    'nsubset;': '\u2282\u20d2',
+    'nsubseteq;': '\u2288',
+    'nsubseteqq;': '\u2ac5\u0338',
+    'nsucc;': '\u2281',
+    'nsucceq;': '\u2ab0\u0338',
+    'nsup;': '\u2285',
+    'nsupE;': '\u2ac6\u0338',
+    'nsupe;': '\u2289',
+    'nsupset;': '\u2283\u20d2',
+    'nsupseteq;': '\u2289',
+    'nsupseteqq;': '\u2ac6\u0338',
+    'ntgl;': '\u2279',
+    'Ntilde': '\xd1',
+    'ntilde': '\xf1',
+    'Ntilde;': '\xd1',
+    'ntilde;': '\xf1',
+    'ntlg;': '\u2278',
+    'ntriangleleft;': '\u22ea',
+    'ntrianglelefteq;': '\u22ec',
+    'ntriangleright;': '\u22eb',
+    'ntrianglerighteq;': '\u22ed',
+    'Nu;': '\u039d',
+    'nu;': '\u03bd',
+    'num;': '#',
+    'numero;': '\u2116',
+    'numsp;': '\u2007',
+    'nvap;': '\u224d\u20d2',
+    'nVDash;': '\u22af',
+    'nVdash;': '\u22ae',
+    'nvDash;': '\u22ad',
+    'nvdash;': '\u22ac',
+    'nvge;': '\u2265\u20d2',
+    'nvgt;': '>\u20d2',
+    'nvHarr;': '\u2904',
+    'nvinfin;': '\u29de',
+    'nvlArr;': '\u2902',
+    'nvle;': '\u2264\u20d2',
+    'nvlt;': '<\u20d2',
+    'nvltrie;': '\u22b4\u20d2',
+    'nvrArr;': '\u2903',
+    'nvrtrie;': '\u22b5\u20d2',
+    'nvsim;': '\u223c\u20d2',
+    'nwarhk;': '\u2923',
+    'nwArr;': '\u21d6',
+    'nwarr;': '\u2196',
+    'nwarrow;': '\u2196',
+    'nwnear;': '\u2927',
+    'Oacute': '\xd3',
+    'oacute': '\xf3',
+    'Oacute;': '\xd3',
+    'oacute;': '\xf3',
+    'oast;': '\u229b',
+    'ocir;': '\u229a',
+    'Ocirc': '\xd4',
+    'ocirc': '\xf4',
+    'Ocirc;': '\xd4',
+    'ocirc;': '\xf4',
+    'Ocy;': '\u041e',
+    'ocy;': '\u043e',
+    'odash;': '\u229d',
+    'Odblac;': '\u0150',
+    'odblac;': '\u0151',
+    'odiv;': '\u2a38',
+    'odot;': '\u2299',
+    'odsold;': '\u29bc',
+    'OElig;': '\u0152',
+    'oelig;': '\u0153',
+    'ofcir;': '\u29bf',
+    'Ofr;': '\U0001d512',
+    'ofr;': '\U0001d52c',
+    'ogon;': '\u02db',
+    'Ograve': '\xd2',
+    'ograve': '\xf2',
+    'Ograve;': '\xd2',
+    'ograve;': '\xf2',
+    'ogt;': '\u29c1',
+    'ohbar;': '\u29b5',
+    'ohm;': '\u03a9',
+    'oint;': '\u222e',
+    'olarr;': '\u21ba',
+    'olcir;': '\u29be',
+    'olcross;': '\u29bb',
+    'oline;': '\u203e',
+    'olt;': '\u29c0',
+    'Omacr;': '\u014c',
+    'omacr;': '\u014d',
+    'Omega;': '\u03a9',
+    'omega;': '\u03c9',
+    'Omicron;': '\u039f',
+    'omicron;': '\u03bf',
+    'omid;': '\u29b6',
+    'ominus;': '\u2296',
+    'Oopf;': '\U0001d546',
+    'oopf;': '\U0001d560',
+    'opar;': '\u29b7',
+    'OpenCurlyDoubleQuote;': '\u201c',
+    'OpenCurlyQuote;': '\u2018',
+    'operp;': '\u29b9',
+    'oplus;': '\u2295',
+    'Or;': '\u2a54',
+    'or;': '\u2228',
+    'orarr;': '\u21bb',
+    'ord;': '\u2a5d',
+    'order;': '\u2134',
+    'orderof;': '\u2134',
+    'ordf': '\xaa',
+    'ordf;': '\xaa',
+    'ordm': '\xba',
+    'ordm;': '\xba',
+    'origof;': '\u22b6',
+    'oror;': '\u2a56',
+    'orslope;': '\u2a57',
+    'orv;': '\u2a5b',
+    'oS;': '\u24c8',
+    'Oscr;': '\U0001d4aa',
+    'oscr;': '\u2134',
+    'Oslash': '\xd8',
+    'oslash': '\xf8',
+    'Oslash;': '\xd8',
+    'oslash;': '\xf8',
+    'osol;': '\u2298',
+    'Otilde': '\xd5',
+    'otilde': '\xf5',
+    'Otilde;': '\xd5',
+    'otilde;': '\xf5',
+    'Otimes;': '\u2a37',
+    'otimes;': '\u2297',
+    'otimesas;': '\u2a36',
+    'Ouml': '\xd6',
+    'ouml': '\xf6',
+    'Ouml;': '\xd6',
+    'ouml;': '\xf6',
+    'ovbar;': '\u233d',
+    'OverBar;': '\u203e',
+    'OverBrace;': '\u23de',
+    'OverBracket;': '\u23b4',
+    'OverParenthesis;': '\u23dc',
+    'par;': '\u2225',
+    'para': '\xb6',
+    'para;': '\xb6',
+    'parallel;': '\u2225',
+    'parsim;': '\u2af3',
+    'parsl;': '\u2afd',
+    'part;': '\u2202',
+    'PartialD;': '\u2202',
+    'Pcy;': '\u041f',
+    'pcy;': '\u043f',
+    'percnt;': '%',
+    'period;': '.',
+    'permil;': '\u2030',
+    'perp;': '\u22a5',
+    'pertenk;': '\u2031',
+    'Pfr;': '\U0001d513',
+    'pfr;': '\U0001d52d',
+    'Phi;': '\u03a6',
+    'phi;': '\u03c6',
+    'phiv;': '\u03d5',
+    'phmmat;': '\u2133',
+    'phone;': '\u260e',
+    'Pi;': '\u03a0',
+    'pi;': '\u03c0',
+    'pitchfork;': '\u22d4',
+    'piv;': '\u03d6',
+    'planck;': '\u210f',
+    'planckh;': '\u210e',
+    'plankv;': '\u210f',
+    'plus;': '+',
+    'plusacir;': '\u2a23',
+    'plusb;': '\u229e',
+    'pluscir;': '\u2a22',
+    'plusdo;': '\u2214',
+    'plusdu;': '\u2a25',
+    'pluse;': '\u2a72',
+    'PlusMinus;': '\xb1',
+    'plusmn': '\xb1',
+    'plusmn;': '\xb1',
+    'plussim;': '\u2a26',
+    'plustwo;': '\u2a27',
+    'pm;': '\xb1',
+    'Poincareplane;': '\u210c',
+    'pointint;': '\u2a15',
+    'Popf;': '\u2119',
+    'popf;': '\U0001d561',
+    'pound': '\xa3',
+    'pound;': '\xa3',
+    'Pr;': '\u2abb',
+    'pr;': '\u227a',
+    'prap;': '\u2ab7',
+    'prcue;': '\u227c',
+    'prE;': '\u2ab3',
+    'pre;': '\u2aaf',
+    'prec;': '\u227a',
+    'precapprox;': '\u2ab7',
+    'preccurlyeq;': '\u227c',
+    'Precedes;': '\u227a',
+    'PrecedesEqual;': '\u2aaf',
+    'PrecedesSlantEqual;': '\u227c',
+    'PrecedesTilde;': '\u227e',
+    'preceq;': '\u2aaf',
+    'precnapprox;': '\u2ab9',
+    'precneqq;': '\u2ab5',
+    'precnsim;': '\u22e8',
+    'precsim;': '\u227e',
+    'Prime;': '\u2033',
+    'prime;': '\u2032',
+    'primes;': '\u2119',
+    'prnap;': '\u2ab9',
+    'prnE;': '\u2ab5',
+    'prnsim;': '\u22e8',
+    'prod;': '\u220f',
+    'Product;': '\u220f',
+    'profalar;': '\u232e',
+    'profline;': '\u2312',
+    'profsurf;': '\u2313',
+    'prop;': '\u221d',
+    'Proportion;': '\u2237',
+    'Proportional;': '\u221d',
+    'propto;': '\u221d',
+    'prsim;': '\u227e',
+    'prurel;': '\u22b0',
+    'Pscr;': '\U0001d4ab',
+    'pscr;': '\U0001d4c5',
+    'Psi;': '\u03a8',
+    'psi;': '\u03c8',
+    'puncsp;': '\u2008',
+    'Qfr;': '\U0001d514',
+    'qfr;': '\U0001d52e',
+    'qint;': '\u2a0c',
+    'Qopf;': '\u211a',
+    'qopf;': '\U0001d562',
+    'qprime;': '\u2057',
+    'Qscr;': '\U0001d4ac',
+    'qscr;': '\U0001d4c6',
+    'quaternions;': '\u210d',
+    'quatint;': '\u2a16',
+    'quest;': '?',
+    'questeq;': '\u225f',
+    'QUOT': '"',
+    'quot': '"',
+    'QUOT;': '"',
+    'quot;': '"',
+    'rAarr;': '\u21db',
+    'race;': '\u223d\u0331',
+    'Racute;': '\u0154',
+    'racute;': '\u0155',
+    'radic;': '\u221a',
+    'raemptyv;': '\u29b3',
+    'Rang;': '\u27eb',
+    'rang;': '\u27e9',
+    'rangd;': '\u2992',
+    'range;': '\u29a5',
+    'rangle;': '\u27e9',
+    'raquo': '\xbb',
+    'raquo;': '\xbb',
+    'Rarr;': '\u21a0',
+    'rArr;': '\u21d2',
+    'rarr;': '\u2192',
+    'rarrap;': '\u2975',
+    'rarrb;': '\u21e5',
+    'rarrbfs;': '\u2920',
+    'rarrc;': '\u2933',
+    'rarrfs;': '\u291e',
+    'rarrhk;': '\u21aa',
+    'rarrlp;': '\u21ac',
+    'rarrpl;': '\u2945',
+    'rarrsim;': '\u2974',
+    'Rarrtl;': '\u2916',
+    'rarrtl;': '\u21a3',
+    'rarrw;': '\u219d',
+    'rAtail;': '\u291c',
+    'ratail;': '\u291a',
+    'ratio;': '\u2236',
+    'rationals;': '\u211a',
+    'RBarr;': '\u2910',
+    'rBarr;': '\u290f',
+    'rbarr;': '\u290d',
+    'rbbrk;': '\u2773',
+    'rbrace;': '}',
+    'rbrack;': ']',
+    'rbrke;': '\u298c',
+    'rbrksld;': '\u298e',
+    'rbrkslu;': '\u2990',
+    'Rcaron;': '\u0158',
+    'rcaron;': '\u0159',
+    'Rcedil;': '\u0156',
+    'rcedil;': '\u0157',
+    'rceil;': '\u2309',
+    'rcub;': '}',
+    'Rcy;': '\u0420',
+    'rcy;': '\u0440',
+    'rdca;': '\u2937',
+    'rdldhar;': '\u2969',
+    'rdquo;': '\u201d',
+    'rdquor;': '\u201d',
+    'rdsh;': '\u21b3',
+    'Re;': '\u211c',
+    'real;': '\u211c',
+    'realine;': '\u211b',
+    'realpart;': '\u211c',
+    'reals;': '\u211d',
+    'rect;': '\u25ad',
+    'REG': '\xae',
+    'reg': '\xae',
+    'REG;': '\xae',
+    'reg;': '\xae',
+    'ReverseElement;': '\u220b',
+    'ReverseEquilibrium;': '\u21cb',
+    'ReverseUpEquilibrium;': '\u296f',
+    'rfisht;': '\u297d',
+    'rfloor;': '\u230b',
+    'Rfr;': '\u211c',
+    'rfr;': '\U0001d52f',
+    'rHar;': '\u2964',
+    'rhard;': '\u21c1',
+    'rharu;': '\u21c0',
+    'rharul;': '\u296c',
+    'Rho;': '\u03a1',
+    'rho;': '\u03c1',
+    'rhov;': '\u03f1',
+    'RightAngleBracket;': '\u27e9',
+    'RightArrow;': '\u2192',
+    'Rightarrow;': '\u21d2',
+    'rightarrow;': '\u2192',
+    'RightArrowBar;': '\u21e5',
+    'RightArrowLeftArrow;': '\u21c4',
+    'rightarrowtail;': '\u21a3',
+    'RightCeiling;': '\u2309',
+    'RightDoubleBracket;': '\u27e7',
+    'RightDownTeeVector;': '\u295d',
+    'RightDownVector;': '\u21c2',
+    'RightDownVectorBar;': '\u2955',
+    'RightFloor;': '\u230b',
+    'rightharpoondown;': '\u21c1',
+    'rightharpoonup;': '\u21c0',
+    'rightleftarrows;': '\u21c4',
+    'rightleftharpoons;': '\u21cc',
+    'rightrightarrows;': '\u21c9',
+    'rightsquigarrow;': '\u219d',
+    'RightTee;': '\u22a2',
+    'RightTeeArrow;': '\u21a6',
+    'RightTeeVector;': '\u295b',
+    'rightthreetimes;': '\u22cc',
+    'RightTriangle;': '\u22b3',
+    'RightTriangleBar;': '\u29d0',
+    'RightTriangleEqual;': '\u22b5',
+    'RightUpDownVector;': '\u294f',
+    'RightUpTeeVector;': '\u295c',
+    'RightUpVector;': '\u21be',
+    'RightUpVectorBar;': '\u2954',
+    'RightVector;': '\u21c0',
+    'RightVectorBar;': '\u2953',
+    'ring;': '\u02da',
+    'risingdotseq;': '\u2253',
+    'rlarr;': '\u21c4',
+    'rlhar;': '\u21cc',
+    'rlm;': '\u200f',
+    'rmoust;': '\u23b1',
+    'rmoustache;': '\u23b1',
+    'rnmid;': '\u2aee',
+    'roang;': '\u27ed',
+    'roarr;': '\u21fe',
+    'robrk;': '\u27e7',
+    'ropar;': '\u2986',
+    'Ropf;': '\u211d',
+    'ropf;': '\U0001d563',
+    'roplus;': '\u2a2e',
+    'rotimes;': '\u2a35',
+    'RoundImplies;': '\u2970',
+    'rpar;': ')',
+    'rpargt;': '\u2994',
+    'rppolint;': '\u2a12',
+    'rrarr;': '\u21c9',
+    'Rrightarrow;': '\u21db',
+    'rsaquo;': '\u203a',
+    'Rscr;': '\u211b',
+    'rscr;': '\U0001d4c7',
+    'Rsh;': '\u21b1',
+    'rsh;': '\u21b1',
+    'rsqb;': ']',
+    'rsquo;': '\u2019',
+    'rsquor;': '\u2019',
+    'rthree;': '\u22cc',
+    'rtimes;': '\u22ca',
+    'rtri;': '\u25b9',
+    'rtrie;': '\u22b5',
+    'rtrif;': '\u25b8',
+    'rtriltri;': '\u29ce',
+    'RuleDelayed;': '\u29f4',
+    'ruluhar;': '\u2968',
+    'rx;': '\u211e',
+    'Sacute;': '\u015a',
+    'sacute;': '\u015b',
+    'sbquo;': '\u201a',
+    'Sc;': '\u2abc',
+    'sc;': '\u227b',
+    'scap;': '\u2ab8',
+    'Scaron;': '\u0160',
+    'scaron;': '\u0161',
+    'sccue;': '\u227d',
+    'scE;': '\u2ab4',
+    'sce;': '\u2ab0',
+    'Scedil;': '\u015e',
+    'scedil;': '\u015f',
+    'Scirc;': '\u015c',
+    'scirc;': '\u015d',
+    'scnap;': '\u2aba',
+    'scnE;': '\u2ab6',
+    'scnsim;': '\u22e9',
+    'scpolint;': '\u2a13',
+    'scsim;': '\u227f',
+    'Scy;': '\u0421',
+    'scy;': '\u0441',
+    'sdot;': '\u22c5',
+    'sdotb;': '\u22a1',
+    'sdote;': '\u2a66',
+    'searhk;': '\u2925',
+    'seArr;': '\u21d8',
+    'searr;': '\u2198',
+    'searrow;': '\u2198',
+    'sect': '\xa7',
+    'sect;': '\xa7',
+    'semi;': ';',
+    'seswar;': '\u2929',
+    'setminus;': '\u2216',
+    'setmn;': '\u2216',
+    'sext;': '\u2736',
+    'Sfr;': '\U0001d516',
+    'sfr;': '\U0001d530',
+    'sfrown;': '\u2322',
+    'sharp;': '\u266f',
+    'SHCHcy;': '\u0429',
+    'shchcy;': '\u0449',
+    'SHcy;': '\u0428',
+    'shcy;': '\u0448',
+    'ShortDownArrow;': '\u2193',
+    'ShortLeftArrow;': '\u2190',
+    'shortmid;': '\u2223',
+    'shortparallel;': '\u2225',
+    'ShortRightArrow;': '\u2192',
+    'ShortUpArrow;': '\u2191',
+    'shy': '\xad',
+    'shy;': '\xad',
+    'Sigma;': '\u03a3',
+    'sigma;': '\u03c3',
+    'sigmaf;': '\u03c2',
+    'sigmav;': '\u03c2',
+    'sim;': '\u223c',
+    'simdot;': '\u2a6a',
+    'sime;': '\u2243',
+    'simeq;': '\u2243',
+    'simg;': '\u2a9e',
+    'simgE;': '\u2aa0',
+    'siml;': '\u2a9d',
+    'simlE;': '\u2a9f',
+    'simne;': '\u2246',
+    'simplus;': '\u2a24',
+    'simrarr;': '\u2972',
+    'slarr;': '\u2190',
+    'SmallCircle;': '\u2218',
+    'smallsetminus;': '\u2216',
+    'smashp;': '\u2a33',
+    'smeparsl;': '\u29e4',
+    'smid;': '\u2223',
+    'smile;': '\u2323',
+    'smt;': '\u2aaa',
+    'smte;': '\u2aac',
+    'smtes;': '\u2aac\ufe00',
+    'SOFTcy;': '\u042c',
+    'softcy;': '\u044c',
+    'sol;': '/',
+    'solb;': '\u29c4',
+    'solbar;': '\u233f',
+    'Sopf;': '\U0001d54a',
+    'sopf;': '\U0001d564',
+    'spades;': '\u2660',
+    'spadesuit;': '\u2660',
+    'spar;': '\u2225',
+    'sqcap;': '\u2293',
+    'sqcaps;': '\u2293\ufe00',
+    'sqcup;': '\u2294',
+    'sqcups;': '\u2294\ufe00',
+    'Sqrt;': '\u221a',
+    'sqsub;': '\u228f',
+    'sqsube;': '\u2291',
+    'sqsubset;': '\u228f',
+    'sqsubseteq;': '\u2291',
+    'sqsup;': '\u2290',
+    'sqsupe;': '\u2292',
+    'sqsupset;': '\u2290',
+    'sqsupseteq;': '\u2292',
+    'squ;': '\u25a1',
+    'Square;': '\u25a1',
+    'square;': '\u25a1',
+    'SquareIntersection;': '\u2293',
+    'SquareSubset;': '\u228f',
+    'SquareSubsetEqual;': '\u2291',
+    'SquareSuperset;': '\u2290',
+    'SquareSupersetEqual;': '\u2292',
+    'SquareUnion;': '\u2294',
+    'squarf;': '\u25aa',
+    'squf;': '\u25aa',
+    'srarr;': '\u2192',
+    'Sscr;': '\U0001d4ae',
+    'sscr;': '\U0001d4c8',
+    'ssetmn;': '\u2216',
+    'ssmile;': '\u2323',
+    'sstarf;': '\u22c6',
+    'Star;': '\u22c6',
+    'star;': '\u2606',
+    'starf;': '\u2605',
+    'straightepsilon;': '\u03f5',
+    'straightphi;': '\u03d5',
+    'strns;': '\xaf',
+    'Sub;': '\u22d0',
+    'sub;': '\u2282',
+    'subdot;': '\u2abd',
+    'subE;': '\u2ac5',
+    'sube;': '\u2286',
+    'subedot;': '\u2ac3',
+    'submult;': '\u2ac1',
+    'subnE;': '\u2acb',
+    'subne;': '\u228a',
+    'subplus;': '\u2abf',
+    'subrarr;': '\u2979',
+    'Subset;': '\u22d0',
+    'subset;': '\u2282',
+    'subseteq;': '\u2286',
+    'subseteqq;': '\u2ac5',
+    'SubsetEqual;': '\u2286',
+    'subsetneq;': '\u228a',
+    'subsetneqq;': '\u2acb',
+    'subsim;': '\u2ac7',
+    'subsub;': '\u2ad5',
+    'subsup;': '\u2ad3',
+    'succ;': '\u227b',
+    'succapprox;': '\u2ab8',
+    'succcurlyeq;': '\u227d',
+    'Succeeds;': '\u227b',
+    'SucceedsEqual;': '\u2ab0',
+    'SucceedsSlantEqual;': '\u227d',
+    'SucceedsTilde;': '\u227f',
+    'succeq;': '\u2ab0',
+    'succnapprox;': '\u2aba',
+    'succneqq;': '\u2ab6',
+    'succnsim;': '\u22e9',
+    'succsim;': '\u227f',
+    'SuchThat;': '\u220b',
+    'Sum;': '\u2211',
+    'sum;': '\u2211',
+    'sung;': '\u266a',
+    'sup1': '\xb9',
+    'sup1;': '\xb9',
+    'sup2': '\xb2',
+    'sup2;': '\xb2',
+    'sup3': '\xb3',
+    'sup3;': '\xb3',
+    'Sup;': '\u22d1',
+    'sup;': '\u2283',
+    'supdot;': '\u2abe',
+    'supdsub;': '\u2ad8',
+    'supE;': '\u2ac6',
+    'supe;': '\u2287',
+    'supedot;': '\u2ac4',
+    'Superset;': '\u2283',
+    'SupersetEqual;': '\u2287',
+    'suphsol;': '\u27c9',
+    'suphsub;': '\u2ad7',
+    'suplarr;': '\u297b',
+    'supmult;': '\u2ac2',
+    'supnE;': '\u2acc',
+    'supne;': '\u228b',
+    'supplus;': '\u2ac0',
+    'Supset;': '\u22d1',
+    'supset;': '\u2283',
+    'supseteq;': '\u2287',
+    'supseteqq;': '\u2ac6',
+    'supsetneq;': '\u228b',
+    'supsetneqq;': '\u2acc',
+    'supsim;': '\u2ac8',
+    'supsub;': '\u2ad4',
+    'supsup;': '\u2ad6',
+    'swarhk;': '\u2926',
+    'swArr;': '\u21d9',
+    'swarr;': '\u2199',
+    'swarrow;': '\u2199',
+    'swnwar;': '\u292a',
+    'szlig': '\xdf',
+    'szlig;': '\xdf',
+    'Tab;': '\t',
+    'target;': '\u2316',
+    'Tau;': '\u03a4',
+    'tau;': '\u03c4',
+    'tbrk;': '\u23b4',
+    'Tcaron;': '\u0164',
+    'tcaron;': '\u0165',
+    'Tcedil;': '\u0162',
+    'tcedil;': '\u0163',
+    'Tcy;': '\u0422',
+    'tcy;': '\u0442',
+    'tdot;': '\u20db',
+    'telrec;': '\u2315',
+    'Tfr;': '\U0001d517',
+    'tfr;': '\U0001d531',
+    'there4;': '\u2234',
+    'Therefore;': '\u2234',
+    'therefore;': '\u2234',
+    'Theta;': '\u0398',
+    'theta;': '\u03b8',
+    'thetasym;': '\u03d1',
+    'thetav;': '\u03d1',
+    'thickapprox;': '\u2248',
+    'thicksim;': '\u223c',
+    'ThickSpace;': '\u205f\u200a',
+    'thinsp;': '\u2009',
+    'ThinSpace;': '\u2009',
+    'thkap;': '\u2248',
+    'thksim;': '\u223c',
+    'THORN': '\xde',
+    'thorn': '\xfe',
+    'THORN;': '\xde',
+    'thorn;': '\xfe',
+    'Tilde;': '\u223c',
+    'tilde;': '\u02dc',
+    'TildeEqual;': '\u2243',
+    'TildeFullEqual;': '\u2245',
+    'TildeTilde;': '\u2248',
+    'times': '\xd7',
+    'times;': '\xd7',
+    'timesb;': '\u22a0',
+    'timesbar;': '\u2a31',
+    'timesd;': '\u2a30',
+    'tint;': '\u222d',
+    'toea;': '\u2928',
+    'top;': '\u22a4',
+    'topbot;': '\u2336',
+    'topcir;': '\u2af1',
+    'Topf;': '\U0001d54b',
+    'topf;': '\U0001d565',
+    'topfork;': '\u2ada',
+    'tosa;': '\u2929',
+    'tprime;': '\u2034',
+    'TRADE;': '\u2122',
+    'trade;': '\u2122',
+    'triangle;': '\u25b5',
+    'triangledown;': '\u25bf',
+    'triangleleft;': '\u25c3',
+    'trianglelefteq;': '\u22b4',
+    'triangleq;': '\u225c',
+    'triangleright;': '\u25b9',
+    'trianglerighteq;': '\u22b5',
+    'tridot;': '\u25ec',
+    'trie;': '\u225c',
+    'triminus;': '\u2a3a',
+    'TripleDot;': '\u20db',
+    'triplus;': '\u2a39',
+    'trisb;': '\u29cd',
+    'tritime;': '\u2a3b',
+    'trpezium;': '\u23e2',
+    'Tscr;': '\U0001d4af',
+    'tscr;': '\U0001d4c9',
+    'TScy;': '\u0426',
+    'tscy;': '\u0446',
+    'TSHcy;': '\u040b',
+    'tshcy;': '\u045b',
+    'Tstrok;': '\u0166',
+    'tstrok;': '\u0167',
+    'twixt;': '\u226c',
+    'twoheadleftarrow;': '\u219e',
+    'twoheadrightarrow;': '\u21a0',
+    'Uacute': '\xda',
+    'uacute': '\xfa',
+    'Uacute;': '\xda',
+    'uacute;': '\xfa',
+    'Uarr;': '\u219f',
+    'uArr;': '\u21d1',
+    'uarr;': '\u2191',
+    'Uarrocir;': '\u2949',
+    'Ubrcy;': '\u040e',
+    'ubrcy;': '\u045e',
+    'Ubreve;': '\u016c',
+    'ubreve;': '\u016d',
+    'Ucirc': '\xdb',
+    'ucirc': '\xfb',
+    'Ucirc;': '\xdb',
+    'ucirc;': '\xfb',
+    'Ucy;': '\u0423',
+    'ucy;': '\u0443',
+    'udarr;': '\u21c5',
+    'Udblac;': '\u0170',
+    'udblac;': '\u0171',
+    'udhar;': '\u296e',
+    'ufisht;': '\u297e',
+    'Ufr;': '\U0001d518',
+    'ufr;': '\U0001d532',
+    'Ugrave': '\xd9',
+    'ugrave': '\xf9',
+    'Ugrave;': '\xd9',
+    'ugrave;': '\xf9',
+    'uHar;': '\u2963',
+    'uharl;': '\u21bf',
+    'uharr;': '\u21be',
+    'uhblk;': '\u2580',
+    'ulcorn;': '\u231c',
+    'ulcorner;': '\u231c',
+    'ulcrop;': '\u230f',
+    'ultri;': '\u25f8',
+    'Umacr;': '\u016a',
+    'umacr;': '\u016b',
+    'uml': '\xa8',
+    'uml;': '\xa8',
+    'UnderBar;': '_',
+    'UnderBrace;': '\u23df',
+    'UnderBracket;': '\u23b5',
+    'UnderParenthesis;': '\u23dd',
+    'Union;': '\u22c3',
+    'UnionPlus;': '\u228e',
+    'Uogon;': '\u0172',
+    'uogon;': '\u0173',
+    'Uopf;': '\U0001d54c',
+    'uopf;': '\U0001d566',
+    'UpArrow;': '\u2191',
+    'Uparrow;': '\u21d1',
+    'uparrow;': '\u2191',
+    'UpArrowBar;': '\u2912',
+    'UpArrowDownArrow;': '\u21c5',
+    'UpDownArrow;': '\u2195',
+    'Updownarrow;': '\u21d5',
+    'updownarrow;': '\u2195',
+    'UpEquilibrium;': '\u296e',
+    'upharpoonleft;': '\u21bf',
+    'upharpoonright;': '\u21be',
+    'uplus;': '\u228e',
+    'UpperLeftArrow;': '\u2196',
+    'UpperRightArrow;': '\u2197',
+    'Upsi;': '\u03d2',
+    'upsi;': '\u03c5',
+    'upsih;': '\u03d2',
+    'Upsilon;': '\u03a5',
+    'upsilon;': '\u03c5',
+    'UpTee;': '\u22a5',
+    'UpTeeArrow;': '\u21a5',
+    'upuparrows;': '\u21c8',
+    'urcorn;': '\u231d',
+    'urcorner;': '\u231d',
+    'urcrop;': '\u230e',
+    'Uring;': '\u016e',
+    'uring;': '\u016f',
+    'urtri;': '\u25f9',
+    'Uscr;': '\U0001d4b0',
+    'uscr;': '\U0001d4ca',
+    'utdot;': '\u22f0',
+    'Utilde;': '\u0168',
+    'utilde;': '\u0169',
+    'utri;': '\u25b5',
+    'utrif;': '\u25b4',
+    'uuarr;': '\u21c8',
+    'Uuml': '\xdc',
+    'uuml': '\xfc',
+    'Uuml;': '\xdc',
+    'uuml;': '\xfc',
+    'uwangle;': '\u29a7',
+    'vangrt;': '\u299c',
+    'varepsilon;': '\u03f5',
+    'varkappa;': '\u03f0',
+    'varnothing;': '\u2205',
+    'varphi;': '\u03d5',
+    'varpi;': '\u03d6',
+    'varpropto;': '\u221d',
+    'vArr;': '\u21d5',
+    'varr;': '\u2195',
+    'varrho;': '\u03f1',
+    'varsigma;': '\u03c2',
+    'varsubsetneq;': '\u228a\ufe00',
+    'varsubsetneqq;': '\u2acb\ufe00',
+    'varsupsetneq;': '\u228b\ufe00',
+    'varsupsetneqq;': '\u2acc\ufe00',
+    'vartheta;': '\u03d1',
+    'vartriangleleft;': '\u22b2',
+    'vartriangleright;': '\u22b3',
+    'Vbar;': '\u2aeb',
+    'vBar;': '\u2ae8',
+    'vBarv;': '\u2ae9',
+    'Vcy;': '\u0412',
+    'vcy;': '\u0432',
+    'VDash;': '\u22ab',
+    'Vdash;': '\u22a9',
+    'vDash;': '\u22a8',
+    'vdash;': '\u22a2',
+    'Vdashl;': '\u2ae6',
+    'Vee;': '\u22c1',
+    'vee;': '\u2228',
+    'veebar;': '\u22bb',
+    'veeeq;': '\u225a',
+    'vellip;': '\u22ee',
+    'Verbar;': '\u2016',
+    'verbar;': '|',
+    'Vert;': '\u2016',
+    'vert;': '|',
+    'VerticalBar;': '\u2223',
+    'VerticalLine;': '|',
+    'VerticalSeparator;': '\u2758',
+    'VerticalTilde;': '\u2240',
+    'VeryThinSpace;': '\u200a',
+    'Vfr;': '\U0001d519',
+    'vfr;': '\U0001d533',
+    'vltri;': '\u22b2',
+    'vnsub;': '\u2282\u20d2',
+    'vnsup;': '\u2283\u20d2',
+    'Vopf;': '\U0001d54d',
+    'vopf;': '\U0001d567',
+    'vprop;': '\u221d',
+    'vrtri;': '\u22b3',
+    'Vscr;': '\U0001d4b1',
+    'vscr;': '\U0001d4cb',
+    'vsubnE;': '\u2acb\ufe00',
+    'vsubne;': '\u228a\ufe00',
+    'vsupnE;': '\u2acc\ufe00',
+    'vsupne;': '\u228b\ufe00',
+    'Vvdash;': '\u22aa',
+    'vzigzag;': '\u299a',
+    'Wcirc;': '\u0174',
+    'wcirc;': '\u0175',
+    'wedbar;': '\u2a5f',
+    'Wedge;': '\u22c0',
+    'wedge;': '\u2227',
+    'wedgeq;': '\u2259',
+    'weierp;': '\u2118',
+    'Wfr;': '\U0001d51a',
+    'wfr;': '\U0001d534',
+    'Wopf;': '\U0001d54e',
+    'wopf;': '\U0001d568',
+    'wp;': '\u2118',
+    'wr;': '\u2240',
+    'wreath;': '\u2240',
+    'Wscr;': '\U0001d4b2',
+    'wscr;': '\U0001d4cc',
+    'xcap;': '\u22c2',
+    'xcirc;': '\u25ef',
+    'xcup;': '\u22c3',
+    'xdtri;': '\u25bd',
+    'Xfr;': '\U0001d51b',
+    'xfr;': '\U0001d535',
+    'xhArr;': '\u27fa',
+    'xharr;': '\u27f7',
+    'Xi;': '\u039e',
+    'xi;': '\u03be',
+    'xlArr;': '\u27f8',
+    'xlarr;': '\u27f5',
+    'xmap;': '\u27fc',
+    'xnis;': '\u22fb',
+    'xodot;': '\u2a00',
+    'Xopf;': '\U0001d54f',
+    'xopf;': '\U0001d569',
+    'xoplus;': '\u2a01',
+    'xotime;': '\u2a02',
+    'xrArr;': '\u27f9',
+    'xrarr;': '\u27f6',
+    'Xscr;': '\U0001d4b3',
+    'xscr;': '\U0001d4cd',
+    'xsqcup;': '\u2a06',
+    'xuplus;': '\u2a04',
+    'xutri;': '\u25b3',
+    'xvee;': '\u22c1',
+    'xwedge;': '\u22c0',
+    'Yacute': '\xdd',
+    'yacute': '\xfd',
+    'Yacute;': '\xdd',
+    'yacute;': '\xfd',
+    'YAcy;': '\u042f',
+    'yacy;': '\u044f',
+    'Ycirc;': '\u0176',
+    'ycirc;': '\u0177',
+    'Ycy;': '\u042b',
+    'ycy;': '\u044b',
+    'yen': '\xa5',
+    'yen;': '\xa5',
+    'Yfr;': '\U0001d51c',
+    'yfr;': '\U0001d536',
+    'YIcy;': '\u0407',
+    'yicy;': '\u0457',
+    'Yopf;': '\U0001d550',
+    'yopf;': '\U0001d56a',
+    'Yscr;': '\U0001d4b4',
+    'yscr;': '\U0001d4ce',
+    'YUcy;': '\u042e',
+    'yucy;': '\u044e',
+    'yuml': '\xff',
+    'Yuml;': '\u0178',
+    'yuml;': '\xff',
+    'Zacute;': '\u0179',
+    'zacute;': '\u017a',
+    'Zcaron;': '\u017d',
+    'zcaron;': '\u017e',
+    'Zcy;': '\u0417',
+    'zcy;': '\u0437',
+    'Zdot;': '\u017b',
+    'zdot;': '\u017c',
+    'zeetrf;': '\u2128',
+    'ZeroWidthSpace;': '\u200b',
+    'Zeta;': '\u0396',
+    'zeta;': '\u03b6',
+    'Zfr;': '\u2128',
+    'zfr;': '\U0001d537',
+    'ZHcy;': '\u0416',
+    'zhcy;': '\u0436',
+    'zigrarr;': '\u21dd',
+    'Zopf;': '\u2124',
+    'zopf;': '\U0001d56b',
+    'Zscr;': '\U0001d4b5',
+    'zscr;': '\U0001d4cf',
+    'zwj;': '\u200d',
+    'zwnj;': '\u200c',
+}
+
 # maps the Unicode codepoint to the HTML entity name
 codepoint2name = {}
 
diff --git a/Lib/html/parser.py b/Lib/html/parser.py
index de504ab544..f8ac82834a 100644
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -10,6 +10,7 @@
 
 import _markupbase
 import re
+import warnings
 
 # Regular expressions used for parsing
 
@@ -113,14 +114,16 @@ class HTMLParser(_markupbase.ParserBase):
 
     CDATA_CONTENT_ELEMENTS = ("script", "style")
 
-    def __init__(self, strict=True):
+    def __init__(self, strict=False):
         """Initialize and reset this instance.
 
-        If strict is set to True (the default), errors are raised when invalid
-        HTML is encountered.  If set to False, an attempt is instead made to
-        continue parsing, making "best guesses" about the intended meaning, in
-        a fashion similar to what browsers typically do.
+        If strict is set to False (the default) the parser will parse invalid
+        markup, otherwise it will raise an error.  Note that the strict mode
+        is deprecated.
         """
+        if strict:
+            warnings.warn("The strict mode is deprecated.",
+                          DeprecationWarning, stacklevel=2)
         self.strict = strict
         self.reset()
 
@@ -271,8 +274,8 @@ class HTMLParser(_markupbase.ParserBase):
     # See also parse_declaration in _markupbase
     def parse_html_declaration(self, i):
         rawdata = self.rawdata
-        if rawdata[i:i+2] != '<!':
-            self.error('unexpected call to parse_html_declaration()')
+        assert rawdata[i:i+2] == '<!', ('unexpected call to '
+                                        'parse_html_declaration()')
         if rawdata[i:i+4] == '<!--':
             # this case is actually already handled in goahead()
             return self.parse_comment(i)
@@ -292,8 +295,8 @@ class HTMLParser(_markupbase.ParserBase):
     # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
     def parse_bogus_comment(self, i, report=1):
         rawdata = self.rawdata
-        if rawdata[i:i+2] not in ('<!', '</'):
-            self.error('unexpected call to parse_comment()')
+        assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
+                                                'parse_comment()')
         pos = rawdata.find('>', i+2)
         if pos == -1:
             return -1
@@ -497,7 +500,6 @@ class HTMLParser(_markupbase.ParserBase):
             self.error("unknown declaration: %r" % (data,))
 
     # Internal -- helper to remove special character quoting
-    entitydefs = None
     def unescape(self, s):
         if '&' not in s:
             return s
@@ -507,24 +509,23 @@ class HTMLParser(_markupbase.ParserBase):
                 if s[0] == "#":
                     s = s[1:]
                     if s[0] in ['x','X']:
-                        c = int(s[1:], 16)
+                        c = int(s[1:].rstrip(';'), 16)
                     else:
-                        c = int(s)
+                        c = int(s.rstrip(';'))
                     return chr(c)
             except ValueError:
-                return '&#'+ s +';'
+                return '&#' + s
             else:
-                # Cannot use name2codepoint directly, because HTMLParser
-                # supports apos, which is not part of HTML 4
-                import html.entities
-                if HTMLParser.entitydefs is None:
-                    entitydefs = HTMLParser.entitydefs = {'apos':"'"}
-                    for k, v in html.entities.name2codepoint.items():
-                        entitydefs[k] = chr(v)
-                try:
-                    return self.entitydefs[s]
-                except KeyError:
-                    return '&'+s+';'
-
-        return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));",
+                from html.entities import html5
+                if s in html5:
+                    return html5[s]
+                elif s.endswith(';'):
+                    return '&' + s
+                for x in range(2, len(s)):
+                    if s[:x] in html5:
+                        return html5[s[:x]] + s[x:]
+                else:
+                    return '&' + s
+
+        return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+;|\w{1,32};?))",
                       replaceEntities, s, flags=re.ASCII)
diff --git a/Lib/http/client.py b/Lib/http/client.py
index 5466d0618d..4663d439e3 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -141,6 +141,9 @@ UNPROCESSABLE_ENTITY = 422
 LOCKED = 423
 FAILED_DEPENDENCY = 424
 UPGRADE_REQUIRED = 426
+PRECONDITION_REQUIRED = 428
+TOO_MANY_REQUESTS = 429
+REQUEST_HEADER_FIELDS_TOO_LARGE = 431
 
 # server error
 INTERNAL_SERVER_ERROR = 500
@@ -151,6 +154,7 @@ GATEWAY_TIMEOUT = 504
 HTTP_VERSION_NOT_SUPPORTED = 505
 INSUFFICIENT_STORAGE = 507
 NOT_EXTENDED = 510
+NETWORK_AUTHENTICATION_REQUIRED = 511
 
 # Mapping status codes to official W3C names
 responses = {
@@ -192,6 +196,9 @@ responses = {
     415: 'Unsupported Media Type',
     416: 'Requested Range Not Satisfiable',
     417: 'Expectation Failed',
+    428: 'Precondition Required',
+    429: 'Too Many Requests',
+    431: 'Request Header Fields Too Large',
 
     500: 'Internal Server Error',
     501: 'Not Implemented',
@@ -199,6 +206,7 @@ responses = {
     503: 'Service Unavailable',
     504: 'Gateway Timeout',
     505: 'HTTP Version Not Supported',
+    511: 'Network Authentication Required',
 }
 
 # maximal amount of data to read at one time in _safe_read
@@ -489,11 +497,17 @@ class HTTPResponse(io.RawIOBase):
             self._close_conn()
             return b""
 
-        if self.chunked:
-            return self._read_chunked(amt)
+        if amt is not None:
+            # Amount is given, so call base class version
+            # (which is implemented in terms of self.readinto)
+            return super(HTTPResponse, self).read(amt)
+        else:
+            # Amount is not given (unbounded read) so we must check self.length
+            # and self.chunked
+
+            if self.chunked:
+                return self._readall_chunked()
 
-        if amt is None:
-            # unbounded read
             if self.length is None:
                 s = self.fp.read()
             else:
@@ -506,83 +520,131 @@ class HTTPResponse(io.RawIOBase):
             self._close_conn()        # we read everything
             return s
 
+    def readinto(self, b):
+        if self.fp is None:
+            return 0
+
+        if self._method == "HEAD":
+            self._close_conn()
+            return 0
+
+        if self.chunked:
+            return self._readinto_chunked(b)
+
         if self.length is not None:
-            if amt > self.length:
+            if len(b) > self.length:
                 # clip the read to the "end of response"
-                amt = self.length
+                b = memoryview(b)[0:self.length]
 
         # we do not use _safe_read() here because this may be a .will_close
         # connection, and the user is reading more bytes than will be provided
         # (for example, reading in 1k chunks)
-        s = self.fp.read(amt)
-        if not s:
+        n = self.fp.readinto(b)
+        if not n:
             # Ideally, we would raise IncompleteRead if the content-length
             # wasn't satisfied, but it might break compatibility.
             self._close_conn()
         elif self.length is not None:
-            self.length -= len(s)
+            self.length -= n
             if not self.length:
                 self._close_conn()
+        return n
 
-        return s
+    def _read_next_chunk_size(self):
+        # Read the next chunk size from the file
+        line = self.fp.readline(_MAXLINE + 1)
+        if len(line) > _MAXLINE:
+            raise LineTooLong("chunk size")
+        i = line.find(b";")
+        if i >= 0:
+            line = line[:i] # strip chunk-extensions
+        try:
+            return int(line, 16)
+        except ValueError:
+            # close the connection as protocol synchronisation is
+            # probably lost
+            self._close_conn()
+            raise
 
-    def _read_chunked(self, amt):
+    def _read_and_discard_trailer(self):
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline(_MAXLINE + 1)
+            if len(line) > _MAXLINE:
+                raise LineTooLong("trailer line")
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line in (b'\r\n', b'\n', b''):
+                break
+
+    def _readall_chunked(self):
         assert self.chunked != _UNKNOWN
         chunk_left = self.chunk_left
         value = []
         while True:
             if chunk_left is None:
-                line = self.fp.readline(_MAXLINE + 1)
-                if len(line) > _MAXLINE:
-                    raise LineTooLong("chunk size")
-                i = line.find(b";")
-                if i >= 0:
-                    line = line[:i] # strip chunk-extensions
                 try:
-                    chunk_left = int(line, 16)
+                    chunk_left = self._read_next_chunk_size()
+                    if chunk_left == 0:
+                        break
                 except ValueError:
-                    # close the connection as protocol synchronisation is
-                    # probably lost
-                    self._close_conn()
                     raise IncompleteRead(b''.join(value))
-                if chunk_left == 0:
-                    break
-            if amt is None:
-                value.append(self._safe_read(chunk_left))
-            elif amt < chunk_left:
-                value.append(self._safe_read(amt))
-                self.chunk_left = chunk_left - amt
-                return b''.join(value)
-            elif amt == chunk_left:
-                value.append(self._safe_read(amt))
+            value.append(self._safe_read(chunk_left))
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        self._read_and_discard_trailer()
+
+        # we read everything; close the "file"
+        self._close_conn()
+
+        return b''.join(value)
+
+    def _readinto_chunked(self, b):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+
+        total_bytes = 0
+        mvb = memoryview(b)
+        while True:
+            if chunk_left is None:
+                try:
+                    chunk_left = self._read_next_chunk_size()
+                    if chunk_left == 0:
+                        break
+                except ValueError:
+                    raise IncompleteRead(bytes(b[0:total_bytes]))
+
+            if len(mvb) < chunk_left:
+                n = self._safe_readinto(mvb)
+                self.chunk_left = chunk_left - n
+                return total_bytes + n
+            elif len(mvb) == chunk_left:
+                n = self._safe_readinto(mvb)
                 self._safe_read(2)  # toss the CRLF at the end of the chunk
                 self.chunk_left = None
-                return b''.join(value)
+                return total_bytes + n
             else:
-                value.append(self._safe_read(chunk_left))
-                amt -= chunk_left
+                temp_mvb = mvb[0:chunk_left]
+                n = self._safe_readinto(temp_mvb)
+                mvb = mvb[n:]
+                total_bytes += n
 
             # we read the whole chunk, get another
             self._safe_read(2)      # toss the CRLF at the end of the chunk
             chunk_left = None
 
-        # read and discard trailer up to the CRLF terminator
-        ### note: we shouldn't have any trailers!
-        while True:
-            line = self.fp.readline(_MAXLINE + 1)
-            if len(line) > _MAXLINE:
-                raise LineTooLong("trailer line")
-            if not line:
-                # a vanishingly small number of sites EOF without
-                # sending the trailer
-                break
-            if line in (b'\r\n', b'\n', b''):
-                break
+        self._read_and_discard_trailer()
 
         # we read everything; close the "file"
         self._close_conn()
 
-        return b''.join(value)
+        return total_bytes
 
     def _safe_read(self, amt):
         """Read the number of bytes requested, compensating for partial reads.
@@ -607,6 +669,22 @@ class HTTPResponse(io.RawIOBase):
             amt -= len(chunk)
         return b"".join(s)
 
+    def _safe_readinto(self, b):
+        """Same as _safe_read, but for reading into a buffer."""
+        total_bytes = 0
+        mvb = memoryview(b)
+        while total_bytes < len(b):
+            if MAXAMOUNT < len(mvb):
+                temp_mvb = mvb[0:MAXAMOUNT]
+                n = self.fp.readinto(temp_mvb)
+            else:
+                n = self.fp.readinto(mvb)
+            if not n:
+                raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b))
+            mvb = mvb[n:]
+            total_bytes += n
+        return total_bytes
+
     def fileno(self):
         return self.fp.fileno()
 
@@ -713,7 +791,7 @@ class HTTPConnection:
         self.send(connect_bytes)
         for header, value in self._tunnel_headers.items():
             header_str = "%s: %s\r\n" % (header, value)
-            header_bytes = header_str.encode("latin1")
+            header_bytes = header_str.encode("latin-1")
             self.send(header_bytes)
         self.send(b'\r\n')
 
@@ -956,7 +1034,7 @@ class HTTPConnection:
         values = list(values)
         for i, one_value in enumerate(values):
             if hasattr(one_value, 'encode'):
-                values[i] = one_value.encode('latin1')
+                values[i] = one_value.encode('latin-1')
             elif isinstance(one_value, int):
                 values[i] = str(one_value).encode('ascii')
         value = b'\r\n\t'.join(values)
diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py
index b6cfc355c9..901e762f0e 100644
--- a/Lib/http/cookiejar.py
+++ b/Lib/http/cookiejar.py
@@ -625,7 +625,7 @@ def request_path(request):
     return path
 
 def request_port(request):
-    host = request.get_host()
+    host = request.host
     i = host.find(':')
     if i >= 0:
         port = host[i+1:]
@@ -949,7 +949,7 @@ class DefaultCookiePolicy(CookiePolicy):
         return True
 
     def set_ok_verifiability(self, cookie, request):
-        if request.is_unverifiable() and is_third_party(request):
+        if request.unverifiable and is_third_party(request):
             if cookie.version > 0 and self.strict_rfc2965_unverifiable:
                 _debug("   third-party RFC 2965 cookie during "
                              "unverifiable transaction")
@@ -1088,7 +1088,7 @@ class DefaultCookiePolicy(CookiePolicy):
         return True
 
     def return_ok_verifiability(self, cookie, request):
-        if request.is_unverifiable() and is_third_party(request):
+        if request.unverifiable and is_third_party(request):
             if cookie.version > 0 and self.strict_rfc2965_unverifiable:
                 _debug("   third-party RFC 2965 cookie during unverifiable "
                        "transaction")
@@ -1100,7 +1100,7 @@ class DefaultCookiePolicy(CookiePolicy):
         return True
 
     def return_ok_secure(self, cookie, request):
-        if cookie.secure and request.get_type() != "https":
+        if cookie.secure and request.type != "https":
             _debug("   secure cookie with non-secure request")
             return False
         return True
diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py
index ddbcbf877b..d2916786f2 100644
--- a/Lib/http/cookies.py
+++ b/Lib/http/cookies.py
@@ -159,7 +159,7 @@ class CookieError(Exception):
 #       _LegalChars       is the list of chars which don't require "'s
 #       _Translator       hash-table for fast quoting
 #
-_LegalChars       = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~"
+_LegalChars       = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~:"
 _Translator       = {
     '\000' : '\\000',  '\001' : '\\001',  '\002' : '\\002',
     '\003' : '\\003',  '\004' : '\\004',  '\005' : '\\005',
diff --git a/Lib/http/server.py b/Lib/http/server.py
index 5569037427..c4ac703d2d 100644
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -100,11 +100,14 @@ import sys
 import time
 import urllib.parse
 import copy
+import argparse
+
 
 # Default error message template
 DEFAULT_ERROR_MESSAGE = """\
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
         "http://www.w3.org/TR/html4/strict.dtd">
+<html>
     <head>
         <meta http-equiv="Content-Type" content="text/html;charset=utf-8">
         <title>Error response</title>
@@ -352,6 +355,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
 
         """
         self.send_response_only(100)
+        self.flush_headers()
         return True
 
     def handle_one_request(self):
@@ -429,7 +433,8 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
             self.wfile.write(content.encode('UTF-8', 'replace'))
 
     def send_response(self, code, message=None):
-        """Send the response header and log the response code.
+        """Add the response header to the headers buffer and log the
+        response code.
 
         Also send two standard headers with the server software
         version and the current date.
@@ -448,16 +453,19 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
             else:
                 message = ''
         if self.request_version != 'HTTP/0.9':
-            self.wfile.write(("%s %d %s\r\n" %
-                              (self.protocol_version, code, message)).encode('latin1', 'strict'))
+            if not hasattr(self, '_headers_buffer'):
+                self._headers_buffer = []
+            self._headers_buffer.append(("%s %d %s\r\n" %
+                    (self.protocol_version, code, message)).encode(
+                        'latin-1', 'strict'))
 
     def send_header(self, keyword, value):
-        """Send a MIME header."""
+        """Send a MIME header to the headers buffer."""
         if self.request_version != 'HTTP/0.9':
             if not hasattr(self, '_headers_buffer'):
                 self._headers_buffer = []
             self._headers_buffer.append(
-                ("%s: %s\r\n" % (keyword, value)).encode('latin1', 'strict'))
+                ("%s: %s\r\n" % (keyword, value)).encode('latin-1', 'strict'))
 
         if keyword.lower() == 'connection':
             if value.lower() == 'close':
@@ -469,6 +477,10 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
         """Send the blank line ending the MIME headers."""
         if self.request_version != 'HTTP/0.9':
             self._headers_buffer.append(b"\r\n")
+            self.flush_headers()
+
+    def flush_headers(self):
+        if hasattr(self, '_headers_buffer'):
             self.wfile.write(b"".join(self._headers_buffer))
             self._headers_buffer = []
 
@@ -514,7 +526,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
         """
 
         sys.stderr.write("%s - - [%s] %s\n" %
-                         (self.client_address[0],
+                         (self.address_string(),
                           self.log_date_time_string(),
                           format%args))
 
@@ -548,15 +560,9 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
                  'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
 
     def address_string(self):
-        """Return the client address formatted for logging.
-
-        This version looks up the full hostname using gethostbyaddr(),
-        and tries to find a name that contains at least one dot.
-
-        """
+        """Return the client address."""
 
-        host, port = self.client_address[:2]
-        return socket.getfqdn(host)
+        return self.client_address[0]
 
     # Essentially static class variables
 
@@ -569,7 +575,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
 
     # Table mapping response codes to messages; entries have the
     # form {code: (shortmessage, longmessage)}.
-    # See RFC 2616.
+    # See RFC 2616 and 6585.
     responses = {
         100: ('Continue', 'Request received, please continue'),
         101: ('Switching Protocols',
@@ -624,6 +630,12 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
               'Cannot satisfy request range.'),
         417: ('Expectation Failed',
               'Expect condition could not be satisfied.'),
+        428: ('Precondition Required',
+              'The origin server requires the request to be conditional.'),
+        429: ('Too Many Requests', 'The user has sent too many requests '
+              'in a given amount of time ("rate limiting").'),
+        431: ('Request Header Fields Too Large', 'The server is unwilling to '
+              'process the request because its header fields are too large.'),
 
         500: ('Internal Server Error', 'Server got itself in trouble'),
         501: ('Not Implemented',
@@ -634,6 +646,8 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
         504: ('Gateway Timeout',
               'The gateway server did not receive a timely response'),
         505: ('HTTP Version Not Supported', 'Cannot fulfill request.'),
+        511: ('Network Authentication Required',
+              'The client needs to authenticate to gain network access.'),
         }
 
 
@@ -722,10 +736,16 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         list.sort(key=lambda a: a.lower())
         r = []
         displaypath = html.escape(urllib.parse.unquote(self.path))
-        r.append('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
-        r.append("<html>\n<title>Directory listing for %s</title>\n" % displaypath)
-        r.append("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
-        r.append("<hr>\n<ul>\n")
+        enc = sys.getfilesystemencoding()
+        title = 'Directory listing for %s' % displaypath
+        r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
+                 '"http://www.w3.org/TR/html4/strict.dtd">')
+        r.append('<html>\n<head>')
+        r.append('<meta http-equiv="Content-Type" '
+                 'content="text/html; charset=%s">' % enc)
+        r.append('<title>%s</title>\n</head>' % title)
+        r.append('<body>\n<h1>%s</h1>' % title)
+        r.append('<hr>\n<ul>')
         for name in list:
             fullname = os.path.join(path, name)
             displayname = linkname = name
@@ -736,11 +756,10 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
             if os.path.islink(fullname):
                 displayname = name + "@"
                 # Note: a link to a directory displays with @ and links with /
-            r.append('<li><a href="%s">%s</a>\n'
+            r.append('<li><a href="%s">%s</a></li>'
                     % (urllib.parse.quote(linkname), html.escape(displayname)))
-        r.append("</ul>\n<hr>\n</body>\n</html>\n")
-        enc = sys.getfilesystemencoding()
-        encoded = ''.join(r).encode(enc)
+        r.append('</ul>\n<hr>\n</body>\n</html>\n')
+        encoded = '\n'.join(r).encode(enc)
         f = io.BytesIO()
         f.write(encoded)
         f.seek(0)
@@ -888,11 +907,7 @@ def nobody_uid():
 
 def executable(path):
     """Test for executable file."""
-    try:
-        st = os.stat(path)
-    except os.error:
-        return False
-    return st.st_mode & 0o111 != 0
+    return os.access(path, os.X_OK)
 
 
 class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
@@ -1008,7 +1023,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
                             scriptname)
             return
         ispy = self.is_python(scriptname)
-        if not ispy:
+        if self.have_fork or not ispy:
             if not self.is_executable(scriptfile):
                 self.send_error(403, "CGI script is not executable (%r)" %
                                 scriptname)
@@ -1029,9 +1044,6 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
         env['SCRIPT_NAME'] = scriptname
         if query:
             env['QUERY_STRING'] = query
-        host = self.address_string()
-        if host != self.client_address[0]:
-            env['REMOTE_HOST'] = host
         env['REMOTE_ADDR'] = self.client_address[0]
         authorization = self.headers.get("authorization")
         if authorization:
@@ -1083,6 +1095,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
             env.setdefault(k, "")
 
         self.send_response(200, "Script output follows")
+        self.flush_headers()
 
         decoded_query = query.replace('+', ' ')
 
@@ -1162,18 +1175,13 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
 
 
 def test(HandlerClass = BaseHTTPRequestHandler,
-         ServerClass = HTTPServer, protocol="HTTP/1.0"):
+         ServerClass = HTTPServer, protocol="HTTP/1.0", port=8000):
     """Test the HTTP request handler class.
 
     This runs an HTTP server on port 8000 (or the first command line
     argument).
 
     """
-
-    if sys.argv[1:]:
-        port = int(sys.argv[1])
-    else:
-        port = 8000
     server_address = ('', port)
 
     HandlerClass.protocol_version = protocol
@@ -1189,4 +1197,15 @@ def test(HandlerClass = BaseHTTPRequestHandler,
         sys.exit(0)
 
 if __name__ == '__main__':
-    test(HandlerClass=SimpleHTTPRequestHandler)
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cgi', action='store_true',
+                       help='Run as CGI Server')
+    parser.add_argument('port', action='store',
+                        default=8000, type=int,
+                        nargs='?',
+                        help='Specify alternate port [default: 8000]')
+    args = parser.parse_args()
+    if args.cgi:
+        test(HandlerClass=CGIHTTPRequestHandler, port=args.port)
+    else:
+        test(HandlerClass=SimpleHTTPRequestHandler, port=args.port)
diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py
index e4c1aff3dc..929d3581c9 100644
--- a/Lib/idlelib/AutoComplete.py
+++ b/Lib/idlelib/AutoComplete.py
@@ -9,9 +9,6 @@ import string
 
 from idlelib.configHandler import idleConf
 
-# This string includes all chars that may be in a file name (without a path
-# separator)
-FILENAME_CHARS = string.ascii_letters + string.digits + os.curdir + "._~#$:-"
 # This string includes all chars that may be in an identifier
 ID_CHARS = string.ascii_letters + string.digits + "_"
 
diff --git a/Lib/idlelib/ColorDelegator.py b/Lib/idlelib/ColorDelegator.py
index e188192994..e4ccb4258c 100644
--- a/Lib/idlelib/ColorDelegator.py
+++ b/Lib/idlelib/ColorDelegator.py
@@ -21,10 +21,11 @@ def make_pat():
     # 1st 'file' colorized normal, 2nd as builtin, 3rd as string
     builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b"
     comment = any("COMMENT", [r"#[^\n]*"])
-    sqstring = r"(\b[rRbB])?'[^'\\\n]*(\\.[^'\\\n]*)*'?"
-    dqstring = r'(\b[rRbB])?"[^"\\\n]*(\\.[^"\\\n]*)*"?'
-    sq3string = r"(\b[rRbB])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
-    dq3string = r'(\b[rRbB])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
+    stringprefix = r"(\br|u|ur|R|U|UR|Ur|uR|b|B|br|Br|bR|BR|rb|rB|Rb|RB)?"
+    sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?"
+    dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?'
+    sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
+    dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
     string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
     return kw + "|" + builtin + "|" + comment + "|" + string +\
            "|" + any("SYNC", [r"\n"])
@@ -149,9 +150,9 @@ class ColorDelegator(Delegator):
             self.stop_colorizing = False
             self.colorizing = True
             if DEBUG: print("colorizing...")
-            t0 = time.clock()
+            t0 = time.perf_counter()
             self.recolorize_main()
-            t1 = time.clock()
+            t1 = time.perf_counter()
             if DEBUG: print("%.3f seconds" % (t1-t0))
         finally:
             self.colorizing = False
diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py
index 16f63c52a4..3397415205 100644
--- a/Lib/idlelib/EditorWindow.py
+++ b/Lib/idlelib/EditorWindow.py
@@ -1,8 +1,9 @@
-import sys
+import imp
+import importlib
 import os
 import re
 import string
-import imp
+import sys
 from tkinter import *
 import tkinter.simpledialog as tkSimpleDialog
 import tkinter.messagebox as tkMessageBox
@@ -27,8 +28,7 @@ def _sphinx_version():
     "Format sys.version_info to produce the Sphinx version string used to install the chm docs"
     major, minor, micro, level, serial = sys.version_info
     release = '%s%s' % (major, minor)
-    if micro:
-        release += '%s' % (micro,)
+    release += '%s' % (micro,)
     if level == 'candidate':
         release += 'rc%s' % (serial,)
     elif level != 'final':
@@ -120,7 +120,7 @@ class EditorWindow(object):
 
     def __init__(self, flist=None, filename=None, key=None, root=None):
         if EditorWindow.help_url is None:
-            dochome =  os.path.join(sys.prefix, 'Doc', 'index.html')
+            dochome =  os.path.join(sys.base_prefix, 'Doc', 'index.html')
             if sys.platform.count('linux'):
                 # look for html docs in a couple of standard places
                 pyver = 'python-docs-' + '%s.%s.%s' % sys.version_info[:3]
@@ -131,13 +131,13 @@ class EditorWindow(object):
                     dochome = os.path.join(basepath, pyver,
                                            'Doc', 'index.html')
             elif sys.platform[:3] == 'win':
-                chmfile = os.path.join(sys.prefix, 'Doc',
+                chmfile = os.path.join(sys.base_prefix, 'Doc',
                                        'Python%s.chm' % _sphinx_version())
                 if os.path.isfile(chmfile):
                     dochome = chmfile
             elif macosxSupport.runningAsOSXApp():
                 # documentation is stored inside the python framework
-                dochome = os.path.join(sys.prefix,
+                dochome = os.path.join(sys.base_prefix,
                         'Resources/English.lproj/Documentation/index.html')
             dochome = os.path.normpath(dochome)
             if os.path.isfile(dochome):
@@ -1041,7 +1041,10 @@ class EditorWindow(object):
 
     def load_extension(self, name):
         try:
-            mod = __import__(name, globals(), locals(), [])
+            try:
+                mod = importlib.import_module('.' + name, package=__package__)
+            except ImportError:
+                mod = importlib.import_module(name)
         except ImportError:
             print("\nFailed to import extension: ", name)
             raise
diff --git a/Lib/idlelib/HyperParser.py b/Lib/idlelib/HyperParser.py
index 4414de72bc..4af4b085c7 100644
--- a/Lib/idlelib/HyperParser.py
+++ b/Lib/idlelib/HyperParser.py
@@ -234,7 +234,7 @@ class HyperParser:
                     # We can't continue after other types of brackets
                     if rawtext[pos] in "'\"":
                         # Scan a string prefix
-                        while pos > 0 and rawtext[pos - 1] in "rRbB":
+                        while pos > 0 and rawtext[pos - 1] in "rRbBuU":
                             pos -= 1
                         last_identifier_pos = pos
                     break
diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py
index c4f14efc27..9fe0701133 100644
--- a/Lib/idlelib/IOBinding.py
+++ b/Lib/idlelib/IOBinding.py
@@ -1,6 +1,6 @@
 import os
 import types
-import pipes
+import shlex
 import sys
 import codecs
 import tempfile
@@ -459,7 +459,7 @@ class IOBinding:
         else: #no printing for this platform
             printPlatform = False
         if printPlatform:  #we can try to print for this platform
-            command = command % pipes.quote(filename)
+            command = command % shlex.quote(filename)
             pipe = os.popen(command, "r")
             # things can get ugly on NT if there is no printer available.
             output = pipe.read().strip()
@@ -486,6 +486,8 @@ class IOBinding:
         ("All files", "*"),
         ]
 
+    defaultextension = '.py' if sys.platform == 'darwin' else ''
+
     def askopenfile(self):
         dir, base = self.defaultfilename("open")
         if not self.opendialog:
@@ -509,8 +511,10 @@ class IOBinding:
     def asksavefile(self):
         dir, base = self.defaultfilename("save")
         if not self.savedialog:
-            self.savedialog = tkFileDialog.SaveAs(master=self.text,
-                                                  filetypes=self.filetypes)
+            self.savedialog = tkFileDialog.SaveAs(
+                    master=self.text,
+                    filetypes=self.filetypes,
+                    defaultextension=self.defaultextension)
         filename = self.savedialog.show(initialdir=dir, initialfile=base)
         return filename
 
diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index f234b64b85..f6e8917d20 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -1,10 +1,23 @@
-What's New in IDLE 3.2.4?
+What's New in IDLE 3.3.1?
+=========================
+
+- Issue #16226: Fix IDLE Path Browser crash.
+  (Patch by Roger Serwy)
+
+- Issue #15853: Prevent IDLE crash on OS X when opening Preferences menu
+  with certain versions of Tk 8.5.  Initial patch by Kevin Walzer.
+
+
+What's New in IDLE 3.3.0?
 =========================
 
 - Issue #7163: Propagate return value of sys.stdout.write.
 
 - Issue #15318: Prevent writing to sys.stdin.
 
+- Issue #4832: Modify IDLE to save files with .py extension by
+  default on Windows and OS X (Tk 8.5) as it already does with X11 Tk.
+
 - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings.
 
 - Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE.
@@ -18,15 +31,10 @@ What's New in IDLE 3.2.4?
 - Issue #14937: Perform auto-completion of filenames in strings even for
   non-ASCII filenames. Likewise for identifiers.
 
-- Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X
-  to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6.
-
-- Issue #15853: Prevent IDLE crash on OS X when opening Preferences menu
-  with certain versions of Tk 8.5.  Initial patch by Kevin Walzer.
+- Issue #8515: Set __file__ when run file in IDLE.
+  Initial patch by Bruce Frederiksen.
 
-
-What's New in IDLE 3.2.3?
-=========================
+- IDLE can be launched as `python -m idlelib`
 
 - Issue #14409: IDLE now properly executes commands in the Shell window
   when it cannot read the normal config files on startup and
@@ -36,6 +44,9 @@ What's New in IDLE 3.2.3?
 - Issue #3573: IDLE hangs when passing invalid command line args
   (directory(ies) instead of file(s)).
 
+- Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X
+  to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6.
+
 
 What's New in IDLE 3.2.1?
 =========================
diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py
index d88a48e344..55bf1aa87c 100644
--- a/Lib/idlelib/PathBrowser.py
+++ b/Lib/idlelib/PathBrowser.py
@@ -1,6 +1,7 @@
 import os
 import sys
 import imp
+import importlib.machinery
 
 from idlelib.TreeWidget import TreeItem
 from idlelib.ClassBrowser import ClassBrowser, ModuleBrowserTreeItem
@@ -70,9 +71,11 @@ class DirBrowserTreeItem(TreeItem):
 
     def listmodules(self, allnames):
         modules = {}
-        suffixes = imp.get_suffixes()
+        suffixes = importlib.machinery.EXTENSION_SUFFIXES[:]
+        suffixes += importlib.machinery.SOURCE_SUFFIXES[:]
+        suffixes += importlib.machinery.BYTECODE_SUFFIXES[:]
         sorted = []
-        for suff, mode, flag in suffixes:
+        for suff in suffixes:
             i = -len(suff)
             for name in allnames[:]:
                 normed_name = os.path.normcase(name)
diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py
index 865472eb19..38ed3af757 100644
--- a/Lib/idlelib/PyShell.py
+++ b/Lib/idlelib/PyShell.py
@@ -481,6 +481,10 @@ class ModifiedInterpreter(InteractiveInterpreter):
 
     def kill_subprocess(self):
         try:
+            self.rpcclt.listening_sock.close()
+        except AttributeError:  # no socket
+            pass
+        try:
             self.rpcclt.close()
         except AttributeError:  # no socket
             pass
@@ -1009,6 +1013,8 @@ class PyShell(OutputWindow):
                 return False
         else:
             nosub = "==== No Subprocess ===="
+            sys.displayhook = rpc.displayhook
+
         self.write("Python %s on %s\n%s\n%s" %
                    (sys.version, sys.platform, self.COPYRIGHT, nosub))
         self.showprompt()
@@ -1231,6 +1237,16 @@ class PyShell(OutputWindow):
         self.set_line_and_column()
 
     def write(self, s, tags=()):
+        if isinstance(s, str) and len(s) and max(s) > '\uffff':
+            # Tk doesn't support outputting non-BMP characters
+            # Let's assume what printed string is not very long,
+            # find first non-BMP character and construct informative
+            # UnicodeEncodeError exception.
+            for start, char in enumerate(s):
+                if char > '\uffff':
+                    break
+            raise UnicodeEncodeError("UCS-2", char, start, start+1,
+                                     'Non-BMP character not supported in Tk')
         try:
             self.text.mark_gravity("iomark", "right")
             count = OutputWindow.write(self, s, tags, "iomark")
diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py
index 18ce9650ae..528adf67a6 100644
--- a/Lib/idlelib/ScriptBinding.py
+++ b/Lib/idlelib/ScriptBinding.py
@@ -150,16 +150,16 @@ class ScriptBinding:
         dirname = os.path.dirname(filename)
         # XXX Too often this discards arguments the user just set...
         interp.runcommand("""if 1:
-            _filename = %r
+            __file__ = {filename!r}
             import sys as _sys
             from os.path import basename as _basename
             if (not _sys.argv or
-                _basename(_sys.argv[0]) != _basename(_filename)):
-                _sys.argv = [_filename]
+                _basename(_sys.argv[0]) != _basename(__file__)):
+                _sys.argv = [__file__]
             import os as _os
-            _os.chdir(%r)
-            del _filename, _sys, _basename, _os
-            \n""" % (filename, dirname))
+            _os.chdir({dirname!r})
+            del _sys, _basename, _os
+            \n""".format(filename=filename, dirname=dirname))
         interp.prepend_syspath(filename)
         # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still
         #         go to __stderr__.  With subprocess, they go to the shell.
diff --git a/Lib/idlelib/__main__.py b/Lib/idlelib/__main__.py
new file mode 100644
index 0000000000..0666f2fd1b
--- /dev/null
+++ b/Lib/idlelib/__main__.py
@@ -0,0 +1,9 @@
+"""
+IDLE main entry point
+
+Run IDLE as python -m idlelib
+"""
+
+
+import idlelib.PyShell
+idlelib.PyShell.main()
diff --git a/Lib/idlelib/configHandler.py b/Lib/idlelib/configHandler.py
index 7fa481d893..5bf2668939 100644
--- a/Lib/idlelib/configHandler.py
+++ b/Lib/idlelib/configHandler.py
@@ -145,7 +145,8 @@ class IdleUserConfParser(IdleConfParser):
             except IOError:
                 os.unlink(fname)
                 cfgFile = open(fname, 'w')
-            self.write(cfgFile)
+            with cfgFile:
+                self.write(cfgFile)
         else:
             self.RemoveFile()
 
diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py
index 17455dbeee..8d8317dd29 100644
--- a/Lib/idlelib/idlever.py
+++ b/Lib/idlelib/idlever.py
@@ -1 +1 @@
-IDLE_VERSION = "3.2.3"
+IDLE_VERSION = "3.3.0"
diff --git a/Lib/idlelib/macosxSupport.py b/Lib/idlelib/macosxSupport.py
index 96904429af..67069fa0f3 100644
--- a/Lib/idlelib/macosxSupport.py
+++ b/Lib/idlelib/macosxSupport.py
@@ -12,12 +12,22 @@ _appbundle = None
 def runningAsOSXApp():
     """
     Returns True if Python is running from within an app on OSX.
-    If so, assume that Python was built with Aqua Tcl/Tk rather than
-    X11 Tcl/Tk.
+    If so, the various OS X customizations will be triggered later (menu
+    fixup, et al).  (Originally, this test was supposed to condition
+    behavior on whether IDLE was running under Aqua Tk rather than
+    under X11 Tk but that does not work since a framework build
+    could be linked with X11.  For several releases, this test actually
+    differentiates between whether IDLE is running from a framework or
+    not.  As a future enhancement, it should be considered whether there
+    should be a difference based on framework and any needed X11 adaptions
+    should be made dependent on a new function that actually tests for X11.)
     """
     global _appbundle
     if _appbundle is None:
-        _appbundle = (sys.platform == 'darwin' and '.app' in sys.executable)
+        _appbundle = sys.platform == 'darwin'
+        if _appbundle:
+            import sysconfig
+            _appbundle = bool(sysconfig.get_config_var('PYTHONFRAMEWORK'))
     return _appbundle
 
 _carbonaquatk = None
diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py
index def43945ae..77cb3ac0a3 100644
--- a/Lib/idlelib/rpc.py
+++ b/Lib/idlelib/rpc.py
@@ -40,6 +40,7 @@ import traceback
 import copyreg
 import types
 import marshal
+import builtins
 
 
 def unpickle_code(ms):
@@ -196,8 +197,12 @@ class SocketIO(object):
                 return ("ERROR", "Unsupported message type: %s" % how)
         except SystemExit:
             raise
+        except KeyboardInterrupt:
+            raise
         except socket.error:
             raise
+        except Exception as ex:
+            return ("CALLEXC", ex)
         except:
             msg = "*** Internal Error: rpc.py:SocketIO.localcall()\n\n"\
                   " Object: %s \n Method: %s \n Args: %s\n"
@@ -257,6 +262,9 @@ class SocketIO(object):
         if how == "ERROR":
             self.debug("decoderesponse: Internal ERROR:", what)
             raise RuntimeError(what)
+        if how == "CALLEXC":
+            self.debug("decoderesponse: Call Exception:", what)
+            raise what
         raise SystemError(how, what)
 
     def decode_interrupthook(self):
@@ -596,3 +604,21 @@ class MethodProxy(object):
 
 # XXX KBK 09Sep03  We need a proper unit test for this module.  Previously
 #                  existing test code was removed at Rev 1.27 (r34098).
+
+def displayhook(value):
+    """Override standard display hook to use non-locale encoding"""
+    if value is None:
+        return
+    # Set '_' to None to avoid recursion
+    builtins._ = None
+    text = repr(value)
+    try:
+        sys.stdout.write(text)
+    except UnicodeEncodeError:
+        # let's use ascii while utf8-bmp codec doesn't present
+        encoding = 'ascii'
+        bytes = text.encode(encoding, 'backslashreplace')
+        text = bytes.decode(encoding, 'strict')
+        sys.stdout.write(text)
+    sys.stdout.write("\n")
+    builtins._ = value
diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py
index 7d0941e80b..d10591246f 100644
--- a/Lib/idlelib/run.py
+++ b/Lib/idlelib/run.py
@@ -7,6 +7,7 @@ import traceback
 import _thread as thread
 import threading
 import queue
+import tkinter
 
 from idlelib import CallTips
 from idlelib import AutoComplete
@@ -41,6 +42,17 @@ else:
         return s
     warnings.formatwarning = idle_formatwarning_subproc
 
+
+tcl = tkinter.Tcl()
+
+
+def handle_tk_events(tcl=tcl):
+    """Process any tk events that are ready to be dispatched if tkinter
+    has been imported, a tcl interpreter has been created and tk has been
+    loaded."""
+    tcl.eval("update")
+
+
 # Thread shared globals: Establish a queue between a subthread (which handles
 # the socket) and the main thread (which runs user code), plus global
 # completion, exit and interruptable (the main thread) flags:
@@ -96,6 +108,7 @@ def main(del_exitfunc=False):
             try:
                 seq, request = rpc.request_queue.get(block=True, timeout=0.05)
             except queue.Empty:
+                handle_tk_events()
                 continue
             method, args, kwargs = request
             ret = method(*args, **kwargs)
@@ -170,7 +183,9 @@ def print_exception():
             print_exc(type(cause), cause, cause.__traceback__)
             print("\nThe above exception was the direct cause "
                   "of the following exception:\n", file=efile)
-        elif context is not None and context not in seen:
+        elif (context is not None and
+              not exc.__suppress_context__ and
+              context not in seen):
             print_exc(type(context), context, context.__traceback__)
             print("\nDuring handling of the above exception, "
                   "another exception occurred:\n", file=efile)
@@ -278,6 +293,7 @@ class MyHandler(rpc.RPCHandler):
         sys.stderr = PyShell.PseudoOutputFile(self.console, "stderr",
                 IOBinding.encoding)
 
+        sys.displayhook = rpc.displayhook
         # page help() text to shell.
         import pydoc # import must be done here to capture i/o binding
         pydoc.pager = pydoc.plainpager
diff --git a/Lib/imaplib.py b/Lib/imaplib.py
index 00a17fbf36..f8c7ffdb92 100644
--- a/Lib/imaplib.py
+++ b/Lib/imaplib.py
@@ -23,7 +23,7 @@ Public functions:       Internaldate2tuple
 __version__ = "2.58"
 
 import binascii, errno, random, re, socket, subprocess, sys, time, calendar
-
+from datetime import datetime, timezone, timedelta
 try:
     import ssl
     HAVE_SSL = True
@@ -249,15 +249,7 @@ class IMAP4:
 
     def read(self, size):
         """Read 'size' bytes from remote."""
-        chunks = []
-        read = 0
-        while read < size:
-            data = self.file.read(min(size-read, 4096))
-            if not data:
-                break
-            read += len(data)
-            chunks.append(data)
-        return b''.join(chunks)
+        return self.file.read(size)
 
 
     def readline(self):
@@ -1179,25 +1171,40 @@ if HAVE_SSL:
 
         """IMAP4 client class over SSL connection
 
-        Instantiate with: IMAP4_SSL([host[, port[, keyfile[, certfile]]]])
+        Instantiate with: IMAP4_SSL([host[, port[, keyfile[, certfile[, ssl_context]]]]])
 
                 host - host's name (default: localhost);
-                port - port number (default: standard IMAP4 SSL port).
+                port - port number (default: standard IMAP4 SSL port);
                 keyfile - PEM formatted file that contains your private key (default: None);
                 certfile - PEM formatted certificate chain file (default: None);
+                ssl_context - a SSLContext object that contains your certificate chain
+                              and private key (default: None)
+                Note: if ssl_context is provided, then parameters keyfile or
+                certfile should not be set otherwise ValueError is raised.
 
         for more documentation see the docstring of the parent class IMAP4.
         """
 
 
-        def __init__(self, host = '', port = IMAP4_SSL_PORT, keyfile = None, certfile = None):
+        def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None):
+            if ssl_context is not None and keyfile is not None:
+                raise ValueError("ssl_context and keyfile arguments are mutually "
+                                 "exclusive")
+            if ssl_context is not None and certfile is not None:
+                raise ValueError("ssl_context and certfile arguments are mutually "
+                                 "exclusive")
+
             self.keyfile = keyfile
             self.certfile = certfile
+            self.ssl_context = ssl_context
             IMAP4.__init__(self, host, port)
 
         def _create_socket(self):
             sock = IMAP4._create_socket(self)
-            return ssl.wrap_socket(sock, self.keyfile, self.certfile)
+            if self.ssl_context:
+                return self.ssl_context.wrap_socket(sock)
+            else:
+                return ssl.wrap_socket(sock, self.keyfile, self.certfile)
 
         def open(self, host='', port=IMAP4_SSL_PORT):
             """Setup connection to remote server on "host:port".
@@ -1310,10 +1317,8 @@ class _Authenticator:
             return b''
         return binascii.a2b_base64(inp)
 
-
-
-Mon2num = {b'Jan': 1, b'Feb': 2, b'Mar': 3, b'Apr': 4, b'May': 5, b'Jun': 6,
-           b'Jul': 7, b'Aug': 8, b'Sep': 9, b'Oct': 10, b'Nov': 11, b'Dec': 12}
+Months = ' Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' ')
+Mon2num = {s.encode():n+1 for n, s in enumerate(Months[1:])}
 
 def Internaldate2tuple(resp):
     """Parse an IMAP4 INTERNALDATE string.
@@ -1381,28 +1386,37 @@ def Time2Internaldate(date_time):
     Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'.  The
     date_time argument can be a number (int or float) representing
     seconds since epoch (as returned by time.time()), a 9-tuple
-    representing local time (as returned by time.localtime()), or a
+    representing local time, an instance of time.struct_time (as
+    returned by time.localtime()), an aware datetime instance or a
     double-quoted string.  In the last case, it is assumed to already
     be in the correct format.
     """
-
     if isinstance(date_time, (int, float)):
-        tt = time.localtime(date_time)
-    elif isinstance(date_time, (tuple, time.struct_time)):
-        tt = date_time
+        dt = datetime.fromtimestamp(date_time,
+                                    timezone.utc).astimezone()
+    elif isinstance(date_time, tuple):
+        try:
+            gmtoff = date_time.tm_gmtoff
+        except AttributeError:
+            if time.daylight:
+                dst = date_time[8]
+                if dst == -1:
+                    dst = time.localtime(time.mktime(date_time))[8]
+                gmtoff = -(time.timezone, time.altzone)[dst]
+            else:
+                gmtoff = -time.timezone
+        delta = timedelta(seconds=gmtoff)
+        dt = datetime(*date_time[:6], tzinfo=timezone(delta))
+    elif isinstance(date_time, datetime):
+        if date_time.tzinfo is None:
+            raise ValueError("date_time must be aware")
+        dt = date_time
     elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'):
         return date_time        # Assume in correct format
     else:
         raise ValueError("date_time not of a known type")
-
-    dt = time.strftime("%d-%b-%Y %H:%M:%S", tt)
-    if dt[0] == '0':
-        dt = ' ' + dt[1:]
-    if time.daylight and tt[-1]:
-        zone = -time.altzone
-    else:
-        zone = -time.timezone
-    return '"' + dt + " %+03d%02d" % divmod(zone//60, 60) + '"'
+    fmt = '"%d-{}-%Y %H:%M:%S %z"'.format(Months[dt.month])
+    return dt.strftime(fmt)
 
 
 
diff --git a/Lib/imp.py b/Lib/imp.py
new file mode 100644
index 0000000000..da9c84ee39
--- /dev/null
+++ b/Lib/imp.py
@@ -0,0 +1,257 @@
+"""This module provides the components needed to build your own __import__
+function.  Undocumented functions are obsolete.
+
+In most cases it is preferred you consider using the importlib module's
+functionality over this module.
+
+"""
+# (Probably) need to stay in _imp
+from _imp import (lock_held, acquire_lock, release_lock,
+                  load_dynamic, get_frozen_object, is_frozen_package,
+                  init_builtin, init_frozen, is_builtin, is_frozen,
+                  _fix_co_filename)
+
+# Directly exposed by this module
+from importlib._bootstrap import new_module
+from importlib._bootstrap import cache_from_source, source_from_cache
+
+
+from importlib import _bootstrap
+from importlib import machinery
+import os
+import sys
+import tokenize
+import warnings
+
+
+# DEPRECATED
+SEARCH_ERROR = 0
+PY_SOURCE = 1
+PY_COMPILED = 2
+C_EXTENSION = 3
+PY_RESOURCE = 4
+PKG_DIRECTORY = 5
+C_BUILTIN = 6
+PY_FROZEN = 7
+PY_CODERESOURCE = 8
+IMP_HOOK = 9
+
+
+def get_magic():
+    """Return the magic number for .pyc or .pyo files."""
+    return _bootstrap._MAGIC_BYTES
+
+
+def get_tag():
+    """Return the magic tag for .pyc or .pyo files."""
+    return sys.implementation.cache_tag
+
+
+def get_suffixes():
+    warnings.warn('imp.get_suffixes() is deprecated; use the constants '
+                  'defined on importlib.machinery instead',
+                  DeprecationWarning, 2)
+    extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
+    source = [(s, 'U', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
+    bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
+
+    return extensions + source + bytecode
+
+
+class NullImporter:
+
+    """Null import object."""
+
+    def __init__(self, path):
+        if path == '':
+            raise ImportError('empty pathname', path='')
+        elif os.path.isdir(path):
+            raise ImportError('existing directory', path=path)
+
+    def find_module(self, fullname):
+        """Always returns None."""
+        return None
+
+
+class _HackedGetData:
+
+    """Compatibiilty support for 'file' arguments of various load_*()
+    functions."""
+
+    def __init__(self, fullname, path, file=None):
+        super().__init__(fullname, path)
+        self.file = file
+
+    def get_data(self, path):
+        """Gross hack to contort loader to deal w/ load_*()'s bad API."""
+        if self.file and path == self.path:
+            with self.file:
+                # Technically should be returning bytes, but
+                # SourceLoader.get_code() just passed what is returned to
+                # compile() which can handle str. And converting to bytes would
+                # require figuring out the encoding to decode to and
+                # tokenize.detect_encoding() only accepts bytes.
+                return self.file.read()
+        else:
+            return super().get_data(path)
+
+
+class _LoadSourceCompatibility(_HackedGetData, _bootstrap.SourceFileLoader):
+
+    """Compatibility support for implementing load_source()."""
+
+
+def load_source(name, pathname, file=None):
+    msg = ('imp.load_source() is deprecated; use '
+           'importlib.machinery.SourceFileLoader(name, pathname).load_module()'
+           ' instead')
+    warnings.warn(msg, DeprecationWarning, 2)
+    return _LoadSourceCompatibility(name, pathname, file).load_module(name)
+
+
+class _LoadCompiledCompatibility(_HackedGetData,
+        _bootstrap.SourcelessFileLoader):
+
+    """Compatibility support for implementing load_compiled()."""
+
+
+def load_compiled(name, pathname, file=None):
+    msg = ('imp.load_compiled() is deprecated; use '
+           'importlib.machinery.SourcelessFileLoader(name, pathname).'
+           'load_module() instead ')
+    warnings.warn(msg, DeprecationWarning, 2)
+    return _LoadCompiledCompatibility(name, pathname, file).load_module(name)
+
+
+def load_package(name, path):
+    msg = ('imp.load_package() is deprecated; use either '
+           'importlib.machinery.SourceFileLoader() or '
+           'importlib.machinery.SourcelessFileLoader() instead')
+    warnings.warn(msg, DeprecationWarning, 2)
+    if os.path.isdir(path):
+        extensions = (machinery.SOURCE_SUFFIXES[:] +
+                      machinery.BYTECODE_SUFFIXES[:])
+        for extension in extensions:
+            path = os.path.join(path, '__init__'+extension)
+            if os.path.exists(path):
+                break
+        else:
+            raise ValueError('{!r} is not a package'.format(path))
+    return _bootstrap.SourceFileLoader(name, path).load_module(name)
+
+
+def load_module(name, file, filename, details):
+    """**DEPRECATED**
+
+    Load a module, given information returned by find_module().
+
+    The module name must include the full package name, if any.
+
+    """
+    suffix, mode, type_ = details
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore')
+        if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
+            raise ValueError('invalid file open mode {!r}'.format(mode))
+        elif file is None and type_ in {PY_SOURCE, PY_COMPILED, C_EXTENSION}:
+            msg = 'file object required for import (type code {})'.format(type_)
+            raise ValueError(msg)
+        elif type_ == PY_SOURCE:
+            return load_source(name, filename, file)
+        elif type_ == PY_COMPILED:
+            return load_compiled(name, filename, file)
+        elif type_ == C_EXTENSION:
+            return load_dynamic(name, filename, file)
+        elif type_ == PKG_DIRECTORY:
+            return load_package(name, filename)
+        elif type_ == C_BUILTIN:
+            return init_builtin(name)
+        elif type_ == PY_FROZEN:
+            return init_frozen(name)
+        else:
+            msg =  "Don't know how to import {} (type code {})".format(name, type_)
+            raise ImportError(msg, name=name)
+
+
+def find_module(name, path=None):
+    """**DEPRECATED**
+
+    Search for a module.
+
+    If path is omitted or None, search for a built-in, frozen or special
+    module and continue search in sys.path. The module name cannot
+    contain '.'; to search for a submodule of a package, pass the
+    submodule name and the package's __path__.
+
+    """
+    if not isinstance(name, str):
+        raise TypeError("'name' must be a str, not {}".format(type(name)))
+    elif not isinstance(path, (type(None), list)):
+        # Backwards-compatibility
+        raise RuntimeError("'list' must be None or a list, "
+                           "not {}".format(type(name)))
+
+    if path is None:
+        if is_builtin(name):
+            return None, None, ('', '', C_BUILTIN)
+        elif is_frozen(name):
+            return None, None, ('', '', PY_FROZEN)
+        else:
+            path = sys.path
+
+    for entry in path:
+        package_directory = os.path.join(entry, name)
+        for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]:
+            package_file_name = '__init__' + suffix
+            file_path = os.path.join(package_directory, package_file_name)
+            if os.path.isfile(file_path):
+                return None, package_directory, ('', '', PKG_DIRECTORY)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            for suffix, mode, type_ in get_suffixes():
+                file_name = name + suffix
+                file_path = os.path.join(entry, file_name)
+                if os.path.isfile(file_path):
+                    break
+            else:
+                continue
+            break  # Break out of outer loop when breaking out of inner loop.
+    else:
+        raise ImportError(_bootstrap._ERR_MSG.format(name), name=name)
+
+    encoding = None
+    if mode == 'U':
+        with open(file_path, 'rb') as file:
+            encoding = tokenize.detect_encoding(file.readline)[0]
+    file = open(file_path, mode, encoding=encoding)
+    return file, file_path, (suffix, mode, type_)
+
+
+_RELOADING = {}
+
+def reload(module):
+    """Reload the module and return it.
+
+    The module must have been successfully imported before.
+
+    """
+    if not module or type(module) != type(sys):
+        raise TypeError("reload() argument must be module")
+    name = module.__name__
+    if name not in sys.modules:
+        msg = "module {} not in sys.modules"
+        raise ImportError(msg.format(name), name=name)
+    if name in _RELOADING:
+        return _RELOADING[name]
+    _RELOADING[name] = module
+    try:
+        parent_name = name.rpartition('.')[0]
+        if parent_name and parent_name not in sys.modules:
+            msg = "parent {!r} not in sys.modules"
+            raise ImportError(msg.format(parentname), name=parent_name)
+        return module.__loader__.load_module(name)
+    finally:
+        try:
+            del _RELOADING[name]
+        except KeyError:
+            pass
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index 2baaf93732..22c90f24a7 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -1,108 +1,74 @@
-"""A pure Python implementation of import.
-
-References on import:
-
-    * Language reference
-          http://docs.python.org/ref/import.html
-    * __import__ function
-          http://docs.python.org/lib/built-in-funcs.html
-    * Packages
-          http://www.python.org/doc/essays/packages.html
-    * PEP 235: Import on Case-Insensitive Platforms
-          http://www.python.org/dev/peps/pep-0235
-    * PEP 275: Import Modules from Zip Archives
-          http://www.python.org/dev/peps/pep-0273
-    * PEP 302: New Import Hooks
-          http://www.python.org/dev/peps/pep-0302/
-    * PEP 328: Imports: Multi-line and Absolute/Relative
-          http://www.python.org/dev/peps/pep-0328
-
-"""
-__all__ = ['__import__', 'import_module']
-
-from . import _bootstrap
-
-import os
-import re
-import tokenize
+"""A pure Python implementation of import."""
+__all__ = ['__import__', 'import_module', 'invalidate_caches']
 
 # Bootstrap help #####################################################
 
-def _case_ok(directory, check):
-    """Check if the directory contains something matching 'check'.
+# Until bootstrapping is complete, DO NOT import any modules that attempt
+# to import importlib._bootstrap (directly or indirectly). Since this
+# partially initialised package would be present in sys.modules, those
+# modules would get an uninitialised copy of the source version, instead
+# of a fully initialised version (either the frozen one or the one
+# initialised below if the frozen one is not available).
+import _imp  # Just the builtin component, NOT the full Python module
+import sys
 
-    No check is done if the file/directory exists or not.
+try:
+    import _frozen_importlib as _bootstrap
+except ImportError:
+    from . import _bootstrap
+    _bootstrap._setup(sys, _imp)
+else:
+    # importlib._bootstrap is the built-in import, ensure we don't create
+    # a second copy of the module.
+    _bootstrap.__name__ = 'importlib._bootstrap'
+    _bootstrap.__package__ = 'importlib'
+    _bootstrap.__file__ = __file__.replace('__init__.py', '_bootstrap.py')
+    sys.modules['importlib._bootstrap'] = _bootstrap
+
+# To simplify imports in test code
+_w_long = _bootstrap._w_long
+_r_long = _bootstrap._r_long
+
+# Fully bootstrapped at this point, import whatever you like, circular
+# dependencies and startup overhead minimisation permitting :)
 
-    """
-    if 'PYTHONCASEOK' in os.environ:
-        return True
-    elif check in os.listdir(directory if directory else os.getcwd()):
-        return True
-    return False
+# Public API #########################################################
 
+from ._bootstrap import __import__
 
-def _w_long(x):
-    """Convert a 32-bit integer to little-endian.
 
-    XXX Temporary until marshal's long functions are exposed.
+def invalidate_caches():
+    """Call the invalidate_caches() method on all meta path finders stored in
+    sys.meta_path (where implemented)."""
+    for finder in sys.meta_path:
+        if hasattr(finder, 'invalidate_caches'):
+            finder.invalidate_caches()
 
-    """
-    x = int(x)
-    int_bytes = []
-    int_bytes.append(x & 0xFF)
-    int_bytes.append((x >> 8) & 0xFF)
-    int_bytes.append((x >> 16) & 0xFF)
-    int_bytes.append((x >> 24) & 0xFF)
-    return bytearray(int_bytes)
 
+def find_loader(name, path=None):
+    """Find the loader for the specified module.
 
-def _r_long(int_bytes):
-    """Convert 4 bytes in little-endian to an integer.
+    First, sys.modules is checked to see if the module was already imported. If
+    so, then sys.modules[name].__loader__ is returned. If that happens to be
+    set to None, then ValueError is raised. If the module is not in
+    sys.modules, then sys.meta_path is searched for a suitable loader with the
+    value of 'path' given to the finders. None is returned if no loader could
+    be found.
 
-    XXX Temporary until marshal's long function are exposed.
+    Dotted names do not have their parent packages implicitly imported. You will
+    most likely need to explicitly import all parent packages in the proper
+    order for a submodule to get the correct loader.
 
     """
-    x = int_bytes[0]
-    x |= int_bytes[1] << 8
-    x |= int_bytes[2] << 16
-    x |= int_bytes[3] << 24
-    return x
-
-
-# Required built-in modules.
-try:
-    import posix as _os
-except ImportError:
     try:
-        import nt as _os
-    except ImportError:
-        try:
-            import os2 as _os
-        except ImportError:
-            raise ImportError('posix, nt, or os2 module required for importlib')
-_bootstrap._os = _os
-import imp, sys, marshal, errno, _io
-_bootstrap.imp = imp
-_bootstrap.sys = sys
-_bootstrap.marshal = marshal
-_bootstrap.errno = errno
-_bootstrap._io = _io
-import _warnings
-_bootstrap._warnings = _warnings
-
-
-from os import sep
-# For os.path.join replacement; pull from Include/osdefs.h:SEP .
-_bootstrap.path_sep = sep
-
-_bootstrap._case_ok = _case_ok
-marshal._w_long = _w_long
-marshal._r_long = _r_long
-
-
-# Public API #########################################################
-
-from ._bootstrap import __import__
+        loader = sys.modules[name].__loader__
+        if loader is None:
+            raise ValueError('{}.__loader__ is None'.format(name))
+        else:
+            return loader
+    except KeyError:
+        pass
+    return _bootstrap._find_module(name, path)
 
 
 def import_module(name, package=None):
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index aa4032c0ff..7e348a4982 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -6,33 +6,93 @@ such it requires the injection of specific modules and attributes in order to
 work. One should use importlib as the public-facing version of this module.
 
 """
-
-# Injected modules are '_warnings', 'imp', 'sys', 'marshal', 'errno', '_io',
-# and '_os' (a.k.a. 'posix', 'nt' or 'os2').
-# Injected attribute is path_sep.
 #
+# IMPORTANT: Whenever making changes to this module, be sure to run
+# a top-level make in order to get the frozen version of the module
+# update. Not doing so, will result in the Makefile to fail for
+# all others who don't have a ./python around to freeze the module
+# in the early stages of compilation.
+#
+
+# See importlib._setup() for what is injected into the global namespace.
+
 # When editing this code be aware that code executed at import time CANNOT
 # reference any injected objects! This includes not only global code but also
 # anything specified at the class level.
 
+# XXX Make sure all public names have no single leading underscore and all
+#     others do.
+
 
 # Bootstrap-related code ######################################################
 
-# XXX Could also expose Modules/getpath.c:joinpath()
-def _path_join(*args):
-    """Replacement for os.path.join."""
-    return path_sep.join(x[:-len(path_sep)] if x.endswith(path_sep) else x
-                            for x in args if x)
+_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin'
 
 
-def _path_exists(path):
-    """Replacement for os.path.exists."""
-    try:
-        _os.stat(path)
-    except OSError:
-        return False
+def _make_relax_case():
+    if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
+        def _relax_case():
+            """True if filenames must be checked case-insensitively."""
+            return b'PYTHONCASEOK' in _os.environ
     else:
-        return True
+        def _relax_case():
+            """True if filenames must be checked case-insensitively."""
+            return False
+    return _relax_case
+
+
+# TODO: Expose from marshal
+def _w_long(x):
+    """Convert a 32-bit integer to little-endian.
+
+    XXX Temporary until marshal's long functions are exposed.
+
+    """
+    x = int(x)
+    int_bytes = []
+    int_bytes.append(x & 0xFF)
+    int_bytes.append((x >> 8) & 0xFF)
+    int_bytes.append((x >> 16) & 0xFF)
+    int_bytes.append((x >> 24) & 0xFF)
+    return bytearray(int_bytes)
+
+
+# TODO: Expose from marshal
+def _r_long(int_bytes):
+    """Convert 4 bytes in little-endian to an integer.
+
+    XXX Temporary until marshal's long function are exposed.
+
+    """
+    x = int_bytes[0]
+    x |= int_bytes[1] << 8
+    x |= int_bytes[2] << 16
+    x |= int_bytes[3] << 24
+    return x
+
+
+def _path_join(*path_parts):
+    """Replacement for os.path.join()."""
+    new_parts = []
+    for part in path_parts:
+        if not part:
+            continue
+        new_parts.append(part)
+        if part[-1] not in path_separators:
+            new_parts.append(path_sep)
+    return ''.join(new_parts[:-1])  # Drop superfluous path separator.
+
+
+def _path_split(path):
+    """Replacement for os.path.split()."""
+    for x in reversed(path):
+        if x in path_separators:
+            sep = x
+            break
+    else:
+        sep = path_sep
+    front, _, tail = path.rpartition(sep)
+    return front, tail
 
 
 def _path_is_mode_type(path, mode):
@@ -58,61 +118,401 @@ def _path_isdir(path):
     return _path_is_mode_type(path, 0o040000)
 
 
-def _path_without_ext(path, ext_type):
-    """Replacement for os.path.splitext()[0]."""
-    for suffix in _suffix_list(ext_type):
-        if path.endswith(suffix):
-            return path[:-len(suffix)]
-    else:
-        raise ValueError("path is not of the specified type")
+def _write_atomic(path, data, mode=0o666):
+    """Best-effort function to write data to a path atomically.
+    Be prepared to handle a FileExistsError if concurrent writing of the
+    temporary file is attempted."""
+    # id() is used to generate a pseudo-random filename.
+    path_tmp = '{}.{}'.format(path, id(path))
+    fd = _os.open(path_tmp,
+                  _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
+    try:
+        # We first write data to a temporary file, and then use os.replace() to
+        # perform an atomic rename.
+        with _io.FileIO(fd, 'wb') as file:
+            file.write(data)
+        _os.replace(path_tmp, path)
+    except OSError:
+        try:
+            _os.unlink(path_tmp)
+        except OSError:
+            pass
+        raise
 
 
-def _path_absolute(path):
-    """Replacement for os.path.abspath."""
-    if not path:
-        path = _os.getcwd()
+def _wrap(new, old):
+    """Simple substitute for functools.update_wrapper."""
+    for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
+        if hasattr(old, replace):
+            setattr(new, replace, getattr(old, replace))
+    new.__dict__.update(old.__dict__)
+
+
+_code_type = type(_wrap.__code__)
+
+
+def new_module(name):
+    """Create a new module.
+
+    The module is not entered into sys.modules.
+
+    """
+    return type(_io)(name)
+
+
+# Module-level locking ########################################################
+
+# A dict mapping module names to weakrefs of _ModuleLock instances
+_module_locks = {}
+# A dict mapping thread ids to _ModuleLock instances
+_blocking_on = {}
+
+
+class _DeadlockError(RuntimeError):
+    pass
+
+
+class _ModuleLock:
+    """A recursive lock implementation which is able to detect deadlocks
+    (e.g. thread 1 trying to take locks A then B, and thread 2 trying to
+    take locks B then A).
+    """
+
+    def __init__(self, name):
+        self.lock = _thread.allocate_lock()
+        self.wakeup = _thread.allocate_lock()
+        self.name = name
+        self.owner = None
+        self.count = 0
+        self.waiters = 0
+
+    def has_deadlock(self):
+        # Deadlock avoidance for concurrent circular imports.
+        me = _thread.get_ident()
+        tid = self.owner
+        while True:
+            lock = _blocking_on.get(tid)
+            if lock is None:
+                return False
+            tid = lock.owner
+            if tid == me:
+                return True
+
+    def acquire(self):
+        """
+        Acquire the module lock.  If a potential deadlock is detected,
+        a _DeadlockError is raised.
+        Otherwise, the lock is always acquired and True is returned.
+        """
+        tid = _thread.get_ident()
+        _blocking_on[tid] = self
+        try:
+            while True:
+                with self.lock:
+                    if self.count == 0 or self.owner == tid:
+                        self.owner = tid
+                        self.count += 1
+                        return True
+                    if self.has_deadlock():
+                        raise _DeadlockError("deadlock detected by %r" % self)
+                    if self.wakeup.acquire(False):
+                        self.waiters += 1
+                # Wait for a release() call
+                self.wakeup.acquire()
+                self.wakeup.release()
+        finally:
+            del _blocking_on[tid]
+
+    def release(self):
+        tid = _thread.get_ident()
+        with self.lock:
+            if self.owner != tid:
+                raise RuntimeError("cannot release un-acquired lock")
+            assert self.count > 0
+            self.count -= 1
+            if self.count == 0:
+                self.owner = None
+                if self.waiters:
+                    self.waiters -= 1
+                    self.wakeup.release()
+
+    def __repr__(self):
+        return "_ModuleLock(%r) at %d" % (self.name, id(self))
+
+
+class _DummyModuleLock:
+    """A simple _ModuleLock equivalent for Python builds without
+    multi-threading support."""
+
+    def __init__(self, name):
+        self.name = name
+        self.count = 0
+
+    def acquire(self):
+        self.count += 1
+        return True
+
+    def release(self):
+        if self.count == 0:
+            raise RuntimeError("cannot release un-acquired lock")
+        self.count -= 1
+
+    def __repr__(self):
+        return "_DummyModuleLock(%r) at %d" % (self.name, id(self))
+
+
+# The following two functions are for consumption by Python/import.c.
+
+def _get_module_lock(name):
+    """Get or create the module lock for a given module name.
+
+    Should only be called with the import lock taken."""
+    lock = None
     try:
-        return _os._getfullpathname(path)
-    except AttributeError:
-        if path.startswith('/'):
-            return path
+        lock = _module_locks[name]()
+    except KeyError:
+        pass
+    if lock is None:
+        if _thread is None:
+            lock = _DummyModuleLock(name)
         else:
-            return _path_join(_os.getcwd(), path)
+            lock = _ModuleLock(name)
+        def cb(_):
+            del _module_locks[name]
+        _module_locks[name] = _weakref.ref(lock, cb)
+    return lock
+
+def _lock_unlock_module(name):
+    """Release the global import lock, and acquires then release the
+    module lock for a given module name.
+    This is used to ensure a module is completely initialized, in the
+    event it is being imported by another thread.
+
+    Should only be called with the import lock taken."""
+    lock = _get_module_lock(name)
+    _imp.release_lock()
+    try:
+        lock.acquire()
+    except _DeadlockError:
+        # Concurrent circular import, we'll accept a partially initialized
+        # module object.
+        pass
+    else:
+        lock.release()
 
+# Frame stripping magic ###############################################
 
-def _wrap(new, old):
-    """Simple substitute for functools.wraps."""
-    for replace in ['__module__', '__name__', '__doc__']:
-        setattr(new, replace, getattr(old, replace))
-    new.__dict__.update(old.__dict__)
+def _call_with_frames_removed(f, *args, **kwds):
+    """remove_importlib_frames in import.c will always remove sequences
+    of importlib frames that end with a call to this function
 
+    Use it instead of a normal call in places where including the importlib
+    frames introduces unwanted noise into the traceback (e.g. when executing
+    module code)
+    """
+    return f(*args, **kwds)
+
+
+# Finder/loader utility code ###############################################
+
+"""Magic word to reject .pyc files generated by other Python versions.
+It should change for each incompatible change to the bytecode.
+
+The value of CR and LF is incorporated so if you ever read or write
+a .pyc file in text mode the magic number will be wrong; also, the
+Apple MPW compiler swaps their values, botching string constants.
+
+The magic numbers must be spaced apart at least 2 values, as the
+-U interpeter flag will cause MAGIC+1 being used. They have been
+odd numbers for some time now.
+
+There were a variety of old schemes for setting the magic number.
+The current working scheme is to increment the previous value by
+10.
+
+Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
+number also includes a new "magic tag", i.e. a human readable string used
+to represent the magic number in __pycache__ directories.  When you change
+the magic number, you must also set a new unique magic tag.  Generally this
+can be named after the Python major version of the magic number bump, but
+it can really be anything, as long as it's different than anything else
+that's come before.  The tags are included in the following table, starting
+with Python 3.2a0.
+
+Known values:
+ Python 1.5:   20121
+ Python 1.5.1: 20121
+    Python 1.5.2: 20121
+    Python 1.6:   50428
+    Python 2.0:   50823
+    Python 2.0.1: 50823
+    Python 2.1:   60202
+    Python 2.1.1: 60202
+    Python 2.1.2: 60202
+    Python 2.2:   60717
+    Python 2.3a0: 62011
+    Python 2.3a0: 62021
+    Python 2.3a0: 62011 (!)
+    Python 2.4a0: 62041
+    Python 2.4a3: 62051
+    Python 2.4b1: 62061
+    Python 2.5a0: 62071
+    Python 2.5a0: 62081 (ast-branch)
+    Python 2.5a0: 62091 (with)
+    Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
+    Python 2.5b3: 62101 (fix wrong code: for x, in ...)
+    Python 2.5b3: 62111 (fix wrong code: x += yield)
+    Python 2.5c1: 62121 (fix wrong lnotab with for loops and
+                         storing constants that should have been removed)
+    Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
+    Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
+    Python 2.6a1: 62161 (WITH_CLEANUP optimization)
+    Python 3000:   3000
+                   3010 (removed UNARY_CONVERT)
+                   3020 (added BUILD_SET)
+                   3030 (added keyword-only parameters)
+                   3040 (added signature annotations)
+                   3050 (print becomes a function)
+                   3060 (PEP 3115 metaclass syntax)
+                   3061 (string literals become unicode)
+                   3071 (PEP 3109 raise changes)
+                   3081 (PEP 3137 make __file__ and __name__ unicode)
+                   3091 (kill str8 interning)
+                   3101 (merge from 2.6a0, see 62151)
+                   3103 (__file__ points to source file)
+    Python 3.0a4: 3111 (WITH_CLEANUP optimization).
+    Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
+    Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
+            change LIST_APPEND and SET_ADD, add MAP_ADD)
+    Python 3.1a0: 3151 (optimize conditional branches:
+            introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
+    Python 3.2a0: 3160 (add SETUP_WITH)
+                  tag: cpython-32
+    Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
+                  tag: cpython-32
+    Python 3.2a2  3180 (add DELETE_DEREF)
+    Python 3.3a0  3190 __class__ super closure changed
+    Python 3.3a0  3200 (__qualname__ added)
+                     3210 (added size modulo 2**32 to the pyc header)
+    Python 3.3a1  3220 (changed PEP 380 implementation)
+    Python 3.3a4  3230 (revert changes to implicit __class__ closure)
+
+MAGIC must change whenever the bytecode emitted by the compiler may no
+longer be understood by older implementations of the eval loop (usually
+due to the addition of new opcodes).
+
+"""
+_RAW_MAGIC_NUMBER = 3230 | ord('\r') << 16 | ord('\n') << 24
+_MAGIC_BYTES = bytes(_RAW_MAGIC_NUMBER >> n & 0xff for n in range(0, 25, 8))
+
+_PYCACHE = '__pycache__'
+
+SOURCE_SUFFIXES = ['.py']  # _setup() adds .pyw as needed.
+
+DEBUG_BYTECODE_SUFFIXES = ['.pyc']
+OPTIMIZED_BYTECODE_SUFFIXES = ['.pyo']
+
+def cache_from_source(path, debug_override=None):
+    """Given the path to a .py file, return the path to its .pyc/.pyo file.
+
+    The .py file does not need to exist; this simply returns the path to the
+    .pyc/.pyo file calculated as if the .py file were imported.  The extension
+    will be .pyc unless sys.flags.optimize is non-zero, then it will be .pyo.
+
+    If debug_override is not None, then it must be a boolean and is used in
+    place of sys.flags.optimize.
+
+    If sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+    """
+    debug = not sys.flags.optimize if debug_override is None else debug_override
+    if debug:
+        suffixes = DEBUG_BYTECODE_SUFFIXES
+    else:
+        suffixes = OPTIMIZED_BYTECODE_SUFFIXES
+    head, tail = _path_split(path)
+    base_filename, sep, _ = tail.partition('.')
+    tag = sys.implementation.cache_tag
+    if tag is None:
+        raise NotImplementedError('sys.implementation.cache_tag is None')
+    filename = ''.join([base_filename, sep, tag, suffixes[0]])
+    return _path_join(head, _PYCACHE, filename)
+
+
+def source_from_cache(path):
+    """Given the path to a .pyc./.pyo file, return the path to its .py file.
+
+    The .pyc/.pyo file does not need to exist; this simply returns the path to
+    the .py file calculated to correspond to the .pyc/.pyo file.  If path does
+    not conform to PEP 3147 format, ValueError will be raised. If
+    sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+    """
+    if sys.implementation.cache_tag is None:
+        raise NotImplementedError('sys.implementation.cache_tag is None')
+    head, pycache_filename = _path_split(path)
+    head, pycache = _path_split(head)
+    if pycache != _PYCACHE:
+        raise ValueError('{} not bottom-level directory in '
+                         '{!r}'.format(_PYCACHE, path))
+    if pycache_filename.count('.') != 2:
+        raise ValueError('expected only 2 dots in '
+                         '{!r}'.format(pycache_filename))
+    base_filename = pycache_filename.partition('.')[0]
+    return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
+
+
+def _get_sourcefile(bytecode_path):
+    """Convert a bytecode file path to a source path (if possible).
+
+    This function exists purely for backwards-compatibility for
+    PyImport_ExecCodeModuleWithFilenames() in the C API.
+
+    """
+    if len(bytecode_path) == 0:
+        return None
+    rest, _, extension = bytecode_path.rparition('.')
+    if not rest or extension.lower()[-3:-1] != '.py':
+        return bytecode_path
+
+    try:
+        source_path = source_from_cache(bytecode_path)
+    except (NotImplementedError, ValueError):
+        source_path = bytcode_path[-1:]
 
-code_type = type(_wrap.__code__)
+    return source_path if _path_isfile(source_stats) else bytecode_path
+
+
+def _verbose_message(message, *args):
+    """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
+    if sys.flags.verbose:
+        if not message.startswith(('#', 'import ')):
+            message = '# ' + message
+        print(message.format(*args), file=sys.stderr)
 
-# Finder/loader utility code ##################################################
 
 def set_package(fxn):
     """Set __package__ on the returned module."""
-    def wrapper(*args, **kwargs):
+    def set_package_wrapper(*args, **kwargs):
         module = fxn(*args, **kwargs)
-        if not hasattr(module, '__package__') or module.__package__ is None:
+        if getattr(module, '__package__', None) is None:
             module.__package__ = module.__name__
             if not hasattr(module, '__path__'):
                 module.__package__ = module.__package__.rpartition('.')[0]
         return module
-    _wrap(wrapper, fxn)
-    return wrapper
+    _wrap(set_package_wrapper, fxn)
+    return set_package_wrapper
 
 
 def set_loader(fxn):
     """Set __loader__ on the returned module."""
-    def wrapper(self, *args, **kwargs):
+    def set_loader_wrapper(self, *args, **kwargs):
         module = fxn(self, *args, **kwargs)
         if not hasattr(module, '__loader__'):
             module.__loader__ = self
         return module
-    _wrap(wrapper, fxn)
-    return wrapper
+    _wrap(set_loader_wrapper, fxn)
+    return set_loader_wrapper
 
 
 def module_for_loader(fxn):
@@ -120,31 +520,54 @@ def module_for_loader(fxn):
 
     The decorated function is passed the module to use instead of the module
     name. The module passed in to the function is either from sys.modules if
-    it already exists or is a new module which has __name__ set and is inserted
-    into sys.modules. If an exception is raised and the decorator created the
-    module it is subsequently removed from sys.modules.
+    it already exists or is a new module. If the module is new, then __name__
+    is set the first argument to the method, __loader__ is set to self, and
+    __package__ is set accordingly (if self.is_package() is defined) will be set
+    before it is passed to the decorated function (if self.is_package() does
+    not work for the module it will be set post-load).
+
+    If an exception is raised and the decorator created the module it is
+    subsequently removed from sys.modules.
 
     The decorator assumes that the decorated function takes the module name as
     the second argument.
 
     """
-    def decorated(self, fullname, *args, **kwargs):
+    def module_for_loader_wrapper(self, fullname, *args, **kwargs):
         module = sys.modules.get(fullname)
-        is_reload = bool(module)
+        is_reload = module is not None
         if not is_reload:
             # This must be done before open() is called as the 'io' module
             # implicitly imports 'locale' and would otherwise trigger an
             # infinite loop.
-            module = imp.new_module(fullname)
+            module = new_module(fullname)
+            # This must be done before putting the module in sys.modules
+            # (otherwise an optimization shortcut in import.c becomes wrong)
+            module.__initializing__ = True
             sys.modules[fullname] = module
+            module.__loader__ = self
+            try:
+                is_package = self.is_package(fullname)
+            except (ImportError, AttributeError):
+                pass
+            else:
+                if is_package:
+                    module.__package__ = fullname
+                else:
+                    module.__package__ = fullname.rpartition('.')[0]
+        else:
+            module.__initializing__ = True
         try:
+            # If __package__ was not set above, __import__() will do it later.
             return fxn(self, module, *args, **kwargs)
         except:
             if not is_reload:
                 del sys.modules[fullname]
             raise
-    _wrap(decorated, fxn)
-    return decorated
+        finally:
+            module.__initializing__ = False
+    _wrap(module_for_loader_wrapper, fxn)
+    return module_for_loader_wrapper
 
 
 def _check_name(method):
@@ -155,38 +578,51 @@ def _check_name(method):
     compared against. If the comparison fails then ImportError is raised.
 
     """
-    def inner(self, name, *args, **kwargs):
-        if self._name != name:
-            raise ImportError("loader cannot handle %s" % name)
+    def _check_name_wrapper(self, name=None, *args, **kwargs):
+        if name is None:
+            name = self.name
+        elif self.name != name:
+            raise ImportError("loader cannot handle %s" % name, name=name)
         return method(self, name, *args, **kwargs)
-    _wrap(inner, method)
-    return inner
+    _wrap(_check_name_wrapper, method)
+    return _check_name_wrapper
 
 
 def _requires_builtin(fxn):
     """Decorator to verify the named module is built-in."""
-    def wrapper(self, fullname):
+    def _requires_builtin_wrapper(self, fullname):
         if fullname not in sys.builtin_module_names:
-            raise ImportError("{0} is not a built-in module".format(fullname))
+            raise ImportError("{} is not a built-in module".format(fullname),
+                              name=fullname)
         return fxn(self, fullname)
-    _wrap(wrapper, fxn)
-    return wrapper
+    _wrap(_requires_builtin_wrapper, fxn)
+    return _requires_builtin_wrapper
 
 
 def _requires_frozen(fxn):
     """Decorator to verify the named module is frozen."""
-    def wrapper(self, fullname):
-        if not imp.is_frozen(fullname):
-            raise ImportError("{0} is not a frozen module".format(fullname))
+    def _requires_frozen_wrapper(self, fullname):
+        if not _imp.is_frozen(fullname):
+            raise ImportError("{} is not a frozen module".format(fullname),
+                              name=fullname)
         return fxn(self, fullname)
-    _wrap(wrapper, fxn)
-    return wrapper
+    _wrap(_requires_frozen_wrapper, fxn)
+    return _requires_frozen_wrapper
+
+
+def _find_module_shim(self, fullname):
+    """Try to find a loader for the specified module by delegating to
+    self.find_loader()."""
+    # Call find_loader(). If it returns a string (indicating this
+    # is a namespace package portion), generate a warning and
+    # return None.
+    loader, portions = self.find_loader(fullname)
+    if loader is None and len(portions):
+        msg = "Not importing directory {}: missing __init__"
+        _warnings.warn(msg.format(portions[0]), ImportWarning)
+    return loader
 
 
-def _suffix_list(suffix_type):
-    """Return a list of file suffixes based on the imp file type."""
-    return [suffix[0] for suffix in imp.get_suffixes()
-            if suffix[2] == suffix_type]
 
 
 # Loaders #####################################################################
@@ -201,6 +637,10 @@ class BuiltinImporter:
     """
 
     @classmethod
+    def module_repr(cls, module):
+        return "<module '{}' (built-in)>".format(module.__name__)
+
+    @classmethod
     def find_module(cls, fullname, path=None):
         """Find the built-in module.
 
@@ -209,7 +649,7 @@ class BuiltinImporter:
         """
         if path is not None:
             return None
-        return cls if imp.is_builtin(fullname) else None
+        return cls if _imp.is_builtin(fullname) else None
 
     @classmethod
     @set_package
@@ -219,7 +659,7 @@ class BuiltinImporter:
         """Load a built-in module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.init_builtin(fullname)
+            return _call_with_frames_removed(_imp.init_builtin, fullname)
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -240,7 +680,7 @@ class BuiltinImporter:
     @classmethod
     @_requires_builtin
     def is_package(cls, fullname):
-        """Return None as built-in module are never packages."""
+        """Return False as built-in modules are never packages."""
         return False
 
 
@@ -254,9 +694,13 @@ class FrozenImporter:
     """
 
     @classmethod
+    def module_repr(cls, m):
+        return "<module '{}' (frozen)>".format(m.__name__)
+
+    @classmethod
     def find_module(cls, fullname, path=None):
         """Find a frozen module."""
-        return cls if imp.is_frozen(fullname) else None
+        return cls if _imp.is_frozen(fullname) else None
 
     @classmethod
     @set_package
@@ -266,7 +710,10 @@ class FrozenImporter:
         """Load a frozen module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.init_frozen(fullname)
+            m = _call_with_frames_removed(_imp.init_frozen, fullname)
+            # Let our own module_repr() method produce a suitable repr.
+            del m.__file__
+            return m
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -276,7 +723,7 @@ class FrozenImporter:
     @_requires_frozen
     def get_code(cls, fullname):
         """Return the code object for the frozen module."""
-        return imp.get_frozen_object(fullname)
+        return _imp.get_frozen_object(fullname)
 
     @classmethod
     @_requires_frozen
@@ -287,40 +734,117 @@ class FrozenImporter:
     @classmethod
     @_requires_frozen
     def is_package(cls, fullname):
-        """Return if the frozen module is a package."""
-        return imp.is_frozen_package(fullname)
+        """Return True if the frozen module is a package."""
+        return _imp.is_frozen_package(fullname)
+
+
+class WindowsRegistryFinder:
+
+    """Meta path finder for modules declared in the Windows registry.
+    """
+
+    REGISTRY_KEY = (
+        "Software\\Python\\PythonCore\\{sys_version}"
+        "\\Modules\\{fullname}")
+    REGISTRY_KEY_DEBUG = (
+        "Software\\Python\\PythonCore\\{sys_version}"
+        "\\Modules\\{fullname}\\Debug")
+    DEBUG_BUILD = False  # Changed in _setup()
+
+    @classmethod
+    def _open_registry(cls, key):
+        try:
+            return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
+        except WindowsError:
+            return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
+
+    @classmethod
+    def _search_registry(cls, fullname):
+        if cls.DEBUG_BUILD:
+            registry_key = cls.REGISTRY_KEY_DEBUG
+        else:
+            registry_key = cls.REGISTRY_KEY
+        key = registry_key.format(fullname=fullname,
+                                  sys_version=sys.version[:3])
+        try:
+            with cls._open_registry(key) as hkey:
+                filepath = _winreg.QueryValue(hkey, "")
+        except WindowsError:
+            return None
+        return filepath
+
+    @classmethod
+    def find_module(cls, fullname, path=None):
+        """Find module named in the registry."""
+        filepath = cls._search_registry(fullname)
+        if filepath is None:
+            return None
+        try:
+            _os.stat(filepath)
+        except OSError:
+            return None
+        for loader, suffixes, _ in _get_supported_file_loaders():
+            if filepath.endswith(tuple(suffixes)):
+                return loader(fullname, filepath)
 
 
 class _LoaderBasics:
 
     """Base class of common code needed by both SourceLoader and
-    _SourcelessFileLoader."""
+    SourcelessFileLoader."""
 
     def is_package(self, fullname):
         """Concrete implementation of InspectLoader.is_package by checking if
         the path returned by get_filename has a filename of '__init__.py'."""
-        filename = self.get_filename(fullname).rpartition(path_sep)[2]
-        return filename.rsplit('.', 1)[0] == '__init__'
+        filename = _path_split(self.get_filename(fullname))[1]
+        filename_base = filename.rsplit('.', 1)[0]
+        tail_name = fullname.rpartition('.')[2]
+        return filename_base == '__init__' and tail_name != '__init__'
 
-    def _bytes_from_bytecode(self, fullname, data, source_mtime):
+    def _bytes_from_bytecode(self, fullname, data, bytecode_path, source_stats):
         """Return the marshalled bytes from bytecode, verifying the magic
-        number and timestamp along the way.
+        number, timestamp and source size along the way.
 
-        If source_mtime is None then skip the timestamp check.
+        If source_stats is None then skip the timestamp check.
 
         """
         magic = data[:4]
         raw_timestamp = data[4:8]
-        if len(magic) != 4 or magic != imp.get_magic():
-            raise ImportError("bad magic number in {}".format(fullname))
+        raw_size = data[8:12]
+        if magic != _MAGIC_BYTES:
+            msg = 'bad magic number in {!r}: {!r}'.format(fullname, magic)
+            raise ImportError(msg, name=fullname, path=bytecode_path)
         elif len(raw_timestamp) != 4:
-            raise EOFError("bad timestamp in {}".format(fullname))
-        elif source_mtime is not None:
-            if marshal._r_long(raw_timestamp) != source_mtime:
-                raise ImportError("bytecode is stale for {}".format(fullname))
+            message = 'bad timestamp in {}'.format(fullname)
+            _verbose_message(message)
+            raise EOFError(message)
+        elif len(raw_size) != 4:
+            message = 'bad size in {}'.format(fullname)
+            _verbose_message(message)
+            raise EOFError(message)
+        if source_stats is not None:
+            try:
+                source_mtime = int(source_stats['mtime'])
+            except KeyError:
+                pass
+            else:
+                if _r_long(raw_timestamp) != source_mtime:
+                    message = 'bytecode is stale for {}'.format(fullname)
+                    _verbose_message(message)
+                    raise ImportError(message, name=fullname,
+                                      path=bytecode_path)
+            try:
+                source_size = source_stats['size'] & 0xFFFFFFFF
+            except KeyError:
+                pass
+            else:
+                if _r_long(raw_size) != source_size:
+                    raise ImportError(
+                        "bytecode is stale for {}".format(fullname),
+                        name=fullname, path=bytecode_path)
         # Can't return the code object as errors from marshal loading need to
         # propagate even when source is available.
-        return data[8:]
+        return data[12:]
 
     @module_for_loader
     def _load_module(self, module, *, sourceless=False):
@@ -330,16 +854,19 @@ class _LoaderBasics:
         code_object = self.get_code(name)
         module.__file__ = self.get_filename(name)
         if not sourceless:
-            module.__cached__ = imp.cache_from_source(module.__file__)
+            try:
+                module.__cached__ = cache_from_source(module.__file__)
+            except NotImplementedError:
+                module.__cached__ = module.__file__
         else:
             module.__cached__ = module.__file__
         module.__package__ = name
         if self.is_package(name):
-            module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
+            module.__path__ = [_path_split(module.__file__)[0]]
         else:
             module.__package__ = module.__package__.rpartition('.')[0]
         module.__loader__ = self
-        exec(code_object, module.__dict__)
+        _call_with_frames_removed(exec, code_object, module.__dict__)
         return module
 
 
@@ -348,11 +875,30 @@ class SourceLoader(_LoaderBasics):
     def path_mtime(self, path):
         """Optional method that returns the modification time (an int) for the
         specified path, where path is a str.
+        """
+        raise NotImplementedError
+
+    def path_stats(self, path):
+        """Optional method returning a metadata dict for the specified path
+        to by the path (str).
+        Possible keys:
+        - 'mtime' (mandatory) is the numeric timestamp of last source
+          code modification;
+        - 'size' (optional) is the size in bytes of the source code.
 
         Implementing this method allows the loader to read bytecode files.
+        """
+        return {'mtime': self.path_mtime(path)}
 
+    def _cache_bytecode(self, source_path, cache_path, data):
+        """Optional method which writes data (bytes) to a file path (a str).
+
+        Implementing this method allows for the writing of bytecode files.
+
+        The source path is needed in order to correctly transfer permissions
         """
-        raise NotImplementedError
+        # For backwards compatibility, we delegate to set_data()
+        return self.set_data(cache_path, data)
 
     def set_data(self, path, data):
         """Optional method which writes data (bytes) to a file path (a str).
@@ -369,28 +915,42 @@ class SourceLoader(_LoaderBasics):
         path = self.get_filename(fullname)
         try:
             source_bytes = self.get_data(path)
-        except IOError:
-            raise ImportError("source not available through get_data()")
-        encoding = tokenize.detect_encoding(_io.BytesIO(source_bytes).readline)
+        except IOError as exc:
+            raise ImportError("source not available through get_data()",
+                              name=fullname) from exc
+        readsource = _io.BytesIO(source_bytes).readline
+        try:
+            encoding = tokenize.detect_encoding(readsource)
+        except SyntaxError as exc:
+            raise ImportError("Failed to detect encoding",
+                              name=fullname) from exc
         newline_decoder = _io.IncrementalNewlineDecoder(None, True)
-        return newline_decoder.decode(source_bytes.decode(encoding[0]))
+        try:
+            return newline_decoder.decode(source_bytes.decode(encoding[0]))
+        except UnicodeDecodeError as exc:
+            raise ImportError("Failed to decode source file",
+                              name=fullname) from exc
 
     def get_code(self, fullname):
         """Concrete implementation of InspectLoader.get_code.
 
-        Reading of bytecode requires path_mtime to be implemented. To write
+        Reading of bytecode requires path_stats to be implemented. To write
         bytecode, set_data must also be implemented.
 
         """
         source_path = self.get_filename(fullname)
-        bytecode_path = imp.cache_from_source(source_path)
         source_mtime = None
-        if bytecode_path is not None:
+        try:
+            bytecode_path = cache_from_source(source_path)
+        except NotImplementedError:
+            bytecode_path = None
+        else:
             try:
-                source_mtime = self.path_mtime(source_path)
+                st = self.path_stats(source_path)
             except NotImplementedError:
                 pass
             else:
+                source_mtime = int(st['mtime'])
                 try:
                     data = self.get_data(bytecode_path)
                 except IOError:
@@ -398,29 +958,37 @@ class SourceLoader(_LoaderBasics):
                 else:
                     try:
                         bytes_data = self._bytes_from_bytecode(fullname, data,
-                                                               source_mtime)
+                                                               bytecode_path,
+                                                               st)
                     except (ImportError, EOFError):
                         pass
                     else:
+                        _verbose_message('{} matches {}', bytecode_path,
+                                        source_path)
                         found = marshal.loads(bytes_data)
-                        if isinstance(found, code_type):
+                        if isinstance(found, _code_type):
+                            _imp._fix_co_filename(found, source_path)
+                            _verbose_message('code object from {}',
+                                            bytecode_path)
                             return found
                         else:
                             msg = "Non-code object in {}"
-                            raise ImportError(msg.format(bytecode_path))
+                            raise ImportError(msg.format(bytecode_path),
+                                              name=fullname, path=bytecode_path)
         source_bytes = self.get_data(source_path)
-        code_object = compile(source_bytes, source_path, 'exec',
-                                dont_inherit=True)
+        code_object = _call_with_frames_removed(compile,
+                          source_bytes, source_path, 'exec',
+                          dont_inherit=True)
+        _verbose_message('code object from {}', source_path)
         if (not sys.dont_write_bytecode and bytecode_path is not None and
-                source_mtime is not None):
-            # If e.g. Jython ever implements imp.cache_from_source to have
-            # their own cached file format, this block of code will most likely
-            # raise an exception.
-            data = bytearray(imp.get_magic())
-            data.extend(marshal._w_long(source_mtime))
+            source_mtime is not None):
+            data = bytearray(_MAGIC_BYTES)
+            data.extend(_w_long(source_mtime))
+            data.extend(_w_long(len(source_bytes)))
             data.extend(marshal.dumps(code_object))
             try:
-                self.set_data(bytecode_path, data)
+                self._cache_bytecode(source_path, bytecode_path, data)
+                _verbose_message('wrote {!r}', bytecode_path)
             except NotImplementedError:
                 pass
         return code_object
@@ -436,7 +1004,7 @@ class SourceLoader(_LoaderBasics):
         return self._load_module(fullname)
 
 
-class _FileLoader:
+class FileLoader:
 
     """Base file loader class which implements the loader protocol methods that
     require file system usage."""
@@ -444,13 +1012,20 @@ class _FileLoader:
     def __init__(self, fullname, path):
         """Cache the module name and the path to the file found by the
         finder."""
-        self._name = fullname
-        self._path = path
+        self.name = fullname
+        self.path = path
+
+    @_check_name
+    def load_module(self, fullname):
+        """Load a module from a file."""
+        # Issue #14857: Avoid the zero-argument form so the implementation
+        # of that form can be updated without breaking the frozen module
+        return super(FileLoader, self).load_module(fullname)
 
     @_check_name
     def get_filename(self, fullname):
         """Return the path to the source file as found by the finder."""
-        return self._path
+        return self.path
 
     def get_data(self, path):
         """Return the data from path as raw bytes."""
@@ -458,52 +1033,56 @@ class _FileLoader:
             return file.read()
 
 
-class _SourceFileLoader(_FileLoader, SourceLoader):
+class SourceFileLoader(FileLoader, SourceLoader):
 
     """Concrete implementation of SourceLoader using the file system."""
 
-    def path_mtime(self, path):
-        """Return the modification time for the path."""
-        return int(_os.stat(path).st_mtime)
+    def path_stats(self, path):
+        """Return the metadata for the path."""
+        st = _os.stat(path)
+        return {'mtime': st.st_mtime, 'size': st.st_size}
 
-    def set_data(self, path, data):
+    def _cache_bytecode(self, source_path, bytecode_path, data):
+        # Adapt between the two APIs
+        try:
+            mode = _os.stat(source_path).st_mode
+        except OSError:
+            mode = 0o666
+        # We always ensure write access so we can update cached files
+        # later even when the source files are read-only on Windows (#6074)
+        mode |= 0o200
+        return self.set_data(bytecode_path, data, _mode=mode)
+
+    def set_data(self, path, data, *, _mode=0o666):
         """Write bytes data to a file."""
-        parent, _, filename = path.rpartition(path_sep)
+        parent, filename = _path_split(path)
         path_parts = []
         # Figure out what directories are missing.
         while parent and not _path_isdir(parent):
-            parent, _, part = parent.rpartition(path_sep)
+            parent, part = _path_split(parent)
             path_parts.append(part)
         # Create needed directories.
         for part in reversed(path_parts):
             parent = _path_join(parent, part)
             try:
                 _os.mkdir(parent)
-            except OSError as exc:
+            except FileExistsError:
                 # Probably another Python process already created the dir.
-                if exc.errno == errno.EEXIST:
-                    continue
-                else:
-                    raise
-            except IOError as exc:
-                # If can't get proper access, then just forget about writing
-                # the data.
-                if exc.errno == errno.EACCES:
-                    return
-                else:
-                    raise
-        try:
-            with _io.FileIO(path, 'wb') as file:
-                file.write(data)
-        except IOError as exc:
-            # Don't worry if you can't write bytecode.
-            if exc.errno == errno.EACCES:
+                continue
+            except OSError as exc:
+                # Could be a permission error, read-only filesystem: just forget
+                # about writing the data.
+                _verbose_message('could not create {!r}: {!r}', parent, exc)
                 return
-            else:
-                raise
+        try:
+            _write_atomic(path, data, _mode)
+            _verbose_message('created {!r}', path)
+        except OSError as exc:
+            # Same as above: just don't write the bytecode.
+            _verbose_message('could not create {!r}: {!r}', path, exc)
 
 
-class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
+class SourcelessFileLoader(FileLoader, _LoaderBasics):
 
     """Loader which handles sourceless file imports."""
 
@@ -513,19 +1092,25 @@ class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
     def get_code(self, fullname):
         path = self.get_filename(fullname)
         data = self.get_data(path)
-        bytes_data = self._bytes_from_bytecode(fullname, data, None)
+        bytes_data = self._bytes_from_bytecode(fullname, data, path, None)
         found = marshal.loads(bytes_data)
-        if isinstance(found, code_type):
+        if isinstance(found, _code_type):
+            _verbose_message('code object from {!r}', path)
             return found
         else:
-            raise ImportError("Non-code object in {}".format(path))
+            raise ImportError("Non-code object in {}".format(path),
+                              name=fullname, path=path)
 
     def get_source(self, fullname):
         """Return None as there is no source code."""
         return None
 
 
-class _ExtensionFileLoader:
+# Filled in by _setup().
+EXTENSION_SUFFIXES = []
+
+
+class ExtensionFileLoader:
 
     """Loader for extension modules.
 
@@ -534,14 +1119,8 @@ class _ExtensionFileLoader:
     """
 
     def __init__(self, name, path):
-        """Initialize the loader.
-
-        If is_pkg is True then an exception is raised as extension modules
-        cannot be the __init__ module for an extension module.
-
-        """
-        self._name = name
-        self._path = path
+        self.name = name
+        self.path = path
 
     @_check_name
     @set_package
@@ -550,297 +1129,528 @@ class _ExtensionFileLoader:
         """Load an extension module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.load_dynamic(fullname, self._path)
+            module = _call_with_frames_removed(_imp.load_dynamic,
+                                               fullname, self.path)
+            _verbose_message('extension module loaded from {!r}', self.path)
+            if self.is_package(fullname) and not hasattr(module, '__path__'):
+                module.__path__ = [_path_split(self.path)[0]]
+            return module
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
             raise
 
-    @_check_name
     def is_package(self, fullname):
-        """Return False as an extension module can never be a package."""
-        return False
+        """Return True if the extension module is a package."""
+        file_name = _path_split(self.path)[1]
+        return any(file_name == '__init__' + suffix
+                   for suffix in EXTENSION_SUFFIXES)
 
-    @_check_name
     def get_code(self, fullname):
         """Return None as an extension module cannot create a code object."""
         return None
 
-    @_check_name
     def get_source(self, fullname):
         """Return None as extension modules have no source code."""
         return None
 
 
+class _NamespacePath:
+    """Represents a namespace package's path.  It uses the module name
+    to find its parent module, and from there it looks up the parent's
+    __path__.  When this changes, the module's own path is recomputed,
+    using path_finder.  For top-level modules, the parent module's path
+    is sys.path."""
+
+    def __init__(self, name, path, path_finder):
+        self._name = name
+        self._path = path
+        self._last_parent_path = tuple(self._get_parent_path())
+        self._path_finder = path_finder
+
+    def _find_parent_path_names(self):
+        """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
+        parent, dot, me = self._name.rpartition('.')
+        if dot == '':
+            # This is a top-level module. sys.path contains the parent path.
+            return 'sys', 'path'
+        # Not a top-level module. parent-module.__path__ contains the
+        #  parent path.
+        return parent, '__path__'
+
+    def _get_parent_path(self):
+        parent_module_name, path_attr_name = self._find_parent_path_names()
+        return getattr(sys.modules[parent_module_name], path_attr_name)
+
+    def _recalculate(self):
+        # If the parent's path has changed, recalculate _path
+        parent_path = tuple(self._get_parent_path()) # Make a copy
+        if parent_path != self._last_parent_path:
+            loader, new_path = self._path_finder(self._name, parent_path)
+            # Note that no changes are made if a loader is returned, but we
+            #  do remember the new parent path
+            if loader is None:
+                self._path = new_path
+            self._last_parent_path = parent_path     # Save the copy
+        return self._path
+
+    def __iter__(self):
+        return iter(self._recalculate())
+
+    def __len__(self):
+        return len(self._recalculate())
+
+    def __repr__(self):
+        return "_NamespacePath({!r})".format(self._path)
+
+    def __contains__(self, item):
+        return item in self._recalculate()
+
+    def append(self, item):
+        self._path.append(item)
+
+
+class NamespaceLoader:
+    def __init__(self, name, path, path_finder):
+        self._path = _NamespacePath(name, path, path_finder)
+
+    @classmethod
+    def module_repr(cls, module):
+        return "<module '{}' (namespace)>".format(module.__name__)
+
+    @module_for_loader
+    def load_module(self, module):
+        """Load a namespace module."""
+        _verbose_message('namespace module loaded with path {!r}', self._path)
+        module.__path__ = self._path
+        return module
+
+
 # Finders #####################################################################
 
 class PathFinder:
 
-    """Meta path finder for sys.(path|path_hooks|path_importer_cache)."""
+    """Meta path finder for sys.path and package __path__ attributes."""
 
     @classmethod
-    def _path_hooks(cls, path, hooks=None):
+    def invalidate_caches(cls):
+        """Call the invalidate_caches() method on all path entry finders
+        stored in sys.path_importer_caches (where implemented)."""
+        for finder in sys.path_importer_cache.values():
+            if hasattr(finder, 'invalidate_caches'):
+                finder.invalidate_caches()
+
+    @classmethod
+    def _path_hooks(cls, path):
         """Search sequence of hooks for a finder for 'path'.
 
         If 'hooks' is false then use sys.path_hooks.
 
         """
-        if not hooks:
-            hooks = sys.path_hooks
-        for hook in hooks:
+        if not sys.path_hooks:
+            _warnings.warn('sys.path_hooks is empty', ImportWarning)
+        for hook in sys.path_hooks:
             try:
                 return hook(path)
             except ImportError:
                 continue
         else:
-            raise ImportError("no path hook found for {0}".format(path))
+            return None
 
     @classmethod
-    def _path_importer_cache(cls, path, default=None):
-        """Get the finder for the path from sys.path_importer_cache.
-
-        If the path is not in the cache, find the appropriate finder and cache
-        it. If None is cached, get the default finder and cache that
-        (if applicable).
+    def _path_importer_cache(cls, path):
+        """Get the finder for the path entry from sys.path_importer_cache.
 
-        Because of NullImporter, some finder should be returned. The only
-        explicit fail case is if None is cached but the path cannot be used for
-        the default hook, for which ImportError is raised.
+        If the path entry is not in the cache, find the appropriate finder
+        and cache it. If no finder is available, store None.
 
         """
+        if path == '':
+            path = '.'
         try:
             finder = sys.path_importer_cache[path]
         except KeyError:
             finder = cls._path_hooks(path)
             sys.path_importer_cache[path] = finder
-        else:
-            if finder is None and default:
-                # Raises ImportError on failure.
-                finder = default(path)
-                sys.path_importer_cache[path] = finder
         return finder
 
     @classmethod
+    def _get_loader(cls, fullname, path):
+        """Find the loader or namespace_path for this module/package name."""
+        # If this ends up being a namespace package, namespace_path is
+        #  the list of paths that will become its __path__
+        namespace_path = []
+        for entry in path:
+            if not isinstance(entry, (str, bytes)):
+                continue
+            finder = cls._path_importer_cache(entry)
+            if finder is not None:
+                if hasattr(finder, 'find_loader'):
+                    loader, portions = finder.find_loader(fullname)
+                else:
+                    loader = finder.find_module(fullname)
+                    portions = []
+                if loader is not None:
+                    # We found a loader: return it immediately.
+                    return loader, namespace_path
+                # This is possibly part of a namespace package.
+                #  Remember these path entries (if any) for when we
+                #  create a namespace package, and continue iterating
+                #  on path.
+                namespace_path.extend(portions)
+        else:
+            return None, namespace_path
+
+    @classmethod
     def find_module(cls, fullname, path=None):
         """Find the module on sys.path or 'path' based on sys.path_hooks and
         sys.path_importer_cache."""
-        if not path:
+        if path is None:
             path = sys.path
-        for entry in path:
-            try:
-                finder = cls._path_importer_cache(entry)
-            except ImportError:
-                continue
-            if finder:
-                loader = finder.find_module(fullname)
-                if loader:
-                    return loader
+        loader, namespace_path = cls._get_loader(fullname, path)
+        if loader is not None:
+            return loader
         else:
-            return None
+            if namespace_path:
+                # We found at least one namespace path.  Return a
+                #  loader which can create the namespace package.
+                return NamespaceLoader(fullname, namespace_path, cls._get_loader)
+            else:
+                return None
 
 
-class _FileFinder:
+class FileFinder:
 
     """File-based finder.
 
-    Constructor takes a list of objects detailing what file extensions their
-    loader supports along with whether it can be used for a package.
+    Interactions with the file system are cached for performance, being
+    refreshed when the directory the finder is handling has been modified.
 
     """
 
     def __init__(self, path, *details):
-        """Initialize with finder details."""
-        packages = []
-        modules = []
-        for detail in details:
-            modules.extend((suffix, detail.loader) for suffix in detail.suffixes)
-            if detail.supports_packages:
-                packages.extend((suffix, detail.loader)
-                                for suffix in detail.suffixes)
-        self.packages = packages
-        self.modules = modules
-        self.path = path
-
-    def find_module(self, fullname):
-        """Try to find a loader for the specified module."""
+        """Initialize with the path to search on and a variable number of
+        2-tuples containing the loader and the file suffixes the loader
+        recognizes."""
+        loaders = []
+        for loader, suffixes in details:
+            loaders.extend((suffix, loader) for suffix in suffixes)
+        self._loaders = loaders
+        # Base (directory) path
+        self.path = path or '.'
+        self._path_mtime = -1
+        self._path_cache = set()
+        self._relaxed_path_cache = set()
+
+    def invalidate_caches(self):
+        """Invalidate the directory mtime."""
+        self._path_mtime = -1
+
+    find_module = _find_module_shim
+
+    def find_loader(self, fullname):
+        """Try to find a loader for the specified module, or the namespace
+        package portions. Returns (loader, list-of-portions)."""
+        is_namespace = False
         tail_module = fullname.rpartition('.')[2]
-        base_path = _path_join(self.path, tail_module)
-        if _path_isdir(base_path) and _case_ok(self.path, tail_module):
-            for suffix, loader in self.packages:
-                init_filename = '__init__' + suffix
-                full_path = _path_join(base_path, init_filename)
-                if (_path_isfile(full_path) and
-                        _case_ok(base_path, init_filename)):
-                    return loader(fullname, full_path)
-            else:
-                msg = "Not importing directory {}: missing __init__"
-                _warnings.warn(msg.format(base_path), ImportWarning)
-        for suffix, loader in self.modules:
-            mod_filename = tail_module + suffix
-            full_path = _path_join(self.path, mod_filename)
-            if _path_isfile(full_path) and _case_ok(self.path, mod_filename):
-                return loader(fullname, full_path)
-        return None
-
-class _SourceFinderDetails:
-
-    loader = _SourceFileLoader
-    supports_packages = True
-
-    def __init__(self):
-        self.suffixes = _suffix_list(imp.PY_SOURCE)
-
-class _SourcelessFinderDetails:
-
-    loader = _SourcelessFileLoader
-    supports_packages = True
+        try:
+            mtime = _os.stat(self.path).st_mtime
+        except OSError:
+            mtime = -1
+        if mtime != self._path_mtime:
+            self._fill_cache()
+            self._path_mtime = mtime
+        # tail_module keeps the original casing, for __file__ and friends
+        if _relax_case():
+            cache = self._relaxed_path_cache
+            cache_module = tail_module.lower()
+        else:
+            cache = self._path_cache
+            cache_module = tail_module
+        # Check if the module is the name of a directory (and thus a package).
+        if cache_module in cache:
+            base_path = _path_join(self.path, tail_module)
+            if _path_isdir(base_path):
+                for suffix, loader in self._loaders:
+                    init_filename = '__init__' + suffix
+                    full_path = _path_join(base_path, init_filename)
+                    if _path_isfile(full_path):
+                        return (loader(fullname, full_path), [base_path])
+                else:
+                    # A namespace package, return the path if we don't also
+                    #  find a module in the next section.
+                    is_namespace = True
+        # Check for a file w/ a proper suffix exists.
+        for suffix, loader in self._loaders:
+            if cache_module + suffix in cache:
+                full_path = _path_join(self.path, tail_module + suffix)
+                if _path_isfile(full_path):
+                    return (loader(fullname, full_path), [])
+        if is_namespace:
+            return (None, [base_path])
+        return (None, [])
+
+    def _fill_cache(self):
+        """Fill the cache of potential modules and packages for this directory."""
+        path = self.path
+        try:
+            contents = _os.listdir(path)
+        except (FileNotFoundError, PermissionError, NotADirectoryError):
+            # Directory has either been removed, turned into a file, or made
+            # unreadable.
+            contents = []
+        # We store two cached versions, to handle runtime changes of the
+        # PYTHONCASEOK environment variable.
+        if not sys.platform.startswith('win'):
+            self._path_cache = set(contents)
+        else:
+            # Windows users can import modules with case-insensitive file
+            # suffixes (for legacy reasons). Make the suffix lowercase here
+            # so it's done once instead of for every import. This is safe as
+            # the specified suffixes to check against are always specified in a
+            # case-sensitive manner.
+            lower_suffix_contents = set()
+            for item in contents:
+                name, dot, suffix = item.partition('.')
+                if dot:
+                    new_name = '{}.{}'.format(name, suffix.lower())
+                else:
+                    new_name = name
+                lower_suffix_contents.add(new_name)
+            self._path_cache = lower_suffix_contents
+        if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
+            self._relaxed_path_cache = set(fn.lower() for fn in contents)
 
-    def __init__(self):
-        self.suffixes = _suffix_list(imp.PY_COMPILED)
+    @classmethod
+    def path_hook(cls, *loader_details):
+        """A class method which returns a closure to use on sys.path_hook
+        which will return an instance using the specified loaders and the path
+        called on the closure.
 
+        If the path called on the closure is not a directory, ImportError is
+        raised.
 
-class _ExtensionFinderDetails:
+        """
+        def path_hook_for_FileFinder(path):
+            """Path hook for importlib.machinery.FileFinder."""
+            if not _path_isdir(path):
+                raise ImportError("only directories are supported", path=path)
+            return cls(path, *loader_details)
 
-    loader = _ExtensionFileLoader
-    supports_packages = False
+        return path_hook_for_FileFinder
 
-    def __init__(self):
-        self.suffixes = _suffix_list(imp.C_EXTENSION)
+    def __repr__(self):
+        return "FileFinder(%r)" % (self.path,)
 
 
 # Import itself ###############################################################
 
-def _file_path_hook(path):
-    """If the path is a directory, return a file-based finder."""
-    if _path_isdir(path):
-        return _FileFinder(path, _ExtensionFinderDetails(),
-                           _SourceFinderDetails(),
-                           _SourcelessFinderDetails())
-    else:
-        raise ImportError("only directories are supported")
+class _ImportLockContext:
 
+    """Context manager for the import lock."""
 
-_DEFAULT_PATH_HOOK = _file_path_hook
+    def __enter__(self):
+        """Acquire the import lock."""
+        _imp.acquire_lock()
 
-class _DefaultPathFinder(PathFinder):
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        """Release the import lock regardless of any raised exceptions."""
+        _imp.release_lock()
 
-    """Subclass of PathFinder that implements implicit semantics for
-    __import__."""
 
-    @classmethod
-    def _path_hooks(cls, path):
-        """Search sys.path_hooks as well as implicit path hooks."""
-        try:
-            return super()._path_hooks(path)
-        except ImportError:
-            implicit_hooks = [_DEFAULT_PATH_HOOK, imp.NullImporter]
-            return super()._path_hooks(path, implicit_hooks)
+def _resolve_name(name, package, level):
+    """Resolve a relative module name to an absolute one."""
+    bits = package.rsplit('.', level - 1)
+    if len(bits) < level:
+        raise ValueError('attempted relative import beyond top-level package')
+    base = bits[0]
+    return '{}.{}'.format(base, name) if name else base
 
-    @classmethod
-    def _path_importer_cache(cls, path):
-        """Use the default path hook when None is stored in
-        sys.path_importer_cache."""
-        return super()._path_importer_cache(path, _DEFAULT_PATH_HOOK)
 
+def _find_module(name, path):
+    """Find a module's loader."""
+    if not sys.meta_path:
+        _warnings.warn('sys.meta_path is empty', ImportWarning)
+    for finder in sys.meta_path:
+        with _ImportLockContext():
+            loader = finder.find_module(name, path)
+        if loader is not None:
+            # The parent import may have already imported this module.
+            if name not in sys.modules:
+                return loader
+            else:
+                return sys.modules[name].__loader__
+    else:
+        return None
 
-class _ImportLockContext:
 
-    """Context manager for the import lock."""
+def _sanity_check(name, package, level):
+    """Verify arguments are "sane"."""
+    if not isinstance(name, str):
+        raise TypeError("module name must be str, not {}".format(type(name)))
+    if level < 0:
+        raise ValueError('level must be >= 0')
+    if package:
+        if not isinstance(package, str):
+            raise TypeError("__package__ not set to a string")
+        elif package not in sys.modules:
+            msg = ("Parent module {!r} not loaded, cannot perform relative "
+                   "import")
+            raise SystemError(msg.format(package))
+    if not name and level == 0:
+        raise ValueError("Empty module name")
 
-    def __enter__(self):
-        """Acquire the import lock."""
-        imp.acquire_lock()
 
-    def __exit__(self, exc_type, exc_value, exc_traceback):
-        """Release the import lock regardless of any raised exceptions."""
-        imp.release_lock()
+_ERR_MSG = 'No module named {!r}'
 
+def _find_and_load_unlocked(name, import_):
+    path = None
+    parent = name.rpartition('.')[0]
+    if parent:
+        if parent not in sys.modules:
+            _call_with_frames_removed(import_, parent)
+        # Crazy side-effects!
+        if name in sys.modules:
+            return sys.modules[name]
+        # Backwards-compatibility; be nicer to skip the dict lookup.
+        parent_module = sys.modules[parent]
+        try:
+            path = parent_module.__path__
+        except AttributeError:
+            msg = (_ERR_MSG + '; {} is not a package').format(name, parent)
+            raise ImportError(msg, name=name)
+    loader = _find_module(name, path)
+    if loader is None:
+        exc = ImportError(_ERR_MSG.format(name), name=name)
+        # TODO(brett): switch to a proper ModuleNotFound exception in Python
+        # 3.4.
+        exc._not_found = True
+        raise exc
+    elif name not in sys.modules:
+        # The parent import may have already imported this module.
+        loader.load_module(name)
+        _verbose_message('import {!r} # {!r}', name, loader)
+    # Backwards-compatibility; be nicer to skip the dict lookup.
+    module = sys.modules[name]
+    if parent:
+        # Set the module as an attribute on its parent.
+        parent_module = sys.modules[parent]
+        setattr(parent_module, name.rpartition('.')[2], module)
+    # Set __package__ if the loader did not.
+    if getattr(module, '__package__', None) is None:
+        try:
+            module.__package__ = module.__name__
+            if not hasattr(module, '__path__'):
+                module.__package__ = module.__package__.rpartition('.')[0]
+        except AttributeError:
+            pass
+    # Set loader if need be.
+    if not hasattr(module, '__loader__'):
+        try:
+            module.__loader__ = loader
+        except AttributeError:
+            pass
+    return module
 
-_IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder]
 
-_ERR_MSG = 'No module named {}'
+def _find_and_load(name, import_):
+    """Find and load the module, and release the import lock."""
+    try:
+        lock = _get_module_lock(name)
+    finally:
+        _imp.release_lock()
+    lock.acquire()
+    try:
+        return _find_and_load_unlocked(name, import_)
+    finally:
+        lock.release()
+
 
 def _gcd_import(name, package=None, level=0):
     """Import and return the module based on its name, the package the call is
     being made from, and the level adjustment.
 
     This function represents the greatest common denominator of functionality
-    between import_module and __import__. This includes settting __package__ if
+    between import_module and __import__. This includes setting __package__ if
     the loader did not.
 
     """
-    if package:
-        if not hasattr(package, 'rindex'):
-            raise ValueError("__package__ not set to a string")
-        elif package not in sys.modules:
-            msg = ("Parent module {0!r} not loaded, cannot perform relative "
-                   "import")
-            raise SystemError(msg.format(package))
-    if not name and level == 0:
-        raise ValueError("Empty module name")
+    _sanity_check(name, package, level)
     if level > 0:
-        dot = len(package)
-        for x in range(level, 1, -1):
-            try:
-                dot = package.rindex('.', 0, dot)
-            except ValueError:
-                raise ValueError("attempted relative import beyond "
-                                 "top-level package")
-        if name:
-            name = "{0}.{1}".format(package[:dot], name)
-        else:
-            name = package[:dot]
-    with _ImportLockContext():
-        try:
-            module = sys.modules[name]
-            if module is None:
-                message = ("import of {} halted; "
-                            "None in sys.modules".format(name))
-                raise ImportError(message)
-            return module
-        except KeyError:
-            pass
-        parent = name.rpartition('.')[0]
-        path = None
-        if parent:
-            if parent not in sys.modules:
-                _gcd_import(parent)
-            # Backwards-compatibility; be nicer to skip the dict lookup.
-            parent_module = sys.modules[parent]
-            try:
-                path = parent_module.__path__
-            except AttributeError:
-                msg = (_ERR_MSG + '; {} is not a package').format(name, parent)
-                raise ImportError(msg)
-        meta_path = sys.meta_path + _IMPLICIT_META_PATH
-        for finder in meta_path:
-            loader = finder.find_module(name, path)
-            if loader is not None:
-                # The parent import may have already imported this module.
-                if name not in sys.modules:
-                    loader.load_module(name)
-                break
-        else:
-            raise ImportError(_ERR_MSG.format(name))
-        # Backwards-compatibility; be nicer to skip the dict lookup.
-        module = sys.modules[name]
-        if parent:
-            # Set the module as an attribute on its parent.
-            setattr(parent_module, name.rpartition('.')[2], module)
-        # Set __package__ if the loader did not.
-        if not hasattr(module, '__package__') or module.__package__ is None:
-            # Watch out for what comes out of sys.modules to not be a module,
-            # e.g. an int.
-            try:
-                module.__package__ = module.__name__
-                if not hasattr(module, '__path__'):
-                    module.__package__ = module.__package__.rpartition('.')[0]
-            except AttributeError:
-                pass
-        return module
+        name = _resolve_name(name, package, level)
+    _imp.acquire_lock()
+    if name not in sys.modules:
+        return _find_and_load(name, _gcd_import)
+    module = sys.modules[name]
+    if module is None:
+        _imp.release_lock()
+        message = ("import of {} halted; "
+                    "None in sys.modules".format(name))
+        raise ImportError(message, name=name)
+    _lock_unlock_module(name)
+    return module
+
+def _handle_fromlist(module, fromlist, import_):
+    """Figure out what __import__ should return.
+
+    The import_ parameter is a callable which takes the name of module to
+    import. It is required to decouple the function from assuming importlib's
+    import implementation is desired.
+
+    """
+    # The hell that is fromlist ...
+    # If a package was imported, try to import stuff from fromlist.
+    if hasattr(module, '__path__'):
+        if '*' in fromlist:
+            fromlist = list(fromlist)
+            fromlist.remove('*')
+            if hasattr(module, '__all__'):
+                fromlist.extend(module.__all__)
+        for x in fromlist:
+            if not hasattr(module, x):
+                from_name = '{}.{}'.format(module.__name__, x)
+                try:
+                    _call_with_frames_removed(import_, from_name)
+                except ImportError as exc:
+                    # Backwards-compatibility dictates we ignore failed
+                    # imports triggered by fromlist for modules that don't
+                    # exist.
+                    # TODO(brett): In Python 3.4, have import raise
+                    #   ModuleNotFound and catch that.
+                    if getattr(exc, '_not_found', False):
+                        if exc.name == from_name:
+                            continue
+                    raise
+    return module
+
+
+def _calc___package__(globals):
+    """Calculate what __package__ should be.
 
+    __package__ is not guaranteed to be defined or could be set to None
+    to represent that its proper value is unknown.
 
-def __import__(name, globals={}, locals={}, fromlist=[], level=0):
+    """
+    package = globals.get('__package__')
+    if package is None:
+        package = globals['__name__']
+        if '__path__' not in globals:
+            package = package.rpartition('.')[0]
+    return package
+
+
+def _get_supported_file_loaders():
+    """Returns a list of file-based module loaders.
+
+    Each item is a tuple (loader, suffixes, allow_packages).
+    """
+    extensions = ExtensionFileLoader, _imp.extension_suffixes()
+    source = SourceFileLoader, SOURCE_SUFFIXES
+    bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
+    return [extensions, source, bytecode]
+
+
+def __import__(name, globals=None, locals=None, fromlist=(), level=0):
     """Import a module.
 
     The 'globals' argument is used to infer where the import is occuring from
@@ -851,40 +1661,117 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0):
     import (e.g. ``from ..pkg import mod`` would have a 'level' of 2).
 
     """
-    if not hasattr(name, 'rpartition'):
-        raise TypeError("module name must be str, not {}".format(type(name)))
     if level == 0:
         module = _gcd_import(name)
     else:
-        # __package__ is not guaranteed to be defined or could be set to None
-        # to represent that it's proper value is unknown
-        package = globals.get('__package__')
-        if package is None:
-            package = globals['__name__']
-            if '__path__' not in globals:
-                package = package.rpartition('.')[0]
+        globals_ = globals if globals is not None else {}
+        package = _calc___package__(globals_)
         module = _gcd_import(name, package, level)
-    # The hell that is fromlist ...
     if not fromlist:
         # Return up to the first dot in 'name'. This is complicated by the fact
         # that 'name' may be relative.
         if level == 0:
-            return sys.modules[name.partition('.')[0]]
+            return _gcd_import(name.partition('.')[0])
         elif not name:
             return module
         else:
+            # Figure out where to slice the module's name up to the first dot
+            # in 'name'.
             cut_off = len(name) - len(name.partition('.')[0])
-            return sys.modules[module.__name__[:-cut_off]]
+            # Slice end needs to be positive to alleviate need to special-case
+            # when ``'.' not in name``.
+            return sys.modules[module.__name__[:len(module.__name__)-cut_off]]
     else:
-        # If a package was imported, try to import stuff from fromlist.
-        if hasattr(module, '__path__'):
-            if '*' in fromlist and hasattr(module, '__all__'):
-                fromlist = list(fromlist)
-                fromlist.remove('*')
-                fromlist.extend(module.__all__)
-            for x in (y for y in fromlist if not hasattr(module,y)):
-                try:
-                    _gcd_import('{0}.{1}'.format(module.__name__, x))
-                except ImportError:
-                    pass
-        return module
+        return _handle_fromlist(module, fromlist, _gcd_import)
+
+
+
+def _setup(sys_module, _imp_module):
+    """Setup importlib by importing needed built-in modules and injecting them
+    into the global namespace.
+
+    As sys is needed for sys.modules access and _imp is needed to load built-in
+    modules, those two modules must be explicitly passed in.
+
+    """
+    global _imp, sys, BYTECODE_SUFFIXES
+    _imp = _imp_module
+    sys = sys_module
+
+    if sys.flags.optimize:
+        BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES
+    else:
+        BYTECODE_SUFFIXES = DEBUG_BYTECODE_SUFFIXES
+
+    module_type = type(sys)
+    for name, module in sys.modules.items():
+        if isinstance(module, module_type):
+            if not hasattr(module, '__loader__'):
+                if name in sys.builtin_module_names:
+                    module.__loader__ = BuiltinImporter
+                elif _imp.is_frozen(name):
+                    module.__loader__ = FrozenImporter
+
+    self_module = sys.modules[__name__]
+    for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+        if builtin_name not in sys.modules:
+            builtin_module = BuiltinImporter.load_module(builtin_name)
+        else:
+            builtin_module = sys.modules[builtin_name]
+        setattr(self_module, builtin_name, builtin_module)
+
+    os_details = ('posix', ['/']), ('nt', ['\\', '/']), ('os2', ['\\', '/'])
+    for builtin_os, path_separators in os_details:
+        # Assumption made in _path_join()
+        assert all(len(sep) == 1 for sep in path_separators)
+        path_sep = path_separators[0]
+        if builtin_os in sys.modules:
+            os_module = sys.modules[builtin_os]
+            break
+        else:
+            try:
+                os_module = BuiltinImporter.load_module(builtin_os)
+                # TODO: rip out os2 code after 3.3 is released as per PEP 11
+                if builtin_os == 'os2' and 'EMX GCC' in sys.version:
+                    path_sep = path_separators[1]
+                break
+            except ImportError:
+                continue
+    else:
+        raise ImportError('importlib requires posix or nt')
+
+    try:
+        thread_module = BuiltinImporter.load_module('_thread')
+    except ImportError:
+        # Python was built without threads
+        thread_module = None
+    weakref_module = BuiltinImporter.load_module('_weakref')
+
+    if builtin_os == 'nt':
+        winreg_module = BuiltinImporter.load_module('winreg')
+        setattr(self_module, '_winreg', winreg_module)
+
+    setattr(self_module, '_os', os_module)
+    setattr(self_module, '_thread', thread_module)
+    setattr(self_module, '_weakref', weakref_module)
+    setattr(self_module, 'path_sep', path_sep)
+    setattr(self_module, 'path_separators', set(path_separators))
+    # Constants
+    setattr(self_module, '_relax_case', _make_relax_case())
+    EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
+    if builtin_os == 'nt':
+        SOURCE_SUFFIXES.append('.pyw')
+        if '_d.pyd' in EXTENSION_SUFFIXES:
+            WindowsRegistryFinder.DEBUG_BUILD = True
+
+
+def _install(sys_module, _imp_module):
+    """Install importlib as the implementation of import."""
+    _setup(sys_module, _imp_module)
+    supported_loaders = _get_supported_file_loaders()
+    sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
+    sys.meta_path.append(BuiltinImporter)
+    sys.meta_path.append(FrozenImporter)
+    if _os.__name__ == 'nt':
+        sys.meta_path.append(WindowsRegistryFinder)
+    sys.meta_path.append(PathFinder)
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index fa343f85a4..387567a450 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -1,44 +1,109 @@
 """Abstract base classes related to import."""
 from . import _bootstrap
 from . import machinery
-from . import util
+try:
+    import _frozen_importlib
+except ImportError as exc:
+    if exc.name != '_frozen_importlib':
+        raise
+    _frozen_importlib = None
 import abc
 import imp
-import io
 import marshal
-import os.path
 import sys
 import tokenize
-import types
 import warnings
 
 
-class Loader(metaclass=abc.ABCMeta):
+def _register(abstract_cls, *classes):
+    for cls in classes:
+        abstract_cls.register(cls)
+        if _frozen_importlib is not None:
+            frozen_cls = getattr(_frozen_importlib, cls.__name__)
+            abstract_cls.register(frozen_cls)
 
-    """Abstract base class for import loaders."""
+
+class Finder(metaclass=abc.ABCMeta):
+
+    """Legacy abstract base class for import finders.
+
+    It may be subclassed for compatibility with legacy third party
+    reimplementations of the import system.  Otherwise, finder
+    implementations should derive from the more specific MetaPathFinder
+    or PathEntryFinder ABCs.
+    """
 
     @abc.abstractmethod
-    def load_module(self, fullname):
-        """Abstract method which when implemented should load a module.
-        The fullname is a str."""
+    def find_module(self, fullname, path=None):
+        """An abstract method that should find a module.
+        The fullname is a str and the optional path is a str or None.
+        Returns a Loader object.
+        """
         raise NotImplementedError
 
 
-class Finder(metaclass=abc.ABCMeta):
+class MetaPathFinder(Finder):
 
-    """Abstract base class for import finders."""
+    """Abstract base class for import finders on sys.meta_path."""
 
     @abc.abstractmethod
-    def find_module(self, fullname, path=None):
-        """Abstract method which when implemented should find a module.
-        The fullname is a str and the optional path is a str or None.
+    def find_module(self, fullname, path):
+        """Abstract method which, when implemented, should find a module.
+        The fullname is a str and the path is a str or None.
         Returns a Loader object.
         """
         raise NotImplementedError
 
-Finder.register(machinery.BuiltinImporter)
-Finder.register(machinery.FrozenImporter)
-Finder.register(machinery.PathFinder)
+    def invalidate_caches(self):
+        """An optional method for clearing the finder's cache, if any.
+        This method is used by importlib.invalidate_caches().
+        """
+        return NotImplemented
+
+_register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter,
+          machinery.PathFinder, machinery.WindowsRegistryFinder)
+
+
+class PathEntryFinder(Finder):
+
+    """Abstract base class for path entry finders used by PathFinder."""
+
+    @abc.abstractmethod
+    def find_loader(self, fullname):
+        """Abstract method which, when implemented, returns a module loader.
+        The fullname is a str.  Returns a 2-tuple of (Loader, portion) where
+        portion is a sequence of file system locations contributing to part of
+        a namespace package.  The sequence may be empty and the loader may be
+        None.
+        """
+        raise NotImplementedError
+
+    find_module = _bootstrap._find_module_shim
+
+    def invalidate_caches(self):
+        """An optional method for clearing the finder's cache, if any.
+        This method is used by PathFinder.invalidate_caches().
+        """
+        return NotImplemented
+
+_register(PathEntryFinder, machinery.FileFinder)
+
+
+class Loader(metaclass=abc.ABCMeta):
+
+    """Abstract base class for import loaders."""
+
+    @abc.abstractmethod
+    def load_module(self, fullname):
+        """Abstract method which when implemented should load a module.
+        The fullname is a str."""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def module_repr(self, module):
+        """Abstract method which when implemented calculates and returns the
+        given module's repr."""
+        raise NotImplementedError
 
 
 class ResourceLoader(Loader):
@@ -84,8 +149,8 @@ class InspectLoader(Loader):
         module.  The fullname is a str.  Returns a str."""
         raise NotImplementedError
 
-InspectLoader.register(machinery.BuiltinImporter)
-InspectLoader.register(machinery.FrozenImporter)
+_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter,
+            machinery.ExtensionFileLoader)
 
 
 class ExecutionLoader(InspectLoader):
@@ -104,6 +169,15 @@ class ExecutionLoader(InspectLoader):
         raise NotImplementedError
 
 
+class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader):
+
+    """Abstract base class partially implementing the ResourceLoader and
+    ExecutionLoader ABCs."""
+
+_register(FileLoader, machinery.SourceFileLoader,
+            machinery.SourcelessFileLoader)
+
+
 class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
 
     """Abstract base class for loading source code (and optionally any
@@ -123,7 +197,20 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
 
     def path_mtime(self, path):
         """Return the (int) modification time for the path (str)."""
-        raise NotImplementedError
+        if self.path_stats.__func__ is SourceLoader.path_stats:
+            raise NotImplementedError
+        return int(self.path_stats(path)['mtime'])
+
+    def path_stats(self, path):
+        """Return a metadata dict for the source pointed to by the path (str).
+        Possible keys:
+        - 'mtime' (mandatory) is the numeric timestamp of last source
+          code modification;
+        - 'size' (optional) is the size in bytes of the source code.
+        """
+        if self.path_mtime.__func__ is SourceLoader.path_mtime:
+            raise NotImplementedError
+        return {'mtime': self.path_mtime(path)}
 
     def set_data(self, path, data):
         """Write the bytes to the path (if possible).
@@ -137,6 +224,7 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
         """
         raise NotImplementedError
 
+_register(SourceLoader, machinery.SourceFileLoader)
 
 class PyLoader(SourceLoader):
 
@@ -195,10 +283,10 @@ class PyLoader(SourceLoader):
                             "use SourceLoader instead. "
                             "See the importlib documentation on how to be "
                             "compatible with Python 3.1 onwards.",
-                        PendingDeprecationWarning)
+                        DeprecationWarning)
         path = self.source_path(fullname)
         if path is None:
-            raise ImportError
+            raise ImportError(name=fullname)
         else:
             return path
 
@@ -226,7 +314,7 @@ class PyPycLoader(PyLoader):
         if path is not None:
             return path
         raise ImportError("no source or bytecode path available for "
-                            "{0!r}".format(fullname))
+                            "{0!r}".format(fullname), name=fullname)
 
     def get_code(self, fullname):
         """Get a code object from source or bytecode."""
@@ -234,7 +322,7 @@ class PyPycLoader(PyLoader):
                             "removal in Python 3.4; use SourceLoader instead. "
                             "If Python 3.1 compatibility is required, see the "
                             "latest documentation for PyLoader.",
-                        PendingDeprecationWarning)
+                        DeprecationWarning)
         source_timestamp = self.source_mtime(fullname)
         # Try to use bytecode if it is available.
         bytecode_path = self.bytecode_path(fullname)
@@ -243,20 +331,30 @@ class PyPycLoader(PyLoader):
             try:
                 magic = data[:4]
                 if len(magic) < 4:
-                    raise ImportError("bad magic number in {}".format(fullname))
+                    raise ImportError(
+                        "bad magic number in {}".format(fullname),
+                        name=fullname, path=bytecode_path)
                 raw_timestamp = data[4:8]
                 if len(raw_timestamp) < 4:
                     raise EOFError("bad timestamp in {}".format(fullname))
-                pyc_timestamp = marshal._r_long(raw_timestamp)
-                bytecode = data[8:]
+                pyc_timestamp = _bootstrap._r_long(raw_timestamp)
+                raw_source_size = data[8:12]
+                if len(raw_source_size) != 4:
+                    raise EOFError("bad file size in {}".format(fullname))
+                # Source size is unused as the ABC does not provide a way to
+                # get the size of the source ahead of reading it.
+                bytecode = data[12:]
                 # Verify that the magic number is valid.
                 if imp.get_magic() != magic:
-                    raise ImportError("bad magic number in {}".format(fullname))
+                    raise ImportError(
+                        "bad magic number in {}".format(fullname),
+                        name=fullname, path=bytecode_path)
                 # Verify that the bytecode is not stale (only matters when
                 # there is source to fall back on.
                 if source_timestamp:
                     if pyc_timestamp < source_timestamp:
-                        raise ImportError("bytecode is stale")
+                        raise ImportError("bytecode is stale", name=fullname,
+                                          path=bytecode_path)
             except (ImportError, EOFError):
                 # If source is available give it a shot.
                 if source_timestamp is not None:
@@ -268,18 +366,20 @@ class PyPycLoader(PyLoader):
                 return marshal.loads(bytecode)
         elif source_timestamp is None:
             raise ImportError("no source or bytecode available to create code "
-                                "object for {0!r}".format(fullname))
+                              "object for {0!r}".format(fullname),
+                              name=fullname)
         # Use the source.
         source_path = self.source_path(fullname)
         if source_path is None:
             message = "a source path must exist to load {0}".format(fullname)
-            raise ImportError(message)
+            raise ImportError(message, name=fullname)
         source = self.get_data(source_path)
         code_object = compile(source, source_path, 'exec', dont_inherit=True)
         # Generate bytecode and write it out.
         if not sys.dont_write_bytecode:
             data = bytearray(imp.get_magic())
-            data.extend(marshal._w_long(source_timestamp))
+            data.extend(_bootstrap._w_long(source_timestamp))
+            data.extend(_bootstrap._w_long(len(source) & 0xFFFFFFFF))
             data.extend(marshal.dumps(code_object))
             self.write_bytecode(fullname, data)
         return code_object
diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py
index 519774440f..ff826e4fae 100644
--- a/Lib/importlib/machinery.py
+++ b/Lib/importlib/machinery.py
@@ -1,5 +1,20 @@
 """The machinery of importlib: finders, loaders, hooks, etc."""
 
+import _imp
+
+from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
+                         OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
+                         EXTENSION_SUFFIXES)
 from ._bootstrap import BuiltinImporter
 from ._bootstrap import FrozenImporter
+from ._bootstrap import WindowsRegistryFinder
 from ._bootstrap import PathFinder
+from ._bootstrap import FileFinder
+from ._bootstrap import SourceFileLoader
+from ._bootstrap import SourcelessFileLoader
+from ._bootstrap import ExtensionFileLoader
+
+
+def all_suffixes():
+    """Returns a list of all recognized module suffixes for this process"""
+    return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES
diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py
deleted file mode 100644
index decc53d8c5..0000000000
--- a/Lib/importlib/test/__main__.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""Run importlib's test suite.
-
-Specifying the ``--builtin`` flag will run tests, where applicable, with
-builtins.__import__ instead of importlib.__import__.
-
-"""
-import importlib
-from importlib.test.import_ import util
-import os.path
-from test.support import run_unittest
-import sys
-import unittest
-
-
-def test_main():
-    if '__pycache__' in __file__:
-        parts = __file__.split(os.path.sep)
-        start_dir = sep.join(parts[:-2])
-    else:
-        start_dir = os.path.dirname(__file__)
-    top_dir = os.path.dirname(os.path.dirname(start_dir))
-    test_loader = unittest.TestLoader()
-    if '--builtin' in sys.argv:
-        util.using___import__ = True
-    run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir))
-
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/importlib/test/benchmark.py b/Lib/importlib/test/benchmark.py
deleted file mode 100644
index b5de6c6b01..0000000000
--- a/Lib/importlib/test/benchmark.py
+++ /dev/null
@@ -1,172 +0,0 @@
-"""Benchmark some basic import use-cases.
-
-The assumption is made that this benchmark is run in a fresh interpreter and
-thus has no external changes made to import-related attributes in sys.
-
-"""
-from . import util
-from .source import util as source_util
-import decimal
-import imp
-import importlib
-import os
-import py_compile
-import sys
-import timeit
-
-
-def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3):
-    """Bench the given statement as many times as necessary until total
-    executions take one second."""
-    stmt = "__import__({!r})".format(name)
-    timer = timeit.Timer(stmt)
-    for x in range(repeat):
-        total_time = 0
-        count = 0
-        while total_time < seconds:
-            try:
-                total_time += timer.timeit(1)
-            finally:
-                cleanup()
-            count += 1
-        else:
-            # One execution too far
-            if total_time > seconds:
-                count -= 1
-        yield count // seconds
-
-def from_cache(seconds, repeat):
-    """sys.modules"""
-    name = '<benchmark import>'
-    module = imp.new_module(name)
-    module.__file__ = '<test>'
-    module.__package__ = ''
-    with util.uncache(name):
-        sys.modules[name] = module
-        for result in bench(name, repeat=repeat, seconds=seconds):
-            yield result
-
-
-def builtin_mod(seconds, repeat):
-    """Built-in module"""
-    name = 'errno'
-    if name in sys.modules:
-        del sys.modules[name]
-    # Relying on built-in importer being implicit.
-    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
-                        seconds=seconds):
-        yield result
-
-
-def source_wo_bytecode(seconds, repeat):
-    """Source w/o bytecode: simple"""
-    sys.dont_write_bytecode = True
-    try:
-        name = '__importlib_test_benchmark__'
-        # Clears out sys.modules and puts an entry at the front of sys.path.
-        with source_util.create_modules(name) as mapping:
-            assert not os.path.exists(imp.cache_from_source(mapping[name]))
-            for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
-                                seconds=seconds):
-                yield result
-    finally:
-        sys.dont_write_bytecode = False
-
-
-def decimal_wo_bytecode(seconds, repeat):
-    """Source w/o bytecode: decimal"""
-    name = 'decimal'
-    decimal_bytecode = imp.cache_from_source(decimal.__file__)
-    if os.path.exists(decimal_bytecode):
-        os.unlink(decimal_bytecode)
-    sys.dont_write_bytecode = True
-    try:
-        for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
-                            seconds=seconds):
-            yield result
-    finally:
-        sys.dont_write_bytecode = False
-
-
-def source_writing_bytecode(seconds, repeat):
-    """Source writing bytecode: simple"""
-    assert not sys.dont_write_bytecode
-    name = '__importlib_test_benchmark__'
-    with source_util.create_modules(name) as mapping:
-        def cleanup():
-            sys.modules.pop(name)
-            os.unlink(imp.cache_from_source(mapping[name]))
-        for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
-            assert not os.path.exists(imp.cache_from_source(mapping[name]))
-            yield result
-
-
-def decimal_writing_bytecode(seconds, repeat):
-    """Source writing bytecode: decimal"""
-    assert not sys.dont_write_bytecode
-    name = 'decimal'
-    def cleanup():
-        sys.modules.pop(name)
-        os.unlink(imp.cache_from_source(decimal.__file__))
-    for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
-        yield result
-
-
-def source_using_bytecode(seconds, repeat):
-    """Bytecode w/ source: simple"""
-    name = '__importlib_test_benchmark__'
-    with source_util.create_modules(name) as mapping:
-        py_compile.compile(mapping[name])
-        assert os.path.exists(imp.cache_from_source(mapping[name]))
-        for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
-                            seconds=seconds):
-            yield result
-
-
-def decimal_using_bytecode(seconds, repeat):
-    """Bytecode w/ source: decimal"""
-    name = 'decimal'
-    py_compile.compile(decimal.__file__)
-    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
-                        seconds=seconds):
-        yield result
-
-
-def main(import_):
-    __builtins__.__import__ = import_
-    benchmarks = (from_cache, builtin_mod,
-                  source_using_bytecode, source_wo_bytecode,
-                  source_writing_bytecode,
-                  decimal_using_bytecode, decimal_writing_bytecode,
-                  decimal_wo_bytecode,)
-    seconds = 1
-    seconds_plural = 's' if seconds > 1 else ''
-    repeat = 3
-    header = "Measuring imports/second over {} second{}, best out of {}\n"
-    print(header.format(seconds, seconds_plural, repeat))
-    for benchmark in benchmarks:
-        print(benchmark.__doc__, "[", end=' ')
-        sys.stdout.flush()
-        results = []
-        for result in benchmark(seconds=seconds, repeat=repeat):
-            results.append(result)
-            print(result, end=' ')
-            sys.stdout.flush()
-        assert not sys.dont_write_bytecode
-        print("]", "best is", format(max(results), ',d'))
-
-
-if __name__ == '__main__':
-    import optparse
-
-    parser = optparse.OptionParser()
-    parser.add_option('-b', '--builtin', dest='builtin', action='store_true',
-                        default=False, help="use the built-in __import__")
-    options, args = parser.parse_args()
-    if args:
-        raise RuntimeError("unrecognized args: {}".format(args))
-    import_ = __import__
-    if not options.builtin:
-        import_ = importlib.__import__
-
-    main(import_)
diff --git a/Lib/importlib/test/extension/test_loader.py b/Lib/importlib/test/extension/test_loader.py
deleted file mode 100644
index 4a783db8a5..0000000000
--- a/Lib/importlib/test/extension/test_loader.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from importlib import _bootstrap
-from . import util as ext_util
-from .. import abc
-from .. import util
-
-import sys
-import unittest
-
-
-class LoaderTests(abc.LoaderTests):
-
-    """Test load_module() for extension modules."""
-
-    def load_module(self, fullname):
-        loader = _bootstrap._ExtensionFileLoader(ext_util.NAME,
-                                                ext_util.FILEPATH)
-        return loader.load_module(fullname)
-
-    def test_module(self):
-        with util.uncache(ext_util.NAME):
-            module = self.load_module(ext_util.NAME)
-            for attr, value in [('__name__', ext_util.NAME),
-                                ('__file__', ext_util.FILEPATH),
-                                ('__package__', '')]:
-                self.assertEqual(getattr(module, attr), value)
-            self.assertTrue(ext_util.NAME in sys.modules)
-            self.assertTrue(isinstance(module.__loader__,
-                                    _bootstrap._ExtensionFileLoader))
-
-    def test_package(self):
-        # Extensions are not found in packages.
-        pass
-
-    def test_lacking_parent(self):
-        # Extensions are not found in packages.
-        pass
-
-    def test_module_reuse(self):
-        with util.uncache(ext_util.NAME):
-            module1 = self.load_module(ext_util.NAME)
-            module2 = self.load_module(ext_util.NAME)
-            self.assertTrue(module1 is module2)
-
-    def test_state_after_failure(self):
-        # No easy way to trigger a failure after a successful import.
-        pass
-
-    def test_unloadable(self):
-        with self.assertRaises(ImportError):
-            self.load_module('asdfjkl;')
-
-
-def test_main():
-    from test.support import run_unittest
-    run_unittest(LoaderTests)
-
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/importlib/test/import_/test_api.py b/Lib/importlib/test/import_/test_api.py
deleted file mode 100644
index 9075d42759..0000000000
--- a/Lib/importlib/test/import_/test_api.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from . import util
-import unittest
-
-
-class APITest(unittest.TestCase):
-
-    """Test API-specific details for __import__ (e.g. raising the right
-    exception when passing in an int for the module name)."""
-
-    def test_name_requires_rparition(self):
-        # Raise TypeError if a non-string is passed in for the module name.
-        with self.assertRaises(TypeError):
-            util.import_(42)
-
-
-def test_main():
-    from test.support import run_unittest
-    run_unittest(APITest)
-
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/importlib/test/import_/test_packages.py b/Lib/importlib/test/import_/test_packages.py
deleted file mode 100644
index faadc32172..0000000000
--- a/Lib/importlib/test/import_/test_packages.py
+++ /dev/null
@@ -1,37 +0,0 @@
-from .. import util
-from . import util as import_util
-import sys
-import unittest
-import importlib
-
-
-class ParentModuleTests(unittest.TestCase):
-
-    """Importing a submodule should import the parent modules."""
-
-    def test_import_parent(self):
-        with util.mock_modules('pkg.__init__', 'pkg.module') as mock:
-            with util.import_state(meta_path=[mock]):
-                module = import_util.import_('pkg.module')
-                self.assertTrue('pkg' in sys.modules)
-
-    def test_bad_parent(self):
-        with util.mock_modules('pkg.module') as mock:
-            with util.import_state(meta_path=[mock]):
-                with self.assertRaises(ImportError):
-                    import_util.import_('pkg.module')
-
-    def test_module_not_package(self):
-        # Try to import a submodule from a non-package should raise ImportError.
-        assert not hasattr(sys, '__path__')
-        with self.assertRaises(ImportError):
-            import_util.import_('sys.no_submodules_here')
-
-
-def test_main():
-    from test.support import run_unittest
-    run_unittest(ParentModuleTests)
-
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/importlib/test/import_/test_path.py b/Lib/importlib/test/import_/test_path.py
deleted file mode 100644
index 2faa23174b..0000000000
--- a/Lib/importlib/test/import_/test_path.py
+++ /dev/null
@@ -1,131 +0,0 @@
-from importlib import _bootstrap
-from importlib import machinery
-from .. import util
-from . import util as import_util
-import imp
-import os
-import sys
-import tempfile
-from test import support
-from types import MethodType
-import unittest
-
-
-class FinderTests(unittest.TestCase):
-
-    """Tests for PathFinder."""
-
-    def test_failure(self):
-        # Test None returned upon not finding a suitable finder.
-        module = '<test module>'
-        with util.import_state():
-            self.assertTrue(machinery.PathFinder.find_module(module) is None)
-
-    def test_sys_path(self):
-        # Test that sys.path is used when 'path' is None.
-        # Implicitly tests that sys.path_importer_cache is used.
-        module = '<test module>'
-        path = '<test path>'
-        importer = util.mock_modules(module)
-        with util.import_state(path_importer_cache={path: importer},
-                               path=[path]):
-            loader = machinery.PathFinder.find_module(module)
-            self.assertTrue(loader is importer)
-
-    def test_path(self):
-        # Test that 'path' is used when set.
-        # Implicitly tests that sys.path_importer_cache is used.
-        module = '<test module>'
-        path = '<test path>'
-        importer = util.mock_modules(module)
-        with util.import_state(path_importer_cache={path: importer}):
-            loader = machinery.PathFinder.find_module(module, [path])
-            self.assertTrue(loader is importer)
-
-    def test_path_hooks(self):
-        # Test that sys.path_hooks is used.
-        # Test that sys.path_importer_cache is set.
-        module = '<test module>'
-        path = '<test path>'
-        importer = util.mock_modules(module)
-        hook = import_util.mock_path_hook(path, importer=importer)
-        with util.import_state(path_hooks=[hook]):
-            loader = machinery.PathFinder.find_module(module, [path])
-            self.assertTrue(loader is importer)
-            self.assertTrue(path in sys.path_importer_cache)
-            self.assertTrue(sys.path_importer_cache[path] is importer)
-
-    def test_path_importer_cache_has_None(self):
-        # Test that if sys.path_importer_cache has None that None is returned.
-        clear_cache = {path: None for path in sys.path}
-        with util.import_state(path_importer_cache=clear_cache):
-            for name in ('asynchat', 'sys', '<test module>'):
-                self.assertTrue(machinery.PathFinder.find_module(name) is None)
-
-    def test_path_importer_cache_has_None_continues(self):
-        # Test that having None in sys.path_importer_cache causes the search to
-        # continue.
-        path = '<test path>'
-        module = '<test module>'
-        importer = util.mock_modules(module)
-        with util.import_state(path=['1', '2'],
-                            path_importer_cache={'1': None, '2': importer}):
-            loader = machinery.PathFinder.find_module(module)
-            self.assertTrue(loader is importer)
-
-
-
-class DefaultPathFinderTests(unittest.TestCase):
-
-    """Test importlib._bootstrap._DefaultPathFinder."""
-
-    def test_implicit_hooks(self):
-        # Test that the implicit path hooks are used.
-        bad_path = '<path>'
-        module = '<module>'
-        assert not os.path.exists(bad_path)
-        existing_path = tempfile.mkdtemp()
-        try:
-            with util.import_state():
-                nothing = _bootstrap._DefaultPathFinder.find_module(module,
-                                                        path=[existing_path])
-                self.assertTrue(nothing is None)
-                self.assertTrue(existing_path in sys.path_importer_cache)
-                result = isinstance(sys.path_importer_cache[existing_path],
-                                    imp.NullImporter)
-                self.assertFalse(result)
-                nothing = _bootstrap._DefaultPathFinder.find_module(module,
-                                                            path=[bad_path])
-                self.assertTrue(nothing is None)
-                self.assertTrue(bad_path in sys.path_importer_cache)
-                self.assertTrue(isinstance(sys.path_importer_cache[bad_path],
-                                           imp.NullImporter))
-        finally:
-            os.rmdir(existing_path)
-
-
-    def test_path_importer_cache_has_None(self):
-        # Test that the default hook is used when sys.path_importer_cache
-        # contains None for a path.
-        module = '<test module>'
-        importer = util.mock_modules(module)
-        path = '<test path>'
-        # XXX Not blackbox.
-        original_hook = _bootstrap._DEFAULT_PATH_HOOK
-        mock_hook = import_util.mock_path_hook(path, importer=importer)
-        _bootstrap._DEFAULT_PATH_HOOK = mock_hook
-        try:
-            with util.import_state(path_importer_cache={path: None}):
-                loader = _bootstrap._DefaultPathFinder.find_module(module,
-                                                                    path=[path])
-                self.assertTrue(loader is importer)
-        finally:
-            _bootstrap._DEFAULT_PATH_HOOK = original_hook
-
-
-def test_main():
-    from test.support import run_unittest
-    run_unittest(FinderTests, DefaultPathFinderTests)
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/importlib/test/regrtest.py b/Lib/importlib/test/regrtest.py
deleted file mode 100644
index b103ae7d0e..0000000000
--- a/Lib/importlib/test/regrtest.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""Run Python's standard test suite using importlib.__import__.
-
-Tests known to fail because of assumptions that importlib (properly)
-invalidates are automatically skipped if the entire test suite is run.
-Otherwise all command-line options valid for test.regrtest are also valid for
-this script.
-
-XXX FAILING
-  * test_import
-    - test_incorrect_code_name
-        file name differing between __file__ and co_filename (r68360 on trunk)
-    - test_import_by_filename
-        exception for trying to import by file name does not match
-
-"""
-import importlib
-import sys
-from test import regrtest
-
-if __name__ == '__main__':
-    __builtins__.__import__ = importlib.__import__
-
-    exclude = ['--exclude',
-                'test_frozen', # Does not expect __loader__ attribute
-                'test_pkg',  # Does not expect __loader__ attribute
-                'test_pydoc', # Does not expect __loader__ attribute
-              ]
-
-    # Switching on --exclude implies running all test but the ones listed, so
-    # only use it when one is not running an explicit test
-    if len(sys.argv) == 1:
-        # No programmatic way to specify tests to exclude
-        sys.argv.extend(exclude)
-
-    regrtest.main(quiet=True, verbose2=True)
diff --git a/Lib/importlib/test/test_api.py b/Lib/importlib/test/test_api.py
deleted file mode 100644
index a151626de7..0000000000
--- a/Lib/importlib/test/test_api.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from . import util
-import imp
-import importlib
-import sys
-import unittest
-
-
-class ImportModuleTests(unittest.TestCase):
-
-    """Test importlib.import_module."""
-
-    def test_module_import(self):
-        # Test importing a top-level module.
-        with util.mock_modules('top_level') as mock:
-            with util.import_state(meta_path=[mock]):
-                module = importlib.import_module('top_level')
-                self.assertEqual(module.__name__, 'top_level')
-
-    def test_absolute_package_import(self):
-        # Test importing a module from a package with an absolute name.
-        pkg_name = 'pkg'
-        pkg_long_name = '{0}.__init__'.format(pkg_name)
-        name = '{0}.mod'.format(pkg_name)
-        with util.mock_modules(pkg_long_name, name) as mock:
-            with util.import_state(meta_path=[mock]):
-                module = importlib.import_module(name)
-                self.assertEqual(module.__name__, name)
-
-    def test_shallow_relative_package_import(self):
-        # Test importing a module from a package through a relative import.
-        pkg_name = 'pkg'
-        pkg_long_name = '{0}.__init__'.format(pkg_name)
-        module_name = 'mod'
-        absolute_name = '{0}.{1}'.format(pkg_name, module_name)
-        relative_name = '.{0}'.format(module_name)
-        with util.mock_modules(pkg_long_name, absolute_name) as mock:
-            with util.import_state(meta_path=[mock]):
-                importlib.import_module(pkg_name)
-                module = importlib.import_module(relative_name, pkg_name)
-                self.assertEqual(module.__name__, absolute_name)
-
-    def test_deep_relative_package_import(self):
-        modules = ['a.__init__', 'a.b.__init__', 'a.c']
-        with util.mock_modules(*modules) as mock:
-            with util.import_state(meta_path=[mock]):
-                importlib.import_module('a')
-                importlib.import_module('a.b')
-                module = importlib.import_module('..c', 'a.b')
-                self.assertEqual(module.__name__, 'a.c')
-
-    def test_absolute_import_with_package(self):
-        # Test importing a module from a package with an absolute name with
-        # the 'package' argument given.
-        pkg_name = 'pkg'
-        pkg_long_name = '{0}.__init__'.format(pkg_name)
-        name = '{0}.mod'.format(pkg_name)
-        with util.mock_modules(pkg_long_name, name) as mock:
-            with util.import_state(meta_path=[mock]):
-                importlib.import_module(pkg_name)
-                module = importlib.import_module(name, pkg_name)
-                self.assertEqual(module.__name__, name)
-
-    def test_relative_import_wo_package(self):
-        # Relative imports cannot happen without the 'package' argument being
-        # set.
-        with self.assertRaises(TypeError):
-            importlib.import_module('.support')
-
-
-    def test_loaded_once(self):
-        # Issue #13591: Modules should only be loaded once when
-        # initializing the parent package attempts to import the
-        # module currently being imported.
-        b_load_count = 0
-        def load_a():
-            importlib.import_module('a.b')
-        def load_b():
-            nonlocal b_load_count
-            b_load_count += 1
-        code = {'a': load_a, 'a.b': load_b}
-        modules = ['a.__init__', 'a.b']
-        with util.mock_modules(*modules, module_code=code) as mock:
-            with util.import_state(meta_path=[mock]):
-                importlib.import_module('a.b')
-        self.assertEqual(b_load_count, 1)
-
-def test_main():
-    from test.support import run_unittest
-    run_unittest(ImportModuleTests)
-
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index 7b44fa1344..1316437102 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -3,3 +3,19 @@
 from ._bootstrap import module_for_loader
 from ._bootstrap import set_loader
 from ._bootstrap import set_package
+from ._bootstrap import _resolve_name
+
+
+def resolve_name(name, package):
+    """Resolve a relative module name to an absolute one."""
+    if not name.startswith('.'):
+        return name
+    elif not package:
+        raise ValueError('{!r} is not a relative name '
+                         '(no leading dot)'.format(name))
+    level = 0
+    for character in name:
+        if character != '.':
+            break
+        level += 1
+    return _resolve_name(name[level:], package, level)
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 203175568b..7a7bb91b1b 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -22,24 +22,29 @@ Here are some of the useful functions provided by this module:
     getouterframes(), getinnerframes() - get info about frames
     currentframe() - get the current stack frame
     stack(), trace() - get info about frames on the stack or in a traceback
+
+    signature() - get a Signature object for the callable
 """
 
 # This module is in the public domain.  No warranties.
 
-__author__ = 'Ka-Ping Yee <ping@lfw.org>'
-__date__ = '1 Jan 2001'
+__author__ = ('Ka-Ping Yee <ping@lfw.org>',
+              'Yury Selivanov <yselivanov@sprymix.com>')
 
-import sys
-import os
-import types
+import imp
+import importlib.machinery
 import itertools
-import string
+import linecache
+import os
 import re
-import imp
+import sys
 import tokenize
-import linecache
+import types
+import warnings
+import functools
+import builtins
 from operator import attrgetter
-from collections import namedtuple
+from collections import namedtuple, OrderedDict
 
 # Create constants for the compiler flags in Include/code.h
 # We try to get them from dis to avoid duplication, but fall
@@ -433,6 +438,8 @@ ModuleInfo = namedtuple('ModuleInfo', 'name suffix mode module_type')
 
 def getmoduleinfo(path):
     """Get the module name, suffix, mode, and module type for a given file."""
+    warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning,
+                  2)
     filename = os.path.basename(path)
     suffixes = [(-len(suffix), suffix, mode, mtype)
                     for suffix, mode, mtype in imp.get_suffixes()]
@@ -443,20 +450,29 @@ def getmoduleinfo(path):
 
 def getmodulename(path):
     """Return the module name for a given file, or None."""
-    info = getmoduleinfo(path)
-    if info: return info[0]
+    fname = os.path.basename(path)
+    # Check for paths that look like an actual module file
+    suffixes = [(-len(suffix), suffix)
+                    for suffix in importlib.machinery.all_suffixes()]
+    suffixes.sort() # try longest suffixes first, in case they overlap
+    for neglen, suffix in suffixes:
+        if fname.endswith(suffix):
+            return fname[:neglen]
+    return None
 
 def getsourcefile(object):
     """Return the filename that can be used to locate an object's source.
     Return None if no way can be identified to get the source.
     """
     filename = getfile(object)
-    if filename[-4:].lower() in ('.pyc', '.pyo'):
-        filename = filename[:-4] + '.py'
-    for suffix, mode, kind in imp.get_suffixes():
-        if 'b' in mode and filename[-len(suffix):].lower() == suffix:
-            # Looks like a binary file.  We want to only return a text file.
-            return None
+    all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:]
+    all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:]
+    if any(filename.endswith(s) for s in all_bytecode_suffixes):
+        filename = (os.path.splitext(filename)[0] +
+                    importlib.machinery.SOURCE_SUFFIXES[0])
+    elif any(filename.endswith(s) for s in
+                 importlib.machinery.EXTENSION_SUFFIXES):
+        return None
     if os.path.exists(filename):
         return filename
     # only return a non-existent filename if the module has a PEP 302 loader
@@ -931,6 +947,43 @@ def formatargvalues(args, varargs, varkw, locals,
         specs.append(formatvarkw(varkw) + formatvalue(locals[varkw]))
     return '(' + ', '.join(specs) + ')'
 
+def _missing_arguments(f_name, argnames, pos, values):
+    names = [repr(name) for name in argnames if name not in values]
+    missing = len(names)
+    if missing == 1:
+        s = names[0]
+    elif missing == 2:
+        s = "{} and {}".format(*names)
+    else:
+        tail = ", {} and {}".format(names[-2:])
+        del names[-2:]
+        s = ", ".join(names) + tail
+    raise TypeError("%s() missing %i required %s argument%s: %s" %
+                    (f_name, missing,
+                      "positional" if pos else "keyword-only",
+                      "" if missing == 1 else "s", s))
+
+def _too_many(f_name, args, kwonly, varargs, defcount, given, values):
+    atleast = len(args) - defcount
+    kwonly_given = len([arg for arg in kwonly if arg in values])
+    if varargs:
+        plural = atleast != 1
+        sig = "at least %d" % (atleast,)
+    elif defcount:
+        plural = True
+        sig = "from %d to %d" % (atleast, len(args))
+    else:
+        plural = len(args) != 1
+        sig = str(len(args))
+    kwonly_sig = ""
+    if kwonly_given:
+        msg = " positional argument%s (and %d keyword-only argument%s)"
+        kwonly_sig = (msg % ("s" if given != 1 else "", kwonly_given,
+                             "s" if kwonly_given != 1 else ""))
+    raise TypeError("%s() takes %s positional argument%s but %d%s %s given" %
+            (f_name, sig, "s" if plural else "", given, kwonly_sig,
+             "was" if given == 1 and not kwonly_given else "were"))
+
 def getcallargs(func, *positional, **named):
     """Get the mapping of arguments to values.
 
@@ -942,65 +995,107 @@ def getcallargs(func, *positional, **named):
     f_name = func.__name__
     arg2value = {}
 
+
     if ismethod(func) and func.__self__ is not None:
         # implicit 'self' (or 'cls' for classmethods) argument
         positional = (func.__self__,) + positional
     num_pos = len(positional)
-    num_total = num_pos + len(named)
     num_args = len(args)
     num_defaults = len(defaults) if defaults else 0
-    for arg, value in zip(args, positional):
-        arg2value[arg] = value
+
+    n = min(num_pos, num_args)
+    for i in range(n):
+        arg2value[args[i]] = positional[i]
     if varargs:
-        if num_pos > num_args:
-            arg2value[varargs] = positional[-(num_pos-num_args):]
-        else:
-            arg2value[varargs] = ()
-    elif 0 < num_args < num_pos:
-        raise TypeError('%s() takes %s %d positional %s (%d given)' % (
-            f_name, 'at most' if defaults else 'exactly', num_args,
-            'arguments' if num_args > 1 else 'argument', num_total))
-    elif num_args == 0 and num_total:
-        if varkw or kwonlyargs:
-            if num_pos:
-                # XXX: We should use num_pos, but Python also uses num_total:
-                raise TypeError('%s() takes exactly 0 positional arguments '
-                                '(%d given)' % (f_name, num_total))
-        else:
-            raise TypeError('%s() takes no arguments (%d given)' %
-                            (f_name, num_total))
-
-    for arg in itertools.chain(args, kwonlyargs):
-        if arg in named:
-            if arg in arg2value:
-                raise TypeError("%s() got multiple values for keyword "
-                                "argument '%s'" % (f_name, arg))
+        arg2value[varargs] = tuple(positional[n:])
+    possible_kwargs = set(args + kwonlyargs)
+    if varkw:
+        arg2value[varkw] = {}
+    for kw, value in named.items():
+        if kw not in possible_kwargs:
+            if not varkw:
+                raise TypeError("%s() got an unexpected keyword argument %r" %
+                                (f_name, kw))
+            arg2value[varkw][kw] = value
+            continue
+        if kw in arg2value:
+            raise TypeError("%s() got multiple values for argument %r" %
+                            (f_name, kw))
+        arg2value[kw] = value
+    if num_pos > num_args and not varargs:
+        _too_many(f_name, args, kwonlyargs, varargs, num_defaults,
+                   num_pos, arg2value)
+    if num_pos < num_args:
+        req = args[:num_args - num_defaults]
+        for arg in req:
+            if arg not in arg2value:
+                _missing_arguments(f_name, req, True, arg2value)
+        for i, arg in enumerate(args[num_args - num_defaults:]):
+            if arg not in arg2value:
+                arg2value[arg] = defaults[i]
+    missing = 0
+    for kwarg in kwonlyargs:
+        if kwarg not in arg2value:
+            if kwarg in kwonlydefaults:
+                arg2value[kwarg] = kwonlydefaults[kwarg]
             else:
-                arg2value[arg] = named.pop(arg)
-    for kwonlyarg in kwonlyargs:
-        if kwonlyarg not in arg2value:
+                missing += 1
+    if missing:
+        _missing_arguments(f_name, kwonlyargs, False, arg2value)
+    return arg2value
+
+ClosureVars = namedtuple('ClosureVars', 'nonlocals globals builtins unbound')
+
+def getclosurevars(func):
+    """
+    Get the mapping of free variables to their current values.
+
+    Returns a named tuple of dicts mapping the current nonlocal, global
+    and builtin references as seen by the body of the function. A final
+    set of unbound names that could not be resolved is also provided.
+    """
+
+    if ismethod(func):
+        func = func.__func__
+
+    if not isfunction(func):
+        raise TypeError("'{!r}' is not a Python function".format(func))
+
+    code = func.__code__
+    # Nonlocal references are named in co_freevars and resolved
+    # by looking them up in __closure__ by positional index
+    if func.__closure__ is None:
+        nonlocal_vars = {}
+    else:
+        nonlocal_vars = {
+            var : cell.cell_contents
+            for var, cell in zip(code.co_freevars, func.__closure__)
+       }
+
+    # Global and builtin references are named in co_names and resolved
+    # by looking them up in __globals__ or __builtins__
+    global_ns = func.__globals__
+    builtin_ns = global_ns.get("__builtins__", builtins.__dict__)
+    if ismodule(builtin_ns):
+        builtin_ns = builtin_ns.__dict__
+    global_vars = {}
+    builtin_vars = {}
+    unbound_names = set()
+    for name in code.co_names:
+        if name in ("None", "True", "False"):
+            # Because these used to be builtins instead of keywords, they
+            # may still show up as name references. We ignore them.
+            continue
+        try:
+            global_vars[name] = global_ns[name]
+        except KeyError:
             try:
-                arg2value[kwonlyarg] = kwonlydefaults[kwonlyarg]
+                builtin_vars[name] = builtin_ns[name]
             except KeyError:
-                raise TypeError("%s() needs keyword-only argument %s" %
-                                (f_name, kwonlyarg))
-    if defaults:    # fill in any missing values with the defaults
-        for arg, value in zip(args[-num_defaults:], defaults):
-            if arg not in arg2value:
-                arg2value[arg] = value
-    if varkw:
-        arg2value[varkw] = named
-    elif named:
-        unexpected = next(iter(named))
-        raise TypeError("%s() got an unexpected keyword argument '%s'" %
-                        (f_name, unexpected))
-    unassigned = num_args - len([arg for arg in args if arg in arg2value])
-    if unassigned:
-        num_required = num_args - num_defaults
-        raise TypeError('%s() takes %s %d %s (%d given)' % (
-            f_name, 'at least' if defaults else 'exactly', num_required,
-            'arguments' if num_required > 1 else 'argument', num_total))
-    return arg2value
+                unbound_names.add(name)
+
+    return ClosureVars(nonlocal_vars, global_vars,
+                       builtin_vars, unbound_names)
 
 # -------------------------------------------------- stack frame extraction
 
@@ -1171,6 +1266,8 @@ def getattr_static(obj, attr, default=_sentinel):
     raise AttributeError(attr)
 
 
+# ------------------------------------------------ generator introspection
+
 GEN_CREATED = 'GEN_CREATED'
 GEN_RUNNING = 'GEN_RUNNING'
 GEN_SUSPENDED = 'GEN_SUSPENDED'
@@ -1192,3 +1289,785 @@ def getgeneratorstate(generator):
     if generator.gi_frame.f_lasti == -1:
         return GEN_CREATED
     return GEN_SUSPENDED
+
+
+def getgeneratorlocals(generator):
+    """
+    Get the mapping of generator local variables to their current values.
+
+    A dict is returned, with the keys the local variable names and values the
+    bound values."""
+
+    if not isgenerator(generator):
+        raise TypeError("'{!r}' is not a Python generator".format(generator))
+
+    frame = getattr(generator, "gi_frame", None)
+    if frame is not None:
+        return generator.gi_frame.f_locals
+    else:
+        return {}
+
+###############################################################################
+### Function Signature Object (PEP 362)
+###############################################################################
+
+
+_WrapperDescriptor = type(type.__call__)
+_MethodWrapper = type(all.__call__)
+
+_NonUserDefinedCallables = (_WrapperDescriptor,
+                            _MethodWrapper,
+                            types.BuiltinFunctionType)
+
+
+def _get_user_defined_method(cls, method_name):
+    try:
+        meth = getattr(cls, method_name)
+    except AttributeError:
+        return
+    else:
+        if not isinstance(meth, _NonUserDefinedCallables):
+            # Once '__signature__' will be added to 'C'-level
+            # callables, this check won't be necessary
+            return meth
+
+
+def signature(obj):
+    '''Get a signature object for the passed callable.'''
+
+    if not callable(obj):
+        raise TypeError('{!r} is not a callable object'.format(obj))
+
+    if isinstance(obj, types.MethodType):
+        # In this case we skip the first parameter of the underlying
+        # function (usually `self` or `cls`).
+        sig = signature(obj.__func__)
+        return sig.replace(parameters=tuple(sig.parameters.values())[1:])
+
+    try:
+        sig = obj.__signature__
+    except AttributeError:
+        pass
+    else:
+        if sig is not None:
+            return sig
+
+    try:
+        # Was this function wrapped by a decorator?
+        wrapped = obj.__wrapped__
+    except AttributeError:
+        pass
+    else:
+        return signature(wrapped)
+
+    if isinstance(obj, types.FunctionType):
+        return Signature.from_function(obj)
+
+    if isinstance(obj, functools.partial):
+        sig = signature(obj.func)
+
+        new_params = OrderedDict(sig.parameters.items())
+
+        partial_args = obj.args or ()
+        partial_keywords = obj.keywords or {}
+        try:
+            ba = sig.bind_partial(*partial_args, **partial_keywords)
+        except TypeError as ex:
+            msg = 'partial object {!r} has incorrect arguments'.format(obj)
+            raise ValueError(msg) from ex
+
+        for arg_name, arg_value in ba.arguments.items():
+            param = new_params[arg_name]
+            if arg_name in partial_keywords:
+                # We set a new default value, because the following code
+                # is correct:
+                #
+                #   >>> def foo(a): print(a)
+                #   >>> print(partial(partial(foo, a=10), a=20)())
+                #   20
+                #   >>> print(partial(partial(foo, a=10), a=20)(a=30))
+                #   30
+                #
+                # So, with 'partial' objects, passing a keyword argument is
+                # like setting a new default value for the corresponding
+                # parameter
+                #
+                # We also mark this parameter with '_partial_kwarg'
+                # flag.  Later, in '_bind', the 'default' value of this
+                # parameter will be added to 'kwargs', to simulate
+                # the 'functools.partial' real call.
+                new_params[arg_name] = param.replace(default=arg_value,
+                                                     _partial_kwarg=True)
+
+            elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and
+                            not param._partial_kwarg):
+                new_params.pop(arg_name)
+
+        return sig.replace(parameters=new_params.values())
+
+    sig = None
+    if isinstance(obj, type):
+        # obj is a class or a metaclass
+
+        # First, let's see if it has an overloaded __call__ defined
+        # in its metaclass
+        call = _get_user_defined_method(type(obj), '__call__')
+        if call is not None:
+            sig = signature(call)
+        else:
+            # Now we check if the 'obj' class has a '__new__' method
+            new = _get_user_defined_method(obj, '__new__')
+            if new is not None:
+                sig = signature(new)
+            else:
+                # Finally, we should have at least __init__ implemented
+                init = _get_user_defined_method(obj, '__init__')
+                if init is not None:
+                    sig = signature(init)
+    elif not isinstance(obj, _NonUserDefinedCallables):
+        # An object with __call__
+        # We also check that the 'obj' is not an instance of
+        # _WrapperDescriptor or _MethodWrapper to avoid
+        # infinite recursion (and even potential segfault)
+        call = _get_user_defined_method(type(obj), '__call__')
+        if call is not None:
+            sig = signature(call)
+
+    if sig is not None:
+        # For classes and objects we skip the first parameter of their
+        # __call__, __new__, or __init__ methods
+        return sig.replace(parameters=tuple(sig.parameters.values())[1:])
+
+    if isinstance(obj, types.BuiltinFunctionType):
+        # Raise a nicer error message for builtins
+        msg = 'no signature found for builtin function {!r}'.format(obj)
+        raise ValueError(msg)
+
+    raise ValueError('callable {!r} is not supported by signature'.format(obj))
+
+
+class _void:
+    '''A private marker - used in Parameter & Signature'''
+
+
+class _empty:
+    pass
+
+
+class _ParameterKind(int):
+    def __new__(self, *args, name):
+        obj = int.__new__(self, *args)
+        obj._name = name
+        return obj
+
+    def __str__(self):
+        return self._name
+
+    def __repr__(self):
+        return '<_ParameterKind: {!r}>'.format(self._name)
+
+
+_POSITIONAL_ONLY        = _ParameterKind(0, name='POSITIONAL_ONLY')
+_POSITIONAL_OR_KEYWORD  = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD')
+_VAR_POSITIONAL         = _ParameterKind(2, name='VAR_POSITIONAL')
+_KEYWORD_ONLY           = _ParameterKind(3, name='KEYWORD_ONLY')
+_VAR_KEYWORD            = _ParameterKind(4, name='VAR_KEYWORD')
+
+
+class Parameter:
+    '''Represents a parameter in a function signature.
+
+    Has the following public attributes:
+
+    * name : str
+        The name of the parameter as a string.
+    * default : object
+        The default value for the parameter if specified.  If the
+        parameter has no default value, this attribute is not set.
+    * annotation
+        The annotation for the parameter if specified.  If the
+        parameter has no annotation, this attribute is not set.
+    * kind : str
+        Describes how argument values are bound to the parameter.
+        Possible values: `Parameter.POSITIONAL_ONLY`,
+        `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`,
+        `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`.
+    '''
+
+    __slots__ = ('_name', '_kind', '_default', '_annotation', '_partial_kwarg')
+
+    POSITIONAL_ONLY         = _POSITIONAL_ONLY
+    POSITIONAL_OR_KEYWORD   = _POSITIONAL_OR_KEYWORD
+    VAR_POSITIONAL          = _VAR_POSITIONAL
+    KEYWORD_ONLY            = _KEYWORD_ONLY
+    VAR_KEYWORD             = _VAR_KEYWORD
+
+    empty = _empty
+
+    def __init__(self, name, kind, *, default=_empty, annotation=_empty,
+                 _partial_kwarg=False):
+
+        if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD,
+                        _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD):
+            raise ValueError("invalid value for 'Parameter.kind' attribute")
+        self._kind = kind
+
+        if default is not _empty:
+            if kind in (_VAR_POSITIONAL, _VAR_KEYWORD):
+                msg = '{} parameters cannot have default values'.format(kind)
+                raise ValueError(msg)
+        self._default = default
+        self._annotation = annotation
+
+        if name is None:
+            if kind != _POSITIONAL_ONLY:
+                raise ValueError("None is not a valid name for a "
+                                 "non-positional-only parameter")
+            self._name = name
+        else:
+            name = str(name)
+            if kind != _POSITIONAL_ONLY and not name.isidentifier():
+                msg = '{!r} is not a valid parameter name'.format(name)
+                raise ValueError(msg)
+            self._name = name
+
+        self._partial_kwarg = _partial_kwarg
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def default(self):
+        return self._default
+
+    @property
+    def annotation(self):
+        return self._annotation
+
+    @property
+    def kind(self):
+        return self._kind
+
+    def replace(self, *, name=_void, kind=_void, annotation=_void,
+                default=_void, _partial_kwarg=_void):
+        '''Creates a customized copy of the Parameter.'''
+
+        if name is _void:
+            name = self._name
+
+        if kind is _void:
+            kind = self._kind
+
+        if annotation is _void:
+            annotation = self._annotation
+
+        if default is _void:
+            default = self._default
+
+        if _partial_kwarg is _void:
+            _partial_kwarg = self._partial_kwarg
+
+        return type(self)(name, kind, default=default, annotation=annotation,
+                          _partial_kwarg=_partial_kwarg)
+
+    def __str__(self):
+        kind = self.kind
+
+        formatted = self._name
+        if kind == _POSITIONAL_ONLY:
+            if formatted is None:
+                formatted = ''
+            formatted = '<{}>'.format(formatted)
+
+        # Add annotation and default value
+        if self._annotation is not _empty:
+            formatted = '{}:{}'.format(formatted,
+                                       formatannotation(self._annotation))
+
+        if self._default is not _empty:
+            formatted = '{}={}'.format(formatted, repr(self._default))
+
+        if kind == _VAR_POSITIONAL:
+            formatted = '*' + formatted
+        elif kind == _VAR_KEYWORD:
+            formatted = '**' + formatted
+
+        return formatted
+
+    def __repr__(self):
+        return '<{} at {:#x} {!r}>'.format(self.__class__.__name__,
+                                           id(self), self.name)
+
+    def __eq__(self, other):
+        return (issubclass(other.__class__, Parameter) and
+                self._name == other._name and
+                self._kind == other._kind and
+                self._default == other._default and
+                self._annotation == other._annotation)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class BoundArguments:
+    '''Result of `Signature.bind` call.  Holds the mapping of arguments
+    to the function's parameters.
+
+    Has the following public attributes:
+
+    * arguments : OrderedDict
+        An ordered mutable mapping of parameters' names to arguments' values.
+        Does not contain arguments' default values.
+    * signature : Signature
+        The Signature object that created this instance.
+    * args : tuple
+        Tuple of positional arguments values.
+    * kwargs : dict
+        Dict of keyword arguments values.
+    '''
+
+    def __init__(self, signature, arguments):
+        self.arguments = arguments
+        self._signature = signature
+
+    @property
+    def signature(self):
+        return self._signature
+
+    @property
+    def args(self):
+        args = []
+        for param_name, param in self._signature.parameters.items():
+            if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
+                                                    param._partial_kwarg):
+                # Keyword arguments mapped by 'functools.partial'
+                # (Parameter._partial_kwarg is True) are mapped
+                # in 'BoundArguments.kwargs', along with VAR_KEYWORD &
+                # KEYWORD_ONLY
+                break
+
+            try:
+                arg = self.arguments[param_name]
+            except KeyError:
+                # We're done here. Other arguments
+                # will be mapped in 'BoundArguments.kwargs'
+                break
+            else:
+                if param.kind == _VAR_POSITIONAL:
+                    # *args
+                    args.extend(arg)
+                else:
+                    # plain argument
+                    args.append(arg)
+
+        return tuple(args)
+
+    @property
+    def kwargs(self):
+        kwargs = {}
+        kwargs_started = False
+        for param_name, param in self._signature.parameters.items():
+            if not kwargs_started:
+                if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
+                                                param._partial_kwarg):
+                    kwargs_started = True
+                else:
+                    if param_name not in self.arguments:
+                        kwargs_started = True
+                        continue
+
+            if not kwargs_started:
+                continue
+
+            try:
+                arg = self.arguments[param_name]
+            except KeyError:
+                pass
+            else:
+                if param.kind == _VAR_KEYWORD:
+                    # **kwargs
+                    kwargs.update(arg)
+                else:
+                    # plain keyword argument
+                    kwargs[param_name] = arg
+
+        return kwargs
+
+    def __eq__(self, other):
+        return (issubclass(other.__class__, BoundArguments) and
+                self.signature == other.signature and
+                self.arguments == other.arguments)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class Signature:
+    '''A Signature object represents the overall signature of a function.
+    It stores a Parameter object for each parameter accepted by the
+    function, as well as information specific to the function itself.
+
+    A Signature object has the following public attributes and methods:
+
+    * parameters : OrderedDict
+        An ordered mapping of parameters' names to the corresponding
+        Parameter objects (keyword-only arguments are in the same order
+        as listed in `code.co_varnames`).
+    * return_annotation : object
+        The annotation for the return type of the function if specified.
+        If the function has no annotation for its return type, this
+        attribute is not set.
+    * bind(*args, **kwargs) -> BoundArguments
+        Creates a mapping from positional and keyword arguments to
+        parameters.
+    * bind_partial(*args, **kwargs) -> BoundArguments
+        Creates a partial mapping from positional and keyword arguments
+        to parameters (simulating 'functools.partial' behavior.)
+    '''
+
+    __slots__ = ('_return_annotation', '_parameters')
+
+    _parameter_cls = Parameter
+    _bound_arguments_cls = BoundArguments
+
+    empty = _empty
+
+    def __init__(self, parameters=None, *, return_annotation=_empty,
+                 __validate_parameters__=True):
+        '''Constructs Signature from the given list of Parameter
+        objects and 'return_annotation'.  All arguments are optional.
+        '''
+
+        if parameters is None:
+            params = OrderedDict()
+        else:
+            if __validate_parameters__:
+                params = OrderedDict()
+                top_kind = _POSITIONAL_ONLY
+
+                for idx, param in enumerate(parameters):
+                    kind = param.kind
+                    if kind < top_kind:
+                        msg = 'wrong parameter order: {} before {}'
+                        msg = msg.format(top_kind, param.kind)
+                        raise ValueError(msg)
+                    else:
+                        top_kind = kind
+
+                    name = param.name
+                    if name is None:
+                        name = str(idx)
+                        param = param.replace(name=name)
+
+                    if name in params:
+                        msg = 'duplicate parameter name: {!r}'.format(name)
+                        raise ValueError(msg)
+                    params[name] = param
+            else:
+                params = OrderedDict(((param.name, param)
+                                                for param in parameters))
+
+        self._parameters = types.MappingProxyType(params)
+        self._return_annotation = return_annotation
+
+    @classmethod
+    def from_function(cls, func):
+        '''Constructs Signature for the given python function'''
+
+        if not isinstance(func, types.FunctionType):
+            raise TypeError('{!r} is not a Python function'.format(func))
+
+        Parameter = cls._parameter_cls
+
+        # Parameter information.
+        func_code = func.__code__
+        pos_count = func_code.co_argcount
+        arg_names = func_code.co_varnames
+        positional = tuple(arg_names[:pos_count])
+        keyword_only_count = func_code.co_kwonlyargcount
+        keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
+        annotations = func.__annotations__
+        defaults = func.__defaults__
+        kwdefaults = func.__kwdefaults__
+
+        if defaults:
+            pos_default_count = len(defaults)
+        else:
+            pos_default_count = 0
+
+        parameters = []
+
+        # Non-keyword-only parameters w/o defaults.
+        non_default_count = pos_count - pos_default_count
+        for name in positional[:non_default_count]:
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_POSITIONAL_OR_KEYWORD))
+
+        # ... w/ defaults.
+        for offset, name in enumerate(positional[non_default_count:]):
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_POSITIONAL_OR_KEYWORD,
+                                        default=defaults[offset]))
+
+        # *args
+        if func_code.co_flags & 0x04:
+            name = arg_names[pos_count + keyword_only_count]
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_VAR_POSITIONAL))
+
+        # Keyword-only parameters.
+        for name in keyword_only:
+            default = _empty
+            if kwdefaults is not None:
+                default = kwdefaults.get(name, _empty)
+
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_KEYWORD_ONLY,
+                                        default=default))
+        # **kwargs
+        if func_code.co_flags & 0x08:
+            index = pos_count + keyword_only_count
+            if func_code.co_flags & 0x04:
+                index += 1
+
+            name = arg_names[index]
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_VAR_KEYWORD))
+
+        return cls(parameters,
+                   return_annotation=annotations.get('return', _empty),
+                   __validate_parameters__=False)
+
+    @property
+    def parameters(self):
+        return self._parameters
+
+    @property
+    def return_annotation(self):
+        return self._return_annotation
+
+    def replace(self, *, parameters=_void, return_annotation=_void):
+        '''Creates a customized copy of the Signature.
+        Pass 'parameters' and/or 'return_annotation' arguments
+        to override them in the new copy.
+        '''
+
+        if parameters is _void:
+            parameters = self.parameters.values()
+
+        if return_annotation is _void:
+            return_annotation = self._return_annotation
+
+        return type(self)(parameters,
+                          return_annotation=return_annotation)
+
+    def __eq__(self, other):
+        if (not issubclass(type(other), Signature) or
+                    self.return_annotation != other.return_annotation or
+                    len(self.parameters) != len(other.parameters)):
+            return False
+
+        other_positions = {param: idx
+                           for idx, param in enumerate(other.parameters.keys())}
+
+        for idx, (param_name, param) in enumerate(self.parameters.items()):
+            if param.kind == _KEYWORD_ONLY:
+                try:
+                    other_param = other.parameters[param_name]
+                except KeyError:
+                    return False
+                else:
+                    if param != other_param:
+                        return False
+            else:
+                try:
+                    other_idx = other_positions[param_name]
+                except KeyError:
+                    return False
+                else:
+                    if (idx != other_idx or
+                                    param != other.parameters[param_name]):
+                        return False
+
+        return True
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def _bind(self, args, kwargs, *, partial=False):
+        '''Private method.  Don't use directly.'''
+
+        arguments = OrderedDict()
+
+        parameters = iter(self.parameters.values())
+        parameters_ex = ()
+        arg_vals = iter(args)
+
+        if partial:
+            # Support for binding arguments to 'functools.partial' objects.
+            # See 'functools.partial' case in 'signature()' implementation
+            # for details.
+            for param_name, param in self.parameters.items():
+                if (param._partial_kwarg and param_name not in kwargs):
+                    # Simulating 'functools.partial' behavior
+                    kwargs[param_name] = param.default
+
+        while True:
+            # Let's iterate through the positional arguments and corresponding
+            # parameters
+            try:
+                arg_val = next(arg_vals)
+            except StopIteration:
+                # No more positional arguments
+                try:
+                    param = next(parameters)
+                except StopIteration:
+                    # No more parameters. That's it. Just need to check that
+                    # we have no `kwargs` after this while loop
+                    break
+                else:
+                    if param.kind == _VAR_POSITIONAL:
+                        # That's OK, just empty *args.  Let's start parsing
+                        # kwargs
+                        break
+                    elif param.name in kwargs:
+                        if param.kind == _POSITIONAL_ONLY:
+                            msg = '{arg!r} parameter is positional only, ' \
+                                  'but was passed as a keyword'
+                            msg = msg.format(arg=param.name)
+                            raise TypeError(msg) from None
+                        parameters_ex = (param,)
+                        break
+                    elif (param.kind == _VAR_KEYWORD or
+                                                param.default is not _empty):
+                        # That's fine too - we have a default value for this
+                        # parameter.  So, lets start parsing `kwargs`, starting
+                        # with the current parameter
+                        parameters_ex = (param,)
+                        break
+                    else:
+                        if partial:
+                            parameters_ex = (param,)
+                            break
+                        else:
+                            msg = '{arg!r} parameter lacking default value'
+                            msg = msg.format(arg=param.name)
+                            raise TypeError(msg) from None
+            else:
+                # We have a positional argument to process
+                try:
+                    param = next(parameters)
+                except StopIteration:
+                    raise TypeError('too many positional arguments') from None
+                else:
+                    if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
+                        # Looks like we have no parameter for this positional
+                        # argument
+                        raise TypeError('too many positional arguments')
+
+                    if param.kind == _VAR_POSITIONAL:
+                        # We have an '*args'-like argument, let's fill it with
+                        # all positional arguments we have left and move on to
+                        # the next phase
+                        values = [arg_val]
+                        values.extend(arg_vals)
+                        arguments[param.name] = tuple(values)
+                        break
+
+                    if param.name in kwargs:
+                        raise TypeError('multiple values for argument '
+                                        '{arg!r}'.format(arg=param.name))
+
+                    arguments[param.name] = arg_val
+
+        # Now, we iterate through the remaining parameters to process
+        # keyword arguments
+        kwargs_param = None
+        for param in itertools.chain(parameters_ex, parameters):
+            if param.kind == _POSITIONAL_ONLY:
+                # This should never happen in case of a properly built
+                # Signature object (but let's have this check here
+                # to ensure correct behaviour just in case)
+                raise TypeError('{arg!r} parameter is positional only, '
+                                'but was passed as a keyword'. \
+                                format(arg=param.name))
+
+            if param.kind == _VAR_KEYWORD:
+                # Memorize that we have a '**kwargs'-like parameter
+                kwargs_param = param
+                continue
+
+            param_name = param.name
+            try:
+                arg_val = kwargs.pop(param_name)
+            except KeyError:
+                # We have no value for this parameter.  It's fine though,
+                # if it has a default value, or it is an '*args'-like
+                # parameter, left alone by the processing of positional
+                # arguments.
+                if (not partial and param.kind != _VAR_POSITIONAL and
+                                                    param.default is _empty):
+                    raise TypeError('{arg!r} parameter lacking default value'. \
+                                    format(arg=param_name)) from None
+
+            else:
+                arguments[param_name] = arg_val
+
+        if kwargs:
+            if kwargs_param is not None:
+                # Process our '**kwargs'-like parameter
+                arguments[kwargs_param.name] = kwargs
+            else:
+                raise TypeError('too many keyword arguments')
+
+        return self._bound_arguments_cls(self, arguments)
+
+    def bind(__bind_self, *args, **kwargs):
+        '''Get a BoundArguments object, that maps the passed `args`
+        and `kwargs` to the function's signature.  Raises `TypeError`
+        if the passed arguments can not be bound.
+        '''
+        return __bind_self._bind(args, kwargs)
+
+    def bind_partial(__bind_self, *args, **kwargs):
+        '''Get a BoundArguments object, that partially maps the
+        passed `args` and `kwargs` to the function's signature.
+        Raises `TypeError` if the passed arguments can not be bound.
+        '''
+        return __bind_self._bind(args, kwargs, partial=True)
+
+    def __str__(self):
+        result = []
+        render_kw_only_separator = True
+        for idx, param in enumerate(self.parameters.values()):
+            formatted = str(param)
+
+            kind = param.kind
+            if kind == _VAR_POSITIONAL:
+                # OK, we have an '*args'-like parameter, so we won't need
+                # a '*' to separate keyword-only arguments
+                render_kw_only_separator = False
+            elif kind == _KEYWORD_ONLY and render_kw_only_separator:
+                # We have a keyword-only parameter to render and we haven't
+                # rendered an '*args'-like parameter before, so add a '*'
+                # separator to the parameters list ("foo(arg1, *, arg2)" case)
+                result.append('*')
+                # This condition should be only triggered once, so
+                # reset the flag
+                render_kw_only_separator = False
+
+            result.append(formatted)
+
+        rendered = '({})'.format(', '.join(result))
+
+        if self.return_annotation is not _empty:
+            anno = formatannotation(self.return_annotation)
+            rendered += ' -> {}'.format(anno)
+
+        return rendered
diff --git a/Lib/io.py b/Lib/io.py
index 63d2b33838..bda4def3da 100644
--- a/Lib/io.py
+++ b/Lib/io.py
@@ -58,6 +58,9 @@ from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
 
 OpenWrapper = _io.open # for compatibility with _pyio
 
+# Pretend this exception was created here.
+UnsupportedOperation.__module__ = "io"
+
 # for seek()
 SEEK_SET = 0
 SEEK_CUR = 1
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
new file mode 100644
index 0000000000..39b3f74f31
--- /dev/null
+++ b/Lib/ipaddress.py
@@ -0,0 +1,2094 @@
+# Copyright 2007 Google Inc.
+#  Licensed to PSF under a Contributor Agreement.
+
+"""A fast, lightweight IPv4/IPv6 manipulation library in Python.
+
+This library is used to create/poke/manipulate IPv4 and IPv6 addresses
+and networks.
+
+"""
+
+__version__ = '1.0'
+
+
+import functools
+
+IPV4LENGTH = 32
+IPV6LENGTH = 128
+
+class AddressValueError(ValueError):
+    """A Value Error related to the address."""
+
+
+class NetmaskValueError(ValueError):
+    """A Value Error related to the netmask."""
+
+
+def ip_address(address):
+    """Take an IP string/int and return an object of the correct type.
+
+    Args:
+        address: A string or integer, the IP address.  Either IPv4 or
+          IPv6 addresses may be supplied; integers less than 2**32 will
+          be considered to be IPv4 by default.
+
+    Returns:
+        An IPv4Address or IPv6Address object.
+
+    Raises:
+        ValueError: if the *address* passed isn't either a v4 or a v6
+          address
+
+    """
+    try:
+        return IPv4Address(address)
+    except (AddressValueError, NetmaskValueError):
+        pass
+
+    try:
+        return IPv6Address(address)
+    except (AddressValueError, NetmaskValueError):
+        pass
+
+    raise ValueError('%r does not appear to be an IPv4 or IPv6 address' %
+                     address)
+
+
+def ip_network(address, strict=True):
+    """Take an IP string/int and return an object of the correct type.
+
+    Args:
+        address: A string or integer, the IP network.  Either IPv4 or
+          IPv6 networks may be supplied; integers less than 2**32 will
+          be considered to be IPv4 by default.
+
+    Returns:
+        An IPv4Network or IPv6Network object.
+
+    Raises:
+        ValueError: if the string passed isn't either a v4 or a v6
+          address. Or if the network has host bits set.
+
+    """
+    try:
+        return IPv4Network(address, strict)
+    except (AddressValueError, NetmaskValueError):
+        pass
+
+    try:
+        return IPv6Network(address, strict)
+    except (AddressValueError, NetmaskValueError):
+        pass
+
+    raise ValueError('%r does not appear to be an IPv4 or IPv6 network' %
+                     address)
+
+
+def ip_interface(address):
+    """Take an IP string/int and return an object of the correct type.
+
+    Args:
+        address: A string or integer, the IP address.  Either IPv4 or
+          IPv6 addresses may be supplied; integers less than 2**32 will
+          be considered to be IPv4 by default.
+
+    Returns:
+        An IPv4Interface or IPv6Interface object.
+
+    Raises:
+        ValueError: if the string passed isn't either a v4 or a v6
+          address.
+
+    Notes:
+        The IPv?Interface classes describe an Address on a particular
+        Network, so they're basically a combination of both the Address
+        and Network classes.
+
+    """
+    try:
+        return IPv4Interface(address)
+    except (AddressValueError, NetmaskValueError):
+        pass
+
+    try:
+        return IPv6Interface(address)
+    except (AddressValueError, NetmaskValueError):
+        pass
+
+    raise ValueError('%r does not appear to be an IPv4 or IPv6 interface' %
+                     address)
+
+
+def v4_int_to_packed(address):
+    """Represent an address as 4 packed bytes in network (big-endian) order.
+
+    Args:
+        address: An integer representation of an IPv4 IP address.
+
+    Returns:
+        The integer address packed as 4 bytes in network (big-endian) order.
+
+    Raises:
+        ValueError: If the integer is negative or too large to be an
+          IPv4 IP address.
+
+    """
+    try:
+        return address.to_bytes(4, 'big')
+    except:
+        raise ValueError("Address negative or too large for IPv4")
+
+
+def v6_int_to_packed(address):
+    """Represent an address as 16 packed bytes in network (big-endian) order.
+
+    Args:
+        address: An integer representation of an IPv6 IP address.
+
+    Returns:
+        The integer address packed as 16 bytes in network (big-endian) order.
+
+    """
+    try:
+        return address.to_bytes(16, 'big')
+    except:
+        raise ValueError("Address negative or too large for IPv6")
+
+
+def _split_optional_netmask(address):
+    """Helper to split the netmask and raise AddressValueError if needed"""
+    addr = str(address).split('/')
+    if len(addr) > 2:
+        raise AddressValueError("Only one '/' permitted in %r" % address)
+    return addr
+
+
+def _find_address_range(addresses):
+    """Find a sequence of IPv#Address.
+
+    Args:
+        addresses: a list of IPv#Address objects.
+
+    Returns:
+        A tuple containing the first and last IP addresses in the sequence.
+
+    """
+    first = last = addresses[0]
+    for ip in addresses[1:]:
+        if ip._ip == last._ip + 1:
+            last = ip
+        else:
+            break
+    return (first, last)
+
+
+def _count_righthand_zero_bits(number, bits):
+    """Count the number of zero bits on the right hand side.
+
+    Args:
+        number: an integer.
+        bits: maximum number of bits to count.
+
+    Returns:
+        The number of zero bits on the right hand side of the number.
+
+    """
+    if number == 0:
+        return bits
+    for i in range(bits):
+        if (number >> i) & 1:
+            return i
+    # All bits of interest were zero, even if there are more in the number
+    return bits
+
+
+def summarize_address_range(first, last):
+    """Summarize a network range given the first and last IP addresses.
+
+    Example:
+        >>> list(summarize_address_range(IPv4Address('192.0.2.0'),
+        ...                              IPv4Address('192.0.2.130')))
+        ...                                #doctest: +NORMALIZE_WHITESPACE
+        [IPv4Network('192.0.2.0/25'), IPv4Network('192.0.2.128/31'),
+         IPv4Network('192.0.2.130/32')]
+
+    Args:
+        first: the first IPv4Address or IPv6Address in the range.
+        last: the last IPv4Address or IPv6Address in the range.
+
+    Returns:
+        An iterator of the summarized IPv(4|6) network objects.
+
+    Raise:
+        TypeError:
+            If the first and last objects are not IP addresses.
+            If the first and last objects are not the same version.
+        ValueError:
+            If the last object is not greater than the first.
+            If the version of the first address is not 4 or 6.
+
+    """
+    if (not (isinstance(first, _BaseAddress) and
+             isinstance(last, _BaseAddress))):
+        raise TypeError('first and last must be IP addresses, not networks')
+    if first.version != last.version:
+        raise TypeError("%s and %s are not of the same version" % (
+                         first, last))
+    if first > last:
+        raise ValueError('last IP address must be greater than first')
+
+    if first.version == 4:
+        ip = IPv4Network
+    elif first.version == 6:
+        ip = IPv6Network
+    else:
+        raise ValueError('unknown IP version')
+
+    ip_bits = first._max_prefixlen
+    first_int = first._ip
+    last_int = last._ip
+    while first_int <= last_int:
+        nbits = min(_count_righthand_zero_bits(first_int, ip_bits),
+                    (last_int - first_int + 1).bit_length() - 1)
+        net = ip('%s/%d' % (first, ip_bits - nbits))
+        yield net
+        first_int += 1 << nbits
+        if first_int - 1 == ip._ALL_ONES:
+            break
+        first = first.__class__(first_int)
+
+
+def _collapse_addresses_recursive(addresses):
+    """Loops through the addresses, collapsing concurrent netblocks.
+
+    Example:
+
+        ip1 = IPv4Network('192.0.2.0/26')
+        ip2 = IPv4Network('192.0.2.64/26')
+        ip3 = IPv4Network('192.0.2.128/26')
+        ip4 = IPv4Network('192.0.2.192/26')
+
+        _collapse_addresses_recursive([ip1, ip2, ip3, ip4]) ->
+          [IPv4Network('192.0.2.0/24')]
+
+        This shouldn't be called directly; it is called via
+          collapse_addresses([]).
+
+    Args:
+        addresses: A list of IPv4Network's or IPv6Network's
+
+    Returns:
+        A list of IPv4Network's or IPv6Network's depending on what we were
+        passed.
+
+    """
+    while True:
+        last_addr = None
+        ret_array = []
+        optimized = False
+
+        for cur_addr in addresses:
+            if not ret_array:
+                last_addr = cur_addr
+                ret_array.append(cur_addr)
+            elif (cur_addr.network_address >= last_addr.network_address and
+                cur_addr.broadcast_address <= last_addr.broadcast_address):
+                optimized = True
+            elif cur_addr == list(last_addr.supernet().subnets())[1]:
+                ret_array[-1] = last_addr = last_addr.supernet()
+                optimized = True
+            else:
+                last_addr = cur_addr
+                ret_array.append(cur_addr)
+
+        addresses = ret_array
+        if not optimized:
+            return addresses
+
+
+def collapse_addresses(addresses):
+    """Collapse a list of IP objects.
+
+    Example:
+        collapse_addresses([IPv4Network('192.0.2.0/25'),
+                            IPv4Network('192.0.2.128/25')]) ->
+                           [IPv4Network('192.0.2.0/24')]
+
+    Args:
+        addresses: An iterator of IPv4Network or IPv6Network objects.
+
+    Returns:
+        An iterator of the collapsed IPv(4|6)Network objects.
+
+    Raises:
+        TypeError: If passed a list of mixed version objects.
+
+    """
+    i = 0
+    addrs = []
+    ips = []
+    nets = []
+
+    # split IP addresses and networks
+    for ip in addresses:
+        if isinstance(ip, _BaseAddress):
+            if ips and ips[-1]._version != ip._version:
+                raise TypeError("%s and %s are not of the same version" % (
+                                 ip, ips[-1]))
+            ips.append(ip)
+        elif ip._prefixlen == ip._max_prefixlen:
+            if ips and ips[-1]._version != ip._version:
+                raise TypeError("%s and %s are not of the same version" % (
+                                 ip, ips[-1]))
+            try:
+                ips.append(ip.ip)
+            except AttributeError:
+                ips.append(ip.network_address)
+        else:
+            if nets and nets[-1]._version != ip._version:
+                raise TypeError("%s and %s are not of the same version" % (
+                                 ip, nets[-1]))
+            nets.append(ip)
+
+    # sort and dedup
+    ips = sorted(set(ips))
+    nets = sorted(set(nets))
+
+    while i < len(ips):
+        (first, last) = _find_address_range(ips[i:])
+        i = ips.index(last) + 1
+        addrs.extend(summarize_address_range(first, last))
+
+    return iter(_collapse_addresses_recursive(sorted(
+        addrs + nets, key=_BaseNetwork._get_networks_key)))
+
+
+def get_mixed_type_key(obj):
+    """Return a key suitable for sorting between networks and addresses.
+
+    Address and Network objects are not sortable by default; they're
+    fundamentally different so the expression
+
+        IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24')
+
+    doesn't make any sense.  There are some times however, where you may wish
+    to have ipaddress sort these for you anyway. If you need to do this, you
+    can use this function as the key= argument to sorted().
+
+    Args:
+      obj: either a Network or Address object.
+    Returns:
+      appropriate key.
+
+    """
+    if isinstance(obj, _BaseNetwork):
+        return obj._get_networks_key()
+    elif isinstance(obj, _BaseAddress):
+        return obj._get_address_key()
+    return NotImplemented
+
+
+class _TotalOrderingMixin:
+    # Helper that derives the other comparison operations from
+    # __lt__ and __eq__
+    # We avoid functools.total_ordering because it doesn't handle
+    # NotImplemented correctly yet (http://bugs.python.org/issue10042)
+    def __eq__(self, other):
+        raise NotImplementedError
+    def __ne__(self, other):
+        equal = self.__eq__(other)
+        if equal is NotImplemented:
+            return NotImplemented
+        return not equal
+    def __lt__(self, other):
+        raise NotImplementedError
+    def __le__(self, other):
+        less = self.__lt__(other)
+        if less is NotImplemented or not less:
+            return self.__eq__(other)
+        return less
+    def __gt__(self, other):
+        less = self.__lt__(other)
+        if less is NotImplemented:
+            return NotImplemented
+        equal = self.__eq__(other)
+        if equal is NotImplemented:
+            return NotImplemented
+        return not (less or equal)
+    def __ge__(self, other):
+        less = self.__lt__(other)
+        if less is NotImplemented:
+            return NotImplemented
+        return not less
+
+class _IPAddressBase(_TotalOrderingMixin):
+
+    """The mother class."""
+
+    @property
+    def exploded(self):
+        """Return the longhand version of the IP address as a string."""
+        return self._explode_shorthand_ip_string()
+
+    @property
+    def compressed(self):
+        """Return the shorthand version of the IP address as a string."""
+        return str(self)
+
+    @property
+    def version(self):
+        msg = '%200s has no version specified' % (type(self),)
+        raise NotImplementedError(msg)
+
+    def _check_int_address(self, address):
+        if address < 0:
+            msg = "%d (< 0) is not permitted as an IPv%d address"
+            raise AddressValueError(msg % (address, self._version))
+        if address > self._ALL_ONES:
+            msg = "%d (>= 2**%d) is not permitted as an IPv%d address"
+            raise AddressValueError(msg % (address, self._max_prefixlen,
+                                           self._version))
+
+    def _check_packed_address(self, address, expected_len):
+        address_len = len(address)
+        if address_len != expected_len:
+            msg = "%r (len %d != %d) is not permitted as an IPv%d address"
+            raise AddressValueError(msg % (address, address_len,
+                                           expected_len, self._version))
+
+    def _ip_int_from_prefix(self, prefixlen=None):
+        """Turn the prefix length netmask into a int for comparison.
+
+        Args:
+            prefixlen: An integer, the prefix length.
+
+        Returns:
+            An integer.
+
+        """
+        if prefixlen is None:
+            prefixlen = self._prefixlen
+        return self._ALL_ONES ^ (self._ALL_ONES >> prefixlen)
+
+    def _prefix_from_ip_int(self, ip_int, mask=32):
+        """Return prefix length from the decimal netmask.
+
+        Args:
+            ip_int: An integer, the IP address.
+            mask: The netmask.  Defaults to 32.
+
+        Returns:
+            An integer, the prefix length.
+
+        """
+        return mask - _count_righthand_zero_bits(ip_int, mask)
+
+    def _ip_string_from_prefix(self, prefixlen=None):
+        """Turn a prefix length into a dotted decimal string.
+
+        Args:
+            prefixlen: An integer, the netmask prefix length.
+
+        Returns:
+            A string, the dotted decimal netmask string.
+
+        """
+        if not prefixlen:
+            prefixlen = self._prefixlen
+        return self._string_from_ip_int(self._ip_int_from_prefix(prefixlen))
+
+
+class _BaseAddress(_IPAddressBase):
+
+    """A generic IP object.
+
+    This IP class contains the version independent methods which are
+    used by single IP addresses.
+
+    """
+
+    def __init__(self, address):
+        if (not isinstance(address, bytes)
+            and '/' in str(address)):
+            raise AddressValueError("Unexpected '/' in %r" % address)
+
+    def __int__(self):
+        return self._ip
+
+    def __eq__(self, other):
+        try:
+            return (self._ip == other._ip
+                    and self._version == other._version)
+        except AttributeError:
+            return NotImplemented
+
+    def __lt__(self, other):
+        if self._version != other._version:
+            raise TypeError('%s and %s are not of the same version' % (
+                             self, other))
+        if not isinstance(other, _BaseAddress):
+            raise TypeError('%s and %s are not of the same type' % (
+                             self, other))
+        if self._ip != other._ip:
+            return self._ip < other._ip
+        return False
+
+    # Shorthand for Integer addition and subtraction. This is not
+    # meant to ever support addition/subtraction of addresses.
+    def __add__(self, other):
+        if not isinstance(other, int):
+            return NotImplemented
+        return self.__class__(int(self) + other)
+
+    def __sub__(self, other):
+        if not isinstance(other, int):
+            return NotImplemented
+        return self.__class__(int(self) - other)
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__class__.__name__, str(self))
+
+    def __str__(self):
+        return str(self._string_from_ip_int(self._ip))
+
+    def __hash__(self):
+        return hash(hex(int(self._ip)))
+
+    def _get_address_key(self):
+        return (self._version, self)
+
+
+class _BaseNetwork(_IPAddressBase):
+
+    """A generic IP network object.
+
+    This IP class contains the version independent methods which are
+    used by networks.
+
+    """
+    def __init__(self, address):
+        self._cache = {}
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__class__.__name__, str(self))
+
+    def __str__(self):
+        return '%s/%d' % (self.network_address, self.prefixlen)
+
+    def hosts(self):
+        """Generate Iterator over usable hosts in a network.
+
+        This is like __iter__ except it doesn't return the network
+        or broadcast addresses.
+
+        """
+        network = int(self.network_address)
+        broadcast = int(self.broadcast_address)
+        for x in range(network + 1, broadcast):
+            yield self._address_class(x)
+
+    def __iter__(self):
+        network = int(self.network_address)
+        broadcast = int(self.broadcast_address)
+        for x in range(network, broadcast + 1):
+            yield self._address_class(x)
+
+    def __getitem__(self, n):
+        network = int(self.network_address)
+        broadcast = int(self.broadcast_address)
+        if n >= 0:
+            if network + n > broadcast:
+                raise IndexError
+            return self._address_class(network + n)
+        else:
+            n += 1
+            if broadcast + n < network:
+                raise IndexError
+            return self._address_class(broadcast + n)
+
+    def __lt__(self, other):
+        if self._version != other._version:
+            raise TypeError('%s and %s are not of the same version' % (
+                             self, other))
+        if not isinstance(other, _BaseNetwork):
+            raise TypeError('%s and %s are not of the same type' % (
+                             self, other))
+        if self.network_address != other.network_address:
+            return self.network_address < other.network_address
+        if self.netmask != other.netmask:
+            return self.netmask < other.netmask
+        return False
+
+    def __eq__(self, other):
+        try:
+            return (self._version == other._version and
+                    self.network_address == other.network_address and
+                    int(self.netmask) == int(other.netmask))
+        except AttributeError:
+            return NotImplemented
+
+    def __hash__(self):
+        return hash(int(self.network_address) ^ int(self.netmask))
+
+    def __contains__(self, other):
+        # always false if one is v4 and the other is v6.
+        if self._version != other._version:
+            return False
+        # dealing with another network.
+        if isinstance(other, _BaseNetwork):
+            return False
+        # dealing with another address
+        else:
+            # address
+            return (int(self.network_address) <= int(other._ip) <=
+                    int(self.broadcast_address))
+
+    def overlaps(self, other):
+        """Tell if self is partly contained in other."""
+        return self.network_address in other or (
+            self.broadcast_address in other or (
+                other.network_address in self or (
+                    other.broadcast_address in self)))
+
+    @property
+    def broadcast_address(self):
+        x = self._cache.get('broadcast_address')
+        if x is None:
+            x = self._address_class(int(self.network_address) |
+                                    int(self.hostmask))
+            self._cache['broadcast_address'] = x
+        return x
+
+    @property
+    def hostmask(self):
+        x = self._cache.get('hostmask')
+        if x is None:
+            x = self._address_class(int(self.netmask) ^ self._ALL_ONES)
+            self._cache['hostmask'] = x
+        return x
+
+    @property
+    def with_prefixlen(self):
+        return '%s/%d' % (self.network_address, self._prefixlen)
+
+    @property
+    def with_netmask(self):
+        return '%s/%s' % (self.network_address, self.netmask)
+
+    @property
+    def with_hostmask(self):
+        return '%s/%s' % (self.network_address, self.hostmask)
+
+    @property
+    def num_addresses(self):
+        """Number of hosts in the current subnet."""
+        return int(self.broadcast_address) - int(self.network_address) + 1
+
+    @property
+    def _address_class(self):
+        # Returning bare address objects (rather than interfaces) allows for
+        # more consistent behaviour across the network address, broadcast
+        # address and individual host addresses.
+        msg = '%200s has no associated address class' % (type(self),)
+        raise NotImplementedError(msg)
+
+    @property
+    def prefixlen(self):
+        return self._prefixlen
+
+    def address_exclude(self, other):
+        """Remove an address from a larger block.
+
+        For example:
+
+            addr1 = ip_network('192.0.2.0/28')
+            addr2 = ip_network('192.0.2.1/32')
+            addr1.address_exclude(addr2) =
+                [IPv4Network('192.0.2.0/32'), IPv4Network('192.0.2.2/31'),
+                IPv4Network('192.0.2.4/30'), IPv4Network('192.0.2.8/29')]
+
+        or IPv6:
+
+            addr1 = ip_network('2001:db8::1/32')
+            addr2 = ip_network('2001:db8::1/128')
+            addr1.address_exclude(addr2) =
+                [ip_network('2001:db8::1/128'),
+                ip_network('2001:db8::2/127'),
+                ip_network('2001:db8::4/126'),
+                ip_network('2001:db8::8/125'),
+                ...
+                ip_network('2001:db8:8000::/33')]
+
+        Args:
+            other: An IPv4Network or IPv6Network object of the same type.
+
+        Returns:
+            An iterator of the IPv(4|6)Network objects which is self
+            minus other.
+
+        Raises:
+            TypeError: If self and other are of difffering address
+              versions, or if other is not a network object.
+            ValueError: If other is not completely contained by self.
+
+        """
+        if not self._version == other._version:
+            raise TypeError("%s and %s are not of the same version" % (
+                             self, other))
+
+        if not isinstance(other, _BaseNetwork):
+            raise TypeError("%s is not a network object" % other)
+
+        if not (other.network_address >= self.network_address and
+                other.broadcast_address <= self.broadcast_address):
+            raise ValueError('%s not contained in %s' % (other, self))
+        if other == self:
+            raise StopIteration
+
+        # Make sure we're comparing the network of other.
+        other = other.__class__('%s/%s' % (other.network_address,
+                                           other.prefixlen))
+
+        s1, s2 = self.subnets()
+        while s1 != other and s2 != other:
+            if (other.network_address >= s1.network_address and
+                other.broadcast_address <= s1.broadcast_address):
+                yield s2
+                s1, s2 = s1.subnets()
+            elif (other.network_address >= s2.network_address and
+                  other.broadcast_address <= s2.broadcast_address):
+                yield s1
+                s1, s2 = s2.subnets()
+            else:
+                # If we got here, there's a bug somewhere.
+                raise AssertionError('Error performing exclusion: '
+                                     's1: %s s2: %s other: %s' %
+                                     (s1, s2, other))
+        if s1 == other:
+            yield s2
+        elif s2 == other:
+            yield s1
+        else:
+            # If we got here, there's a bug somewhere.
+            raise AssertionError('Error performing exclusion: '
+                                 's1: %s s2: %s other: %s' %
+                                 (s1, s2, other))
+
+    def compare_networks(self, other):
+        """Compare two IP objects.
+
+        This is only concerned about the comparison of the integer
+        representation of the network addresses.  This means that the
+        host bits aren't considered at all in this method.  If you want
+        to compare host bits, you can easily enough do a
+        'HostA._ip < HostB._ip'
+
+        Args:
+            other: An IP object.
+
+        Returns:
+            If the IP versions of self and other are the same, returns:
+
+            -1 if self < other:
+              eg: IPv4Network('192.0.2.0/25') < IPv4Network('192.0.2.128/25')
+              IPv6Network('2001:db8::1000/124') <
+                  IPv6Network('2001:db8::2000/124')
+            0 if self == other
+              eg: IPv4Network('192.0.2.0/24') == IPv4Network('192.0.2.0/24')
+              IPv6Network('2001:db8::1000/124') ==
+                  IPv6Network('2001:db8::1000/124')
+            1 if self > other
+              eg: IPv4Network('192.0.2.128/25') > IPv4Network('192.0.2.0/25')
+                  IPv6Network('2001:db8::2000/124') >
+                      IPv6Network('2001:db8::1000/124')
+
+          Raises:
+              TypeError if the IP versions are different.
+
+        """
+        # does this need to raise a ValueError?
+        if self._version != other._version:
+            raise TypeError('%s and %s are not of the same type' % (
+                             self, other))
+        # self._version == other._version below here:
+        if self.network_address < other.network_address:
+            return -1
+        if self.network_address > other.network_address:
+            return 1
+        # self.network_address == other.network_address below here:
+        if self.netmask < other.netmask:
+            return -1
+        if self.netmask > other.netmask:
+            return 1
+        return 0
+
+    def _get_networks_key(self):
+        """Network-only key function.
+
+        Returns an object that identifies this address' network and
+        netmask. This function is a suitable "key" argument for sorted()
+        and list.sort().
+
+        """
+        return (self._version, self.network_address, self.netmask)
+
+    def subnets(self, prefixlen_diff=1, new_prefix=None):
+        """The subnets which join to make the current subnet.
+
+        In the case that self contains only one IP
+        (self._prefixlen == 32 for IPv4 or self._prefixlen == 128
+        for IPv6), yield an iterator with just ourself.
+
+        Args:
+            prefixlen_diff: An integer, the amount the prefix length
+              should be increased by. This should not be set if
+              new_prefix is also set.
+            new_prefix: The desired new prefix length. This must be a
+              larger number (smaller prefix) than the existing prefix.
+              This should not be set if prefixlen_diff is also set.
+
+        Returns:
+            An iterator of IPv(4|6) objects.
+
+        Raises:
+            ValueError: The prefixlen_diff is too small or too large.
+                OR
+            prefixlen_diff and new_prefix are both set or new_prefix
+              is a smaller number than the current prefix (smaller
+              number means a larger network)
+
+        """
+        if self._prefixlen == self._max_prefixlen:
+            yield self
+            return
+
+        if new_prefix is not None:
+            if new_prefix < self._prefixlen:
+                raise ValueError('new prefix must be longer')
+            if prefixlen_diff != 1:
+                raise ValueError('cannot set prefixlen_diff and new_prefix')
+            prefixlen_diff = new_prefix - self._prefixlen
+
+        if prefixlen_diff < 0:
+            raise ValueError('prefix length diff must be > 0')
+        new_prefixlen = self._prefixlen + prefixlen_diff
+
+        if not self._is_valid_netmask(str(new_prefixlen)):
+            raise ValueError(
+                'prefix length diff %d is invalid for netblock %s' % (
+                    new_prefixlen, self))
+
+        first = self.__class__('%s/%s' %
+                                 (self.network_address,
+                                  self._prefixlen + prefixlen_diff))
+
+        yield first
+        current = first
+        while True:
+            broadcast = current.broadcast_address
+            if broadcast == self.broadcast_address:
+                return
+            new_addr = self._address_class(int(broadcast) + 1)
+            current = self.__class__('%s/%s' % (new_addr,
+                                                new_prefixlen))
+
+            yield current
+
+    def supernet(self, prefixlen_diff=1, new_prefix=None):
+        """The supernet containing the current network.
+
+        Args:
+            prefixlen_diff: An integer, the amount the prefix length of
+              the network should be decreased by.  For example, given a
+              /24 network and a prefixlen_diff of 3, a supernet with a
+              /21 netmask is returned.
+
+        Returns:
+            An IPv4 network object.
+
+        Raises:
+            ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have
+              a negative prefix length.
+                OR
+            If prefixlen_diff and new_prefix are both set or new_prefix is a
+              larger number than the current prefix (larger number means a
+              smaller network)
+
+        """
+        if self._prefixlen == 0:
+            return self
+
+        if new_prefix is not None:
+            if new_prefix > self._prefixlen:
+                raise ValueError('new prefix must be shorter')
+            if prefixlen_diff != 1:
+                raise ValueError('cannot set prefixlen_diff and new_prefix')
+            prefixlen_diff = self._prefixlen - new_prefix
+
+        if self.prefixlen - prefixlen_diff < 0:
+            raise ValueError(
+                'current prefixlen is %d, cannot have a prefixlen_diff of %d' %
+                (self.prefixlen, prefixlen_diff))
+        # TODO (pmoody): optimize this.
+        t = self.__class__('%s/%d' % (self.network_address,
+                                      self.prefixlen - prefixlen_diff),
+                                     strict=False)
+        return t.__class__('%s/%d' % (t.network_address, t.prefixlen))
+
+    @property
+    def is_multicast(self):
+        """Test if the address is reserved for multicast use.
+
+        Returns:
+            A boolean, True if the address is a multicast address.
+            See RFC 2373 2.7 for details.
+
+        """
+        return (self.network_address.is_multicast and
+                self.broadcast_address.is_multicast)
+
+    @property
+    def is_reserved(self):
+        """Test if the address is otherwise IETF reserved.
+
+        Returns:
+            A boolean, True if the address is within one of the
+            reserved IPv6 Network ranges.
+
+        """
+        return (self.network_address.is_reserved and
+                self.broadcast_address.is_reserved)
+
+    @property
+    def is_link_local(self):
+        """Test if the address is reserved for link-local.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 4291.
+
+        """
+        return (self.network_address.is_link_local and
+                self.broadcast_address.is_link_local)
+
+    @property
+    def is_private(self):
+        """Test if this address is allocated for private networks.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 4193.
+
+        """
+        return (self.network_address.is_private and
+                self.broadcast_address.is_private)
+
+    @property
+    def is_unspecified(self):
+        """Test if the address is unspecified.
+
+        Returns:
+            A boolean, True if this is the unspecified address as defined in
+            RFC 2373 2.5.2.
+
+        """
+        return (self.network_address.is_unspecified and
+                self.broadcast_address.is_unspecified)
+
+    @property
+    def is_loopback(self):
+        """Test if the address is a loopback address.
+
+        Returns:
+            A boolean, True if the address is a loopback address as defined in
+            RFC 2373 2.5.3.
+
+        """
+        return (self.network_address.is_loopback and
+                self.broadcast_address.is_loopback)
+
+
+class _BaseV4:
+
+    """Base IPv4 object.
+
+    The following methods are used by IPv4 objects in both single IP
+    addresses and networks.
+
+    """
+
+    # Equivalent to 255.255.255.255 or 32 bits of 1's.
+    _ALL_ONES = (2**IPV4LENGTH) - 1
+    _DECIMAL_DIGITS = frozenset('0123456789')
+
+    # the valid octets for host and netmasks. only useful for IPv4.
+    _valid_mask_octets = frozenset((255, 254, 252, 248, 240, 224, 192, 128, 0))
+
+    def __init__(self, address):
+        self._version = 4
+        self._max_prefixlen = IPV4LENGTH
+
+    def _explode_shorthand_ip_string(self):
+        return str(self)
+
+    def _ip_int_from_string(self, ip_str):
+        """Turn the given IP string into an integer for comparison.
+
+        Args:
+            ip_str: A string, the IP ip_str.
+
+        Returns:
+            The IP ip_str as an integer.
+
+        Raises:
+            AddressValueError: if ip_str isn't a valid IPv4 Address.
+
+        """
+        if not ip_str:
+            raise AddressValueError('Address cannot be empty')
+
+        octets = ip_str.split('.')
+        if len(octets) != 4:
+            raise AddressValueError("Expected 4 octets in %r" % ip_str)
+
+        try:
+            return int.from_bytes(map(self._parse_octet, octets), 'big')
+        except ValueError as exc:
+            raise AddressValueError("%s in %r" % (exc, ip_str)) from None
+
+    def _parse_octet(self, octet_str):
+        """Convert a decimal octet into an integer.
+
+        Args:
+            octet_str: A string, the number to parse.
+
+        Returns:
+            The octet as an integer.
+
+        Raises:
+            ValueError: if the octet isn't strictly a decimal from [0..255].
+
+        """
+        if not octet_str:
+            raise ValueError("Empty octet not permitted")
+        # Whitelist the characters, since int() allows a lot of bizarre stuff.
+        if not self._DECIMAL_DIGITS.issuperset(octet_str):
+            msg = "Only decimal digits permitted in %r"
+            raise ValueError(msg % octet_str)
+        # We do the length check second, since the invalid character error
+        # is likely to be more informative for the user
+        if len(octet_str) > 3:
+            msg = "At most 3 characters permitted in %r"
+            raise ValueError(msg % octet_str)
+        # Convert to integer (we know digits are legal)
+        octet_int = int(octet_str, 10)
+        # Any octets that look like they *might* be written in octal,
+        # and which don't look exactly the same in both octal and
+        # decimal are rejected as ambiguous
+        if octet_int > 7 and octet_str[0] == '0':
+            msg = "Ambiguous (octal/decimal) value in %r not permitted"
+            raise ValueError(msg % octet_str)
+        if octet_int > 255:
+            raise ValueError("Octet %d (> 255) not permitted" % octet_int)
+        return octet_int
+
+    def _string_from_ip_int(self, ip_int):
+        """Turns a 32-bit integer into dotted decimal notation.
+
+        Args:
+            ip_int: An integer, the IP address.
+
+        Returns:
+            The IP address as a string in dotted decimal notation.
+
+        """
+        return '.'.join(map(str, ip_int.to_bytes(4, 'big')))
+
+    def _is_valid_netmask(self, netmask):
+        """Verify that the netmask is valid.
+
+        Args:
+            netmask: A string, either a prefix or dotted decimal
+              netmask.
+
+        Returns:
+            A boolean, True if the prefix represents a valid IPv4
+            netmask.
+
+        """
+        mask = netmask.split('.')
+        if len(mask) == 4:
+            try:
+                for x in mask:
+                    if int(x) not in self._valid_mask_octets:
+                        return False
+            except ValueError:
+                # Found something that isn't an integer or isn't valid
+                return False
+            for idx, y in enumerate(mask):
+                if idx > 0 and y > mask[idx - 1]:
+                    return False
+            return True
+        try:
+            netmask = int(netmask)
+        except ValueError:
+            return False
+        return 0 <= netmask <= self._max_prefixlen
+
+    def _is_hostmask(self, ip_str):
+        """Test if the IP string is a hostmask (rather than a netmask).
+
+        Args:
+            ip_str: A string, the potential hostmask.
+
+        Returns:
+            A boolean, True if the IP string is a hostmask.
+
+        """
+        bits = ip_str.split('.')
+        try:
+            parts = [x for x in map(int, bits) if x in self._valid_mask_octets]
+        except ValueError:
+            return False
+        if len(parts) != len(bits):
+            return False
+        if parts[0] < parts[-1]:
+            return True
+        return False
+
+    @property
+    def max_prefixlen(self):
+        return self._max_prefixlen
+
+    @property
+    def version(self):
+        return self._version
+
+
+class IPv4Address(_BaseV4, _BaseAddress):
+
+    """Represent and manipulate single IPv4 Addresses."""
+
+    def __init__(self, address):
+
+        """
+        Args:
+            address: A string or integer representing the IP
+
+              Additionally, an integer can be passed, so
+              IPv4Address('192.0.2.1') == IPv4Address(3221225985).
+              or, more generally
+              IPv4Address(int(IPv4Address('192.0.2.1'))) ==
+                IPv4Address('192.0.2.1')
+
+        Raises:
+            AddressValueError: If ipaddress isn't a valid IPv4 address.
+
+        """
+        _BaseAddress.__init__(self, address)
+        _BaseV4.__init__(self, address)
+
+        # Efficient constructor from integer.
+        if isinstance(address, int):
+            self._check_int_address(address)
+            self._ip = address
+            return
+
+        # Constructing from a packed address
+        if isinstance(address, bytes):
+            self._check_packed_address(address, 4)
+            self._ip = int.from_bytes(address, 'big')
+            return
+
+        # Assume input argument to be string or any object representation
+        # which converts into a formatted IP string.
+        addr_str = str(address)
+        self._ip = self._ip_int_from_string(addr_str)
+
+    @property
+    def packed(self):
+        """The binary representation of this address."""
+        return v4_int_to_packed(self._ip)
+
+    @property
+    def is_reserved(self):
+        """Test if the address is otherwise IETF reserved.
+
+         Returns:
+             A boolean, True if the address is within the
+             reserved IPv4 Network range.
+
+        """
+        reserved_network = IPv4Network('240.0.0.0/4')
+        return self in reserved_network
+
+    @property
+    def is_private(self):
+        """Test if this address is allocated for private networks.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 1918.
+
+        """
+        private_10 = IPv4Network('10.0.0.0/8')
+        private_172 = IPv4Network('172.16.0.0/12')
+        private_192 = IPv4Network('192.168.0.0/16')
+        return (self in private_10 or
+                self in private_172 or
+                self in private_192)
+
+    @property
+    def is_multicast(self):
+        """Test if the address is reserved for multicast use.
+
+        Returns:
+            A boolean, True if the address is multicast.
+            See RFC 3171 for details.
+
+        """
+        multicast_network = IPv4Network('224.0.0.0/4')
+        return self in multicast_network
+
+    @property
+    def is_unspecified(self):
+        """Test if the address is unspecified.
+
+        Returns:
+            A boolean, True if this is the unspecified address as defined in
+            RFC 5735 3.
+
+        """
+        unspecified_address = IPv4Address('0.0.0.0')
+        return self == unspecified_address
+
+    @property
+    def is_loopback(self):
+        """Test if the address is a loopback address.
+
+        Returns:
+            A boolean, True if the address is a loopback per RFC 3330.
+
+        """
+        loopback_network = IPv4Network('127.0.0.0/8')
+        return self in loopback_network
+
+    @property
+    def is_link_local(self):
+        """Test if the address is reserved for link-local.
+
+        Returns:
+            A boolean, True if the address is link-local per RFC 3927.
+
+        """
+        linklocal_network = IPv4Network('169.254.0.0/16')
+        return self in linklocal_network
+
+
+class IPv4Interface(IPv4Address):
+
+    def __init__(self, address):
+        if isinstance(address, (bytes, int)):
+            IPv4Address.__init__(self, address)
+            self.network = IPv4Network(self._ip)
+            self._prefixlen = self._max_prefixlen
+            return
+
+        addr = _split_optional_netmask(address)
+        IPv4Address.__init__(self, addr[0])
+
+        self.network = IPv4Network(address, strict=False)
+        self._prefixlen = self.network._prefixlen
+
+        self.netmask = self.network.netmask
+        self.hostmask = self.network.hostmask
+
+    def __str__(self):
+        return '%s/%d' % (self._string_from_ip_int(self._ip),
+                          self.network.prefixlen)
+
+    def __eq__(self, other):
+        address_equal = IPv4Address.__eq__(self, other)
+        if not address_equal or address_equal is NotImplemented:
+            return address_equal
+        try:
+            return self.network == other.network
+        except AttributeError:
+            # An interface with an associated network is NOT the
+            # same as an unassociated address. That's why the hash
+            # takes the extra info into account.
+            return False
+
+    def __lt__(self, other):
+        address_less = IPv4Address.__lt__(self, other)
+        if address_less is NotImplemented:
+            return NotImplemented
+        try:
+            return self.network < other.network
+        except AttributeError:
+            # We *do* allow addresses and interfaces to be sorted. The
+            # unassociated address is considered less than all interfaces.
+            return False
+
+    def __hash__(self):
+        return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+
+    @property
+    def ip(self):
+        return IPv4Address(self._ip)
+
+    @property
+    def with_prefixlen(self):
+        return '%s/%s' % (self._string_from_ip_int(self._ip),
+                          self._prefixlen)
+
+    @property
+    def with_netmask(self):
+        return '%s/%s' % (self._string_from_ip_int(self._ip),
+                          self.netmask)
+
+    @property
+    def with_hostmask(self):
+        return '%s/%s' % (self._string_from_ip_int(self._ip),
+                          self.hostmask)
+
+
+class IPv4Network(_BaseV4, _BaseNetwork):
+
+    """This class represents and manipulates 32-bit IPv4 network + addresses..
+
+    Attributes: [examples for IPv4Network('192.0.2.0/27')]
+        .network_address: IPv4Address('192.0.2.0')
+        .hostmask: IPv4Address('0.0.0.31')
+        .broadcast_address: IPv4Address('192.0.2.32')
+        .netmask: IPv4Address('255.255.255.224')
+        .prefixlen: 27
+
+    """
+    # Class to use when creating address objects
+    _address_class = IPv4Address
+
+    def __init__(self, address, strict=True):
+
+        """Instantiate a new IPv4 network object.
+
+        Args:
+            address: A string or integer representing the IP [& network].
+              '192.0.2.0/24'
+              '192.0.2.0/255.255.255.0'
+              '192.0.0.2/0.0.0.255'
+              are all functionally the same in IPv4. Similarly,
+              '192.0.2.1'
+              '192.0.2.1/255.255.255.255'
+              '192.0.2.1/32'
+              are also functionaly equivalent. That is to say, failing to
+              provide a subnetmask will create an object with a mask of /32.
+
+              If the mask (portion after the / in the argument) is given in
+              dotted quad form, it is treated as a netmask if it starts with a
+              non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it
+              starts with a zero field (e.g. 0.255.255.255 == /8), with the
+              single exception of an all-zero mask which is treated as a
+              netmask == /0. If no mask is given, a default of /32 is used.
+
+              Additionally, an integer can be passed, so
+              IPv4Network('192.0.2.1') == IPv4Network(3221225985)
+              or, more generally
+              IPv4Interface(int(IPv4Interface('192.0.2.1'))) ==
+                IPv4Interface('192.0.2.1')
+
+        Raises:
+            AddressValueError: If ipaddress isn't a valid IPv4 address.
+            NetmaskValueError: If the netmask isn't valid for
+              an IPv4 address.
+            ValueError: If strict is True and a network address is not
+              supplied.
+
+        """
+
+        _BaseV4.__init__(self, address)
+        _BaseNetwork.__init__(self, address)
+
+        # Constructing from a packed address
+        if isinstance(address, bytes):
+            self.network_address = IPv4Address(address)
+            self._prefixlen = self._max_prefixlen
+            self.netmask = IPv4Address(self._ALL_ONES)
+            #fixme: address/network test here
+            return
+
+        # Efficient constructor from integer.
+        if isinstance(address, int):
+            self.network_address = IPv4Address(address)
+            self._prefixlen = self._max_prefixlen
+            self.netmask = IPv4Address(self._ALL_ONES)
+            #fixme: address/network test here.
+            return
+
+        # Assume input argument to be string or any object representation
+        # which converts into a formatted IP prefix string.
+        addr = _split_optional_netmask(address)
+        self.network_address = IPv4Address(self._ip_int_from_string(addr[0]))
+
+        if len(addr) == 2:
+            mask = addr[1].split('.')
+
+            if len(mask) == 4:
+                # We have dotted decimal netmask.
+                if self._is_valid_netmask(addr[1]):
+                    self.netmask = IPv4Address(self._ip_int_from_string(
+                            addr[1]))
+                elif self._is_hostmask(addr[1]):
+                    self.netmask = IPv4Address(
+                        self._ip_int_from_string(addr[1]) ^ self._ALL_ONES)
+                else:
+                    raise NetmaskValueError('%r is not a valid netmask'
+                                                     % addr[1])
+
+                self._prefixlen = self._prefix_from_ip_int(int(self.netmask))
+            else:
+                # We have a netmask in prefix length form.
+                if not self._is_valid_netmask(addr[1]):
+                    raise NetmaskValueError('%r is not a valid netmask'
+                                                     % addr[1])
+                self._prefixlen = int(addr[1])
+                self.netmask = IPv4Address(self._ip_int_from_prefix(
+                    self._prefixlen))
+        else:
+            self._prefixlen = self._max_prefixlen
+            self.netmask = IPv4Address(self._ip_int_from_prefix(
+                self._prefixlen))
+
+        if strict:
+            if (IPv4Address(int(self.network_address) & int(self.netmask)) !=
+                self.network_address):
+                raise ValueError('%s has host bits set' % self)
+        self.network_address = IPv4Address(int(self.network_address) &
+                                           int(self.netmask))
+
+        if self._prefixlen == (self._max_prefixlen - 1):
+            self.hosts = self.__iter__
+
+
+class _BaseV6:
+
+    """Base IPv6 object.
+
+    The following methods are used by IPv6 objects in both single IP
+    addresses and networks.
+
+    """
+
+    _ALL_ONES = (2**IPV6LENGTH) - 1
+    _HEXTET_COUNT = 8
+    _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef')
+
+    def __init__(self, address):
+        self._version = 6
+        self._max_prefixlen = IPV6LENGTH
+
+    def _ip_int_from_string(self, ip_str):
+        """Turn an IPv6 ip_str into an integer.
+
+        Args:
+            ip_str: A string, the IPv6 ip_str.
+
+        Returns:
+            An int, the IPv6 address
+
+        Raises:
+            AddressValueError: if ip_str isn't a valid IPv6 Address.
+
+        """
+        if not ip_str:
+            raise AddressValueError('Address cannot be empty')
+
+        parts = ip_str.split(':')
+
+        # An IPv6 address needs at least 2 colons (3 parts).
+        _min_parts = 3
+        if len(parts) < _min_parts:
+            msg = "At least %d parts expected in %r" % (_min_parts, ip_str)
+            raise AddressValueError(msg)
+
+        # If the address has an IPv4-style suffix, convert it to hexadecimal.
+        if '.' in parts[-1]:
+            try:
+                ipv4_int = IPv4Address(parts.pop())._ip
+            except AddressValueError as exc:
+                raise AddressValueError("%s in %r" % (exc, ip_str)) from None
+            parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF))
+            parts.append('%x' % (ipv4_int & 0xFFFF))
+
+        # An IPv6 address can't have more than 8 colons (9 parts).
+        # The extra colon comes from using the "::" notation for a single
+        # leading or trailing zero part.
+        _max_parts = self._HEXTET_COUNT + 1
+        if len(parts) > _max_parts:
+            msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str)
+            raise AddressValueError(msg)
+
+        # Disregarding the endpoints, find '::' with nothing in between.
+        # This indicates that a run of zeroes has been skipped.
+        skip_index = None
+        for i in range(1, len(parts) - 1):
+            if not parts[i]:
+                if skip_index is not None:
+                    # Can't have more than one '::'
+                    msg = "At most one '::' permitted in %r" % ip_str
+                    raise AddressValueError(msg)
+                skip_index = i
+
+        # parts_hi is the number of parts to copy from above/before the '::'
+        # parts_lo is the number of parts to copy from below/after the '::'
+        if skip_index is not None:
+            # If we found a '::', then check if it also covers the endpoints.
+            parts_hi = skip_index
+            parts_lo = len(parts) - skip_index - 1
+            if not parts[0]:
+                parts_hi -= 1
+                if parts_hi:
+                    msg = "Leading ':' only permitted as part of '::' in %r"
+                    raise AddressValueError(msg % ip_str)  # ^: requires ^::
+            if not parts[-1]:
+                parts_lo -= 1
+                if parts_lo:
+                    msg = "Trailing ':' only permitted as part of '::' in %r"
+                    raise AddressValueError(msg % ip_str)  # :$ requires ::$
+            parts_skipped = self._HEXTET_COUNT - (parts_hi + parts_lo)
+            if parts_skipped < 1:
+                msg = "Expected at most %d other parts with '::' in %r"
+                raise AddressValueError(msg % (self._HEXTET_COUNT-1, ip_str))
+        else:
+            # Otherwise, allocate the entire address to parts_hi.  The
+            # endpoints could still be empty, but _parse_hextet() will check
+            # for that.
+            if len(parts) != self._HEXTET_COUNT:
+                msg = "Exactly %d parts expected without '::' in %r"
+                raise AddressValueError(msg % (self._HEXTET_COUNT, ip_str))
+            if not parts[0]:
+                msg = "Leading ':' only permitted as part of '::' in %r"
+                raise AddressValueError(msg % ip_str)  # ^: requires ^::
+            if not parts[-1]:
+                msg = "Trailing ':' only permitted as part of '::' in %r"
+                raise AddressValueError(msg % ip_str)  # :$ requires ::$
+            parts_hi = len(parts)
+            parts_lo = 0
+            parts_skipped = 0
+
+        try:
+            # Now, parse the hextets into a 128-bit integer.
+            ip_int = 0
+            for i in range(parts_hi):
+                ip_int <<= 16
+                ip_int |= self._parse_hextet(parts[i])
+            ip_int <<= 16 * parts_skipped
+            for i in range(-parts_lo, 0):
+                ip_int <<= 16
+                ip_int |= self._parse_hextet(parts[i])
+            return ip_int
+        except ValueError as exc:
+            raise AddressValueError("%s in %r" % (exc, ip_str)) from None
+
+    def _parse_hextet(self, hextet_str):
+        """Convert an IPv6 hextet string into an integer.
+
+        Args:
+            hextet_str: A string, the number to parse.
+
+        Returns:
+            The hextet as an integer.
+
+        Raises:
+            ValueError: if the input isn't strictly a hex number from
+              [0..FFFF].
+
+        """
+        # Whitelist the characters, since int() allows a lot of bizarre stuff.
+        if not self._HEX_DIGITS.issuperset(hextet_str):
+            raise ValueError("Only hex digits permitted in %r" % hextet_str)
+        # We do the length check second, since the invalid character error
+        # is likely to be more informative for the user
+        if len(hextet_str) > 4:
+            msg = "At most 4 characters permitted in %r"
+            raise ValueError(msg % hextet_str)
+        # Length check means we can skip checking the integer value
+        return int(hextet_str, 16)
+
+    def _compress_hextets(self, hextets):
+        """Compresses a list of hextets.
+
+        Compresses a list of strings, replacing the longest continuous
+        sequence of "0" in the list with "" and adding empty strings at
+        the beginning or at the end of the string such that subsequently
+        calling ":".join(hextets) will produce the compressed version of
+        the IPv6 address.
+
+        Args:
+            hextets: A list of strings, the hextets to compress.
+
+        Returns:
+            A list of strings.
+
+        """
+        best_doublecolon_start = -1
+        best_doublecolon_len = 0
+        doublecolon_start = -1
+        doublecolon_len = 0
+        for index, hextet in enumerate(hextets):
+            if hextet == '0':
+                doublecolon_len += 1
+                if doublecolon_start == -1:
+                    # Start of a sequence of zeros.
+                    doublecolon_start = index
+                if doublecolon_len > best_doublecolon_len:
+                    # This is the longest sequence of zeros so far.
+                    best_doublecolon_len = doublecolon_len
+                    best_doublecolon_start = doublecolon_start
+            else:
+                doublecolon_len = 0
+                doublecolon_start = -1
+
+        if best_doublecolon_len > 1:
+            best_doublecolon_end = (best_doublecolon_start +
+                                    best_doublecolon_len)
+            # For zeros at the end of the address.
+            if best_doublecolon_end == len(hextets):
+                hextets += ['']
+            hextets[best_doublecolon_start:best_doublecolon_end] = ['']
+            # For zeros at the beginning of the address.
+            if best_doublecolon_start == 0:
+                hextets = [''] + hextets
+
+        return hextets
+
+    def _string_from_ip_int(self, ip_int=None):
+        """Turns a 128-bit integer into hexadecimal notation.
+
+        Args:
+            ip_int: An integer, the IP address.
+
+        Returns:
+            A string, the hexadecimal representation of the address.
+
+        Raises:
+            ValueError: The address is bigger than 128 bits of all ones.
+
+        """
+        if ip_int is None:
+            ip_int = int(self._ip)
+
+        if ip_int > self._ALL_ONES:
+            raise ValueError('IPv6 address is too large')
+
+        hex_str = '%032x' % ip_int
+        hextets = ['%x' % int(hex_str[x:x+4], 16) for x in range(0, 32, 4)]
+
+        hextets = self._compress_hextets(hextets)
+        return ':'.join(hextets)
+
+    def _explode_shorthand_ip_string(self):
+        """Expand a shortened IPv6 address.
+
+        Args:
+            ip_str: A string, the IPv6 address.
+
+        Returns:
+            A string, the expanded IPv6 address.
+
+        """
+        if isinstance(self, IPv6Network):
+            ip_str = str(self.network_address)
+        elif isinstance(self, IPv6Interface):
+            ip_str = str(self.ip)
+        else:
+            ip_str = str(self)
+
+        ip_int = self._ip_int_from_string(ip_str)
+        hex_str = '%032x' % ip_int
+        parts = [hex_str[x:x+4] for x in range(0, 32, 4)]
+        if isinstance(self, (_BaseNetwork, IPv6Interface)):
+            return '%s/%d' % (':'.join(parts), self._prefixlen)
+        return ':'.join(parts)
+
+    @property
+    def max_prefixlen(self):
+        return self._max_prefixlen
+
+    @property
+    def version(self):
+        return self._version
+
+
+class IPv6Address(_BaseV6, _BaseAddress):
+
+    """Represent and manipulate single IPv6 Addresses."""
+
+    def __init__(self, address):
+        """Instantiate a new IPv6 address object.
+
+        Args:
+            address: A string or integer representing the IP
+
+              Additionally, an integer can be passed, so
+              IPv6Address('2001:db8::') ==
+                IPv6Address(42540766411282592856903984951653826560)
+              or, more generally
+              IPv6Address(int(IPv6Address('2001:db8::'))) ==
+                IPv6Address('2001:db8::')
+
+        Raises:
+            AddressValueError: If address isn't a valid IPv6 address.
+
+        """
+        _BaseAddress.__init__(self, address)
+        _BaseV6.__init__(self, address)
+
+        # Efficient constructor from integer.
+        if isinstance(address, int):
+            self._check_int_address(address)
+            self._ip = address
+            return
+
+        # Constructing from a packed address
+        if isinstance(address, bytes):
+            self._check_packed_address(address, 16)
+            self._ip = int.from_bytes(address, 'big')
+            return
+
+        # Assume input argument to be string or any object representation
+        # which converts into a formatted IP string.
+        addr_str = str(address)
+        self._ip = self._ip_int_from_string(addr_str)
+
+    @property
+    def packed(self):
+        """The binary representation of this address."""
+        return v6_int_to_packed(self._ip)
+
+    @property
+    def is_multicast(self):
+        """Test if the address is reserved for multicast use.
+
+        Returns:
+            A boolean, True if the address is a multicast address.
+            See RFC 2373 2.7 for details.
+
+        """
+        multicast_network = IPv6Network('ff00::/8')
+        return self in multicast_network
+
+    @property
+    def is_reserved(self):
+        """Test if the address is otherwise IETF reserved.
+
+        Returns:
+            A boolean, True if the address is within one of the
+            reserved IPv6 Network ranges.
+
+        """
+        reserved_networks = [IPv6Network('::/8'), IPv6Network('100::/8'),
+                             IPv6Network('200::/7'), IPv6Network('400::/6'),
+                             IPv6Network('800::/5'), IPv6Network('1000::/4'),
+                             IPv6Network('4000::/3'), IPv6Network('6000::/3'),
+                             IPv6Network('8000::/3'), IPv6Network('A000::/3'),
+                             IPv6Network('C000::/3'), IPv6Network('E000::/4'),
+                             IPv6Network('F000::/5'), IPv6Network('F800::/6'),
+                             IPv6Network('FE00::/9')]
+
+        return any(self in x for x in reserved_networks)
+
+    @property
+    def is_link_local(self):
+        """Test if the address is reserved for link-local.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 4291.
+
+        """
+        linklocal_network = IPv6Network('fe80::/10')
+        return self in linklocal_network
+
+    @property
+    def is_site_local(self):
+        """Test if the address is reserved for site-local.
+
+        Note that the site-local address space has been deprecated by RFC 3879.
+        Use is_private to test if this address is in the space of unique local
+        addresses as defined by RFC 4193.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 3513 2.5.6.
+
+        """
+        sitelocal_network = IPv6Network('fec0::/10')
+        return self in sitelocal_network
+
+    @property
+    def is_private(self):
+        """Test if this address is allocated for private networks.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 4193.
+
+        """
+        private_network = IPv6Network('fc00::/7')
+        return self in private_network
+
+    @property
+    def is_unspecified(self):
+        """Test if the address is unspecified.
+
+        Returns:
+            A boolean, True if this is the unspecified address as defined in
+            RFC 2373 2.5.2.
+
+        """
+        return self._ip == 0
+
+    @property
+    def is_loopback(self):
+        """Test if the address is a loopback address.
+
+        Returns:
+            A boolean, True if the address is a loopback address as defined in
+            RFC 2373 2.5.3.
+
+        """
+        return self._ip == 1
+
+    @property
+    def ipv4_mapped(self):
+        """Return the IPv4 mapped address.
+
+        Returns:
+            If the IPv6 address is a v4 mapped address, return the
+            IPv4 mapped address. Return None otherwise.
+
+        """
+        if (self._ip >> 32) != 0xFFFF:
+            return None
+        return IPv4Address(self._ip & 0xFFFFFFFF)
+
+    @property
+    def teredo(self):
+        """Tuple of embedded teredo IPs.
+
+        Returns:
+            Tuple of the (server, client) IPs or None if the address
+            doesn't appear to be a teredo address (doesn't start with
+            2001::/32)
+
+        """
+        if (self._ip >> 96) != 0x20010000:
+            return None
+        return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF),
+                IPv4Address(~self._ip & 0xFFFFFFFF))
+
+    @property
+    def sixtofour(self):
+        """Return the IPv4 6to4 embedded address.
+
+        Returns:
+            The IPv4 6to4-embedded address if present or None if the
+            address doesn't appear to contain a 6to4 embedded address.
+
+        """
+        if (self._ip >> 112) != 0x2002:
+            return None
+        return IPv4Address((self._ip >> 80) & 0xFFFFFFFF)
+
+
+class IPv6Interface(IPv6Address):
+
+    def __init__(self, address):
+        if isinstance(address, (bytes, int)):
+            IPv6Address.__init__(self, address)
+            self.network = IPv6Network(self._ip)
+            self._prefixlen = self._max_prefixlen
+            return
+
+        addr = _split_optional_netmask(address)
+        IPv6Address.__init__(self, addr[0])
+        self.network = IPv6Network(address, strict=False)
+        self.netmask = self.network.netmask
+        self._prefixlen = self.network._prefixlen
+        self.hostmask = self.network.hostmask
+
+    def __str__(self):
+        return '%s/%d' % (self._string_from_ip_int(self._ip),
+                          self.network.prefixlen)
+
+    def __eq__(self, other):
+        address_equal = IPv6Address.__eq__(self, other)
+        if not address_equal or address_equal is NotImplemented:
+            return address_equal
+        try:
+            return self.network == other.network
+        except AttributeError:
+            # An interface with an associated network is NOT the
+            # same as an unassociated address. That's why the hash
+            # takes the extra info into account.
+            return False
+
+    def __lt__(self, other):
+        address_less = IPv6Address.__lt__(self, other)
+        if address_less is NotImplemented:
+            return NotImplemented
+        try:
+            return self.network < other.network
+        except AttributeError:
+            # We *do* allow addresses and interfaces to be sorted. The
+            # unassociated address is considered less than all interfaces.
+            return False
+
+    def __hash__(self):
+        return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+
+    @property
+    def ip(self):
+        return IPv6Address(self._ip)
+
+    @property
+    def with_prefixlen(self):
+        return '%s/%s' % (self._string_from_ip_int(self._ip),
+                          self._prefixlen)
+
+    @property
+    def with_netmask(self):
+        return '%s/%s' % (self._string_from_ip_int(self._ip),
+                          self.netmask)
+
+    @property
+    def with_hostmask(self):
+        return '%s/%s' % (self._string_from_ip_int(self._ip),
+                          self.hostmask)
+
+    @property
+    def is_unspecified(self):
+        return self._ip == 0 and self.network.is_unspecified
+
+    @property
+    def is_loopback(self):
+        return self._ip == 1 and self.network.is_loopback
+
+
+class IPv6Network(_BaseV6, _BaseNetwork):
+
+    """This class represents and manipulates 128-bit IPv6 networks.
+
+    Attributes: [examples for IPv6('2001:db8::1000/124')]
+        .network_address: IPv6Address('2001:db8::1000')
+        .hostmask: IPv6Address('::f')
+        .broadcast_address: IPv6Address('2001:db8::100f')
+        .netmask: IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff0')
+        .prefixlen: 124
+
+    """
+
+    # Class to use when creating address objects
+    _address_class = IPv6Address
+
+    def __init__(self, address, strict=True):
+        """Instantiate a new IPv6 Network object.
+
+        Args:
+            address: A string or integer representing the IPv6 network or the
+              IP and prefix/netmask.
+              '2001:db8::/128'
+              '2001:db8:0000:0000:0000:0000:0000:0000/128'
+              '2001:db8::'
+              are all functionally the same in IPv6.  That is to say,
+              failing to provide a subnetmask will create an object with
+              a mask of /128.
+
+              Additionally, an integer can be passed, so
+              IPv6Network('2001:db8::') ==
+                IPv6Network(42540766411282592856903984951653826560)
+              or, more generally
+              IPv6Network(int(IPv6Network('2001:db8::'))) ==
+                IPv6Network('2001:db8::')
+
+            strict: A boolean. If true, ensure that we have been passed
+              A true network address, eg, 2001:db8::1000/124 and not an
+              IP address on a network, eg, 2001:db8::1/124.
+
+        Raises:
+            AddressValueError: If address isn't a valid IPv6 address.
+            NetmaskValueError: If the netmask isn't valid for
+              an IPv6 address.
+            ValueError: If strict was True and a network address was not
+              supplied.
+
+        """
+        _BaseV6.__init__(self, address)
+        _BaseNetwork.__init__(self, address)
+
+        # Efficient constructor from integer.
+        if isinstance(address, int):
+            self.network_address = IPv6Address(address)
+            self._prefixlen = self._max_prefixlen
+            self.netmask = IPv6Address(self._ALL_ONES)
+            return
+
+        # Constructing from a packed address
+        if isinstance(address, bytes):
+            self.network_address = IPv6Address(address)
+            self._prefixlen = self._max_prefixlen
+            self.netmask = IPv6Address(self._ALL_ONES)
+            return
+
+        # Assume input argument to be string or any object representation
+        # which converts into a formatted IP prefix string.
+        addr = _split_optional_netmask(address)
+
+        self.network_address = IPv6Address(self._ip_int_from_string(addr[0]))
+
+        if len(addr) == 2:
+            if self._is_valid_netmask(addr[1]):
+                self._prefixlen = int(addr[1])
+            else:
+                raise NetmaskValueError('%r is not a valid netmask'
+                                                     % addr[1])
+        else:
+            self._prefixlen = self._max_prefixlen
+
+        self.netmask = IPv6Address(self._ip_int_from_prefix(self._prefixlen))
+        if strict:
+            if (IPv6Address(int(self.network_address) & int(self.netmask)) !=
+                self.network_address):
+                raise ValueError('%s has host bits set' % self)
+        self.network_address = IPv6Address(int(self.network_address) &
+                                           int(self.netmask))
+
+        if self._prefixlen == (self._max_prefixlen - 1):
+            self.hosts = self.__iter__
+
+    def _is_valid_netmask(self, prefixlen):
+        """Verify that the netmask/prefixlen is valid.
+
+        Args:
+            prefixlen: A string, the netmask in prefix length format.
+
+        Returns:
+            A boolean, True if the prefix represents a valid IPv6
+            netmask.
+
+        """
+        try:
+            prefixlen = int(prefixlen)
+        except ValueError:
+            return False
+        return 0 <= prefixlen <= self._max_prefixlen
+
+    @property
+    def is_site_local(self):
+        """Test if the address is reserved for site-local.
+
+        Note that the site-local address space has been deprecated by RFC 3879.
+        Use is_private to test if this address is in the space of unique local
+        addresses as defined by RFC 4193.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 3513 2.5.6.
+
+        """
+        return (self.network_address.is_site_local and
+                self.broadcast_address.is_site_local)
diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py
index 0c59edd6ba..288686ab8e 100644
--- a/Lib/json/decoder.py
+++ b/Lib/json/decoder.py
@@ -121,8 +121,7 @@ def py_scanstring(s, end, strict=True,
                 msg = "Invalid \\uXXXX escape"
                 raise ValueError(errmsg(msg, s, end))
             uni = int(esc, 16)
-            # Check for surrogate pair on UCS-4 systems
-            if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
+            if 0xd800 <= uni <= 0xdbff:
                 msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
                 if not s[end + 5:end + 7] == '\\u':
                     raise ValueError(errmsg(msg, s, end))
diff --git a/Lib/lib2to3/__main__.py b/Lib/lib2to3/__main__.py
new file mode 100644
index 0000000000..80688baf27
--- /dev/null
+++ b/Lib/lib2to3/__main__.py
@@ -0,0 +1,4 @@
+import sys
+from .main import main
+
+sys.exit(main("lib2to3.fixes"))
diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py
index afc0467000..b176056199 100644
--- a/Lib/lib2to3/fixer_base.py
+++ b/Lib/lib2to3/fixer_base.py
@@ -27,7 +27,6 @@ class BaseFix(object):
     pattern_tree = None # Tree representation of the pattern
     options = None  # Options object passed to initializer
     filename = None # The filename (set by set_filename)
-    logger = None   # A logger (set by set_filename)
     numbers = itertools.count(1) # For new_name()
     used_names = set() # A set of all used NAMEs
     order = "post" # Does the fixer prefer pre- or post-order traversal
@@ -70,12 +69,11 @@ class BaseFix(object):
                                                                  with_tree=True)
 
     def set_filename(self, filename):
-        """Set the filename, and a logger derived from it.
+        """Set the filename.
 
         The main refactoring tool should call this.
         """
         self.filename = filename
-        self.logger = logging.getLogger(filename)
 
     def match(self, node):
         """Returns match for a given parse tree node.
diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py
index fa4942f392..17cbf0a2f9 100644
--- a/Lib/lib2to3/pytree.py
+++ b/Lib/lib2to3/pytree.py
@@ -109,26 +109,6 @@ class Base(object):
         """
         raise NotImplementedError
 
-    def set_prefix(self, prefix):
-        """
-        Set the prefix for the node (see Leaf class).
-
-        DEPRECATED; use the prefix property directly.
-        """
-        warnings.warn("set_prefix() is deprecated; use the prefix property",
-                      DeprecationWarning, stacklevel=2)
-        self.prefix = prefix
-
-    def get_prefix(self):
-        """
-        Return the prefix for the node (see Leaf class).
-
-        DEPRECATED; use the prefix property directly.
-        """
-        warnings.warn("get_prefix() is deprecated; use the prefix property",
-                      DeprecationWarning, stacklevel=2)
-        return self.prefix
-
     def replace(self, new):
         """Replace this node with a new one in the parent."""
         assert self.parent is not None, str(self)
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
index 38fb8ed9e2..201e193fe2 100644
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -566,7 +566,7 @@ class RefactoringTool(object):
         block_lineno = None
         indent = None
         lineno = 0
-        for line in input.splitlines(True):
+        for line in input.splitlines(keepends=True):
             lineno += 1
             if line.lstrip().startswith(self.PS1):
                 if block is not None:
@@ -610,7 +610,7 @@ class RefactoringTool(object):
                            filename, lineno, err.__class__.__name__, err)
             return block
         if self.refactor_tree(tree, filename):
-            new = str(tree).splitlines(True)
+            new = str(tree).splitlines(keepends=True)
             # Undo the adjustment of the line numbers in wrap_toks() below.
             clipped, new = new[:lineno-1], new[lineno-1:]
             assert clipped == ["\n"] * (lineno-1), clipped
diff --git a/Lib/lib2to3/tests/test_pytree.py b/Lib/lib2to3/tests/test_pytree.py
index ac7d9006aa..a2ab1f3534 100644
--- a/Lib/lib2to3/tests/test_pytree.py
+++ b/Lib/lib2to3/tests/test_pytree.py
@@ -31,23 +31,6 @@ class TestNodes(support.TestCase):
 
     """Unit tests for nodes (Base, Leaf, Node)."""
 
-    if sys.version_info >= (2,6):
-        # warnings.catch_warnings is new in 2.6.
-        def test_deprecated_prefix_methods(self):
-            l = pytree.Leaf(100, "foo")
-            with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter("always", DeprecationWarning)
-                self.assertEqual(l.get_prefix(), "")
-                l.set_prefix("hi")
-            self.assertEqual(l.prefix, "hi")
-            self.assertEqual(len(w), 2)
-            for warning in w:
-                self.assertTrue(warning.category is DeprecationWarning)
-            self.assertEqual(str(w[0].message), "get_prefix() is deprecated; " \
-                                 "use the prefix property")
-            self.assertEqual(str(w[1].message), "set_prefix() is deprecated; " \
-                                 "use the prefix property")
-
     def test_instantiate_base(self):
         if __debug__:
             # Test that instantiating Base() raises an AssertionError
diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py
index 5cb2866d75..fa03f78cce 100644
--- a/Lib/logging/__init__.py
+++ b/Lib/logging/__init__.py
@@ -36,15 +36,9 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR',
            'getLogRecordFactory', 'setLogRecordFactory', 'lastResort']
 
 try:
-    import codecs
-except ImportError:
-    codecs = None
-
-try:
-    import _thread as thread
     import threading
-except ImportError:
-    thread = None
+except ImportError: #pragma: no cover
+    threading = None
 
 __author__  = "Vinay Sajip <vinay_sajip@red-dove.com>"
 __status__  = "production"
@@ -65,16 +59,16 @@ else:
     _srcfile = __file__
 _srcfile = os.path.normcase(_srcfile)
 
-# next bit filched from 1.5.2's inspect.py
-def currentframe():
-    """Return the frame object for the caller's stack frame."""
-    try:
-        raise Exception
-    except:
-        return sys.exc_info()[2].tb_frame.f_back
 
-if hasattr(sys, '_getframe'): currentframe = lambda: sys._getframe(3)
-# done filching
+if hasattr(sys, '_getframe'):
+    currentframe = lambda: sys._getframe(3)
+else: #pragma: no cover
+    def currentframe():
+        """Return the frame object for the caller's stack frame."""
+        try:
+            raise Exception
+        except:
+            return sys.exc_info()[2].tb_frame.f_back
 
 # _srcfile is only used in conjunction with sys._getframe().
 # To provide compatibility with older versions of Python, set _srcfile
@@ -92,22 +86,22 @@ _startTime = time.time()
 #raiseExceptions is used to see if exceptions during handling should be
 #propagated
 #
-raiseExceptions = 1
+raiseExceptions = True
 
 #
 # If you don't want threading information in the log, set this to zero
 #
-logThreads = 1
+logThreads = True
 
 #
 # If you don't want multiprocessing information in the log, set this to zero
 #
-logMultiprocessing = 1
+logMultiprocessing = True
 
 #
 # If you don't want process information in the log, set this to zero
 #
-logProcesses = 1
+logProcesses = True
 
 #---------------------------------------------------------------------------
 #   Level related stuff
@@ -197,9 +191,9 @@ def _checkLevel(level):
 #the lock would already have been acquired - so we need an RLock.
 #The same argument applies to Loggers and Manager.loggerDict.
 #
-if thread:
+if threading:
     _lock = threading.RLock()
-else:
+else: #pragma: no cover
     _lock = None
 
 
@@ -252,7 +246,7 @@ class LogRecord(object):
         # during formatting, we test to see if the arg is present using
         # 'if self.args:'. If the event being logged is e.g. 'Value is %d'
         # and if the passed arg fails 'if self.args:' then no formatting
-        # is done. For example, logger.warn('Value is %d', 0) would log
+        # is done. For example, logger.warning('Value is %d', 0) would log
         # 'Value is %d' instead of 'Value is 0'.
         # For the use case of passing a dictionary, this should not be a
         # problem.
@@ -276,13 +270,13 @@ class LogRecord(object):
         self.created = ct
         self.msecs = (ct - int(ct)) * 1000
         self.relativeCreated = (self.created - _startTime) * 1000
-        if logThreads and thread:
-            self.thread = thread.get_ident()
+        if logThreads and threading:
+            self.thread = threading.get_ident()
             self.threadName = threading.current_thread().name
-        else:
+        else: # pragma: no cover
             self.thread = None
             self.threadName = None
-        if not logMultiprocessing:
+        if not logMultiprocessing: # pragma: no cover
             self.processName = None
         else:
             self.processName = 'MainProcess'
@@ -294,7 +288,7 @@ class LogRecord(object):
                 # for an example
                 try:
                     self.processName = mp.current_process().name
-                except Exception:
+                except Exception: #pragma: no cover
                     pass
         if logProcesses and hasattr(os, 'getpid'):
             self.process = os.getpid()
@@ -466,6 +460,9 @@ class Formatter(object):
         self._fmt = self._style._fmt
         self.datefmt = datefmt
 
+    default_time_format = '%Y-%m-%d %H:%M:%S'
+    default_msec_format = '%s,%03d'
+
     def formatTime(self, record, datefmt=None):
         """
         Return the creation time of the specified LogRecord as formatted text.
@@ -488,8 +485,8 @@ class Formatter(object):
         if datefmt:
             s = time.strftime(datefmt, ct)
         else:
-            t = time.strftime("%Y-%m-%d %H:%M:%S", ct)
-            s = "%s,%03d" % (t, record.msecs) # the use of % here is internal
+            t = time.strftime(self.default_time_format, ct)
+            s = self.default_msec_format % (t, record.msecs)
         return s
 
     def formatException(self, ei):
@@ -642,11 +639,11 @@ class Filter(object):
         yes. If deemed appropriate, the record may be modified in-place.
         """
         if self.nlen == 0:
-            return 1
+            return True
         elif self.name == record.name:
-            return 1
+            return True
         elif record.name.find(self.name, 0, self.nlen) != 0:
-            return 0
+            return False
         return (record.name[self.nlen] == ".")
 
 class Filterer(object):
@@ -686,14 +683,14 @@ class Filterer(object):
 
            Allow filters to be just callables.
         """
-        rv = 1
+        rv = True
         for f in self.filters:
             if hasattr(f, 'filter'):
                 result = f.filter(record)
             else:
                 result = f(record) # assume callable - will raise if not
             if not result:
-                rv = 0
+                rv = False
                 break
         return rv
 
@@ -772,9 +769,9 @@ class Handler(Filterer):
         """
         Acquire a thread lock for serializing access to the underlying I/O.
         """
-        if thread:
+        if threading:
             self.lock = threading.RLock()
-        else:
+        else: #pragma: no cover
             self.lock = None
 
     def acquire(self):
@@ -793,7 +790,7 @@ class Handler(Filterer):
 
     def setLevel(self, level):
         """
-        Set the logging level of this handler.
+        Set the logging level of this handler.  level must be an int or a str.
         """
         self.level = _checkLevel(level)
 
@@ -889,7 +886,7 @@ class Handler(Filterer):
                                           None, sys.stderr)
                 sys.stderr.write('Logged from file %s, line %s\n' % (
                                  record.filename, record.lineno))
-            except IOError:
+            except IOError: #pragma: no cover
                 pass    # see issue 5971
             finally:
                 del ei
@@ -942,7 +939,7 @@ class StreamHandler(Handler):
             stream.write(msg)
             stream.write(self.terminator)
             self.flush()
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit): #pragma: no cover
             raise
         except:
             self.handleError(record)
@@ -951,14 +948,12 @@ class FileHandler(StreamHandler):
     """
     A handler class which writes formatted logging records to disk files.
     """
-    def __init__(self, filename, mode='a', encoding=None, delay=0):
+    def __init__(self, filename, mode='a', encoding=None, delay=False):
         """
         Open the specified file and use it as the stream for logging.
         """
         #keep the absolute path, otherwise derived classes which use this
         #may come a cropper when the current directory changes
-        if codecs is None:
-            encoding = None
         self.baseFilename = os.path.abspath(filename)
         self.mode = mode
         self.encoding = encoding
@@ -990,11 +985,7 @@ class FileHandler(StreamHandler):
         Open the current base file with the (original) mode and encoding.
         Return the resulting stream.
         """
-        if self.encoding is None:
-            stream = open(self.baseFilename, self.mode)
-        else:
-            stream = codecs.open(self.baseFilename, self.mode, self.encoding)
-        return stream
+        return open(self.baseFilename, self.mode, encoding=self.encoding)
 
     def emit(self, record):
         """
@@ -1206,13 +1197,13 @@ class Logger(Filterer):
         self.name = name
         self.level = _checkLevel(level)
         self.parent = None
-        self.propagate = 1
+        self.propagate = True
         self.handlers = []
-        self.disabled = 0
+        self.disabled = False
 
     def setLevel(self, level):
         """
-        Set the logging level of this logger.
+        Set the logging level of this logger.  level must be an int or a str.
         """
         self.level = _checkLevel(level)
 
@@ -1252,7 +1243,10 @@ class Logger(Filterer):
         if self.isEnabledFor(WARNING):
             self._log(WARNING, msg, args, **kwargs)
 
-    warn = warning
+    def warn(self, msg, *args, **kwargs):
+        warnings.warn("The 'warn' method is deprecated, "
+            "use 'warning' instead", DeprecationWarning, 2)
+        self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
         """
@@ -1361,9 +1355,9 @@ class Logger(Filterer):
             #IronPython can use logging.
             try:
                 fn, lno, func, sinfo = self.findCaller(stack_info)
-            except ValueError:
+            except ValueError: # pragma: no cover
                 fn, lno, func = "(unknown file)", 0, "(unknown function)"
-        else:
+        else: # pragma: no cover
             fn, lno, func = "(unknown file)", 0, "(unknown function)"
         if exc_info:
             if not isinstance(exc_info, tuple):
@@ -1475,7 +1469,7 @@ class Logger(Filterer):
         Is this logger enabled for level 'level'?
         """
         if self.manager.disable >= level:
-            return 0
+            return False
         return level >= self.getEffectiveLevel()
 
     def getChild(self, suffix):
@@ -1565,7 +1559,10 @@ class LoggerAdapter(object):
         """
         self.log(WARNING, msg, *args, **kwargs)
 
-    warn = warning
+    def warn(self, msg, *args, **kwargs):
+        warnings.warn("The 'warn' method is deprecated, "
+            "use 'warning' instead", DeprecationWarning, 2)
+        self.warning(msg, *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
         """
@@ -1577,7 +1574,7 @@ class LoggerAdapter(object):
         """
         Delegate an exception call to the underlying logger.
         """
-        kwargs["exc_info"] = 1
+        kwargs["exc_info"] = True
         self.log(ERROR, msg, *args, **kwargs)
 
     def critical(self, msg, *args, **kwargs):
@@ -1660,6 +1657,10 @@ def basicConfig(**kwargs):
     stream    Use the specified stream to initialize the StreamHandler. Note
               that this argument is incompatible with 'filename' - if both
               are present, 'stream' is ignored.
+    handlers  If specified, this should be an iterable of already created
+              handlers, which will be added to the root handler. Any handler
+              in the list which does not have a formatter assigned will be
+              assigned the formatter created in this function.
 
     Note that you could specify a stream created using open(filename, mode)
     rather than passing the filename and mode in. However, it should be
@@ -1667,33 +1668,50 @@ def basicConfig(**kwargs):
     using sys.stdout or sys.stderr), whereas FileHandler closes its stream
     when the handler is closed.
 
-    .. versionchanged: 3.2
+    .. versionchanged:: 3.2
        Added the ``style`` parameter.
+
+    .. versionchanged:: 3.3
+       Added the ``handlers`` parameter. A ``ValueError`` is now thrown for
+       incompatible arguments (e.g. ``handlers`` specified together with
+       ``filename``/``filemode``, or ``filename``/``filemode`` specified
+       together with ``stream``, or ``handlers`` specified together with
+       ``stream``.
     """
     # Add thread safety in case someone mistakenly calls
     # basicConfig() from multiple threads
     _acquireLock()
     try:
         if len(root.handlers) == 0:
-            filename = kwargs.pop("filename", None)
-            if filename:
-                mode = kwargs.pop("filemode", 'a')
-                hdlr = FileHandler(filename, mode)
+            handlers = kwargs.get("handlers")
+            if handlers is None:
+                if "stream" in kwargs and "filename" in kwargs:
+                    raise ValueError("'stream' and 'filename' should not be "
+                                     "specified together")
             else:
-                stream = kwargs.pop("stream", None)
-                hdlr = StreamHandler(stream)
-            fs = kwargs.pop("format", BASIC_FORMAT)
-            dfs = kwargs.pop("datefmt", None)
-            style = kwargs.pop("style", '%')
+                if "stream" in kwargs or "filename" in kwargs:
+                    raise ValueError("'stream' or 'filename' should not be "
+                                     "specified together with 'handlers'")
+            if handlers is None:
+                filename = kwargs.get("filename")
+                if filename:
+                    mode = kwargs.get("filemode", 'a')
+                    h = FileHandler(filename, mode)
+                else:
+                    stream = kwargs.get("stream")
+                    h = StreamHandler(stream)
+                handlers = [h]
+            fs = kwargs.get("format", BASIC_FORMAT)
+            dfs = kwargs.get("datefmt", None)
+            style = kwargs.get("style", '%')
             fmt = Formatter(fs, dfs, style)
-            hdlr.setFormatter(fmt)
-            root.addHandler(hdlr)
-            level = kwargs.pop("level", None)
+            for h in handlers:
+                if h.formatter is None:
+                    h.setFormatter(fmt)
+                root.addHandler(h)
+            level = kwargs.get("level")
             if level is not None:
                 root.setLevel(level)
-            if kwargs:
-                s = ', '.join(kwargs.keys())
-                raise ValueError('Unexpected in keyword arguments: %s' % s)
     finally:
         _releaseLock()
 
@@ -1754,7 +1772,10 @@ def warning(msg, *args, **kwargs):
         basicConfig()
     root.warning(msg, *args, **kwargs)
 
-warn = warning
+def warn(msg, *args, **kwargs):
+    warnings.warn("The 'warn' function is deprecated, "
+        "use 'warning' instead", DeprecationWarning, 2)
+    warning(msg, *args, **kwargs)
 
 def info(msg, *args, **kwargs):
     """
@@ -1839,10 +1860,10 @@ class NullHandler(Handler):
     package.
     """
     def handle(self, record):
-        pass
+        """Stub."""
 
     def emit(self, record):
-        pass
+        """Stub."""
 
     def createLock(self):
         self.lock = None
diff --git a/Lib/logging/config.py b/Lib/logging/config.py
index 373da2b076..5ef5c913a9 100644
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -24,13 +24,13 @@ Copyright (C) 2001-2010 Vinay Sajip. All Rights Reserved.
 To use, simply 'import logging' and log away!
 """
 
-import sys, logging, logging.handlers, socket, struct, os, traceback, re
-import types, io
+import sys, logging, logging.handlers, socket, struct, traceback, re
+import io
 
 try:
     import _thread as thread
     import threading
-except ImportError:
+except ImportError: #pragma: no cover
     thread = None
 
 from socketserver import ThreadingTCPServer, StreamRequestHandler
@@ -98,9 +98,6 @@ def _resolve(name):
 def _strip_spaces(alist):
     return map(lambda x: x.strip(), alist)
 
-def _encoded(s):
-    return s if isinstance(s, str) else s.encode('utf-8')
-
 def _create_formatters(cp):
     """Create and return formatters"""
     flist = cp["formatters"]["keys"]
@@ -215,7 +212,7 @@ def _install_loggers(cp, handlers, disable_existing):
     #avoid disabling child loggers of explicitly
     #named loggers. With a sorted list it is easier
     #to find the child loggers.
-    existing.sort(key=_encoded)
+    existing.sort()
     #We'll keep the list of existing loggers
     #which are children of named loggers here...
     child_loggers = []
@@ -588,7 +585,7 @@ class DictConfigurator(BaseConfigurator):
                 #avoid disabling child loggers of explicitly
                 #named loggers. With a sorted list it is easier
                 #to find the child loggers.
-                existing.sort(key=_encoded)
+                existing.sort()
                 #We'll keep the list of existing loggers
                 #which are children of named loggers here...
                 child_loggers = []
@@ -786,7 +783,7 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
     and which you can join() when appropriate. To stop the server, call
     stopListening().
     """
-    if not thread:
+    if not thread: #pragma: no cover
         raise NotImplementedError("listen() needs threading to work")
 
     class ConfigStreamHandler(StreamRequestHandler):
@@ -804,7 +801,6 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
             struct.pack(">L", n), followed by the config file.
             Uses fileConfig() to do the grunt work.
             """
-            import tempfile
             try:
                 conn = self.connection
                 chunk = conn.recv(4)
@@ -825,7 +821,7 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
                         file = io.StringIO(chunk)
                         try:
                             fileConfig(file)
-                        except (KeyboardInterrupt, SystemExit):
+                        except (KeyboardInterrupt, SystemExit): #pragma: no cover
                             raise
                         except:
                             traceback.print_exc()
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
index 8349d3a7fb..f286cd61d4 100644
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py
@@ -24,18 +24,14 @@ To use, simply 'import logging.handlers' and log away!
 """
 
 import errno, logging, socket, os, pickle, struct, time, re
+from codecs import BOM_UTF8
 from stat import ST_DEV, ST_INO, ST_MTIME
 import queue
 try:
     import threading
-except ImportError:
+except ImportError: #pragma: no cover
     threading = None
 
-try:
-    import codecs
-except ImportError:
-    codecs = None
-
 #
 # Some constants...
 #
@@ -55,15 +51,15 @@ class BaseRotatingHandler(logging.FileHandler):
     Not meant to be instantiated directly.  Instead, use RotatingFileHandler
     or TimedRotatingFileHandler.
     """
-    def __init__(self, filename, mode, encoding=None, delay=0):
+    def __init__(self, filename, mode, encoding=None, delay=False):
         """
         Use the specified filename for streamed logging
         """
-        if codecs is None:
-            encoding = None
         logging.FileHandler.__init__(self, filename, mode, encoding, delay)
         self.mode = mode
         self.encoding = encoding
+        self.namer = None
+        self.rotator = None
 
     def emit(self, record):
         """
@@ -76,17 +72,55 @@ class BaseRotatingHandler(logging.FileHandler):
             if self.shouldRollover(record):
                 self.doRollover()
             logging.FileHandler.emit(self, record)
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit): #pragma: no cover
             raise
         except:
             self.handleError(record)
 
+    def rotation_filename(self, default_name):
+        """
+        Modify the filename of a log file when rotating.
+
+        This is provided so that a custom filename can be provided.
+
+        The default implementation calls the 'namer' attribute of the
+        handler, if it's callable, passing the default name to
+        it. If the attribute isn't callable (the default is None), the name
+        is returned unchanged.
+
+        :param default_name: The default name for the log file.
+        """
+        if not callable(self.namer):
+            result = default_name
+        else:
+            result = self.namer(default_name)
+        return result
+
+    def rotate(self, source, dest):
+        """
+        When rotating, rotate the current log.
+
+        The default implementation calls the 'rotator' attribute of the
+        handler, if it's callable, passing the source and dest arguments to
+        it. If the attribute isn't callable (the default is None), the source
+        is simply renamed to the destination.
+
+        :param source: The source filename. This is normally the base
+                       filename, e.g. 'test.log'
+        :param dest:   The destination filename. This is normally
+                       what the source is rotated to, e.g. 'test.log.1'.
+        """
+        if not callable(self.rotator):
+            os.rename(source, dest)
+        else:
+            self.rotator(source, dest)
+
 class RotatingFileHandler(BaseRotatingHandler):
     """
     Handler for logging to a set of files, which switches from one file
     to the next when the current file reaches a certain size.
     """
-    def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0):
+    def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False):
         """
         Open the specified file and use it as the stream for logging.
 
@@ -127,16 +161,17 @@ class RotatingFileHandler(BaseRotatingHandler):
             self.stream = None
         if self.backupCount > 0:
             for i in range(self.backupCount - 1, 0, -1):
-                sfn = "%s.%d" % (self.baseFilename, i)
-                dfn = "%s.%d" % (self.baseFilename, i + 1)
+                sfn = self.rotation_filename("%s.%d" % (self.baseFilename, i))
+                dfn = self.rotation_filename("%s.%d" % (self.baseFilename,
+                                                        i + 1))
                 if os.path.exists(sfn):
                     if os.path.exists(dfn):
                         os.remove(dfn)
                     os.rename(sfn, dfn)
-            dfn = self.baseFilename + ".1"
+            dfn = self.rotation_filename(self.baseFilename + ".1")
             if os.path.exists(dfn):
                 os.remove(dfn)
-            os.rename(self.baseFilename, dfn)
+            self.rotate(self.baseFilename, dfn)
         self.stream = self._open()
 
     def shouldRollover(self, record):
@@ -183,19 +218,19 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
         if self.when == 'S':
             self.interval = 1 # one second
             self.suffix = "%Y-%m-%d_%H-%M-%S"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$"
+            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$"
         elif self.when == 'M':
             self.interval = 60 # one minute
             self.suffix = "%Y-%m-%d_%H-%M"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$"
+            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(\.\w+)?$"
         elif self.when == 'H':
             self.interval = 60 * 60 # one hour
             self.suffix = "%Y-%m-%d_%H"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}$"
+            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(\.\w+)?$"
         elif self.when == 'D' or self.when == 'MIDNIGHT':
             self.interval = 60 * 60 * 24 # one day
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}$"
+            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
         elif self.when.startswith('W'):
             self.interval = 60 * 60 * 24 * 7 # one week
             if len(self.when) != 2:
@@ -204,7 +239,7 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 raise ValueError("Invalid day specified for weekly rollover: %s" % self.when)
             self.dayOfWeek = int(self.when[1])
             self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}$"
+            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
         else:
             raise ValueError("Invalid rollover interval specified: %s" % self.when)
 
@@ -337,10 +372,11 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
                 else:
                     addend = -3600
                 timeTuple = time.localtime(t + addend)
-        dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
+        dfn = self.rotation_filename(self.baseFilename + "." +
+                                     time.strftime(self.suffix, timeTuple))
         if os.path.exists(dfn):
             os.remove(dfn)
-        os.rename(self.baseFilename, dfn)
+        self.rotate(self.baseFilename, dfn)
         if self.backupCount > 0:
             for s in self.getFilesToDelete():
                 os.remove(s)
@@ -379,7 +415,7 @@ class WatchedFileHandler(logging.FileHandler):
     This handler is based on a suggestion and patch by Chad J.
     Schroeder.
     """
-    def __init__(self, filename, mode='a', encoding=None, delay=0):
+    def __init__(self, filename, mode='a', encoding=None, delay=False):
         logging.FileHandler.__init__(self, filename, mode, encoding, delay)
         self.dev, self.ino = -1, -1
         self._statstream()
@@ -438,15 +474,15 @@ class SocketHandler(logging.Handler):
         """
         Initializes the handler with a specific host address and port.
 
-        The attribute 'closeOnError' is set to 1 - which means that if
-        a socket error occurs, the socket is silently closed and then
-        reopened on the next logging call.
+        When the attribute *closeOnError* is set to True - if a socket error
+        occurs, the socket is silently closed and then reopened on the next
+        logging call.
         """
         logging.Handler.__init__(self)
         self.host = host
         self.port = port
         self.sock = None
-        self.closeOnError = 0
+        self.closeOnError = False
         self.retryTime = None
         #
         # Exponential backoff parameters.
@@ -463,8 +499,12 @@ class SocketHandler(logging.Handler):
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         if hasattr(s, 'settimeout'):
             s.settimeout(timeout)
-        s.connect((self.host, self.port))
-        return s
+        try:
+            s.connect((self.host, self.port))
+            return s
+        except socket.error:
+            s.close()
+            raise
 
     def createSocket(self):
         """
@@ -477,7 +517,7 @@ class SocketHandler(logging.Handler):
         # is the first time back after a disconnect, or
         # we've waited long enough.
         if self.retryTime is None:
-            attempt = 1
+            attempt = True
         else:
             attempt = (now >= self.retryTime)
         if attempt:
@@ -510,14 +550,14 @@ class SocketHandler(logging.Handler):
             try:
                 if hasattr(self.sock, "sendall"):
                     self.sock.sendall(s)
-                else:
+                else: #pragma: no cover
                     sentsofar = 0
                     left = len(s)
                     while left > 0:
                         sent = self.sock.send(s[sentsofar:])
                         sentsofar = sentsofar + sent
                         left = left - sent
-            except socket.error:
+            except socket.error: #pragma: no cover
                 self.sock.close()
                 self.sock = None  # so we can call createSocket next time
 
@@ -567,7 +607,7 @@ class SocketHandler(logging.Handler):
         try:
             s = self.makePickle(record)
             self.send(s)
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit): #pragma: no cover
             raise
         except:
             self.handleError(record)
@@ -601,7 +641,7 @@ class DatagramHandler(SocketHandler):
         Initializes the handler with a specific host address and port.
         """
         SocketHandler.__init__(self, host, port)
-        self.closeOnError = 0
+        self.closeOnError = False
 
     def makeSocket(self):
         """
@@ -742,10 +782,10 @@ class SysLogHandler(logging.Handler):
         self.socktype = socktype
 
         if isinstance(address, str):
-            self.unixsocket = 1
+            self.unixsocket = True
             self._connect_unixsocket(address)
         else:
-            self.unixsocket = 0
+            self.unixsocket = False
             self.socket = socket.socket(socket.AF_INET, socktype)
             if socktype == socket.SOCK_STREAM:
                 self.socket.connect(address)
@@ -778,8 +818,7 @@ class SysLogHandler(logging.Handler):
         """
         self.acquire()
         try:
-            if self.unixsocket:
-                self.socket.close()
+            self.socket.close()
             logging.Handler.close(self)
         finally:
             self.release()
@@ -794,6 +833,7 @@ class SysLogHandler(logging.Handler):
         """
         return self.priority_map.get(levelName, "warning")
 
+    ident = ''          # prepended to all messages
     append_nul = True   # some old syslog daemons expect a NUL terminator
 
     def emit(self, record):
@@ -804,6 +844,8 @@ class SysLogHandler(logging.Handler):
         exception information is present, it is NOT sent to the server.
         """
         msg = self.format(record)
+        if self.ident:
+            msg = self.ident + msg
         if self.append_nul:
             msg += '\000'
         """
@@ -827,7 +869,7 @@ class SysLogHandler(logging.Handler):
                 self.socket.sendto(msg, self.address)
             else:
                 self.socket.sendall(msg)
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit): #pragma: no cover
             raise
         except:
             self.handleError(record)
@@ -837,7 +879,7 @@ class SMTPHandler(logging.Handler):
     A handler class which sends an SMTP email for each logging event.
     """
     def __init__(self, mailhost, fromaddr, toaddrs, subject,
-                 credentials=None, secure=None):
+                 credentials=None, secure=None, timeout=5.0):
         """
         Initialize the handler.
 
@@ -851,6 +893,8 @@ class SMTPHandler(logging.Handler):
         will be either an empty tuple, or a single-value tuple with the name
         of a keyfile, or a 2-value tuple with the names of the keyfile and
         certificate file. (This tuple is passed to the `starttls` method).
+        A timeout in seconds can be specified for the SMTP connection (the
+        default is one second).
         """
         logging.Handler.__init__(self)
         if isinstance(mailhost, tuple):
@@ -867,7 +911,7 @@ class SMTPHandler(logging.Handler):
         self.toaddrs = toaddrs
         self.subject = subject
         self.secure = secure
-        self._timeout = 5.0
+        self.timeout = timeout
 
     def getSubject(self, record):
         """
@@ -890,7 +934,7 @@ class SMTPHandler(logging.Handler):
             port = self.mailport
             if not port:
                 port = smtplib.SMTP_PORT
-            smtp = smtplib.SMTP(self.mailhost, port, timeout=self._timeout)
+            smtp = smtplib.SMTP(self.mailhost, port, timeout=self.timeout)
             msg = self.format(record)
             msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % (
                             self.fromaddr,
@@ -905,7 +949,7 @@ class SMTPHandler(logging.Handler):
                 smtp.login(self.username, self.password)
             smtp.sendmail(self.fromaddr, self.toaddrs, msg)
             smtp.quit()
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit): #pragma: no cover
             raise
         except:
             self.handleError(record)
@@ -992,7 +1036,7 @@ class NTEventLogHandler(logging.Handler):
                 type = self.getEventType(record)
                 msg = self.format(record)
                 self._welu.ReportEvent(self.appname, id, cat, type, [msg])
-            except (KeyboardInterrupt, SystemExit):
+            except (KeyboardInterrupt, SystemExit): #pragma: no cover
                 raise
             except:
                 self.handleError(record)
@@ -1075,9 +1119,11 @@ class HTTPHandler(logging.Handler):
                 s = ('u%s:%s' % self.credentials).encode('utf-8')
                 s = 'Basic ' + base64.b64encode(s).strip()
                 h.putheader('Authorization', s)
-            h.endheaders(data if self.method == "POST" else None)
+            h.endheaders()
+            if self.method == "POST":
+                h.send(data.encode('utf-8'))
             h.getresponse()    #can't do anything with the result
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit): #pragma: no cover
             raise
         except:
             self.handleError(record)
@@ -1259,7 +1305,7 @@ class QueueHandler(logging.Handler):
         """
         try:
             self.enqueue(self.prepare(record))
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit): #pragma: no cover
             raise
         except:
             self.handleError(record)
@@ -1356,6 +1402,16 @@ if threading:
                 except queue.Empty:
                     break
 
+        def enqueue_sentinel(self):
+            """
+            This is used to enqueue the sentinel record.
+
+            The base implementation uses put_nowait. You may want to override this
+            method if you want to use timeouts or work with custom queue
+            implementations.
+            """
+            self.queue.put_nowait(self._sentinel)
+
         def stop(self):
             """
             Stop the listener.
@@ -1365,6 +1421,6 @@ if threading:
             may be some records still left on the queue, which won't be processed.
             """
             self._stop.set()
-            self.queue.put_nowait(self._sentinel)
+            self.enqueue_sentinel()
             self._thread.join()
             self._thread = None
diff --git a/Lib/lzma.py b/Lib/lzma.py
new file mode 100644
index 0000000000..1a1b065f8e
--- /dev/null
+++ b/Lib/lzma.py
@@ -0,0 +1,454 @@
+"""Interface to the liblzma compression library.
+
+This module provides a class for reading and writing compressed files,
+classes for incremental (de)compression, and convenience functions for
+one-shot (de)compression.
+
+These classes and functions support both the XZ and legacy LZMA
+container formats, as well as raw compressed data streams.
+"""
+
+__all__ = [
+    "CHECK_NONE", "CHECK_CRC32", "CHECK_CRC64", "CHECK_SHA256",
+    "CHECK_ID_MAX", "CHECK_UNKNOWN",
+    "FILTER_LZMA1", "FILTER_LZMA2", "FILTER_DELTA", "FILTER_X86", "FILTER_IA64",
+    "FILTER_ARM", "FILTER_ARMTHUMB", "FILTER_POWERPC", "FILTER_SPARC",
+    "FORMAT_AUTO", "FORMAT_XZ", "FORMAT_ALONE", "FORMAT_RAW",
+    "MF_HC3", "MF_HC4", "MF_BT2", "MF_BT3", "MF_BT4",
+    "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME",
+
+    "LZMACompressor", "LZMADecompressor", "LZMAFile", "LZMAError",
+    "open", "compress", "decompress", "is_check_supported",
+]
+
+import builtins
+import io
+from _lzma import *
+from _lzma import _encode_filter_properties, _decode_filter_properties
+
+
+_MODE_CLOSED   = 0
+_MODE_READ     = 1
+_MODE_READ_EOF = 2
+_MODE_WRITE    = 3
+
+_BUFFER_SIZE = 8192
+
+
+class LZMAFile(io.BufferedIOBase):
+
+    """A file object providing transparent LZMA (de)compression.
+
+    An LZMAFile can act as a wrapper for an existing file object, or
+    refer directly to a named file on disk.
+
+    Note that LZMAFile provides a *binary* file interface - data read
+    is returned as bytes, and data to be written must be given as bytes.
+    """
+
+    def __init__(self, filename=None, mode="r", *,
+                 format=None, check=-1, preset=None, filters=None):
+        """Open an LZMA-compressed file in binary mode.
+
+        filename can be either an actual file name (given as a str or
+        bytes object), in which case the named file is opened, or it can
+        be an existing file object to read from or write to.
+
+        mode can be "r" for reading (default), "w" for (over)writing, or
+        "a" for appending. These can equivalently be given as "rb", "wb",
+        and "ab" respectively.
+
+        format specifies the container format to use for the file.
+        If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the
+        default is FORMAT_XZ.
+
+        check specifies the integrity check to use. This argument can
+        only be used when opening a file for writing. For FORMAT_XZ,
+        the default is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not
+        support integrity checks - for these formats, check must be
+        omitted, or be CHECK_NONE.
+
+        When opening a file for reading, the *preset* argument is not
+        meaningful, and should be omitted. The *filters* argument should
+        also be omitted, except when format is FORMAT_RAW (in which case
+        it is required).
+
+        When opening a file for writing, the settings used by the
+        compressor can be specified either as a preset compression
+        level (with the *preset* argument), or in detail as a custom
+        filter chain (with the *filters* argument). For FORMAT_XZ and
+        FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset
+        level. For FORMAT_RAW, the caller must always specify a filter
+        chain; the raw compressor does not support preset compression
+        levels.
+
+        preset (if provided) should be an integer in the range 0-9,
+        optionally OR-ed with the constant PRESET_EXTREME.
+
+        filters (if provided) should be a sequence of dicts. Each dict
+        should have an entry for "id" indicating ID of the filter, plus
+        additional entries for options to the filter.
+        """
+        self._fp = None
+        self._closefp = False
+        self._mode = _MODE_CLOSED
+        self._pos = 0
+        self._size = -1
+
+        if mode in ("r", "rb"):
+            if check != -1:
+                raise ValueError("Cannot specify an integrity check "
+                                 "when opening a file for reading")
+            if preset is not None:
+                raise ValueError("Cannot specify a preset compression "
+                                 "level when opening a file for reading")
+            if format is None:
+                format = FORMAT_AUTO
+            mode_code = _MODE_READ
+            # Save the args to pass to the LZMADecompressor initializer.
+            # If the file contains multiple compressed streams, each
+            # stream will need a separate decompressor object.
+            self._init_args = {"format":format, "filters":filters}
+            self._decompressor = LZMADecompressor(**self._init_args)
+            self._buffer = None
+        elif mode in ("w", "wb", "a", "ab"):
+            if format is None:
+                format = FORMAT_XZ
+            mode_code = _MODE_WRITE
+            self._compressor = LZMACompressor(format=format, check=check,
+                                              preset=preset, filters=filters)
+        else:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if isinstance(filename, (str, bytes)):
+            if "b" not in mode:
+                mode += "b"
+            self._fp = builtins.open(filename, mode)
+            self._closefp = True
+            self._mode = mode_code
+        elif hasattr(filename, "read") or hasattr(filename, "write"):
+            self._fp = filename
+            self._mode = mode_code
+        else:
+            raise TypeError("filename must be a str or bytes object, or a file")
+
+    def close(self):
+        """Flush and close the file.
+
+        May be called more than once without error. Once the file is
+        closed, any other operation on it will raise a ValueError.
+        """
+        if self._mode == _MODE_CLOSED:
+            return
+        try:
+            if self._mode in (_MODE_READ, _MODE_READ_EOF):
+                self._decompressor = None
+                self._buffer = None
+            elif self._mode == _MODE_WRITE:
+                self._fp.write(self._compressor.flush())
+                self._compressor = None
+        finally:
+            try:
+                if self._closefp:
+                    self._fp.close()
+            finally:
+                self._fp = None
+                self._closefp = False
+                self._mode = _MODE_CLOSED
+
+    @property
+    def closed(self):
+        """True if this file is closed."""
+        return self._mode == _MODE_CLOSED
+
+    def fileno(self):
+        """Return the file descriptor for the underlying file."""
+        self._check_not_closed()
+        return self._fp.fileno()
+
+    def seekable(self):
+        """Return whether the file supports seeking."""
+        return self.readable() and self._fp.seekable()
+
+    def readable(self):
+        """Return whether the file was opened for reading."""
+        self._check_not_closed()
+        return self._mode in (_MODE_READ, _MODE_READ_EOF)
+
+    def writable(self):
+        """Return whether the file was opened for writing."""
+        self._check_not_closed()
+        return self._mode == _MODE_WRITE
+
+    # Mode-checking helper functions.
+
+    def _check_not_closed(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+    def _check_can_read(self):
+        if not self.readable():
+            raise io.UnsupportedOperation("File not open for reading")
+
+    def _check_can_write(self):
+        if not self.writable():
+            raise io.UnsupportedOperation("File not open for writing")
+
+    def _check_can_seek(self):
+        if not self.readable():
+            raise io.UnsupportedOperation("Seeking is only supported "
+                                          "on files open for reading")
+        if not self._fp.seekable():
+            raise io.UnsupportedOperation("The underlying file object "
+                                          "does not support seeking")
+
+    # Fill the readahead buffer if it is empty. Returns False on EOF.
+    def _fill_buffer(self):
+        # Depending on the input data, our call to the decompressor may not
+        # return any data. In this case, try again after reading another block.
+        while True:
+            if self._buffer:
+                return True
+
+            if self._decompressor.unused_data:
+                rawblock = self._decompressor.unused_data
+            else:
+                rawblock = self._fp.read(_BUFFER_SIZE)
+
+            if not rawblock:
+                if self._decompressor.eof:
+                    self._mode = _MODE_READ_EOF
+                    self._size = self._pos
+                    return False
+                else:
+                    raise EOFError("Compressed file ended before the "
+                                   "end-of-stream marker was reached")
+
+            # Continue to next stream.
+            if self._decompressor.eof:
+                self._decompressor = LZMADecompressor(**self._init_args)
+
+            self._buffer = self._decompressor.decompress(rawblock)
+
+    # Read data until EOF.
+    # If return_data is false, consume the data without returning it.
+    def _read_all(self, return_data=True):
+        blocks = []
+        while self._fill_buffer():
+            if return_data:
+                blocks.append(self._buffer)
+            self._pos += len(self._buffer)
+            self._buffer = None
+        if return_data:
+            return b"".join(blocks)
+
+    # Read a block of up to n bytes.
+    # If return_data is false, consume the data without returning it.
+    def _read_block(self, n, return_data=True):
+        blocks = []
+        while n > 0 and self._fill_buffer():
+            if n < len(self._buffer):
+                data = self._buffer[:n]
+                self._buffer = self._buffer[n:]
+            else:
+                data = self._buffer
+                self._buffer = None
+            if return_data:
+                blocks.append(data)
+            self._pos += len(data)
+            n -= len(data)
+        if return_data:
+            return b"".join(blocks)
+
+    def peek(self, size=-1):
+        """Return buffered data without advancing the file position.
+
+        Always returns at least one byte of data, unless at EOF.
+        The exact number of bytes returned is unspecified.
+        """
+        self._check_can_read()
+        if self._mode == _MODE_READ_EOF or not self._fill_buffer():
+            return b""
+        return self._buffer
+
+    def read(self, size=-1):
+        """Read up to size uncompressed bytes from the file.
+
+        If size is negative or omitted, read until EOF is reached.
+        Returns b"" if the file is already at EOF.
+        """
+        self._check_can_read()
+        if self._mode == _MODE_READ_EOF or size == 0:
+            return b""
+        elif size < 0:
+            return self._read_all()
+        else:
+            return self._read_block(size)
+
+    def read1(self, size=-1):
+        """Read up to size uncompressed bytes, while trying to avoid
+        making multiple reads from the underlying stream.
+
+        Returns b"" if the file is at EOF.
+        """
+        # Usually, read1() calls _fp.read() at most once. However, sometimes
+        # this does not give enough data for the decompressor to make progress.
+        # In this case we make multiple reads, to avoid returning b"".
+        self._check_can_read()
+        if (size == 0 or self._mode == _MODE_READ_EOF or
+            not self._fill_buffer()):
+            return b""
+        if 0 < size < len(self._buffer):
+            data = self._buffer[:size]
+            self._buffer = self._buffer[size:]
+        else:
+            data = self._buffer
+            self._buffer = None
+        self._pos += len(data)
+        return data
+
+    def write(self, data):
+        """Write a bytes object to the file.
+
+        Returns the number of uncompressed bytes written, which is
+        always len(data). Note that due to buffering, the file on disk
+        may not reflect the data written until close() is called.
+        """
+        self._check_can_write()
+        compressed = self._compressor.compress(data)
+        self._fp.write(compressed)
+        self._pos += len(data)
+        return len(data)
+
+    # Rewind the file to the beginning of the data stream.
+    def _rewind(self):
+        self._fp.seek(0, 0)
+        self._mode = _MODE_READ
+        self._pos = 0
+        self._decompressor = LZMADecompressor(**self._init_args)
+        self._buffer = None
+
+    def seek(self, offset, whence=0):
+        """Change the file position.
+
+        The new position is specified by offset, relative to the
+        position indicated by whence. Possible values for whence are:
+
+            0: start of stream (default): offset must not be negative
+            1: current stream position
+            2: end of stream; offset must not be positive
+
+        Returns the new file position.
+
+        Note that seeking is emulated, sp depending on the parameters,
+        this operation may be extremely slow.
+        """
+        self._check_can_seek()
+
+        # Recalculate offset as an absolute file position.
+        if whence == 0:
+            pass
+        elif whence == 1:
+            offset = self._pos + offset
+        elif whence == 2:
+            # Seeking relative to EOF - we need to know the file's size.
+            if self._size < 0:
+                self._read_all(return_data=False)
+            offset = self._size + offset
+        else:
+            raise ValueError("Invalid value for whence: {}".format(whence))
+
+        # Make it so that offset is the number of bytes to skip forward.
+        if offset < self._pos:
+            self._rewind()
+        else:
+            offset -= self._pos
+
+        # Read and discard data until we reach the desired position.
+        if self._mode != _MODE_READ_EOF:
+            self._read_block(offset, return_data=False)
+
+        return self._pos
+
+    def tell(self):
+        """Return the current file position."""
+        self._check_not_closed()
+        return self._pos
+
+
+def open(filename, mode="rb", *,
+         format=None, check=-1, preset=None, filters=None,
+         encoding=None, errors=None, newline=None):
+    """Open an LZMA-compressed file in binary or text mode.
+
+    filename can be either an actual file name (given as a str or bytes object),
+    in which case the named file is opened, or it can be an existing file object
+    to read from or write to.
+
+    The mode argument can be "r", "rb" (default), "w", "wb", "a", or "ab" for
+    binary mode, or "rt", "wt" or "at" for text mode.
+
+    The format, check, preset and filters arguments specify the compression
+    settings, as for LZMACompressor, LZMADecompressor and LZMAFile.
+
+    For binary mode, this function is equivalent to the LZMAFile constructor:
+    LZMAFile(filename, mode, ...). In this case, the encoding, errors and
+    newline arguments must not be provided.
+
+    For text mode, a LZMAFile object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
+
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    lz_mode = mode.replace("t", "")
+    binary_file = LZMAFile(filename, lz_mode, format=format, check=check,
+                           preset=preset, filters=filters)
+
+    if "t" in mode:
+        return io.TextIOWrapper(binary_file, encoding, errors, newline)
+    else:
+        return binary_file
+
+
+def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None):
+    """Compress a block of data.
+
+    Refer to LZMACompressor's docstring for a description of the
+    optional arguments *format*, *check*, *preset* and *filters*.
+
+    For incremental compression, use an LZMACompressor object instead.
+    """
+    comp = LZMACompressor(format, check, preset, filters)
+    return comp.compress(data) + comp.flush()
+
+
+def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None):
+    """Decompress a block of data.
+
+    Refer to LZMADecompressor's docstring for a description of the
+    optional arguments *format*, *check* and *filters*.
+
+    For incremental decompression, use a LZMADecompressor object instead.
+    """
+    results = []
+    while True:
+        decomp = LZMADecompressor(format, memlimit, filters)
+        results.append(decomp.decompress(data))
+        if not decomp.eof:
+            raise LZMAError("Compressed data ended before the "
+                            "end-of-stream marker was reached")
+        if not decomp.unused_data:
+            return b"".join(results)
+        # There is unused data left over. Proceed to next stream.
+        data = decomp.unused_data
diff --git a/Lib/mailbox.py b/Lib/mailbox.py
index c73fb95fe2..d3bf3fd80b 100644
--- a/Lib/mailbox.py
+++ b/Lib/mailbox.py
@@ -1157,8 +1157,7 @@ class MH(Mailbox):
     def get_sequences(self):
         """Return a name-to-key-list dictionary to define each sequence."""
         results = {}
-        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
-        try:
+        with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f:
             all_keys = set(self.keys())
             for line in f:
                 try:
@@ -1177,13 +1176,11 @@ class MH(Mailbox):
                 except ValueError:
                     raise FormatError('Invalid sequence specification: %s' %
                                       line.rstrip())
-        finally:
-            f.close()
         return results
 
     def set_sequences(self, sequences):
         """Set sequences using the given name-to-key-list dictionary."""
-        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII')
         try:
             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
             for name, keys in sequences.items():
@@ -1523,9 +1520,10 @@ class Message(email.message.Message):
 
     def _become_message(self, message):
         """Assume the non-format-specific state of message."""
-        for name in ('_headers', '_unixfrom', '_payload', '_charset',
-                     'preamble', 'epilogue', 'defects', '_default_type'):
-            self.__dict__[name] = message.__dict__[name]
+        type_specific = getattr(message, '_type_specific_attributes', [])
+        for name in message.__dict__:
+            if name not in type_specific:
+                self.__dict__[name] = message.__dict__[name]
 
     def _explain_to(self, message):
         """Copy format-specific state to message insofar as possible."""
@@ -1538,6 +1536,8 @@ class Message(email.message.Message):
 class MaildirMessage(Message):
     """Message with Maildir-specific properties."""
 
+    _type_specific_attributes = ['_subdir', '_info', '_date']
+
     def __init__(self, message=None):
         """Initialize a MaildirMessage instance."""
         self._subdir = 'new'
@@ -1645,6 +1645,8 @@ class MaildirMessage(Message):
 class _mboxMMDFMessage(Message):
     """Message with mbox- or MMDF-specific properties."""
 
+    _type_specific_attributes = ['_from']
+
     def __init__(self, message=None):
         """Initialize an mboxMMDFMessage instance."""
         self.set_from('MAILER-DAEMON', True)
@@ -1760,6 +1762,8 @@ class mboxMessage(_mboxMMDFMessage):
 class MHMessage(Message):
     """Message with MH-specific properties."""
 
+    _type_specific_attributes = ['_sequences']
+
     def __init__(self, message=None):
         """Initialize an MHMessage instance."""
         self._sequences = []
@@ -1830,6 +1834,8 @@ class MHMessage(Message):
 class BabylMessage(Message):
     """Message with Babyl-specific properties."""
 
+    _type_specific_attributes = ['_labels', '_visible']
+
     def __init__(self, message=None):
         """Initialize an BabylMessage instance."""
         self._labels = []
diff --git a/Lib/mailcap.py b/Lib/mailcap.py
index 4ae13d7ed7..99f4958bf7 100644
--- a/Lib/mailcap.py
+++ b/Lib/mailcap.py
@@ -33,10 +33,10 @@ def getcaps():
 
 def listmailcapfiles():
     """Return a list of all mailcap files found on the system."""
-    # XXX Actually, this is Unix-specific
+    # This is mostly a Unix thing, but we use the OS path separator anyway
     if 'MAILCAPS' in os.environ:
-        str = os.environ['MAILCAPS']
-        mailcaps = str.split(':')
+        pathstr = os.environ['MAILCAPS']
+        mailcaps = pathstr.split(os.pathsep)
     else:
         if 'HOME' in os.environ:
             home = os.environ['HOME']
diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index 8ce02f9200..3f0bd0e719 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -249,7 +249,6 @@ class MimeTypes:
                     yield ctype
                 i += 1
 
-        default_encoding = sys.getdefaultencoding()
         with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT,
                              r'MIME\Database\Content Type') as mimedb:
             for ctype in enum_types(mimedb):
@@ -434,6 +433,8 @@ def _default_mime_types():
         '.ksh'    : 'text/plain',
         '.latex'  : 'application/x-latex',
         '.m1v'    : 'video/mpeg',
+        '.m3u'    : 'application/vnd.apple.mpegurl',
+        '.m3u8'   : 'application/vnd.apple.mpegurl',
         '.man'    : 'application/x-troff-man',
         '.me'     : 'application/x-troff-me',
         '.mht'    : 'message/rfc822',
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
index f033ba98ef..f90a4327e6 100644
--- a/Lib/modulefinder.py
+++ b/Lib/modulefinder.py
@@ -1,16 +1,14 @@
 """Find modules used by a script, using introspection."""
 
-from __future__ import generators
 import dis
 import imp
+import importlib.machinery
 import marshal
 import os
 import sys
 import types
 import struct
 
-READ_MODE = "rU"
-
 # XXX Clean up once str8's cstor matches bytes.
 LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')])
 IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')])
@@ -29,9 +27,7 @@ packagePathMap = {}
 
 # A Public interface
 def AddPackagePath(packagename, path):
-    paths = packagePathMap.get(packagename, [])
-    paths.append(path)
-    packagePathMap[packagename] = paths
+    packagePathMap.setdefault(packagename, []).append(path)
 
 replacePackageMap = {}
 
@@ -106,14 +102,14 @@ class ModuleFinder:
 
     def run_script(self, pathname):
         self.msg(2, "run_script", pathname)
-        with open(pathname, READ_MODE) as fp:
+        with open(pathname) as fp:
             stuff = ("", "r", imp.PY_SOURCE)
             self.load_module('__main__', fp, pathname, stuff)
 
     def load_file(self, pathname):
         dir, name = os.path.split(pathname)
         name, ext = os.path.splitext(name)
-        with open(pathname, READ_MODE) as fp:
+        with open(pathname) as fp:
             stuff = (ext, "r", imp.PY_SOURCE)
             self.load_module(name, fp, pathname, stuff)
 
@@ -227,8 +223,9 @@ class ModuleFinder:
         # But we must also collect Python extension modules - although
         # we cannot separate normal dlls from Python extensions.
         suffixes = []
-        for triple in imp.get_suffixes():
-            suffixes.append(triple[0])
+        suffixes += importlib.machinery.EXTENSION_SUFFIXES[:]
+        suffixes += importlib.machinery.SOURCE_SUFFIXES[:]
+        suffixes += importlib.machinery.BYTECODE_SUFFIXES[:]
         for dir in m.__path__:
             try:
                 names = os.listdir(dir)
@@ -270,7 +267,8 @@ class ModuleFinder:
         try:
             m = self.load_module(fqname, fp, pathname, stuff)
         finally:
-            if fp: fp.close()
+            if fp:
+                fp.close()
         if parent:
             setattr(parent, partname, m)
         self.msgout(3, "import_module ->", m)
@@ -662,4 +660,4 @@ if __name__ == '__main__':
     try:
         mf = test()
     except KeyboardInterrupt:
-        print("\n[interrupt]")
+        print("\n[interrupted]")
diff --git a/Lib/multiprocessing/__init__.py b/Lib/multiprocessing/__init__.py
index e6e16c8322..1f3e67c9b8 100644
--- a/Lib/multiprocessing/__init__.py
+++ b/Lib/multiprocessing/__init__.py
@@ -13,32 +13,7 @@
 #
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 __version__ = '0.70a1'
@@ -48,8 +23,8 @@ __all__ = [
     'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger',
     'allow_connection_pickling', 'BufferTooShort', 'TimeoutError',
     'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
-    'Event', 'Queue', 'JoinableQueue', 'Pool', 'Value', 'Array',
-    'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING',
+    'Event', 'Barrier', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool',
+    'Value', 'Array', 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING',
     ]
 
 __author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)'
@@ -161,7 +136,9 @@ def allow_connection_pickling():
     '''
     Install support for sending connections and sockets between processes
     '''
-    from multiprocessing import reduction
+    # This is undocumented.  In previous versions of multiprocessing
+    # its only effect was to make socket objects inheritable on Windows.
+    import multiprocessing.connection
 
 #
 # Definitions depending on native semaphores
@@ -209,6 +186,13 @@ def Event():
     from multiprocessing.synchronize import Event
     return Event()
 
+def Barrier(parties, action=None, timeout=None):
+    '''
+    Returns a barrier object
+    '''
+    from multiprocessing.synchronize import Barrier
+    return Barrier(parties, action, timeout)
+
 def Queue(maxsize=0):
     '''
     Returns a queue object
@@ -223,6 +207,13 @@ def JoinableQueue(maxsize=0):
     from multiprocessing.queues import JoinableQueue
     return JoinableQueue(maxsize)
 
+def SimpleQueue():
+    '''
+    Returns a queue object
+    '''
+    from multiprocessing.queues import SimpleQueue
+    return SimpleQueue()
+
 def Pool(processes=None, initializer=None, initargs=(), maxtasksperchild=None):
     '''
     Returns a process pool object
@@ -244,19 +235,19 @@ def RawArray(typecode_or_type, size_or_initializer):
     from multiprocessing.sharedctypes import RawArray
     return RawArray(typecode_or_type, size_or_initializer)
 
-def Value(typecode_or_type, *args, **kwds):
+def Value(typecode_or_type, *args, lock=True):
     '''
     Returns a synchronized shared object
     '''
     from multiprocessing.sharedctypes import Value
-    return Value(typecode_or_type, *args, **kwds)
+    return Value(typecode_or_type, *args, lock=lock)
 
-def Array(typecode_or_type, size_or_initializer, **kwds):
+def Array(typecode_or_type, size_or_initializer, *, lock=True):
     '''
     Returns a synchronized shared array
     '''
     from multiprocessing.sharedctypes import Array
-    return Array(typecode_or_type, size_or_initializer, **kwds)
+    return Array(typecode_or_type, size_or_initializer, lock=lock)
 
 #
 #
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index a5dc2a8424..1d65f46a5b 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -4,49 +4,34 @@
 # multiprocessing/connection.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
-__all__ = [ 'Client', 'Listener', 'Pipe' ]
+__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
 
+import io
 import os
 import sys
+import pickle
+import select
 import socket
+import struct
 import errno
 import time
 import tempfile
 import itertools
 
 import _multiprocessing
-from multiprocessing import current_process, AuthenticationError
+from multiprocessing import current_process, AuthenticationError, BufferTooShort
 from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
-from multiprocessing.forking import duplicate, close
-
+from multiprocessing.forking import ForkingPickler
+try:
+    import _winapi
+    from _winapi import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE
+except ImportError:
+    if sys.platform == 'win32':
+        raise
+    _winapi = None
 
 #
 #
@@ -122,6 +107,309 @@ def address_type(address):
         raise ValueError('address type of %r unrecognized' % address)
 
 #
+# Connection classes
+#
+
+class _ConnectionBase:
+    _handle = None
+
+    def __init__(self, handle, readable=True, writable=True):
+        handle = handle.__index__()
+        if handle < 0:
+            raise ValueError("invalid handle")
+        if not readable and not writable:
+            raise ValueError(
+                "at least one of `readable` and `writable` must be True")
+        self._handle = handle
+        self._readable = readable
+        self._writable = writable
+
+    # XXX should we use util.Finalize instead of a __del__?
+
+    def __del__(self):
+        if self._handle is not None:
+            self._close()
+
+    def _check_closed(self):
+        if self._handle is None:
+            raise IOError("handle is closed")
+
+    def _check_readable(self):
+        if not self._readable:
+            raise IOError("connection is write-only")
+
+    def _check_writable(self):
+        if not self._writable:
+            raise IOError("connection is read-only")
+
+    def _bad_message_length(self):
+        if self._writable:
+            self._readable = False
+        else:
+            self.close()
+        raise IOError("bad message length")
+
+    @property
+    def closed(self):
+        """True if the connection is closed"""
+        return self._handle is None
+
+    @property
+    def readable(self):
+        """True if the connection is readable"""
+        return self._readable
+
+    @property
+    def writable(self):
+        """True if the connection is writable"""
+        return self._writable
+
+    def fileno(self):
+        """File descriptor or handle of the connection"""
+        self._check_closed()
+        return self._handle
+
+    def close(self):
+        """Close the connection"""
+        if self._handle is not None:
+            try:
+                self._close()
+            finally:
+                self._handle = None
+
+    def send_bytes(self, buf, offset=0, size=None):
+        """Send the bytes data from a bytes-like object"""
+        self._check_closed()
+        self._check_writable()
+        m = memoryview(buf)
+        # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
+        if m.itemsize > 1:
+            m = memoryview(bytes(m))
+        n = len(m)
+        if offset < 0:
+            raise ValueError("offset is negative")
+        if n < offset:
+            raise ValueError("buffer length < offset")
+        if size is None:
+            size = n - offset
+        elif size < 0:
+            raise ValueError("size is negative")
+        elif offset + size > n:
+            raise ValueError("buffer length < offset + size")
+        self._send_bytes(m[offset:offset + size])
+
+    def send(self, obj):
+        """Send a (picklable) object"""
+        self._check_closed()
+        self._check_writable()
+        buf = io.BytesIO()
+        ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
+        self._send_bytes(buf.getbuffer())
+
+    def recv_bytes(self, maxlength=None):
+        """
+        Receive bytes data as a bytes object.
+        """
+        self._check_closed()
+        self._check_readable()
+        if maxlength is not None and maxlength < 0:
+            raise ValueError("negative maxlength")
+        buf = self._recv_bytes(maxlength)
+        if buf is None:
+            self._bad_message_length()
+        return buf.getvalue()
+
+    def recv_bytes_into(self, buf, offset=0):
+        """
+        Receive bytes data into a writeable buffer-like object.
+        Return the number of bytes read.
+        """
+        self._check_closed()
+        self._check_readable()
+        with memoryview(buf) as m:
+            # Get bytesize of arbitrary buffer
+            itemsize = m.itemsize
+            bytesize = itemsize * len(m)
+            if offset < 0:
+                raise ValueError("negative offset")
+            elif offset > bytesize:
+                raise ValueError("offset too large")
+            result = self._recv_bytes()
+            size = result.tell()
+            if bytesize < offset + size:
+                raise BufferTooShort(result.getvalue())
+            # Message can fit in dest
+            result.seek(0)
+            result.readinto(m[offset // itemsize :
+                              (offset + size) // itemsize])
+            return size
+
+    def recv(self):
+        """Receive a (picklable) object"""
+        self._check_closed()
+        self._check_readable()
+        buf = self._recv_bytes()
+        return pickle.loads(buf.getbuffer())
+
+    def poll(self, timeout=0.0):
+        """Whether there is any input available to be read"""
+        self._check_closed()
+        self._check_readable()
+        return self._poll(timeout)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self.close()
+
+
+if _winapi:
+
+    class PipeConnection(_ConnectionBase):
+        """
+        Connection class based on a Windows named pipe.
+        Overlapped I/O is used, so the handles must have been created
+        with FILE_FLAG_OVERLAPPED.
+        """
+        _got_empty_message = False
+
+        def _close(self, _CloseHandle=_winapi.CloseHandle):
+            _CloseHandle(self._handle)
+
+        def _send_bytes(self, buf):
+            ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
+            try:
+                if err == _winapi.ERROR_IO_PENDING:
+                    waitres = _winapi.WaitForMultipleObjects(
+                        [ov.event], False, INFINITE)
+                    assert waitres == WAIT_OBJECT_0
+            except:
+                ov.cancel()
+                raise
+            finally:
+                nwritten, err = ov.GetOverlappedResult(True)
+            assert err == 0
+            assert nwritten == len(buf)
+
+        def _recv_bytes(self, maxsize=None):
+            if self._got_empty_message:
+                self._got_empty_message = False
+                return io.BytesIO()
+            else:
+                bsize = 128 if maxsize is None else min(maxsize, 128)
+                try:
+                    ov, err = _winapi.ReadFile(self._handle, bsize,
+                                                overlapped=True)
+                    try:
+                        if err == _winapi.ERROR_IO_PENDING:
+                            waitres = _winapi.WaitForMultipleObjects(
+                                [ov.event], False, INFINITE)
+                            assert waitres == WAIT_OBJECT_0
+                    except:
+                        ov.cancel()
+                        raise
+                    finally:
+                        nread, err = ov.GetOverlappedResult(True)
+                        if err == 0:
+                            f = io.BytesIO()
+                            f.write(ov.getbuffer())
+                            return f
+                        elif err == _winapi.ERROR_MORE_DATA:
+                            return self._get_more_data(ov, maxsize)
+                except IOError as e:
+                    if e.winerror == _winapi.ERROR_BROKEN_PIPE:
+                        raise EOFError
+                    else:
+                        raise
+            raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
+
+        def _poll(self, timeout):
+            if (self._got_empty_message or
+                        _winapi.PeekNamedPipe(self._handle)[0] != 0):
+                return True
+            return bool(wait([self], timeout))
+
+        def _get_more_data(self, ov, maxsize):
+            buf = ov.getbuffer()
+            f = io.BytesIO()
+            f.write(buf)
+            left = _winapi.PeekNamedPipe(self._handle)[1]
+            assert left > 0
+            if maxsize is not None and len(buf) + left > maxsize:
+                self._bad_message_length()
+            ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
+            rbytes, err = ov.GetOverlappedResult(True)
+            assert err == 0
+            assert rbytes == left
+            f.write(ov.getbuffer())
+            return f
+
+
+class Connection(_ConnectionBase):
+    """
+    Connection class based on an arbitrary file descriptor (Unix only), or
+    a socket handle (Windows).
+    """
+
+    if _winapi:
+        def _close(self, _close=_multiprocessing.closesocket):
+            _close(self._handle)
+        _write = _multiprocessing.send
+        _read = _multiprocessing.recv
+    else:
+        def _close(self, _close=os.close):
+            _close(self._handle)
+        _write = os.write
+        _read = os.read
+
+    def _send(self, buf, write=_write):
+        remaining = len(buf)
+        while True:
+            n = write(self._handle, buf)
+            remaining -= n
+            if remaining == 0:
+                break
+            buf = buf[n:]
+
+    def _recv(self, size, read=_read):
+        buf = io.BytesIO()
+        handle = self._handle
+        remaining = size
+        while remaining > 0:
+            chunk = read(handle, remaining)
+            n = len(chunk)
+            if n == 0:
+                if remaining == size:
+                    raise EOFError
+                else:
+                    raise IOError("got end of file during message")
+            buf.write(chunk)
+            remaining -= n
+        return buf
+
+    def _send_bytes(self, buf):
+        # For wire compatibility with 3.2 and lower
+        n = len(buf)
+        self._send(struct.pack("!i", n))
+        # The condition is necessary to avoid "broken pipe" errors
+        # when sending a 0-length buffer if the other end closed the pipe.
+        if n > 0:
+            self._send(buf)
+
+    def _recv_bytes(self, maxsize=None):
+        buf = self._recv(4)
+        size, = struct.unpack("!i", buf.getvalue())
+        if maxsize is not None and size > maxsize:
+            return None
+        return self._recv(size)
+
+    def _poll(self, timeout):
+        r = wait([self], timeout)
+        return bool(r)
+
+
+#
 # Public functions
 #
 
@@ -154,6 +442,8 @@ class Listener(object):
 
         Returns a `Connection` object.
         '''
+        if self._listener is None:
+            raise IOError('listener is closed')
         c = self._listener.accept()
         if self._authkey:
             deliver_challenge(c, self._authkey)
@@ -164,11 +454,19 @@ class Listener(object):
         '''
         Close the bound socket or named pipe of `self`.
         '''
-        return self._listener.close()
+        if self._listener is not None:
+            self._listener.close()
+            self._listener = None
 
     address = property(lambda self: self._listener._address)
     last_accepted = property(lambda self: self._listener._last_accepted)
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self.close()
+
 
 def Client(address, family=None, authkey=None):
     '''
@@ -201,19 +499,16 @@ if sys.platform != 'win32':
             s1, s2 = socket.socketpair()
             s1.setblocking(True)
             s2.setblocking(True)
-            c1 = _multiprocessing.Connection(os.dup(s1.fileno()))
-            c2 = _multiprocessing.Connection(os.dup(s2.fileno()))
-            s1.close()
-            s2.close()
+            c1 = Connection(s1.detach())
+            c2 = Connection(s2.detach())
         else:
             fd1, fd2 = os.pipe()
-            c1 = _multiprocessing.Connection(fd1, writable=False)
-            c2 = _multiprocessing.Connection(fd2, readable=False)
+            c1 = Connection(fd1, writable=False)
+            c2 = Connection(fd2, readable=False)
 
         return c1, c2
 
 else:
-    from _multiprocessing import win32
 
     def Pipe(duplex=True):
         '''
@@ -221,35 +516,35 @@ else:
         '''
         address = arbitrary_address('AF_PIPE')
         if duplex:
-            openmode = win32.PIPE_ACCESS_DUPLEX
-            access = win32.GENERIC_READ | win32.GENERIC_WRITE
+            openmode = _winapi.PIPE_ACCESS_DUPLEX
+            access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
             obsize, ibsize = BUFSIZE, BUFSIZE
         else:
-            openmode = win32.PIPE_ACCESS_INBOUND
-            access = win32.GENERIC_WRITE
+            openmode = _winapi.PIPE_ACCESS_INBOUND
+            access = _winapi.GENERIC_WRITE
             obsize, ibsize = 0, BUFSIZE
 
-        h1 = win32.CreateNamedPipe(
-            address, openmode,
-            win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
-            win32.PIPE_WAIT,
-            1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL
+        h1 = _winapi.CreateNamedPipe(
+            address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
+            _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
+            _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
+            _winapi.PIPE_WAIT,
+            1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
             )
-        h2 = win32.CreateFile(
-            address, access, 0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
+        h2 = _winapi.CreateFile(
+            address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
+            _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
             )
-        win32.SetNamedPipeHandleState(
-            h2, win32.PIPE_READMODE_MESSAGE, None, None
+        _winapi.SetNamedPipeHandleState(
+            h2, _winapi.PIPE_READMODE_MESSAGE, None, None
             )
 
-        try:
-            win32.ConnectNamedPipe(h1, win32.NULL)
-        except WindowsError as e:
-            if e.args[0] != win32.ERROR_PIPE_CONNECTED:
-                raise
+        overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
+        _, err = overlapped.GetOverlappedResult(True)
+        assert err == 0
 
-        c1 = _multiprocessing.PipeConnection(h1, writable=duplex)
-        c2 = _multiprocessing.PipeConnection(h2, readable=duplex)
+        c1 = PipeConnection(h1, writable=duplex)
+        c2 = PipeConnection(h2, readable=duplex)
 
         return c1, c2
 
@@ -264,12 +559,15 @@ class SocketListener(object):
     def __init__(self, address, family, backlog=1):
         self._socket = socket.socket(getattr(socket, family))
         try:
-            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            # SO_REUSEADDR has different semantics on Windows (issue #2550).
+            if os.name == 'posix':
+                self._socket.setsockopt(socket.SOL_SOCKET,
+                                        socket.SO_REUSEADDR, 1)
             self._socket.setblocking(True)
             self._socket.bind(address)
             self._socket.listen(backlog)
             self._address = self._socket.getsockname()
-        except socket.error:
+        except OSError:
             self._socket.close()
             raise
         self._family = family
@@ -285,10 +583,7 @@ class SocketListener(object):
     def accept(self):
         s, self._last_accepted = self._socket.accept()
         s.setblocking(True)
-        fd = duplicate(s.fileno())
-        conn = _multiprocessing.Connection(fd)
-        s.close()
-        return conn
+        return Connection(s.detach())
 
     def close(self):
         self._socket.close()
@@ -303,24 +598,8 @@ def SocketClient(address):
     family = address_type(address)
     with socket.socket( getattr(socket, family) ) as s:
         s.setblocking(True)
-        t = _init_timeout()
-
-        while 1:
-            try:
-                s.connect(address)
-            except socket.error as e:
-                if e.args[0] != errno.ECONNREFUSED or _check_timeout(t):
-                    debug('failed to connect to address %s', address)
-                    raise
-                time.sleep(0.01)
-            else:
-                break
-        else:
-            raise
-
-        fd = duplicate(s.fileno())
-    conn = _multiprocessing.Connection(fd)
-    return conn
+        s.connect(address)
+        return Connection(s.detach())
 
 #
 # Definitions for connections based on named pipes
@@ -334,48 +613,55 @@ if sys.platform == 'win32':
         '''
         def __init__(self, address, backlog=None):
             self._address = address
-            handle = win32.CreateNamedPipe(
-                address, win32.PIPE_ACCESS_DUPLEX,
-                win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
-                win32.PIPE_WAIT,
-                win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
-                win32.NMPWAIT_WAIT_FOREVER, win32.NULL
-                )
-            self._handle_queue = [handle]
-            self._last_accepted = None
+            self._handle_queue = [self._new_handle(first=True)]
 
+            self._last_accepted = None
             sub_debug('listener created with address=%r', self._address)
-
             self.close = Finalize(
                 self, PipeListener._finalize_pipe_listener,
                 args=(self._handle_queue, self._address), exitpriority=0
                 )
 
-        def accept(self):
-            newhandle = win32.CreateNamedPipe(
-                self._address, win32.PIPE_ACCESS_DUPLEX,
-                win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
-                win32.PIPE_WAIT,
-                win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
-                win32.NMPWAIT_WAIT_FOREVER, win32.NULL
+        def _new_handle(self, first=False):
+            flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
+            if first:
+                flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
+            return _winapi.CreateNamedPipe(
+                self._address, flags,
+                _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
+                _winapi.PIPE_WAIT,
+                _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
+                _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
                 )
-            self._handle_queue.append(newhandle)
+
+        def accept(self):
+            self._handle_queue.append(self._new_handle())
             handle = self._handle_queue.pop(0)
             try:
-                win32.ConnectNamedPipe(handle, win32.NULL)
-            except WindowsError as e:
+                ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
+            except OSError as e:
+                if e.winerror != _winapi.ERROR_NO_DATA:
+                    raise
                 # ERROR_NO_DATA can occur if a client has already connected,
                 # written data and then disconnected -- see Issue 14725.
-                if e.args[0] not in (win32.ERROR_PIPE_CONNECTED,
-                                     win32.ERROR_NO_DATA):
+            else:
+                try:
+                    res = _winapi.WaitForMultipleObjects(
+                        [ov.event], False, INFINITE)
+                except:
+                    ov.cancel()
+                    _winapi.CloseHandle(handle)
                     raise
-            return _multiprocessing.PipeConnection(handle)
+                finally:
+                    _, err = ov.GetOverlappedResult(True)
+                    assert err == 0
+            return PipeConnection(handle)
 
         @staticmethod
         def _finalize_pipe_listener(queue, address):
             sub_debug('closing listener with address=%r', address)
             for handle in queue:
-                close(handle)
+                _winapi.CloseHandle(handle)
 
     def PipeClient(address):
         '''
@@ -384,24 +670,25 @@ if sys.platform == 'win32':
         t = _init_timeout()
         while 1:
             try:
-                win32.WaitNamedPipe(address, 1000)
-                h = win32.CreateFile(
-                    address, win32.GENERIC_READ | win32.GENERIC_WRITE,
-                    0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
+                _winapi.WaitNamedPipe(address, 1000)
+                h = _winapi.CreateFile(
+                    address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
+                    0, _winapi.NULL, _winapi.OPEN_EXISTING,
+                    _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
                     )
             except WindowsError as e:
-                if e.args[0] not in (win32.ERROR_SEM_TIMEOUT,
-                                     win32.ERROR_PIPE_BUSY) or _check_timeout(t):
+                if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
+                                      _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
                     raise
             else:
                 break
         else:
             raise
 
-        win32.SetNamedPipeHandleState(
-            h, win32.PIPE_READMODE_MESSAGE, None, None
+        _winapi.SetNamedPipeHandleState(
+            h, _winapi.PIPE_READMODE_MESSAGE, None, None
             )
-        return _multiprocessing.PipeConnection(h)
+        return PipeConnection(h)
 
 #
 # Authentication stuff
@@ -458,10 +745,10 @@ class ConnectionWrapper(object):
         return self._loads(s)
 
 def _xml_dumps(obj):
-    return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8')
+    return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
 
 def _xml_loads(s):
-    (obj,), method = xmlrpclib.loads(s.decode('utf8'))
+    (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
     return obj
 
 class XmlListener(Listener):
@@ -475,3 +762,163 @@ def XmlClient(*args, **kwds):
     global xmlrpclib
     import xmlrpc.client as xmlrpclib
     return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
+
+#
+# Wait
+#
+
+if sys.platform == 'win32':
+
+    def _exhaustive_wait(handles, timeout):
+        # Return ALL handles which are currently signalled.  (Only
+        # returning the first signalled might create starvation issues.)
+        L = list(handles)
+        ready = []
+        while L:
+            res = _winapi.WaitForMultipleObjects(L, False, timeout)
+            if res == WAIT_TIMEOUT:
+                break
+            elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
+                res -= WAIT_OBJECT_0
+            elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
+                res -= WAIT_ABANDONED_0
+            else:
+                raise RuntimeError('Should not get here')
+            ready.append(L[res])
+            L = L[res+1:]
+            timeout = 0
+        return ready
+
+    _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
+
+    def wait(object_list, timeout=None):
+        '''
+        Wait till an object in object_list is ready/readable.
+
+        Returns list of those objects in object_list which are ready/readable.
+        '''
+        if timeout is None:
+            timeout = INFINITE
+        elif timeout < 0:
+            timeout = 0
+        else:
+            timeout = int(timeout * 1000 + 0.5)
+
+        object_list = list(object_list)
+        waithandle_to_obj = {}
+        ov_list = []
+        ready_objects = set()
+        ready_handles = set()
+
+        try:
+            for o in object_list:
+                try:
+                    fileno = getattr(o, 'fileno')
+                except AttributeError:
+                    waithandle_to_obj[o.__index__()] = o
+                else:
+                    # start an overlapped read of length zero
+                    try:
+                        ov, err = _winapi.ReadFile(fileno(), 0, True)
+                    except OSError as e:
+                        err = e.winerror
+                        if err not in _ready_errors:
+                            raise
+                    if err == _winapi.ERROR_IO_PENDING:
+                        ov_list.append(ov)
+                        waithandle_to_obj[ov.event] = o
+                    else:
+                        # If o.fileno() is an overlapped pipe handle and
+                        # err == 0 then there is a zero length message
+                        # in the pipe, but it HAS NOT been consumed.
+                        ready_objects.add(o)
+                        timeout = 0
+
+            ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
+        finally:
+            # request that overlapped reads stop
+            for ov in ov_list:
+                ov.cancel()
+
+            # wait for all overlapped reads to stop
+            for ov in ov_list:
+                try:
+                    _, err = ov.GetOverlappedResult(True)
+                except OSError as e:
+                    err = e.winerror
+                    if err not in _ready_errors:
+                        raise
+                if err != _winapi.ERROR_OPERATION_ABORTED:
+                    o = waithandle_to_obj[ov.event]
+                    ready_objects.add(o)
+                    if err == 0:
+                        # If o.fileno() is an overlapped pipe handle then
+                        # a zero length message HAS been consumed.
+                        if hasattr(o, '_got_empty_message'):
+                            o._got_empty_message = True
+
+        ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
+        return [o for o in object_list if o in ready_objects]
+
+else:
+
+    if hasattr(select, 'poll'):
+        def _poll(fds, timeout):
+            if timeout is not None:
+                timeout = int(timeout) * 1000  # timeout is in milliseconds
+            fd_map = {}
+            pollster = select.poll()
+            for fd in fds:
+                pollster.register(fd, select.POLLIN)
+                if hasattr(fd, 'fileno'):
+                    fd_map[fd.fileno()] = fd
+                else:
+                    fd_map[fd] = fd
+            ls = []
+            for fd, event in pollster.poll(timeout):
+                if event & select.POLLNVAL:
+                    raise ValueError('invalid file descriptor %i' % fd)
+                ls.append(fd_map[fd])
+            return ls
+    else:
+        def _poll(fds, timeout):
+            return select.select(fds, [], [], timeout)[0]
+
+
+    def wait(object_list, timeout=None):
+        '''
+        Wait till an object in object_list is ready/readable.
+
+        Returns list of those objects in object_list which are ready/readable.
+        '''
+        if timeout is not None:
+            if timeout <= 0:
+                return _poll(object_list, 0)
+            else:
+                deadline = time.time() + timeout
+        while True:
+            try:
+                return _poll(object_list, timeout)
+            except OSError as e:
+                if e.errno != errno.EINTR:
+                    raise
+            if timeout is not None:
+                timeout = deadline - time.time()
+
+#
+# Make connection and socket objects sharable if possible
+#
+
+if sys.platform == 'win32':
+    from . import reduction
+    ForkingPickler.register(socket.socket, reduction.reduce_socket)
+    ForkingPickler.register(Connection, reduction.reduce_connection)
+    ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
+else:
+    try:
+        from . import reduction
+    except ImportError:
+        pass
+    else:
+        ForkingPickler.register(socket.socket, reduction.reduce_socket)
+        ForkingPickler.register(Connection, reduction.reduce_connection)
diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py
index 101c3cba4d..e31fc61572 100644
--- a/Lib/multiprocessing/dummy/__init__.py
+++ b/Lib/multiprocessing/dummy/__init__.py
@@ -35,7 +35,7 @@
 __all__ = [
     'Process', 'current_process', 'active_children', 'freeze_support',
     'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
-    'Event', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue'
+    'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue'
     ]
 
 #
@@ -46,12 +46,10 @@ import threading
 import sys
 import weakref
 import array
-import itertools
 
-from multiprocessing import TimeoutError, cpu_count
 from multiprocessing.dummy.connection import Pipe
 from threading import Lock, RLock, Semaphore, BoundedSemaphore
-from threading import Event
+from threading import Event, Condition, Barrier
 from queue import Queue
 
 #
@@ -85,17 +83,6 @@ class DummyProcess(threading.Thread):
 #
 #
 
-class Condition(threading._Condition):
-    # XXX
-    if sys.version_info < (3, 0):
-        notify_all = threading._Condition.notify_all.__func__
-    else:
-        notify_all = threading._Condition.notify_all
-
-#
-#
-#
-
 Process = DummyProcess
 current_process = threading.current_thread
 current_process()._children = weakref.WeakKeyDictionary()
diff --git a/Lib/multiprocessing/dummy/connection.py b/Lib/multiprocessing/dummy/connection.py
index af105794f1..874ec8e432 100644
--- a/Lib/multiprocessing/dummy/connection.py
+++ b/Lib/multiprocessing/dummy/connection.py
@@ -53,6 +53,12 @@ class Listener(object):
 
     address = property(lambda self: self._backlog_queue)
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self.close()
+
 
 def Client(address):
     _in, _out = Queue(), Queue()
@@ -85,3 +91,9 @@ class Connection(object):
 
     def close(self):
         pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self.close()
diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py
index 8dc4b005fc..0bb21c469d 100644
--- a/Lib/multiprocessing/forking.py
+++ b/Lib/multiprocessing/forking.py
@@ -4,32 +4,7 @@
 # multiprocessing/forking.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 import os
@@ -39,7 +14,7 @@ import errno
 
 from multiprocessing import util, process
 
-__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close', 'ForkingPickler']
+__all__ = ['Popen', 'assert_spawning', 'duplicate', 'close', 'ForkingPickler']
 
 #
 # Check that the current thread is spawning a child process
@@ -56,18 +31,18 @@ def assert_spawning(self):
 # Try making some callable types picklable
 #
 
-from pickle import _Pickler as Pickler
+from pickle import Pickler
+from copyreg import dispatch_table
+
 class ForkingPickler(Pickler):
-    dispatch = Pickler.dispatch.copy()
+    _extra_reducers = {}
+    def __init__(self, *args):
+        Pickler.__init__(self, *args)
+        self.dispatch_table = dispatch_table.copy()
+        self.dispatch_table.update(self._extra_reducers)
     @classmethod
     def register(cls, type, reduce):
-        def dispatcher(self, obj):
-            rv = reduce(obj)
-            if isinstance(rv, str):
-                self.save_global(obj, rv)
-            else:
-                self.save_reduce(obj=obj, *rv)
-        cls.dispatch[type] = dispatcher
+        cls._extra_reducers[type] = reduce
 
 def _reduce_method(m):
     if m.__self__ is None:
@@ -101,9 +76,6 @@ else:
 #
 
 if sys.platform != 'win32':
-    import time
-
-    exit = os._exit
     duplicate = os.dup
     close = os.close
 
@@ -119,14 +91,23 @@ if sys.platform != 'win32':
             sys.stderr.flush()
             self.returncode = None
 
+            r, w = os.pipe()
+            self.sentinel = r
+
             self.pid = os.fork()
             if self.pid == 0:
+                os.close(r)
                 if 'random' in sys.modules:
                     import random
                     random.seed()
                 code = process_obj._bootstrap()
                 os._exit(code)
 
+            # `w` will be closed when the child exits, at which point `r`
+            # will become ready for reading (using e.g. select()).
+            os.close(w)
+            util.Finalize(self, os.close, (r,))
+
         def poll(self, flag=os.WNOHANG):
             if self.returncode is None:
                 while True:
@@ -149,26 +130,20 @@ if sys.platform != 'win32':
             return self.returncode
 
         def wait(self, timeout=None):
-            if timeout is None:
-                return self.poll(0)
-            deadline = time.time() + timeout
-            delay = 0.0005
-            while 1:
-                res = self.poll()
-                if res is not None:
-                    break
-                remaining = deadline - time.time()
-                if remaining <= 0:
-                    break
-                delay = min(delay * 2, remaining, 0.05)
-                time.sleep(delay)
-            return res
+            if self.returncode is None:
+                if timeout is not None:
+                    from .connection import wait
+                    if not wait([self.sentinel], timeout):
+                        return None
+                # This shouldn't block if wait() returned successfully.
+                return self.poll(os.WNOHANG if timeout == 0.0 else 0)
+            return self.returncode
 
         def terminate(self):
             if self.returncode is None:
                 try:
                     os.kill(self.pid, signal.SIGTERM)
-                except OSError as e:
+                except OSError:
                     if self.wait(timeout=0.1) is None:
                         raise
 
@@ -183,12 +158,9 @@ if sys.platform != 'win32':
 else:
     import _thread
     import msvcrt
-    import _subprocess
-    import time
+    import _winapi
 
-    from pickle import dump, load, HIGHEST_PROTOCOL
-    from _multiprocessing import win32, Connection, PipeConnection
-    from .util import Finalize
+    from pickle import load, HIGHEST_PROTOCOL
 
     def dump(obj, file, protocol=None):
         ForkingPickler(file, protocol).dump(obj)
@@ -201,8 +173,7 @@ else:
     WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
     WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
 
-    exit = win32.ExitProcess
-    close = win32.CloseHandle
+    close = _winapi.CloseHandle
 
     #
     # _python_exe is the assumed path to the python executable.
@@ -224,11 +195,11 @@ else:
 
     def duplicate(handle, target_process=None, inheritable=False):
         if target_process is None:
-            target_process = _subprocess.GetCurrentProcess()
-        return _subprocess.DuplicateHandle(
-            _subprocess.GetCurrentProcess(), handle, target_process,
-            0, inheritable, _subprocess.DUPLICATE_SAME_ACCESS
-            ).Detach()
+            target_process = _winapi.GetCurrentProcess()
+        return _winapi.DuplicateHandle(
+            _winapi.GetCurrentProcess(), handle, target_process,
+            0, inheritable, _winapi.DUPLICATE_SAME_ACCESS
+            )
 
     #
     # We define a Popen class similar to the one from subprocess, but
@@ -242,6 +213,9 @@ else:
         _tls = _thread._local()
 
         def __init__(self, process_obj):
+            cmd = ' '.join('"%s"' % x for x in get_command_line())
+            prep_data = get_preparation_data(process_obj._name)
+
             # create pipe for communication with child
             rfd, wfd = os.pipe()
 
@@ -249,30 +223,31 @@ else:
             rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
             os.close(rfd)
 
-            # start process
-            cmd = get_command_line() + [rhandle]
-            cmd = ' '.join('"%s"' % x for x in cmd)
-            hp, ht, pid, tid = _subprocess.CreateProcess(
-                _python_exe, cmd, None, None, 1, 0, None, None, None
-                )
-            ht.Close()
-            close(rhandle)
-
-            # set attributes of self
-            self.pid = pid
-            self.returncode = None
-            self._handle = hp
-
-            # send information to child
-            prep_data = get_preparation_data(process_obj._name)
-            to_child = os.fdopen(wfd, 'wb')
-            Popen._tls.process_handle = int(hp)
-            try:
-                dump(prep_data, to_child, HIGHEST_PROTOCOL)
-                dump(process_obj, to_child, HIGHEST_PROTOCOL)
-            finally:
-                del Popen._tls.process_handle
-                to_child.close()
+            with open(wfd, 'wb', closefd=True) as to_child:
+                # start process
+                try:
+                    hp, ht, pid, tid = _winapi.CreateProcess(
+                        _python_exe, cmd + (' %s' % rhandle),
+                        None, None, 1, 0, None, None, None
+                        )
+                    _winapi.CloseHandle(ht)
+                finally:
+                    close(rhandle)
+
+                # set attributes of self
+                self.pid = pid
+                self.returncode = None
+                self._handle = hp
+                self.sentinel = int(hp)
+                util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
+
+                # send information to child
+                Popen._tls.process_handle = int(hp)
+                try:
+                    dump(prep_data, to_child, HIGHEST_PROTOCOL)
+                    dump(process_obj, to_child, HIGHEST_PROTOCOL)
+                finally:
+                    del Popen._tls.process_handle
 
         @staticmethod
         def thread_is_spawning():
@@ -285,13 +260,13 @@ else:
         def wait(self, timeout=None):
             if self.returncode is None:
                 if timeout is None:
-                    msecs = _subprocess.INFINITE
+                    msecs = _winapi.INFINITE
                 else:
                     msecs = max(0, int(timeout * 1000 + 0.5))
 
-                res = _subprocess.WaitForSingleObject(int(self._handle), msecs)
-                if res == _subprocess.WAIT_OBJECT_0:
-                    code = _subprocess.GetExitCodeProcess(self._handle)
+                res = _winapi.WaitForSingleObject(int(self._handle), msecs)
+                if res == _winapi.WAIT_OBJECT_0:
+                    code = _winapi.GetExitCodeProcess(self._handle)
                     if code == TERMINATE:
                         code = -signal.SIGTERM
                     self.returncode = code
@@ -304,9 +279,9 @@ else:
         def terminate(self):
             if self.returncode is None:
                 try:
-                    _subprocess.TerminateProcess(int(self._handle), TERMINATE)
-                except WindowsError:
-                    if self.wait(timeout=0.1) is None:
+                    _winapi.TerminateProcess(int(self._handle), TERMINATE)
+                except OSError:
+                    if self.wait(timeout=1.0) is None:
                         raise
 
     #
@@ -356,7 +331,8 @@ else:
             return [sys.executable, '--multiprocessing-fork']
         else:
             prog = 'from multiprocessing.forking import main; main()'
-            return [_python_exe, '-c', prog, '--multiprocessing-fork']
+            opts = util._args_from_interpreter_flags()
+            return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
 
 
     def main():
@@ -378,7 +354,7 @@ else:
         from_parent.close()
 
         exitcode = self._bootstrap()
-        exit(exitcode)
+        sys.exit(exitcode)
 
 
     def get_preparation_data(name):
@@ -411,22 +387,6 @@ else:
 
         return d
 
-    #
-    # Make (Pipe)Connection picklable
-    #
-
-    def reduce_connection(conn):
-        if not Popen.thread_is_spawning():
-            raise RuntimeError(
-                'By default %s objects can only be shared between processes\n'
-                'using inheritance' % type(conn).__name__
-                )
-        return type(conn), (Popen.duplicate_for_child(conn.fileno()),
-                            conn.readable, conn.writable)
-
-    ForkingPickler.register(Connection, reduce_connection)
-    ForkingPickler.register(PipeConnection, reduce_connection)
-
 #
 # Prepare current process
 #
diff --git a/Lib/multiprocessing/heap.py b/Lib/multiprocessing/heap.py
index 0a25ef05c7..4e93c12f81 100644
--- a/Lib/multiprocessing/heap.py
+++ b/Lib/multiprocessing/heap.py
@@ -4,37 +4,11 @@
 # multiprocessing/heap.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 import bisect
 import mmap
-import tempfile
 import os
 import sys
 import threading
@@ -52,7 +26,7 @@ __all__ = ['BufferWrapper']
 
 if sys.platform == 'win32':
 
-    from _multiprocessing import win32
+    import _winapi
 
     class Arena(object):
 
@@ -62,7 +36,7 @@ if sys.platform == 'win32':
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter))
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            assert _winapi.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +46,7 @@ if sys.platform == 'win32':
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS
 
 else:
 
@@ -231,7 +205,7 @@ class Heap(object):
             self._lock.release()
 
 #
-# Class representing a chunk of an mmap -- can be inherited
+# Class representing a chunk of an mmap -- can be inherited by child process
 #
 
 class BufferWrapper(object):
@@ -244,11 +218,6 @@ class BufferWrapper(object):
         self._state = (block, size)
         Finalize(self, BufferWrapper._heap.free, args=(block,))
 
-    def get_address(self):
+    def create_memoryview(self):
         (arena, start, stop), size = self._state
-        address, length = _multiprocessing.address_of_buffer(arena.buffer)
-        assert size <= length
-        return address + start
-
-    def get_size(self):
-        return self._state[1]
+        return memoryview(arena.buffer)[start:start+size]
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index 5588ead116..1ab147e29e 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -5,32 +5,7 @@
 # multiprocessing/managers.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
@@ -39,19 +14,16 @@ __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
 # Imports
 #
 
-import os
 import sys
-import weakref
 import threading
 import array
 import queue
 
 from traceback import format_exc
-from pickle import PicklingError
 from multiprocessing import Process, current_process, active_children, Pool, util, connection
 from multiprocessing.process import AuthenticationString
-from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler
-from multiprocessing.util import Finalize, info
+from multiprocessing.forking import Popen, ForkingPickler
+from time import time as _time
 
 #
 # Register some things for pickling
@@ -168,28 +140,38 @@ class Server(object):
         self.id_to_obj = {'0': (None, ())}
         self.id_to_refcount = {}
         self.mutex = threading.RLock()
-        self.stop = 0
 
     def serve_forever(self):
         '''
         Run the server forever
         '''
+        self.stop_event = threading.Event()
         current_process()._manager_server = self
         try:
+            accepter = threading.Thread(target=self.accepter)
+            accepter.daemon = True
+            accepter.start()
             try:
-                while 1:
-                    try:
-                        c = self.listener.accept()
-                    except (OSError, IOError):
-                        continue
-                    t = threading.Thread(target=self.handle_request, args=(c,))
-                    t.daemon = True
-                    t.start()
+                while not self.stop_event.is_set():
+                    self.stop_event.wait(1)
             except (KeyboardInterrupt, SystemExit):
                 pass
         finally:
-            self.stop = 999
-            self.listener.close()
+            if sys.stdout != sys.__stdout__:
+                util.debug('resetting stdout, stderr')
+                sys.stdout = sys.__stdout__
+                sys.stderr = sys.__stderr__
+            sys.exit(0)
+
+    def accepter(self):
+        while True:
+            try:
+                c = self.listener.accept()
+            except (OSError, IOError):
+                continue
+            t = threading.Thread(target=self.handle_request, args=(c,))
+            t.daemon = True
+            t.start()
 
     def handle_request(self, c):
         '''
@@ -236,7 +218,7 @@ class Server(object):
         send = conn.send
         id_to_obj = self.id_to_obj
 
-        while not self.stop:
+        while not self.stop_event.is_set():
 
             try:
                 methodname = obj = None
@@ -346,32 +328,13 @@ class Server(object):
         Shutdown this process
         '''
         try:
-            try:
-                util.debug('manager received shutdown message')
-                c.send(('#RETURN', None))
-
-                if sys.stdout != sys.__stdout__:
-                    util.debug('resetting stdout, stderr')
-                    sys.stdout = sys.__stdout__
-                    sys.stderr = sys.__stderr__
-
-                util._run_finalizers(0)
-
-                for p in active_children():
-                    util.debug('terminating a child process of manager')
-                    p.terminate()
-
-                for p in active_children():
-                    util.debug('terminating a child process of manager')
-                    p.join()
-
-                util._run_finalizers()
-                util.info('manager exiting with exitcode 0')
-            except:
-                import traceback
-                traceback.print_exc()
+            util.debug('manager received shutdown message')
+            c.send(('#RETURN', None))
+        except:
+            import traceback
+            traceback.print_exc()
         finally:
-            exit(0)
+            self.stop_event.set()
 
     def create(self, c, typeid, *args, **kwds):
         '''
@@ -483,10 +446,6 @@ class BaseManager(object):
         self._serializer = serializer
         self._Listener, self._Client = listener_client[serializer]
 
-    def __reduce__(self):
-        return type(self).from_address, \
-               (self._address, self._authkey, self._serializer)
-
     def get_server(self):
         '''
         Return server object with serve_forever() method and address attribute
@@ -576,7 +535,10 @@ class BaseManager(object):
         '''
         Join the manager process (if it has been spawned)
         '''
-        self._process.join(timeout)
+        if self._process is not None:
+            self._process.join(timeout)
+            if not self._process.is_alive():
+                self._process = None
 
     def _debug_info(self):
         '''
@@ -599,6 +561,9 @@ class BaseManager(object):
             conn.close()
 
     def __enter__(self):
+        if self._state.value == State.INITIAL:
+            self.start()
+        assert self._state.value == State.STARTED
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -620,7 +585,7 @@ class BaseManager(object):
             except Exception:
                 pass
 
-            process.join(timeout=0.2)
+            process.join(timeout=1.0)
             if process.is_alive():
                 util.info('manager still alive')
                 if hasattr(process, 'terminate'):
@@ -982,8 +947,9 @@ class IteratorProxy(BaseProxy):
 
 class AcquirerProxy(BaseProxy):
     _exposed_ = ('acquire', 'release')
-    def acquire(self, blocking=True):
-        return self._callmethod('acquire', (blocking,))
+    def acquire(self, blocking=True, timeout=None):
+        args = (blocking,) if timeout is None else (blocking, timeout)
+        return self._callmethod('acquire', args)
     def release(self):
         return self._callmethod('release')
     def __enter__(self):
@@ -1000,6 +966,24 @@ class ConditionProxy(AcquirerProxy):
         return self._callmethod('notify')
     def notify_all(self):
         return self._callmethod('notify_all')
+    def wait_for(self, predicate, timeout=None):
+        result = predicate()
+        if result:
+            return result
+        if timeout is not None:
+            endtime = _time() + timeout
+        else:
+            endtime = None
+            waittime = None
+        while not result:
+            if endtime is not None:
+                waittime = endtime - _time()
+                if waittime <= 0:
+                    break
+            self.wait(waittime)
+            result = predicate()
+        return result
+
 
 class EventProxy(BaseProxy):
     _exposed_ = ('is_set', 'set', 'clear', 'wait')
@@ -1012,6 +996,26 @@ class EventProxy(BaseProxy):
     def wait(self, timeout=None):
         return self._callmethod('wait', (timeout,))
 
+
+class BarrierProxy(BaseProxy):
+    _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset')
+    def wait(self, timeout=None):
+        return self._callmethod('wait', (timeout,))
+    def abort(self):
+        return self._callmethod('abort')
+    def reset(self):
+        return self._callmethod('reset')
+    @property
+    def parties(self):
+        return self._callmethod('__getattribute__', ('parties',))
+    @property
+    def n_waiting(self):
+        return self._callmethod('__getattribute__', ('n_waiting',))
+    @property
+    def broken(self):
+        return self._callmethod('__getattribute__', ('broken',))
+
+
 class NamespaceProxy(BaseProxy):
     _exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
     def __getattr__(self, key):
@@ -1041,12 +1045,11 @@ class ValueProxy(BaseProxy):
 
 
 BaseListProxy = MakeProxyType('BaseListProxy', (
-    '__add__', '__contains__', '__delitem__', '__delslice__',
-    '__getitem__', '__getslice__', '__len__', '__mul__',
-    '__reversed__', '__rmul__', '__setitem__', '__setslice__',
+    '__add__', '__contains__', '__delitem__', '__getitem__', '__len__',
+    '__mul__', '__reversed__', '__rmul__', '__setitem__',
     'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove',
     'reverse', 'sort', '__imul__'
-    ))                  # XXX __getslice__ and __setslice__ unneeded in Py3.0
+    ))
 class ListProxy(BaseListProxy):
     def __iadd__(self, value):
         self._callmethod('extend', (value,))
@@ -1064,17 +1067,18 @@ DictProxy = MakeProxyType('DictProxy', (
 
 
 ArrayProxy = MakeProxyType('ArrayProxy', (
-    '__len__', '__getitem__', '__setitem__', '__getslice__', '__setslice__'
-    ))                  # XXX __getslice__ and __setslice__ unneeded in Py3.0
+    '__len__', '__getitem__', '__setitem__'
+    ))
 
 
 PoolProxy = MakeProxyType('PoolProxy', (
     'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',
-    'map', 'map_async', 'terminate'
+    'map', 'map_async', 'starmap', 'starmap_async', 'terminate'
     ))
 PoolProxy._method_to_typeid_ = {
     'apply_async': 'AsyncResult',
     'map_async': 'AsyncResult',
+    'starmap_async': 'AsyncResult',
     'imap': 'Iterator',
     'imap_unordered': 'Iterator'
     }
@@ -1103,6 +1107,7 @@ SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
 SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
                      AcquirerProxy)
 SyncManager.register('Condition', threading.Condition, ConditionProxy)
+SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
 SyncManager.register('Pool', Pool, PoolProxy)
 SyncManager.register('list', list, ListProxy)
 SyncManager.register('dict', dict, DictProxy)
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py
index 7502ff89f4..7f73b441c2 100644
--- a/Lib/multiprocessing/pool.py
+++ b/Lib/multiprocessing/pool.py
@@ -4,32 +4,7 @@
 # multiprocessing/pool.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 __all__ = ['Pool']
@@ -64,6 +39,9 @@ job_counter = itertools.count()
 def mapstar(args):
     return list(map(*args))
 
+def starmapstar(args):
+    return list(itertools.starmap(args[0], args[1]))
+
 #
 # Code run by worker processes
 #
@@ -247,14 +225,30 @@ class Pool(object):
         Apply `func` to each element in `iterable`, collecting the results
         in a list that is returned.
         '''
-        assert self._state == RUN
-        return self.map_async(func, iterable, chunksize).get()
+        return self._map_async(func, iterable, mapstar, chunksize).get()
+
+    def starmap(self, func, iterable, chunksize=None):
+        '''
+        Like `map()` method but the elements of the `iterable` are expected to
+        be iterables as well and will be unpacked as arguments. Hence
+        `func` and (a, b) becomes func(a, b).
+        '''
+        return self._map_async(func, iterable, starmapstar, chunksize).get()
+
+    def starmap_async(self, func, iterable, chunksize=None, callback=None,
+            error_callback=None):
+        '''
+        Asynchronous version of `starmap()` method.
+        '''
+        return self._map_async(func, iterable, starmapstar, chunksize,
+                               callback, error_callback)
 
     def imap(self, func, iterable, chunksize=1):
         '''
         Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
         '''
-        assert self._state == RUN
+        if self._state != RUN:
+            raise ValueError("Pool not running")
         if chunksize == 1:
             result = IMapIterator(self._cache)
             self._taskqueue.put((((result._job, i, func, (x,), {})
@@ -272,7 +266,8 @@ class Pool(object):
         '''
         Like `imap()` method but ordering of results is arbitrary.
         '''
-        assert self._state == RUN
+        if self._state != RUN:
+            raise ValueError("Pool not running")
         if chunksize == 1:
             result = IMapUnorderedIterator(self._cache)
             self._taskqueue.put((((result._job, i, func, (x,), {})
@@ -291,7 +286,8 @@ class Pool(object):
         '''
         Asynchronous version of `apply()` method.
         '''
-        assert self._state == RUN
+        if self._state != RUN:
+            raise ValueError("Pool not running")
         result = ApplyResult(self._cache, callback, error_callback)
         self._taskqueue.put(([(result._job, None, func, args, kwds)], None))
         return result
@@ -301,7 +297,16 @@ class Pool(object):
         '''
         Asynchronous version of `map()` method.
         '''
-        assert self._state == RUN
+        return self._map_async(func, iterable, mapstar, chunksize, callback,
+            error_callback)
+
+    def _map_async(self, func, iterable, mapper, chunksize=None, callback=None,
+            error_callback=None):
+        '''
+        Helper function to implement map, starmap and their async counterparts.
+        '''
+        if self._state != RUN:
+            raise ValueError("Pool not running")
         if not hasattr(iterable, '__len__'):
             iterable = list(iterable)
 
@@ -315,7 +320,7 @@ class Pool(object):
         task_batches = Pool._get_tasks(func, iterable, chunksize)
         result = MapResult(self._cache, chunksize, len(iterable), callback,
                            error_callback=error_callback)
-        self._taskqueue.put((((result._job, i, mapstar, (x,), {})
+        self._taskqueue.put((((result._job, i, mapper, (x,), {})
                               for i, x in enumerate(task_batches)), None))
         return result
 
@@ -519,6 +524,12 @@ class Pool(object):
                     debug('cleaning up worker %d' % p.pid)
                     p.join()
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.terminate()
+
 #
 # Class whose instances are returned by `Pool.apply_async()`
 #
@@ -526,32 +537,26 @@ class Pool(object):
 class ApplyResult(object):
 
     def __init__(self, cache, callback, error_callback):
-        self._cond = threading.Condition(threading.Lock())
+        self._event = threading.Event()
         self._job = next(job_counter)
         self._cache = cache
-        self._ready = False
         self._callback = callback
         self._error_callback = error_callback
         cache[self._job] = self
 
     def ready(self):
-        return self._ready
+        return self._event.is_set()
 
     def successful(self):
-        assert self._ready
+        assert self.ready()
         return self._success
 
     def wait(self, timeout=None):
-        self._cond.acquire()
-        try:
-            if not self._ready:
-                self._cond.wait(timeout)
-        finally:
-            self._cond.release()
+        self._event.wait(timeout)
 
     def get(self, timeout=None):
         self.wait(timeout)
-        if not self._ready:
+        if not self.ready():
             raise TimeoutError
         if self._success:
             return self._value
@@ -564,12 +569,7 @@ class ApplyResult(object):
             self._callback(self._value)
         if self._error_callback and not self._success:
             self._error_callback(self._value)
-        self._cond.acquire()
-        try:
-            self._ready = True
-            self._cond.notify()
-        finally:
-            self._cond.release()
+        self._event.set()
         del self._cache[self._job]
 
 #
@@ -586,7 +586,7 @@ class MapResult(ApplyResult):
         self._chunksize = chunksize
         if chunksize <= 0:
             self._number_left = 0
-            self._ready = True
+            self._event.set()
             del cache[self._job]
         else:
             self._number_left = length//chunksize + bool(length % chunksize)
@@ -600,24 +600,14 @@ class MapResult(ApplyResult):
                 if self._callback:
                     self._callback(self._value)
                 del self._cache[self._job]
-                self._cond.acquire()
-                try:
-                    self._ready = True
-                    self._cond.notify()
-                finally:
-                    self._cond.release()
+                self._event.set()
         else:
             self._success = False
             self._value = result
             if self._error_callback:
                 self._error_callback(self._value)
             del self._cache[self._job]
-            self._cond.acquire()
-            try:
-                self._ready = True
-                self._cond.notify()
-            finally:
-                self._cond.release()
+            self._event.set()
 
 #
 # Class whose instances are returned by `Pool.imap()`
diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py
index 3262b50f50..893507bcd5 100644
--- a/Lib/multiprocessing/process.py
+++ b/Lib/multiprocessing/process.py
@@ -4,32 +4,7 @@
 # multiprocessing/process.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 __all__ = ['Process', 'current_process', 'active_children']
@@ -92,12 +67,16 @@ class Process(object):
     '''
     _Popen = None
 
-    def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
+    def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
+                 *, daemon=None):
         assert group is None, 'group argument must be None for now'
         count = next(_current_process._counter)
         self._identity = _current_process._identity + (count,)
         self._authkey = _current_process._authkey
-        self._daemonic = _current_process._daemonic
+        if daemon is not None:
+            self._daemonic = daemon
+        else:
+            self._daemonic = _current_process._daemonic
         self._tempdir = _current_process._tempdir
         self._parent_pid = os.getpid()
         self._popen = None
@@ -130,6 +109,7 @@ class Process(object):
         else:
             from .forking import Popen
         self._popen = Popen(self)
+        self._sentinel = self._popen.sentinel
         _current_process._children.add(self)
 
     def terminate(self):
@@ -216,6 +196,17 @@ class Process(object):
 
     pid = ident
 
+    @property
+    def sentinel(self):
+        '''
+        Return a file descriptor (Unix) or handle (Windows) suitable for
+        waiting for process termination.
+        '''
+        try:
+            return self._sentinel
+        except AttributeError:
+            raise ValueError("process not started")
+
     def __repr__(self):
         if self is _current_process:
             status = 'started'
diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py
index 51d991245c..37271fb4eb 100644
--- a/Lib/multiprocessing/queues.py
+++ b/Lib/multiprocessing/queues.py
@@ -4,32 +4,7 @@
 # multiprocessing/queues.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
@@ -39,12 +14,12 @@ import os
 import threading
 import collections
 import time
-import atexit
 import weakref
+import errno
 
 from queue import Empty, Full
 import _multiprocessing
-from multiprocessing import Pipe
+from multiprocessing.connection import Pipe
 from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition
 from multiprocessing.util import debug, info, Finalize, register_after_fork
 from multiprocessing.forking import assert_spawning
@@ -67,6 +42,8 @@ class Queue(object):
         else:
             self._wlock = Lock()
         self._sem = BoundedSemaphore(maxsize)
+        # For use by concurrent.futures
+        self._ignore_epipe = False
 
         self._after_fork()
 
@@ -75,11 +52,11 @@ class Queue(object):
 
     def __getstate__(self):
         assert_spawning(self)
-        return (self._maxsize, self._reader, self._writer,
+        return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
                 self._rlock, self._wlock, self._sem, self._opid)
 
     def __setstate__(self, state):
-        (self._maxsize, self._reader, self._writer,
+        (self._ignore_epipe, self._maxsize, self._reader, self._writer,
          self._rlock, self._wlock, self._sem, self._opid) = state
         self._after_fork()
 
@@ -182,7 +159,7 @@ class Queue(object):
         self._thread = threading.Thread(
             target=Queue._feed,
             args=(self._buffer, self._notempty, self._send,
-                  self._wlock, self._writer.close),
+                  self._wlock, self._writer.close, self._ignore_epipe),
             name='QueueFeederThread'
             )
         self._thread.daemon = True
@@ -233,7 +210,7 @@ class Queue(object):
             notempty.release()
 
     @staticmethod
-    def _feed(buffer, notempty, send, writelock, close):
+    def _feed(buffer, notempty, send, writelock, close, ignore_epipe):
         debug('starting thread to feed data to pipe')
         from .util import is_exiting
 
@@ -275,6 +252,8 @@ class Queue(object):
                 except IndexError:
                     pass
         except Exception as e:
+            if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
+                return
             # Since this runs in a daemon thread the resources it uses
             # may be become unusable while the process is cleaning up.
             # We ignore errors which happen after the process has
@@ -356,6 +335,7 @@ class SimpleQueue(object):
     def __init__(self):
         self._reader, self._writer = Pipe(duplex=False)
         self._rlock = Lock()
+        self._poll = self._reader.poll
         if sys.platform == 'win32':
             self._wlock = None
         else:
@@ -363,7 +343,7 @@ class SimpleQueue(object):
         self._make_methods()
 
     def empty(self):
-        return not self._reader.poll()
+        return not self._poll()
 
     def __getstate__(self):
         assert_spawning(self)
diff --git a/Lib/multiprocessing/reduction.py b/Lib/multiprocessing/reduction.py
index 6e5e5bc9de..656fa8ff6b 100644
--- a/Lib/multiprocessing/reduction.py
+++ b/Lib/multiprocessing/reduction.py
@@ -5,53 +5,29 @@
 # multiprocessing/reduction.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
-__all__ = []
+__all__ = ['reduce_socket', 'reduce_connection', 'send_handle', 'recv_handle']
 
 import os
 import sys
 import socket
 import threading
+import struct
+import signal
 
-import _multiprocessing
 from multiprocessing import current_process
-from multiprocessing.forking import Popen, duplicate, close, ForkingPickler
 from multiprocessing.util import register_after_fork, debug, sub_debug
-from multiprocessing.connection import Client, Listener
+from multiprocessing.util import is_exiting, sub_warning
 
 
 #
 #
 #
 
-if not(sys.platform == 'win32' or hasattr(_multiprocessing, 'recvfd')):
+if not(sys.platform == 'win32' or (hasattr(socket, 'CMSG_LEN') and
+                                   hasattr(socket, 'SCM_RIGHTS'))):
     raise ImportError('pickling of connections not supported')
 
 #
@@ -59,157 +35,246 @@ if not(sys.platform == 'win32' or hasattr(_multiprocessing, 'recvfd')):
 #
 
 if sys.platform == 'win32':
-    import _subprocess
-    from _multiprocessing import win32
-
-    def send_handle(conn, handle, destination_pid):
-        process_handle = win32.OpenProcess(
-            win32.PROCESS_ALL_ACCESS, False, destination_pid
-            )
-        try:
-            new_handle = duplicate(handle, process_handle)
-            conn.send(new_handle)
-        finally:
-            close(process_handle)
-
-    def recv_handle(conn):
-        return conn.recv()
+    # Windows
+    __all__ += ['reduce_pipe_connection']
+    import _winapi
 
-else:
     def send_handle(conn, handle, destination_pid):
-        _multiprocessing.sendfd(conn.fileno(), handle)
+        dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid)
+        conn.send(dh)
 
     def recv_handle(conn):
-        return _multiprocessing.recvfd(conn.fileno())
-
-#
-# Support for a per-process server thread which caches pickled handles
-#
-
-_cache = set()
-
-def _reset(obj):
-    global _lock, _listener, _cache
-    for h in _cache:
-        close(h)
-    _cache.clear()
-    _lock = threading.Lock()
-    _listener = None
-
-_reset(None)
-register_after_fork(_reset, _reset)
-
-def _get_listener():
-    global _listener
-
-    if _listener is None:
-        _lock.acquire()
-        try:
-            if _listener is None:
-                debug('starting listener and thread for sending handles')
-                _listener = Listener(authkey=current_process().authkey)
-                t = threading.Thread(target=_serve)
-                t.daemon = True
-                t.start()
-        finally:
-            _lock.release()
-
-    return _listener
-
-def _serve():
-    from .util import is_exiting, sub_warning
-
-    while 1:
-        try:
-            conn = _listener.accept()
-            handle_wanted, destination_pid = conn.recv()
-            _cache.remove(handle_wanted)
-            send_handle(conn, handle_wanted, destination_pid)
-            close(handle_wanted)
-            conn.close()
-        except:
-            if not is_exiting():
-                import traceback
-                sub_warning(
-                    'thread for sharing handles raised exception :\n' +
-                    '-'*79 + '\n' + traceback.format_exc() + '-'*79
-                    )
-
-#
-# Functions to be used for pickling/unpickling objects with handles
-#
-
-def reduce_handle(handle):
-    if Popen.thread_is_spawning():
-        return (None, Popen.duplicate_for_child(handle), True)
-    dup_handle = duplicate(handle)
-    _cache.add(dup_handle)
-    sub_debug('reducing handle %d', handle)
-    return (_get_listener().address, dup_handle, False)
-
-def rebuild_handle(pickled_data):
-    address, handle, inherited = pickled_data
-    if inherited:
-        return handle
-    sub_debug('rebuilding handle %d', handle)
-    conn = Client(address, authkey=current_process().authkey)
-    conn.send((handle, os.getpid()))
-    new_handle = recv_handle(conn)
-    conn.close()
-    return new_handle
+        return conn.recv().detach()
+
+    class DupHandle(object):
+        def __init__(self, handle, access, pid=None):
+            # duplicate handle for process with given pid
+            if pid is None:
+                pid = os.getpid()
+            proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid)
+            try:
+                self._handle = _winapi.DuplicateHandle(
+                    _winapi.GetCurrentProcess(),
+                    handle, proc, access, False, 0)
+            finally:
+                _winapi.CloseHandle(proc)
+            self._access = access
+            self._pid = pid
+
+        def detach(self):
+            # retrieve handle from process which currently owns it
+            if self._pid == os.getpid():
+                return self._handle
+            proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False,
+                                       self._pid)
+            try:
+                return _winapi.DuplicateHandle(
+                    proc, self._handle, _winapi.GetCurrentProcess(),
+                    self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE)
+            finally:
+                _winapi.CloseHandle(proc)
+
+    class DupSocket(object):
+        def __init__(self, sock):
+            new_sock = sock.dup()
+            def send(conn, pid):
+                share = new_sock.share(pid)
+                conn.send_bytes(share)
+            self._id = resource_sharer.register(send, new_sock.close)
+
+        def detach(self):
+            conn = resource_sharer.get_connection(self._id)
+            try:
+                share = conn.recv_bytes()
+                return socket.fromshare(share)
+            finally:
+                conn.close()
+
+    def reduce_socket(s):
+        return rebuild_socket, (DupSocket(s),)
+
+    def rebuild_socket(ds):
+        return ds.detach()
+
+    def reduce_connection(conn):
+        handle = conn.fileno()
+        with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
+            ds = DupSocket(s)
+            return rebuild_connection, (ds, conn.readable, conn.writable)
+
+    def rebuild_connection(ds, readable, writable):
+        from .connection import Connection
+        sock = ds.detach()
+        return Connection(sock.detach(), readable, writable)
 
-#
-# Register `_multiprocessing.Connection` with `ForkingPickler`
-#
-
-def reduce_connection(conn):
-    rh = reduce_handle(conn.fileno())
-    return rebuild_connection, (rh, conn.readable, conn.writable)
-
-def rebuild_connection(reduced_handle, readable, writable):
-    handle = rebuild_handle(reduced_handle)
-    return _multiprocessing.Connection(
-        handle, readable=readable, writable=writable
-        )
-
-ForkingPickler.register(_multiprocessing.Connection, reduce_connection)
+    def reduce_pipe_connection(conn):
+        access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
+                  (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
+        dh = DupHandle(conn.fileno(), access)
+        return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
 
-#
-# Register `socket.socket` with `ForkingPickler`
-#
+    def rebuild_pipe_connection(dh, readable, writable):
+        from .connection import PipeConnection
+        handle = dh.detach()
+        return PipeConnection(handle, readable, writable)
 
-def fromfd(fd, family, type_, proto=0):
-    s = socket.fromfd(fd, family, type_, proto)
-    if s.__class__ is not socket.socket:
-        s = socket.socket(_sock=s)
-    return s
+else:
+    # Unix
 
-def reduce_socket(s):
-    reduced_handle = reduce_handle(s.fileno())
-    return rebuild_socket, (reduced_handle, s.family, s.type, s.proto)
+    # On MacOSX we should acknowledge receipt of fds -- see Issue14669
+    ACKNOWLEDGE = sys.platform == 'darwin'
 
-def rebuild_socket(reduced_handle, family, type_, proto):
-    fd = rebuild_handle(reduced_handle)
-    _sock = fromfd(fd, family, type_, proto)
-    close(fd)
-    return _sock
+    def send_handle(conn, handle, destination_pid):
+        with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
+            s.sendmsg([b'x'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS,
+                                struct.pack("@i", handle))])
+        if ACKNOWLEDGE and conn.recv_bytes() != b'ACK':
+            raise RuntimeError('did not receive acknowledgement of fd')
 
-ForkingPickler.register(socket.socket, reduce_socket)
+    def recv_handle(conn):
+        size = struct.calcsize("@i")
+        with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
+            msg, ancdata, flags, addr = s.recvmsg(1, socket.CMSG_LEN(size))
+            try:
+                if ACKNOWLEDGE:
+                    conn.send_bytes(b'ACK')
+                cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+                if (cmsg_level == socket.SOL_SOCKET and
+                    cmsg_type == socket.SCM_RIGHTS):
+                    return struct.unpack("@i", cmsg_data[:size])[0]
+            except (ValueError, IndexError, struct.error):
+                pass
+            raise RuntimeError('Invalid data received')
+
+    class DupFd(object):
+        def __init__(self, fd):
+            new_fd = os.dup(fd)
+            def send(conn, pid):
+                send_handle(conn, new_fd, pid)
+            def close():
+                os.close(new_fd)
+            self._id = resource_sharer.register(send, close)
+
+        def detach(self):
+            conn = resource_sharer.get_connection(self._id)
+            try:
+                return recv_handle(conn)
+            finally:
+                conn.close()
+
+    def reduce_socket(s):
+        df = DupFd(s.fileno())
+        return rebuild_socket, (df, s.family, s.type, s.proto)
+
+    def rebuild_socket(df, family, type, proto):
+        fd = df.detach()
+        s = socket.fromfd(fd, family, type, proto)
+        os.close(fd)
+        return s
+
+    def reduce_connection(conn):
+        df = DupFd(conn.fileno())
+        return rebuild_connection, (df, conn.readable, conn.writable)
+
+    def rebuild_connection(df, readable, writable):
+        from .connection import Connection
+        fd = df.detach()
+        return Connection(fd, readable, writable)
 
 #
-# Register `_multiprocessing.PipeConnection` with `ForkingPickler`
+# Server which shares registered resources with clients
 #
 
-if sys.platform == 'win32':
-
-    def reduce_pipe_connection(conn):
-        rh = reduce_handle(conn.fileno())
-        return rebuild_pipe_connection, (rh, conn.readable, conn.writable)
-
-    def rebuild_pipe_connection(reduced_handle, readable, writable):
-        handle = rebuild_handle(reduced_handle)
-        return _multiprocessing.PipeConnection(
-            handle, readable=readable, writable=writable
-            )
-
-    ForkingPickler.register(_multiprocessing.PipeConnection, reduce_pipe_connection)
+class ResourceSharer(object):
+    def __init__(self):
+        self._key = 0
+        self._cache = {}
+        self._old_locks = []
+        self._lock = threading.Lock()
+        self._listener = None
+        self._address = None
+        self._thread = None
+        register_after_fork(self, ResourceSharer._afterfork)
+
+    def register(self, send, close):
+        with self._lock:
+            if self._address is None:
+                self._start()
+            self._key += 1
+            self._cache[self._key] = (send, close)
+            return (self._address, self._key)
+
+    @staticmethod
+    def get_connection(ident):
+        from .connection import Client
+        address, key = ident
+        c = Client(address, authkey=current_process().authkey)
+        c.send((key, os.getpid()))
+        return c
+
+    def stop(self, timeout=None):
+        from .connection import Client
+        with self._lock:
+            if self._address is not None:
+                c = Client(self._address, authkey=current_process().authkey)
+                c.send(None)
+                c.close()
+                self._thread.join(timeout)
+                if self._thread.is_alive():
+                    sub_warn('ResourceSharer thread did not stop when asked')
+                self._listener.close()
+                self._thread = None
+                self._address = None
+                self._listener = None
+                for key, (send, close) in self._cache.items():
+                    close()
+                self._cache.clear()
+
+    def _afterfork(self):
+        for key, (send, close) in self._cache.items():
+            close()
+        self._cache.clear()
+        # If self._lock was locked at the time of the fork, it may be broken
+        # -- see issue 6721.  Replace it without letting it be gc'ed.
+        self._old_locks.append(self._lock)
+        self._lock = threading.Lock()
+        if self._listener is not None:
+            self._listener.close()
+        self._listener = None
+        self._address = None
+        self._thread = None
+
+    def _start(self):
+        from .connection import Listener
+        assert self._listener is None
+        debug('starting listener and thread for sending handles')
+        self._listener = Listener(authkey=current_process().authkey)
+        self._address = self._listener.address
+        t = threading.Thread(target=self._serve)
+        t.daemon = True
+        t.start()
+        self._thread = t
+
+    def _serve(self):
+        if hasattr(signal, 'pthread_sigmask'):
+            signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
+        while 1:
+            try:
+                conn = self._listener.accept()
+                msg = conn.recv()
+                if msg is None:
+                    break
+                key, destination_pid = msg
+                send, close = self._cache.pop(key)
+                send(conn, destination_pid)
+                close()
+                conn.close()
+            except:
+                if not is_exiting():
+                    import traceback
+                    sub_warning(
+                        'thread for sharing handles raised exception :\n' +
+                        '-'*79 + '\n' + traceback.format_exc() + '-'*79
+                        )
+
+resource_sharer = ResourceSharer()
diff --git a/Lib/multiprocessing/sharedctypes.py b/Lib/multiprocessing/sharedctypes.py
index 1e694da49d..a358ed4f12 100644
--- a/Lib/multiprocessing/sharedctypes.py
+++ b/Lib/multiprocessing/sharedctypes.py
@@ -4,35 +4,9 @@
 # multiprocessing/sharedctypes.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
-import sys
 import ctypes
 import weakref
 
@@ -89,7 +63,7 @@ def RawArray(typecode_or_type, size_or_initializer):
         result.__init__(*size_or_initializer)
         return result
 
-def Value(typecode_or_type, *args, lock=None):
+def Value(typecode_or_type, *args, lock=True):
     '''
     Return a synchronization wrapper for a Value
     '''
@@ -102,13 +76,10 @@ def Value(typecode_or_type, *args, lock=None):
         raise AttributeError("'%r' has no method 'acquire'" % lock)
     return synchronized(obj, lock)
 
-def Array(typecode_or_type, size_or_initializer, **kwds):
+def Array(typecode_or_type, size_or_initializer, *, lock=True):
     '''
     Return a synchronization wrapper for a RawArray
     '''
-    lock = kwds.pop('lock', None)
-    if kwds:
-        raise ValueError('unrecognized keyword argument(s): %s' % list(kwds.keys()))
     obj = RawArray(typecode_or_type, size_or_initializer)
     if lock is False:
         return obj
@@ -158,7 +129,8 @@ def rebuild_ctype(type_, wrapper, length):
     if length is not None:
         type_ = type_ * length
     ForkingPickler.register(type_, reduce_ctype)
-    obj = type_.from_address(wrapper.get_address())
+    buf = wrapper.create_memoryview()
+    obj = type_.from_buffer(buf)
     obj._wrapper = wrapper
     return obj
 
diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py
index 70ae82569c..22eabe55b8 100644
--- a/Lib/multiprocessing/synchronize.py
+++ b/Lib/multiprocessing/synchronize.py
@@ -4,32 +4,7 @@
 # multiprocessing/synchronize.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
 __all__ = [
@@ -37,15 +12,13 @@ __all__ = [
     ]
 
 import threading
-import os
 import sys
 
-from time import time as _time, sleep as _sleep
-
 import _multiprocessing
 from multiprocessing.process import current_process
-from multiprocessing.util import Finalize, register_after_fork, debug
+from multiprocessing.util import register_after_fork, debug
 from multiprocessing.forking import assert_spawning, Popen
+from time import time as _time
 
 # Try to import the mp.synchronize module cleanly, if it fails
 # raise ImportError for platforms lacking a working sem_open implementation.
@@ -243,7 +216,7 @@ class Condition(object):
 
         try:
             # wait for notification or timeout
-            ret = self._wait_semaphore.acquire(True, timeout)
+            return self._wait_semaphore.acquire(True, timeout)
         finally:
             # indicate that this thread has woken
             self._woken_count.release()
@@ -251,7 +224,6 @@ class Condition(object):
             # reacquire lock
             for i in range(count):
                 self._lock.acquire()
-            return ret
 
     def notify(self):
         assert self._lock._semlock._is_mine(), 'lock is not owned'
@@ -293,6 +265,24 @@ class Condition(object):
             while self._wait_semaphore.acquire(False):
                 pass
 
+    def wait_for(self, predicate, timeout=None):
+        result = predicate()
+        if result:
+            return result
+        if timeout is not None:
+            endtime = _time() + timeout
+        else:
+            endtime = None
+            waittime = None
+        while not result:
+            if endtime is not None:
+                waittime = endtime - _time()
+                if waittime <= 0:
+                    break
+            self.wait(waittime)
+            result = predicate()
+        return result
+
 #
 # Event
 #
@@ -343,3 +333,43 @@ class Event(object):
             return False
         finally:
             self._cond.release()
+
+#
+# Barrier
+#
+
+class Barrier(threading.Barrier):
+
+    def __init__(self, parties, action=None, timeout=None):
+        import struct
+        from multiprocessing.heap import BufferWrapper
+        wrapper = BufferWrapper(struct.calcsize('i') * 2)
+        cond = Condition()
+        self.__setstate__((parties, action, timeout, cond, wrapper))
+        self._state = 0
+        self._count = 0
+
+    def __setstate__(self, state):
+        (self._parties, self._action, self._timeout,
+         self._cond, self._wrapper) = state
+        self._array = self._wrapper.create_memoryview().cast('i')
+
+    def __getstate__(self):
+        return (self._parties, self._action, self._timeout,
+                self._cond, self._wrapper)
+
+    @property
+    def _state(self):
+        return self._array[0]
+
+    @_state.setter
+    def _state(self, value):
+        self._array[0] = value
+
+    @property
+    def _count(self):
+        return self._array[1]
+
+    @_count.setter
+    def _count(self, value):
+        self._array[1] = value
diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py
index 61b05335ac..72385a8fa3 100644
--- a/Lib/multiprocessing/util.py
+++ b/Lib/multiprocessing/util.py
@@ -4,39 +4,18 @@
 # multiprocessing/util.py
 #
 # Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-#    used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
 #
 
+import sys
+import functools
+import os
 import itertools
 import weakref
 import atexit
 import threading        # we want threading to install it's
                         # cleanup function before multiprocessing does
+from subprocess import _args_from_interpreter_flags
 
 from multiprocessing.process import current_process, active_children
 
@@ -84,7 +63,7 @@ def get_logger():
     Returns logger used by multiprocessing
     '''
     global _logger
-    import logging, atexit
+    import logging
 
     logging._acquireLock()
     try:
@@ -183,10 +162,15 @@ class Finalize(object):
         self._args = args
         self._kwargs = kwargs or {}
         self._key = (exitpriority, next(_finalizer_counter))
+        self._pid = os.getpid()
 
         _finalizer_registry[self._key] = self
 
-    def __call__(self, wr=None):
+    def __call__(self, wr=None,
+                 # Need to bind these locally because the globals can have
+                 # been cleared at shutdown
+                 _finalizer_registry=_finalizer_registry,
+                 sub_debug=sub_debug, getpid=os.getpid):
         '''
         Run the callback unless it has already been called or cancelled
         '''
@@ -195,9 +179,13 @@ class Finalize(object):
         except KeyError:
             sub_debug('finalizer no longer registered')
         else:
-            sub_debug('finalizer calling %s with args %s and kwargs %s',
-                     self._callback, self._args, self._kwargs)
-            res = self._callback(*self._args, **self._kwargs)
+            if self._pid != getpid():
+                sub_debug('finalizer ignored because different process')
+                res = None
+            else:
+                sub_debug('finalizer calling %s with args %s and kwargs %s',
+                          self._callback, self._args, self._kwargs)
+                res = self._callback(*self._args, **self._kwargs)
             self._weakref = self._callback = self._args = \
                             self._kwargs = self._key = None
             return res
@@ -299,16 +287,21 @@ def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers,
         info('process shutting down')
         debug('running all "atexit" finalizers with priority >= 0')
         _run_finalizers(0)
+
         if current_process() is not None:
             # We check if the current process is None here because if
-            # it's None, any call to ``active_children()`` will raise an
-            # AttributeError (active_children winds up trying to get
-            # attributes from util._current_process).  This happens in a
-            # variety of shutdown circumstances that are not well-understood
-            # because module-scope variables are not apparently supposed to
-            # be destroyed until after this function is called.  However,
-            # they are indeed destroyed before this function is called.  See
-            # issues #9775 and #15881.  Also related: #4106, #9205, and #9207.
+            # it's None, any call to ``active_children()`` will raise
+            # an AttributeError (active_children winds up trying to
+            # get attributes from util._current_process).  One
+            # situation where this can happen is if someone has
+            # manipulated sys.modules, causing this module to be
+            # garbage collected.  The destructor for the module type
+            # then replaces all values in the module dict with None.
+            # For instance, after setuptools runs a test it replaces
+            # sys.modules with a copy created earlier.  See issues
+            # #9775 and #15881.  Also related: #4106, #9205, and
+            # #9207.
+
             for p in active_children():
                 if p._daemonic:
                     info('calling terminate() for daemon %s', p.name)
diff --git a/Lib/nntplib.py b/Lib/nntplib.py
index 32bffd8e27..2de6ebd1b5 100644
--- a/Lib/nntplib.py
+++ b/Lib/nntplib.py
@@ -166,7 +166,7 @@ def decode_header(header_str):
             parts.append(v.decode(enc or 'ascii'))
         else:
             parts.append(v)
-    return ' '.join(parts)
+    return ''.join(parts)
 
 def _parse_overview_fmt(lines):
     """Parse a list of string representing the response to LIST OVERVIEW.FMT
@@ -351,6 +351,20 @@ class _NNTPBase:
         # Log in and encryption setup order is left to subclasses.
         self.authenticated = False
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        is_connected = lambda: hasattr(self, "file")
+        if is_connected():
+            try:
+                self.quit()
+            except (socket.error, EOFError):
+                pass
+            finally:
+                if is_connected():
+                    self._close()
+
     def getwelcome(self):
         """Get the welcome message from the server
         (this is read and squirreled away by __init__()).
@@ -819,7 +833,7 @@ class _NNTPBase:
         - list: list of (name,title) strings"""
         warnings.warn("The XGTITLE extension is not actively used, "
                       "use descriptions() instead",
-                      PendingDeprecationWarning, 2)
+                      DeprecationWarning, 2)
         line_pat = re.compile('^([^ \t]+)[ \t]+(.*)$')
         resp, raw_lines = self._longcmdstring('XGTITLE ' + group, file)
         lines = []
@@ -837,7 +851,7 @@ class _NNTPBase:
         path: directory path to article
         """
         warnings.warn("The XPATH extension is not actively used",
-                      PendingDeprecationWarning, 2)
+                      DeprecationWarning, 2)
 
         resp = self._shortcmd('XPATH {0}'.format(id))
         if not resp.startswith('223'):
diff --git a/Lib/numbers.py b/Lib/numbers.py
index ecfad7cef8..b206457dfc 100644
--- a/Lib/numbers.py
+++ b/Lib/numbers.py
@@ -5,7 +5,7 @@
 
 TODO: Fill out more detailed documentation on the operators."""
 
-from abc import ABCMeta, abstractmethod, abstractproperty
+from abc import ABCMeta, abstractmethod
 
 __all__ = ["Number", "Complex", "Real", "Rational", "Integral"]
 
@@ -50,7 +50,8 @@ class Complex(Number):
         """True if self != 0. Called for bool(self)."""
         return self != 0
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def real(self):
         """Retrieve the real component of this number.
 
@@ -58,7 +59,8 @@ class Complex(Number):
         """
         raise NotImplementedError
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def imag(self):
         """Retrieve the imaginary component of this number.
 
@@ -272,11 +274,13 @@ class Rational(Real):
 
     __slots__ = ()
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def numerator(self):
         raise NotImplementedError
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def denominator(self):
         raise NotImplementedError
 
diff --git a/Lib/opcode.py b/Lib/opcode.py
index 8e15d13e98..a639fe322b 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -6,7 +6,7 @@ operate on bytecodes (e.g. peephole optimizers).
 
 __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
            "haslocal", "hascompare", "hasfree", "opname", "opmap",
-           "HAVE_ARGUMENT", "EXTENDED_ARG"]
+           "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"]
 
 cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is',
         'is not', 'exception match', 'BAD')
@@ -18,6 +18,7 @@ hasjabs = []
 haslocal = []
 hascompare = []
 hasfree = []
+hasnargs = []
 
 opmap = {}
 opname = [''] * 256
@@ -43,7 +44,6 @@ def jabs_op(name, op):
 # Instruction opcodes for compiled code
 # Blank lines correspond to available opcodes
 
-def_op('STOP_CODE', 0)
 def_op('POP_TOP', 1)
 def_op('ROT_TWO', 2)
 def_op('ROT_THREE', 3)
@@ -88,6 +88,7 @@ def_op('STORE_LOCALS', 69)
 
 def_op('PRINT_EXPR', 70)
 def_op('LOAD_BUILD_CLASS', 71)
+def_op('YIELD_FROM', 72)
 
 def_op('INPLACE_LSHIFT', 75)
 def_op('INPLACE_RSHIFT', 76)
@@ -152,6 +153,7 @@ haslocal.append(126)
 
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
 def_op('CALL_FUNCTION', 131)    # #args + (#kwargs << 8)
+hasnargs.append(131)
 def_op('MAKE_FUNCTION', 132)    # Number of args with default values
 def_op('BUILD_SLICE', 133)      # Number of items
 def_op('MAKE_CLOSURE', 134)
@@ -165,8 +167,11 @@ def_op('DELETE_DEREF', 138)
 hasfree.append(138)
 
 def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
+hasnargs.append(140)
 def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)
+hasnargs.append(141)
 def_op('CALL_FUNCTION_VAR_KW', 142)  # #args + (#kwargs << 8)
+hasnargs.append(142)
 
 jrel_op('SETUP_WITH', 143)
 
diff --git a/Lib/optparse.py b/Lib/optparse.py
index d97a1f7aca..37764d317a 100644
--- a/Lib/optparse.py
+++ b/Lib/optparse.py
@@ -86,10 +86,16 @@ def _repr(self):
 #   Id: errors.py 509 2006-04-20 00:58:24Z gward
 
 try:
-    from gettext import gettext
+    from gettext import gettext, ngettext
 except ImportError:
     def gettext(message):
         return message
+
+    def ngettext(singular, plural, n):
+        if n == 1:
+            return singular
+        return plural
+
 _ = gettext
 
 
@@ -411,11 +417,8 @@ def _parse_num(val, type):
 def _parse_int(val):
     return _parse_num(val, int)
 
-def _parse_long(val):
-    return _parse_num(val, int)
-
 _builtin_cvt = { "int" : (_parse_int, _("integer")),
-                 "long" : (_parse_long, _("long integer")),
+                 "long" : (_parse_int, _("integer")),
                  "float" : (float, _("floating-point")),
                  "complex" : (complex, _("complex")) }
 
@@ -1483,11 +1486,10 @@ class OptionParser (OptionContainer):
         if option.takes_value():
             nargs = option.nargs
             if len(rargs) < nargs:
-                if nargs == 1:
-                    self.error(_("%s option requires an argument") % opt)
-                else:
-                    self.error(_("%s option requires %d arguments")
-                               % (opt, nargs))
+                self.error(ngettext(
+                    "%(option)s option requires %(number)d argument",
+                    "%(option)s option requires %(number)d arguments",
+                    nargs) % {"option": opt, "number": nargs})
             elif nargs == 1:
                 value = rargs.pop(0)
             else:
@@ -1522,11 +1524,10 @@ class OptionParser (OptionContainer):
 
                 nargs = option.nargs
                 if len(rargs) < nargs:
-                    if nargs == 1:
-                        self.error(_("%s option requires an argument") % opt)
-                    else:
-                        self.error(_("%s option requires %d arguments")
-                                   % (opt, nargs))
+                    self.error(ngettext(
+                        "%(option)s option requires %(number)d argument",
+                        "%(option)s option requires %(number)d arguments",
+                        nargs) % {"option": opt, "number": nargs})
                 elif nargs == 1:
                     value = rargs.pop(0)
                 else:
diff --git a/Lib/os.py b/Lib/os.py
index 81e037af3c..e4ea1424b9 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -24,13 +24,18 @@ and opendir), and leave all pathname manipulation to os.path
 #'
 
 import sys, errno
+import stat as st
 
 _names = sys.builtin_module_names
 
 # Note:  more names are added to __all__ later.
 __all__ = ["altsep", "curdir", "pardir", "sep", "pathsep", "linesep",
-           "defpath", "name", "path", "devnull",
-           "SEEK_SET", "SEEK_CUR", "SEEK_END"]
+           "defpath", "name", "path", "devnull", "SEEK_SET", "SEEK_CUR",
+           "SEEK_END", "fsencode", "fsdecode", "get_exec_path", "fdopen",
+           "popen", "extsep"]
+
+def _exists(name):
+    return name in globals()
 
 def _get_exports_list(module):
     try:
@@ -38,19 +43,23 @@ def _get_exports_list(module):
     except AttributeError:
         return [n for n in dir(module) if n[0] != '_']
 
+# Any new dependencies of the os module and/or changes in path separator
+# requires updating importlib as well.
 if 'posix' in _names:
     name = 'posix'
     linesep = '\n'
     from posix import *
     try:
         from posix import _exit
+        __all__.append('_exit')
     except ImportError:
         pass
     import posixpath as path
 
-    import posix
-    __all__.extend(_get_exports_list(posix))
-    del posix
+    try:
+        from posix import _have_functions
+    except ImportError:
+        pass
 
 elif 'nt' in _names:
     name = 'nt'
@@ -58,6 +67,7 @@ elif 'nt' in _names:
     from nt import *
     try:
         from nt import _exit
+        __all__.append('_exit')
     except ImportError:
         pass
     import ntpath as path
@@ -66,12 +76,18 @@ elif 'nt' in _names:
     __all__.extend(_get_exports_list(nt))
     del nt
 
+    try:
+        from nt import _have_functions
+    except ImportError:
+        pass
+
 elif 'os2' in _names:
     name = 'os2'
     linesep = '\r\n'
     from os2 import *
     try:
         from os2 import _exit
+        __all__.append('_exit')
     except ImportError:
         pass
     if sys.version.find('EMX GCC') == -1:
@@ -84,12 +100,18 @@ elif 'os2' in _names:
     __all__.extend(_get_exports_list(os2))
     del os2
 
+    try:
+        from os2 import _have_functions
+    except ImportError:
+        pass
+
 elif 'ce' in _names:
     name = 'ce'
     linesep = '\r\n'
     from ce import *
     try:
         from ce import _exit
+        __all__.append('_exit')
     except ImportError:
         pass
     # We can use the standard Windows path.
@@ -99,6 +121,11 @@ elif 'ce' in _names:
     __all__.extend(_get_exports_list(ce))
     del ce
 
+    try:
+        from ce import _have_functions
+    except ImportError:
+        pass
+
 else:
     raise ImportError('no os specific module found')
 
@@ -108,8 +135,97 @@ from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep,
 
 del _names
 
+
+if _exists("_have_functions"):
+    _globals = globals()
+    def _add(str, fn):
+        if (fn in _globals) and (str in _have_functions):
+            _set.add(_globals[fn])
+
+    _set = set()
+    _add("HAVE_FACCESSAT",  "access")
+    _add("HAVE_FCHMODAT",   "chmod")
+    _add("HAVE_FCHOWNAT",   "chown")
+    _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_FUTIMESAT",  "utime")
+    _add("HAVE_LINKAT",     "link")
+    _add("HAVE_MKDIRAT",    "mkdir")
+    _add("HAVE_MKFIFOAT",   "mkfifo")
+    _add("HAVE_MKNODAT",    "mknod")
+    _add("HAVE_OPENAT",     "open")
+    _add("HAVE_READLINKAT", "readlink")
+    _add("HAVE_RENAMEAT",   "rename")
+    _add("HAVE_SYMLINKAT",  "symlink")
+    _add("HAVE_UNLINKAT",   "unlink")
+    _add("HAVE_UNLINKAT",   "rmdir")
+    _add("HAVE_UTIMENSAT",  "utime")
+    supports_dir_fd = _set
+
+    _set = set()
+    _add("HAVE_FACCESSAT",  "access")
+    supports_effective_ids = _set
+
+    _set = set()
+    _add("HAVE_FCHDIR",     "chdir")
+    _add("HAVE_FCHMOD",     "chmod")
+    _add("HAVE_FCHOWN",     "chown")
+    _add("HAVE_FDOPENDIR",  "listdir")
+    _add("HAVE_FEXECVE",    "execve")
+    _set.add(stat) # fstat always works
+    _add("HAVE_FTRUNCATE",  "truncate")
+    _add("HAVE_FUTIMENS",   "utime")
+    _add("HAVE_FUTIMES",    "utime")
+    _add("HAVE_FPATHCONF",  "pathconf")
+    if _exists("statvfs") and _exists("fstatvfs"): # mac os x10.3
+        _add("HAVE_FSTATVFS", "statvfs")
+    supports_fd = _set
+
+    _set = set()
+    _add("HAVE_FACCESSAT",  "access")
+    # Some platforms don't support lchmod().  Often the function exists
+    # anyway, as a stub that always returns ENOSUP or perhaps EOPNOTSUPP.
+    # (No, I don't know why that's a good design.)  ./configure will detect
+    # this and reject it--so HAVE_LCHMOD still won't be defined on such
+    # platforms.  This is Very Helpful.
+    #
+    # However, sometimes platforms without a working lchmod() *do* have
+    # fchmodat().  (Examples: Linux kernel 3.2 with glibc 2.15,
+    # OpenIndiana 3.x.)  And fchmodat() has a flag that theoretically makes
+    # it behave like lchmod().  So in theory it would be a suitable
+    # replacement for lchmod().  But when lchmod() doesn't work, fchmodat()'s
+    # flag doesn't work *either*.  Sadly ./configure isn't sophisticated
+    # enough to detect this condition--it only determines whether or not
+    # fchmodat() minimally works.
+    #
+    # Therefore we simply ignore fchmodat() when deciding whether or not
+    # os.chmod supports follow_symlinks.  Just checking lchmod() is
+    # sufficient.  After all--if you have a working fchmodat(), your
+    # lchmod() almost certainly works too.
+    #
+    # _add("HAVE_FCHMODAT",   "chmod")
+    _add("HAVE_FCHOWNAT",   "chown")
+    _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_LCHFLAGS",   "chflags")
+    _add("HAVE_LCHMOD",     "chmod")
+    if _exists("lchown"): # mac os x10.3
+        _add("HAVE_LCHOWN", "chown")
+    _add("HAVE_LINKAT",     "link")
+    _add("HAVE_LUTIMES",    "utime")
+    _add("HAVE_LSTAT",      "stat")
+    _add("HAVE_FSTATAT",    "stat")
+    _add("HAVE_UTIMENSAT",  "utime")
+    _add("MS_WINDOWS",      "stat")
+    supports_follow_symlinks = _set
+
+    del _set
+    del _have_functions
+    del _globals
+    del _add
+
+
 # Python uses fixed values for the SEEK_ constants; they are mapped
 # to native constants if necessary in posixmodule.c
+# Other possible SEEK values are directly imported from posixmodule.c
 SEEK_SET = 0
 SEEK_CUR = 1
 SEEK_END = 2
@@ -120,8 +236,6 @@ def _get_masked_mode(mode):
     umask(mask)
     return mode & ~mask
 
-#'
-
 # Super directory utilities.
 # (Inspired by Eric Raymond; the doc strings are mostly his)
 
@@ -154,7 +268,6 @@ def makedirs(name, mode=0o777, exist_ok=False):
     try:
         mkdir(name, mode)
     except OSError as e:
-        import stat as st
         dir_exists = path.isdir(name)
         expected_mode = _get_masked_mode(mode)
         if dir_exists:
@@ -166,6 +279,9 @@ def makedirs(name, mode=0o777, exist_ok=False):
             actual_mode = -1
         if not (e.errno == errno.EEXIST and exist_ok and dir_exists and
                 actual_mode == expected_mode):
+            if dir_exists and actual_mode != expected_mode:
+                e.strerror += ' (mode %o != expected mode %o)' % (
+                        actual_mode, expected_mode)
             raise
 
 def removedirs(name):
@@ -303,13 +419,107 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
     for name in dirs:
         new_path = join(top, name)
         if followlinks or not islink(new_path):
-            for x in walk(new_path, topdown, onerror, followlinks):
-                yield x
+            yield from walk(new_path, topdown, onerror, followlinks)
     if not topdown:
         yield top, dirs, nondirs
 
 __all__.append("walk")
 
+if {open, stat} <= supports_dir_fd and {listdir, stat} <= supports_fd:
+
+    def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None):
+        """Directory tree generator.
+
+        This behaves exactly like walk(), except that it yields a 4-tuple
+
+            dirpath, dirnames, filenames, dirfd
+
+        `dirpath`, `dirnames` and `filenames` are identical to walk() output,
+        and `dirfd` is a file descriptor referring to the directory `dirpath`.
+
+        The advantage of fwalk() over walk() is that it's safe against symlink
+        races (when follow_symlinks is False).
+
+        If dir_fd is not None, it should be a file descriptor open to a directory,
+          and top should be relative; top will then be relative to that directory.
+          (dir_fd is always supported for fwalk.)
+
+        Caution:
+        Since fwalk() yields file descriptors, those are only valid until the
+        next iteration step, so you should dup() them if you want to keep them
+        for a longer period.
+
+        Example:
+
+        import os
+        for root, dirs, files, rootfd in os.fwalk('python/Lib/email'):
+            print(root, "consumes", end="")
+            print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]),
+                  end="")
+            print("bytes in", len(files), "non-directory files")
+            if 'CVS' in dirs:
+                dirs.remove('CVS')  # don't visit CVS directories
+        """
+        # Note: To guard against symlink races, we use the standard
+        # lstat()/open()/fstat() trick.
+        orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd)
+        topfd = open(top, O_RDONLY, dir_fd=dir_fd)
+        try:
+            if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and
+                                    path.samestat(orig_st, stat(topfd)))):
+                yield from _fwalk(topfd, top, topdown, onerror, follow_symlinks)
+        finally:
+            close(topfd)
+
+    def _fwalk(topfd, toppath, topdown, onerror, follow_symlinks):
+        # Note: This uses O(depth of the directory tree) file descriptors: if
+        # necessary, it can be adapted to only require O(1) FDs, see issue
+        # #13734.
+
+        names = listdir(topfd)
+        dirs, nondirs = [], []
+        for name in names:
+            try:
+                # Here, we don't use AT_SYMLINK_NOFOLLOW to be consistent with
+                # walk() which reports symlinks to directories as directories.
+                # We do however check for symlinks before recursing into
+                # a subdirectory.
+                if st.S_ISDIR(stat(name, dir_fd=topfd).st_mode):
+                    dirs.append(name)
+                else:
+                    nondirs.append(name)
+            except FileNotFoundError:
+                try:
+                    # Add dangling symlinks, ignore disappeared files
+                    if st.S_ISLNK(stat(name, dir_fd=topfd, follow_symlinks=False)
+                                .st_mode):
+                        nondirs.append(name)
+                except FileNotFoundError:
+                    continue
+
+        if topdown:
+            yield toppath, dirs, nondirs, topfd
+
+        for name in dirs:
+            try:
+                orig_st = stat(name, dir_fd=topfd, follow_symlinks=follow_symlinks)
+                dirfd = open(name, O_RDONLY, dir_fd=topfd)
+            except error as err:
+                if onerror is not None:
+                    onerror(err)
+                return
+            try:
+                if follow_symlinks or path.samestat(orig_st, stat(dirfd)):
+                    dirpath = path.join(toppath, name)
+                    yield from _fwalk(dirfd, dirpath, topdown, onerror, follow_symlinks)
+            finally:
+                close(dirfd)
+
+        if not topdown:
+            yield toppath, dirs, nondirs, topfd
+
+    __all__.append("fwalk")
+
 # Make sure os.environ exists, at least
 try:
     environ
@@ -446,7 +656,7 @@ def get_exec_path(env=None):
 
 
 # Change environ to automatically call putenv(), unsetenv if they exist.
-from _abcoll import MutableMapping  # Can't use collections (bootstrap)
+from collections.abc import MutableMapping
 
 class _Environ(MutableMapping):
     def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv):
@@ -610,15 +820,14 @@ def _fscodec():
 fsencode, fsdecode = _fscodec()
 del _fscodec
 
-def _exists(name):
-    return name in globals()
-
 # Supply spawn*() (probably only for Unix)
 if _exists("fork") and not _exists("spawnv") and _exists("execv"):
 
     P_WAIT = 0
     P_NOWAIT = P_NOWAITO = 1
 
+    __all__.extend(["P_WAIT", "P_NOWAIT", "P_NOWAITO"])
+
     # XXX Should we support P_DETACH?  I suppose it could fork()**2
     # and close the std I/O streams.  Also, P_OVERLAY is the same
     # as execv*()?
@@ -779,7 +988,7 @@ def popen(cmd, mode="r", buffering=-1):
         raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
     if mode not in ("r", "w"):
         raise ValueError("invalid mode %r" % mode)
-    if buffering == 0 or buffering == None:
+    if buffering == 0 or buffering is None:
         raise ValueError("popen() does not support unbuffered streams")
     import subprocess, io
     if mode == "r":
diff --git a/Lib/pdb.py b/Lib/pdb.py
index 8c7c12b92c..e6d7f8ff70 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -73,6 +73,7 @@ import cmd
 import bdb
 import dis
 import code
+import glob
 import pprint
 import signal
 import inspect
@@ -155,6 +156,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         # Try to load readline if it exists
         try:
             import readline
+            # remove some common file name delimiters
+            readline.set_completer_delims(' \t\n`@#$%^&*()=+[{]}\\|;:\'",<>?')
         except ImportError:
             pass
         self.allow_kbdint = False
@@ -445,6 +448,61 @@ class Pdb(bdb.Bdb, cmd.Cmd):
     def error(self, msg):
         print('***', msg, file=self.stdout)
 
+    # Generic completion functions.  Individual complete_foo methods can be
+    # assigned below to one of these functions.
+
+    def _complete_location(self, text, line, begidx, endidx):
+        # Complete a file/module/function location for break/tbreak/clear.
+        if line.strip().endswith((':', ',')):
+            # Here comes a line number or a condition which we can't complete.
+            return []
+        # First, try to find matching functions (i.e. expressions).
+        try:
+            ret = self._complete_expression(text, line, begidx, endidx)
+        except Exception:
+            ret = []
+        # Then, try to complete file names as well.
+        globs = glob.glob(text + '*')
+        for fn in globs:
+            if os.path.isdir(fn):
+                ret.append(fn + '/')
+            elif os.path.isfile(fn) and fn.lower().endswith(('.py', '.pyw')):
+                ret.append(fn + ':')
+        return ret
+
+    def _complete_bpnumber(self, text, line, begidx, endidx):
+        # Complete a breakpoint number.  (This would be more helpful if we could
+        # display additional info along with the completions, such as file/line
+        # of the breakpoint.)
+        return [str(i) for i, bp in enumerate(bdb.Breakpoint.bpbynumber)
+                if bp is not None and str(i).startswith(text)]
+
+    def _complete_expression(self, text, line, begidx, endidx):
+        # Complete an arbitrary expression.
+        if not self.curframe:
+            return []
+        # Collect globals and locals.  It is usually not really sensible to also
+        # complete builtins, and they clutter the namespace quite heavily, so we
+        # leave them out.
+        ns = self.curframe.f_globals.copy()
+        ns.update(self.curframe_locals)
+        if '.' in text:
+            # Walk an attribute chain up to the last part, similar to what
+            # rlcompleter does.  This will bail if any of the parts are not
+            # simple attribute access, which is what we want.
+            dotted = text.split('.')
+            try:
+                obj = ns[dotted[0]]
+                for part in dotted[1:-1]:
+                    obj = getattr(obj, part)
+            except (KeyError, AttributeError):
+                return []
+            prefix = '.'.join(dotted[:-1]) + '.'
+            return [prefix + n for n in dir(obj) if n.startswith(dotted[-1])]
+        else:
+            # Complete a simple name.
+            return [n for n in ns.keys() if n.startswith(text)]
+
     # Command definitions, called by cmdloop()
     # The argument is the remaining string on the command line
     # Return true to exit from the command loop
@@ -526,6 +584,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
             self.commands_defining = False
             self.prompt = prompt_back
 
+    complete_commands = _complete_bpnumber
+
     def do_break(self, arg, temporary = 0):
         """b(reak) [ ([filename:]lineno | function) [, condition] ]
         Without argument, list all breaks.
@@ -628,6 +688,9 @@ class Pdb(bdb.Bdb, cmd.Cmd):
 
     do_b = do_break
 
+    complete_break = _complete_location
+    complete_b = _complete_location
+
     def do_tbreak(self, arg):
         """tbreak [ ([filename:]lineno | function) [, condition] ]
         Same arguments as break, but sets a temporary breakpoint: it
@@ -635,6 +698,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         """
         self.do_break(arg, 1)
 
+    complete_tbreak = _complete_location
+
     def lineinfo(self, identifier):
         failed = (None, None, None)
         # Input is identifier, may be in single quotes
@@ -704,6 +769,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
                 bp.enable()
                 self.message('Enabled %s' % bp)
 
+    complete_enable = _complete_bpnumber
+
     def do_disable(self, arg):
         """disable bpnumber [bpnumber ...]
         Disables the breakpoints given as a space separated list of
@@ -722,6 +789,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
                 bp.disable()
                 self.message('Disabled %s' % bp)
 
+    complete_disable = _complete_bpnumber
+
     def do_condition(self, arg):
         """condition bpnumber [condition]
         Set a new condition for the breakpoint, an expression which
@@ -745,6 +814,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
             else:
                 self.message('New condition set for breakpoint %d.' % bp.number)
 
+    complete_condition = _complete_bpnumber
+
     def do_ignore(self, arg):
         """ignore bpnumber [count]
         Set the ignore count for the given breakpoint number.  If
@@ -776,6 +847,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
                 self.message('Will stop next time breakpoint %d is reached.'
                              % bp.number)
 
+    complete_ignore = _complete_bpnumber
+
     def do_clear(self, arg):
         """cl(ear) filename:lineno\ncl(ear) [bpnumber [bpnumber...]]
         With a space separated list of breakpoint numbers, clear
@@ -824,6 +897,9 @@ class Pdb(bdb.Bdb, cmd.Cmd):
                 self.message('Deleted %s' % bp)
     do_cl = do_clear # 'c' is already an abbreviation for 'continue'
 
+    complete_clear = _complete_location
+    complete_cl = _complete_location
+
     def do_where(self, arg):
         """w(here)
         Print a stack trace, with the most recent frame at the bottom.
@@ -1014,6 +1090,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         sys.settrace(self.trace_dispatch)
         self.lastcmd = p.lastcmd
 
+    complete_debug = _complete_expression
+
     def do_quit(self, arg):
         """q(uit)\nexit
         Quit from the debugger. The program being executed is aborted.
@@ -1100,6 +1178,10 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         except:
             pass
 
+    complete_print = _complete_expression
+    complete_p = _complete_expression
+    complete_pp = _complete_expression
+
     def do_list(self, arg):
         """l(ist) [first [,last] | .]
 
@@ -1180,6 +1262,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
             return
         self._print_lines(lines, lineno)
 
+    complete_source = _complete_expression
+
     def _print_lines(self, lines, start, breaks=(), frame=None):
         """Print a range of lines."""
         if frame:
@@ -1234,6 +1318,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         # None of the above...
         self.message(type(value))
 
+    complete_whatis = _complete_expression
+
     def do_display(self, arg):
         """display [expression]
 
@@ -1251,6 +1337,8 @@ class Pdb(bdb.Bdb, cmd.Cmd):
             self.displaying.setdefault(self.curframe, {})[arg] = val
             self.message('display %s: %r' % (arg, val))
 
+    complete_display = _complete_expression
+
     def do_undisplay(self, arg):
         """undisplay [expression]
 
@@ -1266,6 +1354,10 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         else:
             self.displaying.pop(self.curframe, None)
 
+    def complete_undisplay(self, text, line, begidx, endidx):
+        return [e for e in self.displaying.get(self.curframe, {})
+                if e.startswith(text)]
+
     def do_interact(self, arg):
         """interact
 
@@ -1320,6 +1412,9 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         if args[0] in self.aliases:
             del self.aliases[args[0]]
 
+    def complete_unalias(self, text, line, begidx, endidx):
+        return [a for a in self.aliases if a.startswith(text)]
+
     # List of all the commands making the program resume execution.
     commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return',
                          'do_quit', 'do_jump']
diff --git a/Lib/pickle.py b/Lib/pickle.py
index a690ccd8bc..e81a3790c3 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -23,8 +23,6 @@ Misc variables:
 
 """
 
-__version__ = "$Revision$"       # Code version
-
 from types import FunctionType, BuiltinFunctionType
 from copyreg import dispatch_table
 from copyreg import _extension_registry, _inverted_registry, _extension_cache
@@ -299,8 +297,8 @@ class _Pickler:
             f(self, obj) # Call unbound method with explicit self
             return
 
-        # Check copyreg.dispatch_table
-        reduce = dispatch_table.get(t)
+        # Check private dispatch table if any, or else copyreg.dispatch_table
+        reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
         if reduce:
             rv = reduce(obj)
         else:
@@ -377,7 +375,7 @@ class _Pickler:
             # allowing protocol 0 and 1 to work normally.  For this to
             # work, the function returned by __reduce__ should be
             # called __newobj__, and its first argument should be a
-            # new-style class.  The implementation for __newobj__
+            # class.  The implementation for __newobj__
             # should be as follows, although pickle has no way to
             # verify this:
             #
@@ -440,6 +438,14 @@ class _Pickler:
         self.write(NONE)
     dispatch[type(None)] = save_none
 
+    def save_ellipsis(self, obj):
+        self.save_global(Ellipsis, 'Ellipsis')
+    dispatch[type(Ellipsis)] = save_ellipsis
+
+    def save_notimplemented(self, obj):
+        self.save_global(NotImplemented, 'NotImplemented')
+    dispatch[type(NotImplemented)] = save_notimplemented
+
     def save_bool(self, obj):
         if self.proto >= 2:
             self.write(obj and NEWTRUE or NEWFALSE)
@@ -1345,7 +1351,7 @@ def _test():
     return doctest.testmod()
 
 if __name__ == "__main__":
-    import sys, argparse
+    import argparse
     parser = argparse.ArgumentParser(
         description='display contents of the pickle files')
     parser.add_argument(
diff --git a/Lib/pickletools.py b/Lib/pickletools.py
index ec6cc53ae3..66f4eddbc9 100644
--- a/Lib/pickletools.py
+++ b/Lib/pickletools.py
@@ -510,10 +510,7 @@ def read_decimalnl_short(f):
     elif s == b"01":
         return True
 
-    try:
-        return int(s)
-    except OverflowError:
-        return int(s)
+    return int(s)
 
 def read_decimalnl_long(f):
     r"""
@@ -1642,6 +1639,8 @@ opcodes = [
       is pushed on the stack.
 
       NOTE:  checks for __safe_for_unpickling__ went away in Python 2.3.
+      NOTE:  the distinction between old-style and new-style classes does
+             not make sense in Python 3.
       """),
 
     I(name='OBJ',
diff --git a/Lib/pipes.py b/Lib/pipes.py
index 4297053816..f1a16f63de 100644
--- a/Lib/pipes.py
+++ b/Lib/pipes.py
@@ -60,7 +60,9 @@ To create a new template object initialized to a given one:
 import re
 import os
 import tempfile
-import string
+# we import the quote function rather than the module for backward compat
+# (quote used to be an undocumented but used function in pipes)
+from shlex import quote
 
 __all__ = ["Template"]
 
@@ -243,22 +245,3 @@ def makepipeline(infile, steps, outfile):
         cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd
     #
     return cmdlist
-
-
-# Reliably quote a string as a single argument for /bin/sh
-
-# Safe unquoted
-_safechars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./')
-
-def quote(file):
-    """Return a shell-escaped version of the file string."""
-    for c in file:
-        if c not in _safechars:
-            break
-    else:
-        if not file:
-            return "''"
-        return file
-    # use single quotes, and put single quotes into double quotes
-    # the string $'b is then quoted as '$'"'"'b'
-    return "'" + file.replace("'", "'\"'\"'") + "'"
diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
index 51da0b1bb5..a5de04d257 100644
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -2,8 +2,10 @@
 
 import os
 import sys
+import importlib
 import imp
 import os.path
+from warnings import warn
 from types import ModuleType
 
 __all__ = [
@@ -21,7 +23,7 @@ def read_code(stream):
     if magic != imp.get_magic():
         return None
 
-    stream.read(4) # Skip timestamp
+    stream.read(8) # Skip timestamp and size
     return marshal.load(stream)
 
 
@@ -155,6 +157,49 @@ def iter_importer_modules(importer, prefix=''):
 
 iter_importer_modules = simplegeneric(iter_importer_modules)
 
+# Implement a file walker for the normal importlib path hook
+def _iter_file_finder_modules(importer, prefix=''):
+    if importer.path is None or not os.path.isdir(importer.path):
+        return
+
+    yielded = {}
+    import inspect
+    try:
+        filenames = os.listdir(importer.path)
+    except OSError:
+        # ignore unreadable directories like import does
+        filenames = []
+    filenames.sort()  # handle packages before same-named modules
+
+    for fn in filenames:
+        modname = inspect.getmodulename(fn)
+        if modname=='__init__' or modname in yielded:
+            continue
+
+        path = os.path.join(importer.path, fn)
+        ispkg = False
+
+        if not modname and os.path.isdir(path) and '.' not in fn:
+            modname = fn
+            try:
+                dircontents = os.listdir(path)
+            except OSError:
+                # ignore unreadable directories like import does
+                dircontents = []
+            for fn in dircontents:
+                subname = inspect.getmodulename(fn)
+                if subname=='__init__':
+                    ispkg = True
+                    break
+            else:
+                continue    # not a package
+
+        if modname and '.' not in modname:
+            yielded[modname] = 1
+            yield prefix + modname, ispkg
+
+iter_importer_modules.register(
+    importlib.machinery.FileFinder, _iter_file_finder_modules)
 
 class ImpImporter:
     """PEP 302 Importer that wraps Python's "classic" import algorithm
@@ -168,6 +213,8 @@ class ImpImporter:
     """
 
     def __init__(self, path=None):
+        warn("This emulation is deprecated, use 'importlib' instead",
+             DeprecationWarning)
         self.path = path
 
     def find_module(self, fullname, path=None):
@@ -232,6 +279,8 @@ class ImpLoader:
     code = source = None
 
     def __init__(self, fullname, file, filename, etc):
+        warn("This emulation is deprecated, use 'importlib' instead",
+             DeprecationWarning)
         self.file = file
         self.filename = filename
         self.fullname = fullname
@@ -256,7 +305,7 @@ class ImpLoader:
         if self.file and self.file.closed:
             mod_type = self.etc[2]
             if mod_type==imp.PY_SOURCE:
-                self.file = open(self.filename, 'rU')
+                self.file = open(self.filename, 'r')
             elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
                 self.file = open(self.filename, 'rb')
 
@@ -301,7 +350,7 @@ class ImpLoader:
                     self.file.close()
             elif mod_type==imp.PY_COMPILED:
                 if os.path.exists(self.filename[:-1]):
-                    f = open(self.filename[:-1], 'rU')
+                    f = open(self.filename[:-1], 'r')
                     self.source = f.read()
                     f.close()
             elif mod_type==imp.PKG_DIRECTORY:
@@ -315,9 +364,9 @@ class ImpLoader:
     def get_filename(self, fullname=None):
         fullname = self._fix_name(fullname)
         mod_type = self.etc[2]
-        if self.etc[2]==imp.PKG_DIRECTORY:
+        if mod_type==imp.PKG_DIRECTORY:
             return self._get_delegate().get_filename()
-        elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
+        elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
             return self.filename
         return None
 
@@ -366,10 +415,6 @@ def get_importer(path_item):
     The returned importer is cached in sys.path_importer_cache
     if it was newly created by a path hook.
 
-    If there is no importer, a wrapper around the basic import
-    machinery is returned. This wrapper is never inserted into
-    the importer cache (None is inserted instead).
-
     The cache (or part of it) can be cleared manually if a
     rescan of sys.path_hooks is necessary.
     """
@@ -379,18 +424,12 @@ def get_importer(path_item):
         for path_hook in sys.path_hooks:
             try:
                 importer = path_hook(path_item)
+                sys.path_importer_cache.setdefault(path_item, importer)
                 break
             except ImportError:
                 pass
         else:
             importer = None
-        sys.path_importer_cache.setdefault(path_item, importer)
-
-    if importer is None:
-        try:
-            importer = ImpImporter(path_item)
-        except ImportError:
-            importer = None
     return importer
 
 
@@ -398,55 +437,37 @@ def iter_importers(fullname=""):
     """Yield PEP 302 importers for the given module name
 
     If fullname contains a '.', the importers will be for the package
-    containing fullname, otherwise they will be importers for sys.meta_path,
-    sys.path, and Python's "classic" import machinery, in that order.  If
-    the named module is in a package, that package is imported as a side
-    effect of invoking this function.
-
-    Non PEP 302 mechanisms (e.g. the Windows registry) used by the
-    standard import machinery to find files in alternative locations
-    are partially supported, but are searched AFTER sys.path. Normally,
-    these locations are searched BEFORE sys.path, preventing sys.path
-    entries from shadowing them.
+    containing fullname, otherwise they will be all registered top level
+    importers (i.e. those on both sys.meta_path and sys.path_hooks).
 
-    For this to cause a visible difference in behaviour, there must
-    be a module or package name that is accessible via both sys.path
-    and one of the non PEP 302 file system mechanisms. In this case,
-    the emulation will find the former version, while the builtin
-    import mechanism will find the latter.
+    If the named module is in a package, that package is imported as a side
+    effect of invoking this function.
 
-    Items of the following types can be affected by this discrepancy:
-        imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
+    If no module name is specified, all top level importers are produced.
     """
     if fullname.startswith('.'):
-        raise ImportError("Relative module names not supported")
+        msg = "Relative module name {!r} not supported".format(fullname)
+        raise ImportError(msg)
     if '.' in fullname:
         # Get the containing package's __path__
-        pkg = '.'.join(fullname.split('.')[:-1])
-        if pkg not in sys.modules:
-            __import__(pkg)
-        path = getattr(sys.modules[pkg], '__path__', None) or []
+        pkg_name = fullname.rpartition(".")[0]
+        pkg = importlib.import_module(pkg)
+        path = getattr(sys.modules[pkg], '__path__', None)
+        if path is None:
+            return
     else:
         for importer in sys.meta_path:
             yield importer
         path = sys.path
     for item in path:
         yield get_importer(item)
-    if '.' not in fullname:
-        yield ImpImporter()
 
 def get_loader(module_or_name):
     """Get a PEP 302 "loader" object for module_or_name
 
-    If the module or package is accessible via the normal import
-    mechanism, a wrapper around the relevant part of that machinery
-    is returned.  Returns None if the module cannot be found or imported.
+    Returns None if the module cannot be found or imported.
     If the named module is not already imported, its containing package
     (if any) is imported, in order to establish the package __path__.
-
-    This function uses iter_importers(), and is thus subject to the same
-    limitations regarding platform-specific special import locations such
-    as the Windows registry.
     """
     if module_or_name in sys.modules:
         module_or_name = sys.modules[module_or_name]
@@ -460,20 +481,33 @@ def get_loader(module_or_name):
         fullname = module_or_name
     return find_loader(fullname)
 
+
 def find_loader(fullname):
     """Find a PEP 302 "loader" object for fullname
 
-    If fullname contains dots, path must be the containing package's __path__.
-    Returns None if the module cannot be found or imported. This function uses
-    iter_importers(), and is thus subject to the same limitations regarding
-    platform-specific special import locations such as the Windows registry.
+    This is s convenience wrapper around :func:`importlib.find_loader` that
+    sets the *path* argument correctly when searching for submodules, and
+    also ensures parent packages (if any) are imported before searching for
+    submodules.
     """
-    for importer in iter_importers(fullname):
-        loader = importer.find_module(fullname)
-        if loader is not None:
-            return loader
-
-    return None
+    if fullname.startswith('.'):
+        msg = "Relative module name {!r} not supported".format(fullname)
+        raise ImportError(msg)
+    path = None
+    pkg_name = fullname.rpartition(".")[0]
+    if pkg_name:
+        pkg = importlib.import_module(pkg_name)
+        path = getattr(pkg, "__path__", None)
+        if path is None:
+            return None
+    try:
+        return importlib.find_loader(fullname, path)
+    except (ImportError, AttributeError, TypeError, ValueError) as ex:
+        # This hack fixes an impedance mismatch between pkgutil and
+        # importlib, where the latter raises other errors for cases where
+        # pkgutil previously raised ImportError
+        msg = "Error while finding loader for {!r} ({}: {})"
+        raise ImportError(msg.format(fullname, type(ex), ex)) from ex
 
 
 def extend_path(path, name):
@@ -514,21 +548,41 @@ def extend_path(path, name):
         # frozen package.  Return the path unchanged in that case.
         return path
 
-    pname = os.path.join(*name.split('.')) # Reconstitute as relative path
     sname_pkg = name + ".pkg"
-    init_py = "__init__.py"
 
     path = path[:] # Start with a copy of the existing path
 
-    for dir in sys.path:
-        if not isinstance(dir, str) or not os.path.isdir(dir):
+    parent_package, _, final_name = name.rpartition('.')
+    if parent_package:
+        try:
+            search_path = sys.modules[parent_package].__path__
+        except (KeyError, AttributeError):
+            # We can't do anything: find_loader() returns None when
+            # passed a dotted name.
+            return path
+    else:
+        search_path = sys.path
+
+    for dir in search_path:
+        if not isinstance(dir, str):
             continue
-        subdir = os.path.join(dir, pname)
-        # XXX This may still add duplicate entries to path on
-        # case-insensitive filesystems
-        initfile = os.path.join(subdir, init_py)
-        if subdir not in path and os.path.isfile(initfile):
-            path.append(subdir)
+
+        finder = get_importer(dir)
+        if finder is not None:
+            # Is this finder PEP 420 compliant?
+            if hasattr(finder, 'find_loader'):
+                loader, portions = finder.find_loader(final_name)
+            else:
+                # No, no need to call it
+                loader = None
+                portions = []
+
+            for portion in portions:
+                # XXX This may still add duplicate entries to path on
+                # case-insensitive filesystems
+                if portion not in path:
+                    path.append(portion)
+
         # XXX Is this the right thing for subpackages like zope.app?
         # It looks for a file named "zope.app.pkg"
         pkgfile = os.path.join(dir, sname_pkg)
diff --git a/Lib/plat-generic/regen b/Lib/plat-generic/regen
index a20cdc1518..c96167dcb0 100755
--- a/Lib/plat-generic/regen
+++ b/Lib/plat-generic/regen
@@ -1,3 +1,3 @@
 #! /bin/sh
 set -v
-python$EXE ../../Tools/scripts/h2py.py -i '(u_long)' /usr/include/netinet/in.h
+eval $PYTHON_FOR_BUILD ../../Tools/scripts/h2py.py -i "'(u_long)'" /usr/include/netinet/in.h
diff --git a/Lib/plat-linux2/CDROM.py b/Lib/plat-linux/CDROM.py
index 434093684c..434093684c 100644
--- a/Lib/plat-linux2/CDROM.py
+++ b/Lib/plat-linux/CDROM.py
diff --git a/Lib/plat-linux2/DLFCN.py b/Lib/plat-linux/DLFCN.py
index dd10ac4ead..dd10ac4ead 100644
--- a/Lib/plat-linux2/DLFCN.py
+++ b/Lib/plat-linux/DLFCN.py
diff --git a/Lib/plat-linux2/IN.py b/Lib/plat-linux/IN.py
index d7d30024c2..d7d30024c2 100644
--- a/Lib/plat-linux2/IN.py
+++ b/Lib/plat-linux/IN.py
diff --git a/Lib/plat-linux2/TYPES.py b/Lib/plat-linux/TYPES.py
index e7a324b25a..e7a324b25a 100644
--- a/Lib/plat-linux2/TYPES.py
+++ b/Lib/plat-linux/TYPES.py
diff --git a/Lib/plat-linux2/regen b/Lib/plat-linux/regen
index c76950e232..c76950e232 100755
--- a/Lib/plat-linux2/regen
+++ b/Lib/plat-linux/regen
diff --git a/Lib/platform.py b/Lib/platform.py
index 686a045845..2b8a24a84e 100755
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -111,6 +111,7 @@ __copyright__ = """
 
 __version__ = '1.0.7'
 
+import collections
 import sys, os, re, subprocess
 
 ### Globals & Constants
@@ -130,15 +131,15 @@ except AttributeError:
 
 ### Platform specific APIs
 
-_libc_search = re.compile(r'(__libc_init)'
-                          '|'
-                          '(GLIBC_([0-9.]+))'
-                          '|'
-                          '(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII)
+_libc_search = re.compile(b'(__libc_init)'
+                          b'|'
+                          b'(GLIBC_([0-9.]+))'
+                          b'|'
+                          br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII)
 
 def libc_ver(executable=sys.executable,lib='',version='',
 
-             chunksize=2048):
+             chunksize=16384):
 
     """ Tries to determine the libc version that the file executable
         (which defaults to the Python interpreter) is linked against.
@@ -159,17 +160,22 @@ def libc_ver(executable=sys.executable,lib='',version='',
         # able to open symlinks for reading
         executable = os.path.realpath(executable)
     f = open(executable,'rb')
-    binary = f.read(chunksize).decode('latin-1')
+    binary = f.read(chunksize)
     pos = 0
     while 1:
-        m = _libc_search.search(binary,pos)
+        if b'libc' in binary or b'GLIBC' in binary:
+            m = _libc_search.search(binary,pos)
+        else:
+            m = None
         if not m:
-            binary = f.read(chunksize).decode('latin-1')
+            binary = f.read(chunksize)
             if not binary:
                 break
             pos = 0
             continue
-        libcinit,glibc,glibcversion,so,threads,soversion = m.groups()
+        libcinit,glibc,glibcversion,so,threads,soversion = [
+            s.decode('latin1') if s is not None else s
+            for s in m.groups()]
         if libcinit and not lib:
             lib = 'libc'
         elif glibc:
@@ -255,7 +261,7 @@ _release_version = re.compile(r'([^0-9]+)'
 _supported_dists = (
     'SuSE', 'debian', 'fedora', 'redhat', 'centos',
     'mandrake', 'mandriva', 'rocks', 'slackware', 'yellowdog', 'gentoo',
-    'UnitedLinux', 'turbolinux')
+    'UnitedLinux', 'turbolinux', 'arch', 'mageia')
 
 def _parse_release_file(firstline):
 
@@ -357,92 +363,13 @@ def dist(distname='',version='',id='',
                               supported_dists=supported_dists,
                               full_distribution_name=0)
 
-class _popen:
-
-    """ Fairly portable (alternative) popen implementation.
-
-        This is mostly needed in case os.popen() is not available, or
-        doesn't work as advertised, e.g. in Win9X GUI programs like
-        PythonWin or IDLE.
-
-        Writing to the pipe is currently not supported.
-
-    """
-    tmpfile = ''
-    pipe = None
-    bufsize = None
-    mode = 'r'
-
-    def __init__(self,cmd,mode='r',bufsize=None):
-
-        if mode != 'r':
-            raise ValueError('popen()-emulation only supports read mode')
-        import tempfile
-        self.tmpfile = tmpfile = tempfile.mktemp()
-        os.system(cmd + ' > %s' % tmpfile)
-        self.pipe = open(tmpfile,'rb')
-        self.bufsize = bufsize
-        self.mode = mode
-
-    def read(self):
-
-        return self.pipe.read()
-
-    def readlines(self):
-
-        if self.bufsize is not None:
-            return self.pipe.readlines()
-
-    def close(self,
-
-              remove=os.unlink,error=os.error):
-
-        if self.pipe:
-            rc = self.pipe.close()
-        else:
-            rc = 255
-        if self.tmpfile:
-            try:
-                remove(self.tmpfile)
-            except error:
-                pass
-        return rc
-
-    # Alias
-    __del__ = close
-
 def popen(cmd, mode='r', bufsize=-1):
 
     """ Portable popen() interface.
     """
-    # Find a working popen implementation preferring win32pipe.popen
-    # over os.popen over _popen
-    popen = None
-    if os.environ.get('OS','') == 'Windows_NT':
-        # On NT win32pipe should work; on Win9x it hangs due to bugs
-        # in the MS C lib (see MS KnowledgeBase article Q150956)
-        try:
-            import win32pipe
-        except ImportError:
-            pass
-        else:
-            popen = win32pipe.popen
-    if popen is None:
-        if hasattr(os,'popen'):
-            popen = os.popen
-            # Check whether it works... it doesn't in GUI programs
-            # on Windows platforms
-            if sys.platform == 'win32': # XXX Others too ?
-                try:
-                    popen('')
-                except os.error:
-                    popen = _popen
-        else:
-            popen = _popen
-    if bufsize is None:
-        return popen(cmd,mode)
-    else:
-        return popen(cmd,mode,bufsize)
+    import warnings
+    warnings.warn('use os.popen instead', DeprecationWarning, stacklevel=2)
+    return os.popen(cmd, mode, bufsize)
 
 def _norm_version(version, build=''):
 
@@ -779,7 +706,7 @@ def _mac_ver_xml():
     pl = plistlib.readPlist(fn)
     release = pl['ProductVersion']
     versioninfo=('', '', '')
-    machine = os.uname()[4]
+    machine = os.uname().machine
     if machine in ('ppc', 'Power Macintosh'):
         # for compatibility with the gestalt based code
         machine = 'PowerPC'
@@ -1004,9 +931,10 @@ def _syscmd_file(target,default=''):
     try:
         proc = subprocess.Popen(['file', target],
                 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
     except (AttributeError,os.error):
         return default
-    output = proc.communicate()[0].decode("latin-1")
+    output = proc.communicate()[0].decode('latin-1')
     rc = proc.wait()
     if not output or rc:
         return default
@@ -1107,6 +1035,9 @@ def architecture(executable=sys.executable,bits='',linkage=''):
 
 ### Portable uname() interface
 
+uname_result = collections.namedtuple("uname_result",
+                    "system node release version machine processor")
+
 _uname_cache = None
 
 def uname():
@@ -1241,7 +1172,7 @@ def uname():
         system = 'Windows'
         release = 'Vista'
 
-    _uname_cache = system,node,release,version,machine,processor
+    _uname_cache = uname_result(system,node,release,version,machine,processor)
     return _uname_cache
 
 ### Direct interfaces to some of the uname() return values
@@ -1253,7 +1184,7 @@ def system():
         An empty string is returned if the value cannot be determined.
 
     """
-    return uname()[0]
+    return uname().system
 
 def node():
 
@@ -1263,7 +1194,7 @@ def node():
         An empty string is returned if the value cannot be determined.
 
     """
-    return uname()[1]
+    return uname().node
 
 def release():
 
@@ -1272,7 +1203,7 @@ def release():
         An empty string is returned if the value cannot be determined.
 
     """
-    return uname()[2]
+    return uname().release
 
 def version():
 
@@ -1281,7 +1212,7 @@ def version():
         An empty string is returned if the value cannot be determined.
 
     """
-    return uname()[3]
+    return uname().version
 
 def machine():
 
@@ -1290,7 +1221,7 @@ def machine():
         An empty string is returned if the value cannot be determined.
 
     """
-    return uname()[4]
+    return uname().machine
 
 def processor():
 
@@ -1302,7 +1233,7 @@ def processor():
         e.g.  NetBSD does this.
 
     """
-    return uname()[5]
+    return uname().processor
 
 ### Various APIs for extracting information from sys.version
 
diff --git a/Lib/plistlib.py b/Lib/plistlib.py
index 2e7e5126d7..41fd8f2efd 100644
--- a/Lib/plistlib.py
+++ b/Lib/plistlib.py
@@ -237,20 +237,26 @@ class PlistWriter(DumbXMLWriter):
         self.endElement("data")
 
     def writeDict(self, d):
-        self.beginElement("dict")
-        items = sorted(d.items())
-        for key, value in items:
-            if not isinstance(key, str):
-                raise TypeError("keys must be strings")
-            self.simpleElement("key", key)
-            self.writeValue(value)
-        self.endElement("dict")
+        if d:
+            self.beginElement("dict")
+            items = sorted(d.items())
+            for key, value in items:
+                if not isinstance(key, str):
+                    raise TypeError("keys must be strings")
+                self.simpleElement("key", key)
+                self.writeValue(value)
+            self.endElement("dict")
+        else:
+            self.simpleElement("dict")
 
     def writeArray(self, array):
-        self.beginElement("array")
-        for value in array:
-            self.writeValue(value)
-        self.endElement("array")
+        if array:
+            self.beginElement("array")
+            for value in array:
+                self.writeValue(value)
+            self.endElement("array")
+        else:
+            self.simpleElement("array")
 
 
 class _InternalDict(dict):
@@ -266,13 +272,13 @@ class _InternalDict(dict):
             raise AttributeError(attr)
         from warnings import warn
         warn("Attribute access from plist dicts is deprecated, use d[key] "
-             "notation instead", PendingDeprecationWarning, 2)
+             "notation instead", DeprecationWarning, 2)
         return value
 
     def __setattr__(self, attr, value):
         from warnings import warn
         warn("Attribute access from plist dicts is deprecated, use d[key] "
-             "notation instead", PendingDeprecationWarning, 2)
+             "notation instead", DeprecationWarning, 2)
         self[attr] = value
 
     def __delattr__(self, attr):
@@ -282,14 +288,14 @@ class _InternalDict(dict):
             raise AttributeError(attr)
         from warnings import warn
         warn("Attribute access from plist dicts is deprecated, use d[key] "
-             "notation instead", PendingDeprecationWarning, 2)
+             "notation instead", DeprecationWarning, 2)
 
 class Dict(_InternalDict):
 
     def __init__(self, **kwargs):
         from warnings import warn
         warn("The plistlib.Dict class is deprecated, use builtin dict instead",
-             PendingDeprecationWarning, 2)
+             DeprecationWarning, 2)
         super().__init__(**kwargs)
 
 
@@ -302,7 +308,7 @@ class Plist(_InternalDict):
     def __init__(self, **kwargs):
         from warnings import warn
         warn("The Plist class is deprecated, use the readPlist() and "
-             "writePlist() functions instead", PendingDeprecationWarning, 2)
+             "writePlist() functions instead", DeprecationWarning, 2)
         super().__init__(**kwargs)
 
     def fromFile(cls, pathOrFile):
diff --git a/Lib/poplib.py b/Lib/poplib.py
index 84ea88de46..d42d9dd320 100644
--- a/Lib/poplib.py
+++ b/Lib/poplib.py
@@ -250,15 +250,18 @@ class POP3:
 
     def quit(self):
         """Signoff: commit changes on server, unlock mailbox, close connection."""
-        try:
-            resp = self._shortcmd('QUIT')
-        except error_proto as val:
-            resp = val
-        self.file.close()
-        self.sock.close()
-        del self.file, self.sock
+        resp = self._shortcmd('QUIT')
+        self.close()
         return resp
 
+    def close(self):
+        """Close the connection without assuming anything about it."""
+        if self.file is not None:
+            self.file.close()
+        if self.sock is not None:
+            self.sock.close()
+        self.file = self.sock = None
+
     #__del__ = quit
 
 
diff --git a/Lib/posixpath.py b/Lib/posixpath.py
index bc76c90dde..b1e1a9255e 100644
--- a/Lib/posixpath.py
+++ b/Lib/posixpath.py
@@ -88,7 +88,8 @@ def join(a, *p):
                           for s in (a, ) + p)
         if valid_types:
             # Must have a mixture of text and binary data
-            raise TypeError("Can't mix strings and bytes in path components.")
+            raise TypeError("Can't mix strings and bytes in path "
+                            "components.") from None
         raise
     return path
 
diff --git a/Lib/profile.py b/Lib/profile.py
index 297e32d64a..743e77d7bf 100755
--- a/Lib/profile.py
+++ b/Lib/profile.py
@@ -83,26 +83,6 @@ def runctx(statement, globals, locals, filename=None, sort=-1):
     else:
         return prof.print_stats(sort)
 
-if hasattr(os, "times"):
-    def _get_time_times(timer=os.times):
-        t = timer()
-        return t[0] + t[1]
-
-# Using getrusage(3) is better than clock(3) if available:
-# on some systems (e.g. FreeBSD), getrusage has a higher resolution
-# Furthermore, on a POSIX system, returns microseconds, which
-# wrap around after 36min.
-_has_res = 0
-try:
-    import resource
-    resgetrusage = lambda: resource.getrusage(resource.RUSAGE_SELF)
-    def _get_time_resource(timer=resgetrusage):
-        t = timer()
-        return t[0] + t[1]
-    _has_res = 1
-except ImportError:
-    pass
-
 class Profile:
     """Profiler class.
 
@@ -155,20 +135,8 @@ class Profile:
         self.bias = bias     # Materialize in local dict for lookup speed.
 
         if not timer:
-            if _has_res:
-                self.timer = resgetrusage
-                self.dispatcher = self.trace_dispatch
-                self.get_time = _get_time_resource
-            elif hasattr(time, 'clock'):
-                self.timer = self.get_time = time.clock
-                self.dispatcher = self.trace_dispatch_i
-            elif hasattr(os, 'times'):
-                self.timer = os.times
-                self.dispatcher = self.trace_dispatch
-                self.get_time = _get_time_times
-            else:
-                self.timer = self.get_time = time.time
-                self.dispatcher = self.trace_dispatch_i
+            self.timer = self.get_time = time.process_time
+            self.dispatcher = self.trace_dispatch_i
         else:
             self.timer = timer
             t = self.timer() # test out timer function
diff --git a/Lib/pstats.py b/Lib/pstats.py
index bfbaa41386..6a77605e14 100644
--- a/Lib/pstats.py
+++ b/Lib/pstats.py
@@ -678,13 +678,14 @@ if __name__ == '__main__':
                 return stop
             return None
 
-    import sys
     if len(sys.argv) > 1:
         initprofile = sys.argv[1]
     else:
         initprofile = None
     try:
         browser = ProfileBrowser(initprofile)
+        for profile in sys.argv[2:]:
+            browser.do_add(profile)
         print("Welcome to the profile statistics browser.", file=browser.stream)
         browser.cmdloop()
         print("Goodbye.", file=browser.stream)
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index 5adb70a293..62d69ad448 100644
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -110,9 +110,11 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
     """
     with tokenize.open(file) as f:
         try:
-            timestamp = int(os.fstat(f.fileno()).st_mtime)
+            st = os.fstat(f.fileno())
         except AttributeError:
-            timestamp = int(os.stat(file).st_mtime)
+            st = os.stat(file)
+        timestamp = int(st.st_mtime)
+        size = st.st_size & 0xFFFFFFFF
         codestring = f.read()
     try:
         codeobject = builtins.compile(codestring, dfile or file, 'exec',
@@ -139,6 +141,7 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
     with open(cfile, 'wb') as fc:
         fc.write(b'\0\0\0\0')
         wr_long(fc, timestamp)
+        wr_long(fc, size)
         marshal.dump(codeobject, fc)
         fc.flush()
         fc.seek(0, 0)
diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py
index 65e9fbedfd..9ec05ee80b 100644
--- a/Lib/pyclbr.py
+++ b/Lib/pyclbr.py
@@ -39,8 +39,10 @@ Instances of this class have the following instance variables:
         lineno -- the line in the file on which the class statement occurred
 """
 
+import io
+import os
 import sys
-import imp
+import importlib
 import tokenize
 from token import NAME, DEDENT, OP
 from operator import itemgetter
@@ -135,19 +137,24 @@ def _readmodule(module, path, inpackage=None):
     # Search the path for the module
     f = None
     if inpackage is not None:
-        f, fname, (_s, _m, ty) = imp.find_module(module, path)
+        search_path = path
     else:
-        f, fname, (_s, _m, ty) = imp.find_module(module, path + sys.path)
-    if ty == imp.PKG_DIRECTORY:
-        dict['__path__'] = [fname]
-        path = [fname] + path
-        f, fname, (_s, _m, ty) = imp.find_module('__init__', [fname])
+        search_path = path + sys.path
+    loader = importlib.find_loader(fullmodule, search_path)
+    fname = loader.get_filename(fullmodule)
     _modules[fullmodule] = dict
-    if ty != imp.PY_SOURCE:
+    if loader.is_package(fullmodule):
+        dict['__path__'] = [os.path.dirname(fname)]
+    try:
+        source = loader.get_source(fullmodule)
+        if source is None:
+            return dict
+    except (AttributeError, ImportError):
         # not Python source, can't do anything with this module
-        f.close()
         return dict
 
+    f = io.StringIO(source)
+
     stack = [] # stack of (class, indent) pairs
 
     g = tokenize.generate_tokens(f.readline)
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 37616fb3ed..fa531e9051 100755
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Generate Python documentation in HTML or text for interactive use.
 
-In the Python interpreter, do "from pydoc import help" to provide online
+In the Python interpreter, do "from pydoc import help" to provide
 help.  Calling help(thing) on a Python object documents the object.
 
 Or, at the shell command line outside of Python:
@@ -22,11 +22,6 @@ Run "pydoc -b" to start an HTTP server on an arbitrary unused port and
 open a Web browser to interactively browse documentation.  The -p option
 can be used with the -b option to explicitly specify the server port.
 
-For platforms without a command line, "pydoc -g" starts the HTTP server
-and also pops up a little window for controlling it.  This option is
-deprecated, since the server can now be controlled directly from HTTP
-clients.
-
 Run "pydoc -w <name>" to write out the HTML documentation for a module
 to a file named "<name>.html".
 
@@ -42,7 +37,6 @@ __all__ = ['help']
 __author__ = "Ka-Ping Yee <ping@lfw.org>"
 __date__ = "26 February 2001"
 
-__version__ = "$Revision$"
 __credits__ = """Guido van Rossum, for an excellent programming language.
 Tommy Burnette, the original creator of manpy.
 Paul Prescod, for all his work on onlinehelp.
@@ -59,6 +53,7 @@ Richard Chamberlain, for the first implementation of textdoc.
 
 import builtins
 import imp
+import importlib.machinery
 import inspect
 import io
 import os
@@ -168,12 +163,12 @@ def _split_list(s, predicate):
 
 def visiblename(name, all=None, obj=None):
     """Decide whether to show documentation on a variable."""
-    # Certain special names are redundant.
-    _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__',
-                     '__module__', '__name__', '__slots__', '__package__',
-                     '__cached__', '__author__', '__credits__', '__date__',
-                     '__version__')
-    if name in _hidden_names: return 0
+    # Certain special names are redundant or internal.
+    if name in {'__author__', '__builtins__', '__cached__', '__credits__',
+                '__date__', '__doc__', '__file__', '__initializing__',
+                '__loader__', '__module__', '__name__', '__package__',
+                '__path__', '__qualname__', '__slots__', '__version__'}:
+        return 0
     # Private names are hidden, but special names are displayed.
     if name.startswith('__') and name.endswith('__'): return 1
     # Namedtuples have public fields and methods with a single leading underscore
@@ -226,20 +221,34 @@ def synopsis(filename, cache={}):
     mtime = os.stat(filename).st_mtime
     lastupdate, result = cache.get(filename, (None, None))
     if lastupdate is None or lastupdate < mtime:
-        info = inspect.getmoduleinfo(filename)
         try:
             file = tokenize.open(filename)
         except IOError:
             # module can't be opened, so skip it
             return None
-        if info and 'b' in info[2]: # binary modules have to be imported
-            try: module = imp.load_module('__temp__', file, filename, info[1:])
-            except: return None
+        binary_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:]
+        binary_suffixes += importlib.machinery.EXTENSION_SUFFIXES[:]
+        if any(filename.endswith(x) for x in binary_suffixes):
+            # binary modules have to be imported
+            file.close()
+            if any(filename.endswith(x) for x in
+                    importlib.machinery.BYTECODE_SUFFIXES):
+                loader = importlib.machinery.SourcelessFileLoader('__temp__',
+                                                                  filename)
+            else:
+                loader = importlib.machinery.ExtensionFileLoader('__temp__',
+                                                                 filename)
+            try:
+                module = loader.load_module('__temp__')
+            except:
+                return None
             result = (module.__doc__ or '').splitlines()[0]
             del sys.modules['__temp__']
-        else: # text modules can be directly examined
+        else:
+            # text modules can be directly examined
             result = source_synopsis(file)
             file.close()
+
         cache[filename] = (mtime, result)
     return result
 
@@ -305,9 +314,8 @@ def safeimport(path, forceload=0, cache={}):
         elif exc is SyntaxError:
             # A SyntaxError occurred before we could execute the module.
             raise ErrorDuringImport(value.filename, info)
-        elif exc is ImportError and extract_tb(tb)[-1][2]=='safeimport':
-            # The import error occurred directly in this function,
-            # which means there is no such module in the path.
+        elif exc is ImportError and value.name == path:
+            # No such module in the path.
             return None
         else:
             # Some other error occurred during the importing process.
@@ -361,7 +369,7 @@ class Doc:
 
         docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS)
 
-        basedir = os.path.join(sys.exec_prefix, "lib",
+        basedir = os.path.join(sys.base_exec_prefix, "lib",
                                "python%d.%d" %  sys.version_info[:2])
         if (isinstance(object, type(os)) and
             (object.__name__ in ('errno', 'exceptions', 'gc', 'imp',
@@ -963,6 +971,9 @@ class HTMLDoc(Doc):
         modpkgs = []
         if shadowed is None: shadowed = {}
         for importer, name, ispkg in pkgutil.iter_modules([dir]):
+            if any((0xD800 <= ord(ch) <= 0xDFFF) for ch in name):
+                # ignore a module if its name contains a surrogate character
+                continue
             modpkgs.append((name, '', ispkg, name in shadowed))
             shadowed[name] = 1
 
@@ -1827,7 +1838,7 @@ has the same effect as typing a particular string at the help> prompt.
 
     def intro(self):
         self.output.write('''
-Welcome to Python %s!  This is the online help utility.
+Welcome to Python %s!  This is the interactive help utility.
 
 If this is your first time using Python, you should definitely check out
 the tutorial on the Internet at http://docs.python.org/%s/tutorial/.
@@ -2018,14 +2029,6 @@ class ModuleScanner:
             if self.quit:
                 break
 
-            # XXX Skipping this file is a workaround for a bug
-            # that causes python to crash with a segfault.
-            # http://bugs.python.org/issue9319
-            #
-            # TODO Remove this once the bug is fixed.
-            if modname in {'test.badsyntax_pep3120', 'badsyntax_pep3120'}:
-                continue
-
             if key is None:
                 callback(None, modname, '')
             else:
@@ -2037,7 +2040,7 @@ class ModuleScanner:
                 if hasattr(loader, 'get_source'):
                     try:
                         source = loader.get_source(modname)
-                    except UnicodeDecodeError:
+                    except Exception:
                         if onerror:
                             onerror(modname)
                         continue
@@ -2074,272 +2077,6 @@ def apropos(key):
         warnings.filterwarnings('ignore') # ignore problems during import
         ModuleScanner().run(callback, key, onerror=onerror)
 
-# --------------------------------------------------- Web browser interface
-
-def serve(port, callback=None, completer=None):
-    import http.server, email.message, select
-
-    msg = 'the pydoc.serve() function is deprecated'
-    warnings.warn(msg, DeprecationWarning, stacklevel=2)
-
-    class DocHandler(http.server.BaseHTTPRequestHandler):
-        def send_document(self, title, contents):
-            try:
-                self.send_response(200)
-                self.send_header('Content-Type', 'text/html; charset=UTF-8')
-                self.end_headers()
-                self.wfile.write(html.page(title, contents).encode('utf-8'))
-            except IOError: pass
-
-        def do_GET(self):
-            path = self.path
-            if path[-5:] == '.html': path = path[:-5]
-            if path[:1] == '/': path = path[1:]
-            if path and path != '.':
-                try:
-                    obj = locate(path, forceload=1)
-                except ErrorDuringImport as value:
-                    self.send_document(path, html.escape(str(value)))
-                    return
-                if obj:
-                    self.send_document(describe(obj), html.document(obj, path))
-                else:
-                    self.send_document(path,
-'no Python documentation found for %s' % repr(path))
-            else:
-                heading = html.heading(
-'<big><big><strong>Python: Index of Modules</strong></big></big>',
-'#ffffff', '#7799ee')
-                def bltinlink(name):
-                    return '<a href="%s.html">%s</a>' % (name, name)
-                names = [x for x in sys.builtin_module_names if x != '__main__']
-                contents = html.multicolumn(names, bltinlink)
-                indices = ['<p>' + html.bigsection(
-                    'Built-in Modules', '#ffffff', '#ee77aa', contents)]
-
-                seen = {}
-                for dir in sys.path:
-                    indices.append(html.index(dir, seen))
-                contents = heading + ' '.join(indices) + '''<p align=right>
-<font color="#909090" face="helvetica, arial"><strong>
-pydoc</strong> by Ka-Ping Yee &lt;ping@lfw.org&gt;</font>'''
-                self.send_document('Index of Modules', contents)
-
-        def log_message(self, *args): pass
-
-    class DocServer(http.server.HTTPServer):
-        def __init__(self, port, callback):
-            host = 'localhost'
-            self.address = (host, port)
-            self.url = 'http://%s:%d/' % (host, port)
-            self.callback = callback
-            self.base.__init__(self, self.address, self.handler)
-
-        def serve_until_quit(self):
-            import select
-            self.quit = False
-            while not self.quit:
-                rd, wr, ex = select.select([self.socket.fileno()], [], [], 1)
-                if rd: self.handle_request()
-            self.server_close()
-
-        def server_activate(self):
-            self.base.server_activate(self)
-            if self.callback: self.callback(self)
-
-    DocServer.base = http.server.HTTPServer
-    DocServer.handler = DocHandler
-    DocHandler.MessageClass = email.message.Message
-    try:
-        try:
-            DocServer(port, callback).serve_until_quit()
-        except (KeyboardInterrupt, select.error):
-            pass
-    finally:
-        if completer: completer()
-
-# ----------------------------------------------------- graphical interface
-
-def gui():
-    """Graphical interface (starts Web server and pops up a control window)."""
-
-    msg = ('the pydoc.gui() function and "pydoc -g" option are deprecated\n',
-           'use "pydoc.browse() function and "pydoc -b" option instead.')
-    warnings.warn(msg, DeprecationWarning, stacklevel=2)
-
-    class GUI:
-        def __init__(self, window, port=7464):
-            self.window = window
-            self.server = None
-            self.scanner = None
-
-            import tkinter
-            self.server_frm = tkinter.Frame(window)
-            self.title_lbl = tkinter.Label(self.server_frm,
-                text='Starting server...\n ')
-            self.open_btn = tkinter.Button(self.server_frm,
-                text='open browser', command=self.open, state='disabled')
-            self.quit_btn = tkinter.Button(self.server_frm,
-                text='quit serving', command=self.quit, state='disabled')
-
-            self.search_frm = tkinter.Frame(window)
-            self.search_lbl = tkinter.Label(self.search_frm, text='Search for')
-            self.search_ent = tkinter.Entry(self.search_frm)
-            self.search_ent.bind('<Return>', self.search)
-            self.stop_btn = tkinter.Button(self.search_frm,
-                text='stop', pady=0, command=self.stop, state='disabled')
-            if sys.platform == 'win32':
-                # Trying to hide and show this button crashes under Windows.
-                self.stop_btn.pack(side='right')
-
-            self.window.title('pydoc')
-            self.window.protocol('WM_DELETE_WINDOW', self.quit)
-            self.title_lbl.pack(side='top', fill='x')
-            self.open_btn.pack(side='left', fill='x', expand=1)
-            self.quit_btn.pack(side='right', fill='x', expand=1)
-            self.server_frm.pack(side='top', fill='x')
-
-            self.search_lbl.pack(side='left')
-            self.search_ent.pack(side='right', fill='x', expand=1)
-            self.search_frm.pack(side='top', fill='x')
-            self.search_ent.focus_set()
-
-            font = ('helvetica', sys.platform == 'win32' and 8 or 10)
-            self.result_lst = tkinter.Listbox(window, font=font, height=6)
-            self.result_lst.bind('<Button-1>', self.select)
-            self.result_lst.bind('<Double-Button-1>', self.goto)
-            self.result_scr = tkinter.Scrollbar(window,
-                orient='vertical', command=self.result_lst.yview)
-            self.result_lst.config(yscrollcommand=self.result_scr.set)
-
-            self.result_frm = tkinter.Frame(window)
-            self.goto_btn = tkinter.Button(self.result_frm,
-                text='go to selected', command=self.goto)
-            self.hide_btn = tkinter.Button(self.result_frm,
-                text='hide results', command=self.hide)
-            self.goto_btn.pack(side='left', fill='x', expand=1)
-            self.hide_btn.pack(side='right', fill='x', expand=1)
-
-            self.window.update()
-            self.minwidth = self.window.winfo_width()
-            self.minheight = self.window.winfo_height()
-            self.bigminheight = (self.server_frm.winfo_reqheight() +
-                                 self.search_frm.winfo_reqheight() +
-                                 self.result_lst.winfo_reqheight() +
-                                 self.result_frm.winfo_reqheight())
-            self.bigwidth, self.bigheight = self.minwidth, self.bigminheight
-            self.expanded = 0
-            self.window.wm_geometry('%dx%d' % (self.minwidth, self.minheight))
-            self.window.wm_minsize(self.minwidth, self.minheight)
-            self.window.tk.willdispatch()
-
-            import threading
-            threading.Thread(
-                target=serve, args=(port, self.ready, self.quit)).start()
-
-        def ready(self, server):
-            self.server = server
-            self.title_lbl.config(
-                text='Python documentation server at\n' + server.url)
-            self.open_btn.config(state='normal')
-            self.quit_btn.config(state='normal')
-
-        def open(self, event=None, url=None):
-            url = url or self.server.url
-            import webbrowser
-            webbrowser.open(url)
-
-        def quit(self, event=None):
-            if self.server:
-                self.server.quit = 1
-            self.window.quit()
-
-        def search(self, event=None):
-            key = self.search_ent.get()
-            self.stop_btn.pack(side='right')
-            self.stop_btn.config(state='normal')
-            self.search_lbl.config(text='Searching for "%s"...' % key)
-            self.search_ent.forget()
-            self.search_lbl.pack(side='left')
-            self.result_lst.delete(0, 'end')
-            self.goto_btn.config(state='disabled')
-            self.expand()
-
-            import threading
-            if self.scanner:
-                self.scanner.quit = 1
-            self.scanner = ModuleScanner()
-            threading.Thread(target=self.scanner.run,
-                             args=(self.update, key, self.done)).start()
-
-        def update(self, path, modname, desc):
-            if modname[-9:] == '.__init__':
-                modname = modname[:-9] + ' (package)'
-            self.result_lst.insert('end',
-                modname + ' - ' + (desc or '(no description)'))
-
-        def stop(self, event=None):
-            if self.scanner:
-                self.scanner.quit = 1
-                self.scanner = None
-
-        def done(self):
-            self.scanner = None
-            self.search_lbl.config(text='Search for')
-            self.search_lbl.pack(side='left')
-            self.search_ent.pack(side='right', fill='x', expand=1)
-            if sys.platform != 'win32': self.stop_btn.forget()
-            self.stop_btn.config(state='disabled')
-
-        def select(self, event=None):
-            self.goto_btn.config(state='normal')
-
-        def goto(self, event=None):
-            selection = self.result_lst.curselection()
-            if selection:
-                modname = self.result_lst.get(selection[0]).split()[0]
-                self.open(url=self.server.url + modname + '.html')
-
-        def collapse(self):
-            if not self.expanded: return
-            self.result_frm.forget()
-            self.result_scr.forget()
-            self.result_lst.forget()
-            self.bigwidth = self.window.winfo_width()
-            self.bigheight = self.window.winfo_height()
-            self.window.wm_geometry('%dx%d' % (self.minwidth, self.minheight))
-            self.window.wm_minsize(self.minwidth, self.minheight)
-            self.expanded = 0
-
-        def expand(self):
-            if self.expanded: return
-            self.result_frm.pack(side='bottom', fill='x')
-            self.result_scr.pack(side='right', fill='y')
-            self.result_lst.pack(side='top', fill='both', expand=1)
-            self.window.wm_geometry('%dx%d' % (self.bigwidth, self.bigheight))
-            self.window.wm_minsize(self.minwidth, self.bigminheight)
-            self.expanded = 1
-
-        def hide(self, event=None):
-            self.stop()
-            self.collapse()
-
-    import tkinter
-    try:
-        root = tkinter.Tk()
-        # Tk will crash if pythonw.exe has an XP .manifest
-        # file and the root has is not destroyed explicitly.
-        # If the problem is ever fixed in Tk, the explicit
-        # destroy can go.
-        try:
-            gui = GUI(root)
-            root.mainloop()
-        finally:
-            root.destroy()
-    except KeyboardInterrupt:
-        pass
-
-
 # --------------------------------------- enhanced Web browser interface
 
 def _start_server(urlhandler, port):
@@ -2796,15 +2533,12 @@ def cli():
         sys.path.insert(0, '.')
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'bgk:p:w')
+        opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w')
         writing = False
         start_server = False
         open_browser = False
         port = None
         for opt, val in opts:
-            if opt == '-g':
-                gui()
-                return
             if opt == '-b':
                 start_server = True
                 open_browser = True
@@ -2817,8 +2551,8 @@ def cli():
             if opt == '-w':
                 writing = True
 
-        if start_server == True:
-            if port == None:
+        if start_server:
+            if port is None:
                 port = 0
             browse(port, open_browser=open_browser)
             return
@@ -2865,9 +2599,6 @@ def cli():
     to interactively browse documentation.  The -p option can be used with
     the -b option to explicitly specify the server port.
 
-{cmd} -g
-    Deprecated.
-
 {cmd} -w <name> ...
     Write out the HTML documentation for a module to a file in the current
     directory.  If <name> contains a '{sep}', it is treated as a filename; if
diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py
index 08a9e7c01e..62a54a91b8 100644
--- a/Lib/pydoc_data/topics.py
+++ b/Lib/pydoc_data/topics.py
@@ -1,16 +1,17 @@
-# Autogenerated by Sphinx on Thu Feb 23 18:37:54 2012
+# -*- coding: utf-8 -*-
+# Autogenerated by Sphinx on Sat Aug 25 12:12:45 2012
 topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n   assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n   if __debug__:\n      if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n   if __debug__:\n      if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names.  In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O).  The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime.  Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal.  The value for the built-in\nvariable is determined when the interpreter starts.\n',
  'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n   assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n   target_list     ::= target ("," target)* [","]\n   target          ::= identifier\n              | "(" target_list ")"\n              | "[" target_list "]"\n              | attributeref\n              | subscription\n              | slicing\n              | "*" target\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable.  The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list, optionally enclosed in\nparentheses or square brackets, is recursively defined as follows.\n\n* If the target list is a single target: The object is assigned to\n  that target.\n\n* If the target list is a comma-separated list of targets: The object\n  must be an iterable with the same number of items as there are\n  targets in the target list, and the items are assigned, from left to\n  right, to the corresponding targets.\n\n  * If the target list contains one target prefixed with an asterisk,\n    called a "starred" target: The object must be a sequence with at\n    least as many items as there are targets in the target list, minus\n    one.  The first items of the sequence are assigned, from left to\n    right, to the targets before the starred target.  The final items\n    of the sequence are assigned to the targets after the starred\n    target.  A list of the remaining items in the sequence is then\n    assigned to the starred target (the list can be empty).\n\n  * Else: The object must be a sequence with the same number of items\n    as there are targets in the target list, and the items are\n    assigned, from left to right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n  * If the name does not occur in a ``global`` or ``nonlocal``\n    statement in the current code block: the name is bound to the\n    object in the current local namespace.\n\n  * Otherwise: the name is bound to the object in the global namespace\n    or the outer namespace determined by ``nonlocal``, respectively.\n\n  The name is rebound if it was already bound.  This may cause the\n  reference count for the object previously bound to the name to reach\n  zero, causing the object to be deallocated and its destructor (if it\n  has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n  brackets: The object must be an iterable with the same number of\n  items as there are targets in the target list, and its items are\n  assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n  the reference is evaluated.  It should yield an object with\n  assignable attributes; if this is not the case, ``TypeError`` is\n  raised.  That object is then asked to assign the assigned object to\n  the given attribute; if it cannot perform the assignment, it raises\n  an exception (usually but not necessarily ``AttributeError``).\n\n  Note: If the object is a class instance and the attribute reference\n  occurs on both sides of the assignment operator, the RHS expression,\n  ``a.x`` can access either an instance attribute or (if no instance\n  attribute exists) a class attribute.  The LHS target ``a.x`` is\n  always set as an instance attribute, creating it if necessary.\n  Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n  same attribute: if the RHS expression refers to a class attribute,\n  the LHS creates a new instance attribute as the target of the\n  assignment:\n\n     class Cls:\n         x = 3             # class variable\n     inst = Cls()\n     inst.x = inst.x + 1   # writes inst.x as 4 leaving Cls.x as 3\n\n  This description does not necessarily apply to descriptor\n  attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n  reference is evaluated.  It should yield either a mutable sequence\n  object (such as a list) or a mapping object (such as a dictionary).\n  Next, the subscript expression is evaluated.\n\n  If the primary is a mutable sequence object (such as a list), the\n  subscript must yield an integer.  If it is negative, the sequence\'s\n  length is added to it.  The resulting value must be a nonnegative\n  integer less than the sequence\'s length, and the sequence is asked\n  to assign the assigned object to its item with that index.  If the\n  index is out of range, ``IndexError`` is raised (assignment to a\n  subscripted sequence cannot add new items to a list).\n\n  If the primary is a mapping object (such as a dictionary), the\n  subscript must have a type compatible with the mapping\'s key type,\n  and the mapping is then asked to create a key/datum pair which maps\n  the subscript to the assigned object.  This can either replace an\n  existing key/value pair with the same key value, or insert a new\n  key/value pair (if no key with the same value existed).\n\n  For user-defined objects, the ``__setitem__()`` method is called\n  with appropriate arguments.\n\n* If the target is a slicing: The primary expression in the reference\n  is evaluated.  It should yield a mutable sequence object (such as a\n  list).  The assigned object should be a sequence object of the same\n  type.  Next, the lower and upper bound expressions are evaluated,\n  insofar they are present; defaults are zero and the sequence\'s\n  length.  The bounds should evaluate to integers. If either bound is\n  negative, the sequence\'s length is added to it.  The resulting\n  bounds are clipped to lie between zero and the sequence\'s length,\n  inclusive.  Finally, the sequence object is asked to replace the\n  slice with the items of the assigned sequence.  The length of the\n  slice may be different from the length of the assigned sequence,\n  thus changing the length of the target sequence, if the object\n  allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe!  For instance, the\nfollowing program prints ``[0, 2]``:\n\n   x = [0, 1]\n   i = 0\n   i, x[i] = 1, 2\n   print(x)\n\nSee also:\n\n   **PEP 3132** - Extended Iterable Unpacking\n      The specification for the ``*target`` feature.\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n   augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n   augtarget                 ::= identifier | attributeref | subscription | slicing\n   augop                     ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n             | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
  'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name.  See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them.  The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name.  For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``.  This transformation is independent of\nthe syntactical context in which the identifier is used.  If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen.  If the class name\nconsists only of underscores, no transformation is done.\n',
  'atom-literals': "\nLiterals\n********\n\nPython supports string and bytes literals and various numeric\nliterals:\n\n   literal ::= stringliteral | bytesliteral\n               | integer | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\nbytes, integer, floating point number, complex number) with the given\nvalue.  The value may be approximated in the case of floating point\nand imaginary (complex) literals.  See section *Literals* for details.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value.  Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n",
- 'attribute-access': '\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control over attribute access.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary). *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should call the base class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\nobject.__dir__(self)\n\n   Called when ``dir()`` is called on the object.  A list must be\n   returned.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to an object instance, ``a.x`` is transformed into the\n   call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a class, ``A.x`` is transformed into the call:\n   ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of classes have a dictionary for attribute\nstorage.  This wastes space for objects having very few instance\nvariables.  The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable.  Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   class, *__slots__* reserves space for the declared variables and\n   prevents the automatic creation of *__dict__* and *__weakref__* for\n   each instance.\n\n\nNotes on using *__slots__*\n--------------------------\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``int``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n',
+ 'attribute-access': '\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control over attribute access.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary). *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should call the base class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\nobject.__dir__(self)\n\n   Called when ``dir()`` is called on the object. A sequence must be\n   returned. ``dir()`` converts the returned sequence to a list and\n   sorts it.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to an object instance, ``a.x`` is transformed into the\n   call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a class, ``A.x`` is transformed into the call:\n   ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of classes have a dictionary for attribute\nstorage.  This wastes space for objects having very few instance\nvariables.  The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable.  Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   class, *__slots__* reserves space for the declared variables and\n   prevents the automatic creation of *__dict__* and *__weakref__* for\n   each instance.\n\n\nNotes on using *__slots__*\n--------------------------\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``int``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n',
  'attribute-references': '\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n   attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, which most objects do.  This object is then\nasked to produce the attribute whose name is the identifier (which can\nbe customized by overriding the ``__getattr__()`` method).  If this\nattribute is not available, the exception ``AttributeError`` is\nraised.  Otherwise, the type and value of the object produced is\ndetermined by the object.  Multiple evaluations of the same attribute\nreference may yield different objects.\n',
  'augassign': '\nAugmented assignment statements\n*******************************\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n   augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n   augtarget                 ::= identifier | attributeref | subscription | slicing\n   augop                     ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n             | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
- 'binary': '\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels.  Note that some of these operations also apply to certain non-\nnumeric types.  Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n   m_expr ::= u_expr | m_expr "*" u_expr | m_expr "//" u_expr | m_expr "/" u_expr\n              | m_expr "%" u_expr\n   a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe ``*`` (multiplication) operator yields the product of its\narguments.  The arguments must either both be numbers, or one argument\nmust be an integer and the other must be a sequence. In the former\ncase, the numbers are converted to a common type and then multiplied\ntogether.  In the latter case, sequence repetition is performed; a\nnegative repetition factor yields an empty sequence.\n\nThe ``/`` (division) and ``//`` (floor division) operators yield the\nquotient of their arguments.  The numeric arguments are first\nconverted to a common type. Integer division yields a float, while\nfloor division of integers results in an integer; the result is that\nof mathematical division with the \'floor\' function applied to the\nresult.  Division by zero raises the ``ZeroDivisionError`` exception.\n\nThe ``%`` (modulo) operator yields the remainder from the division of\nthe first argument by the second.  The numeric arguments are first\nconverted to a common type.  A zero right argument raises the\n``ZeroDivisionError`` exception.  The arguments may be floating point\nnumbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals\n``4*0.7 + 0.34``.)  The modulo operator always yields a result with\nthe same sign as its second operand (or zero); the absolute value of\nthe result is strictly smaller than the absolute value of the second\noperand [1].\n\nThe floor division and modulo operators are connected by the following\nidentity: ``x == (x//y)*y + (x%y)``.  Floor division and modulo are\nalso connected with the built-in function ``divmod()``: ``divmod(x, y)\n== (x//y, x%y)``. [2].\n\nIn addition to performing the modulo operation on numbers, the ``%``\noperator is also overloaded by string objects to perform old-style\nstring formatting (also known as interpolation).  The syntax for\nstring formatting is described in the Python Library Reference,\nsection *Old String Formatting Operations*.\n\nThe floor division operator, the modulo operator, and the ``divmod()``\nfunction are not defined for complex numbers.  Instead, convert to a\nfloating point number using the ``abs()`` function if appropriate.\n\nThe ``+`` (addition) operator yields the sum of its arguments.  The\narguments must either both be numbers or both sequences of the same\ntype.  In the former case, the numbers are converted to a common type\nand then added together.  In the latter case, the sequences are\nconcatenated.\n\nThe ``-`` (subtraction) operator yields the difference of its\narguments.  The numeric arguments are first converted to a common\ntype.\n',
+ 'binary': '\nBinary arithmetic operations\n****************************\n\nThe binary arithmetic operations have the conventional priority\nlevels.  Note that some of these operations also apply to certain non-\nnumeric types.  Apart from the power operator, there are only two\nlevels, one for multiplicative operators and one for additive\noperators:\n\n   m_expr ::= u_expr | m_expr "*" u_expr | m_expr "//" u_expr | m_expr "/" u_expr\n              | m_expr "%" u_expr\n   a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n\nThe ``*`` (multiplication) operator yields the product of its\narguments.  The arguments must either both be numbers, or one argument\nmust be an integer and the other must be a sequence. In the former\ncase, the numbers are converted to a common type and then multiplied\ntogether.  In the latter case, sequence repetition is performed; a\nnegative repetition factor yields an empty sequence.\n\nThe ``/`` (division) and ``//`` (floor division) operators yield the\nquotient of their arguments.  The numeric arguments are first\nconverted to a common type. Integer division yields a float, while\nfloor division of integers results in an integer; the result is that\nof mathematical division with the \'floor\' function applied to the\nresult.  Division by zero raises the ``ZeroDivisionError`` exception.\n\nThe ``%`` (modulo) operator yields the remainder from the division of\nthe first argument by the second.  The numeric arguments are first\nconverted to a common type.  A zero right argument raises the\n``ZeroDivisionError`` exception.  The arguments may be floating point\nnumbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals\n``4*0.7 + 0.34``.)  The modulo operator always yields a result with\nthe same sign as its second operand (or zero); the absolute value of\nthe result is strictly smaller than the absolute value of the second\noperand [1].\n\nThe floor division and modulo operators are connected by the following\nidentity: ``x == (x//y)*y + (x%y)``.  Floor division and modulo are\nalso connected with the built-in function ``divmod()``: ``divmod(x, y)\n== (x//y, x%y)``. [2].\n\nIn addition to performing the modulo operation on numbers, the ``%``\noperator is also overloaded by string objects to perform old-style\nstring formatting (also known as interpolation).  The syntax for\nstring formatting is described in the Python Library Reference,\nsection *printf-style String Formatting*.\n\nThe floor division operator, the modulo operator, and the ``divmod()``\nfunction are not defined for complex numbers.  Instead, convert to a\nfloating point number using the ``abs()`` function if appropriate.\n\nThe ``+`` (addition) operator yields the sum of its arguments.  The\narguments must either both be numbers or both sequences of the same\ntype.  In the former case, the numbers are converted to a common type\nand then added together.  In the latter case, the sequences are\nconcatenated.\n\nThe ``-`` (subtraction) operator yields the difference of its\narguments.  The numeric arguments are first converted to a common\ntype.\n',
  'bitwise': '\nBinary bitwise operations\n*************************\n\nEach of the three bitwise operations has a different priority level:\n\n   and_expr ::= shift_expr | and_expr "&" shift_expr\n   xor_expr ::= and_expr | xor_expr "^" and_expr\n   or_expr  ::= xor_expr | or_expr "|" xor_expr\n\nThe ``&`` operator yields the bitwise AND of its arguments, which must\nbe integers.\n\nThe ``^`` operator yields the bitwise XOR (exclusive OR) of its\narguments, which must be integers.\n\nThe ``|`` operator yields the bitwise (inclusive) OR of its arguments,\nwhich must be integers.\n',
  'bltin-code-objects': '\nCode Objects\n************\n\nCode objects are used by the implementation to represent "pseudo-\ncompiled" executable Python code such as a function body. They differ\nfrom function objects because they don\'t contain a reference to their\nglobal execution environment.  Code objects are returned by the built-\nin ``compile()`` function and can be extracted from function objects\nthrough their ``__code__`` attribute. See also the ``code`` module.\n\nA code object can be executed or evaluated by passing it (instead of a\nsource string) to the ``exec()`` or ``eval()``  built-in functions.\n\nSee *The standard type hierarchy* for more information.\n',
- 'bltin-ellipsis-object': '\nThe Ellipsis Object\n*******************\n\nThis object is commonly used by slicing (see *Slicings*).  It supports\nno special operations.  There is exactly one ellipsis object, named\n``Ellipsis`` (a built-in name).\n\nIt is written as ``Ellipsis`` or ``...``.\n',
- 'bltin-null-object': "\nThe Null Object\n***************\n\nThis object is returned by functions that don't explicitly return a\nvalue.  It supports no special operations.  There is exactly one null\nobject, named ``None`` (a built-in name).\n\nIt is written as ``None``.\n",
+ 'bltin-ellipsis-object': '\nThe Ellipsis Object\n*******************\n\nThis object is commonly used by slicing (see *Slicings*).  It supports\nno special operations.  There is exactly one ellipsis object, named\n``Ellipsis`` (a built-in name).  ``type(Ellipsis)()`` produces the\n``Ellipsis`` singleton.\n\nIt is written as ``Ellipsis`` or ``...``.\n',
+ 'bltin-null-object': "\nThe Null Object\n***************\n\nThis object is returned by functions that don't explicitly return a\nvalue.  It supports no special operations.  There is exactly one null\nobject, named ``None`` (a built-in name).  ``type(None)()`` produces\nthe same singleton.\n\nIt is written as ``None``.\n",
  'bltin-type-objects': "\nType Objects\n************\n\nType objects represent the various object types.  An object's type is\naccessed by the built-in function ``type()``.  There are no special\noperations on types.  The standard module ``types`` defines names for\nall standard built-in types.\n\nTypes are written like this: ``<class 'int'>``.\n",
  'booleans': '\nBoolean operations\n******************\n\n   or_test  ::= and_test | or_test "or" and_test\n   and_test ::= not_test | and_test "and" not_test\n   not_test ::= comparison | "not" not_test\n\nIn the context of Boolean operations, and also when expressions are\nused by control flow statements, the following values are interpreted\nas false: ``False``, ``None``, numeric zero of all types, and empty\nstrings and containers (including strings, tuples, lists,\ndictionaries, sets and frozensets).  All other values are interpreted\nas true.  User-defined objects can customize their truth value by\nproviding a ``__bool__()`` method.\n\nThe operator ``not`` yields ``True`` if its argument is false,\n``False`` otherwise.\n\nThe expression ``x and y`` first evaluates *x*; if *x* is false, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\nThe expression ``x or y`` first evaluates *x*; if *x* is true, its\nvalue is returned; otherwise, *y* is evaluated and the resulting value\nis returned.\n\n(Note that neither ``and`` nor ``or`` restrict the value and type they\nreturn to ``False`` and ``True``, but rather return the last evaluated\nargument.  This is sometimes useful, e.g., if ``s`` is a string that\nshould be replaced by a default value if it is empty, the expression\n``s or \'foo\'`` yields the desired value.  Because ``not`` has to\ninvent a value anyway, it does not bother to return a value of the\nsame type as its argument, so e.g., ``not \'foo\'`` yields ``False``,\nnot ``\'\'``.)\n',
  'break': '\nThe ``break`` statement\n***********************\n\n   break_stmt ::= "break"\n\n``break`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition\nwithin that loop.\n\nIt terminates the nearest enclosing loop, skipping the optional\n``else`` clause if the loop has one.\n\nIf a ``for`` loop is terminated by ``break``, the loop control target\nkeeps its current value.\n\nWhen ``break`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nleaving the loop.\n',
@@ -18,13 +19,13 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss
  'calls': '\nCalls\n*****\n\nA call calls a callable object (e.g., a function) with a possibly\nempty series of arguments:\n\n   call                 ::= primary "(" [argument_list [","] | comprehension] ")"\n   argument_list        ::= positional_arguments ["," keyword_arguments]\n                       ["," "*" expression] ["," keyword_arguments]\n                       ["," "**" expression]\n                     | keyword_arguments ["," "*" expression]\n                       ["," keyword_arguments] ["," "**" expression]\n                     | "*" expression ["," keyword_arguments] ["," "**" expression]\n                     | "**" expression\n   positional_arguments ::= expression ("," expression)*\n   keyword_arguments    ::= keyword_item ("," keyword_item)*\n   keyword_item         ::= identifier "=" expression\n\nA trailing comma may be present after the positional and keyword\narguments but does not affect the semantics.\n\nThe primary must evaluate to a callable object (user-defined\nfunctions, built-in functions, methods of built-in objects, class\nobjects, methods of class instances, and all objects having a\n``__call__()`` method are callable).  All argument expressions are\nevaluated before the call is attempted.  Please refer to section\n*Function definitions* for the syntax of formal parameter lists.\n\nIf keyword arguments are present, they are first converted to\npositional arguments, as follows.  First, a list of unfilled slots is\ncreated for the formal parameters.  If there are N positional\narguments, they are placed in the first N slots.  Next, for each\nkeyword argument, the identifier is used to determine the\ncorresponding slot (if the identifier is the same as the first formal\nparameter name, the first slot is used, and so on).  If the slot is\nalready filled, a ``TypeError`` exception is raised. Otherwise, the\nvalue of the argument is placed in the slot, filling it (even if the\nexpression is ``None``, it fills the slot).  When all arguments have\nbeen processed, the slots that are still unfilled are filled with the\ncorresponding default value from the function definition.  (Default\nvalues are calculated, once, when the function is defined; thus, a\nmutable object such as a list or dictionary used as default value will\nbe shared by all calls that don\'t specify an argument value for the\ncorresponding slot; this should usually be avoided.)  If there are any\nunfilled slots for which no default value is specified, a\n``TypeError`` exception is raised.  Otherwise, the list of filled\nslots is used as the argument list for the call.\n\n**CPython implementation detail:** An implementation may provide\nbuilt-in functions whose positional parameters do not have names, even\nif they are \'named\' for the purpose of documentation, and which\ntherefore cannot be supplied by keyword.  In CPython, this is the case\nfor functions implemented in C that use ``PyArg_ParseTuple()`` to\nparse their arguments.\n\nIf there are more positional arguments than there are formal parameter\nslots, a ``TypeError`` exception is raised, unless a formal parameter\nusing the syntax ``*identifier`` is present; in this case, that formal\nparameter receives a tuple containing the excess positional arguments\n(or an empty tuple if there were no excess positional arguments).\n\nIf any keyword argument does not correspond to a formal parameter\nname, a ``TypeError`` exception is raised, unless a formal parameter\nusing the syntax ``**identifier`` is present; in this case, that\nformal parameter receives a dictionary containing the excess keyword\narguments (using the keywords as keys and the argument values as\ncorresponding values), or a (new) empty dictionary if there were no\nexcess keyword arguments.\n\nIf the syntax ``*expression`` appears in the function call,\n``expression`` must evaluate to an iterable.  Elements from this\niterable are treated as if they were additional positional arguments;\nif there are positional arguments *x1*, ..., *xN*, and ``expression``\nevaluates to a sequence *y1*, ..., *yM*, this is equivalent to a call\nwith M+N positional arguments *x1*, ..., *xN*, *y1*, ..., *yM*.\n\nA consequence of this is that although the ``*expression`` syntax may\nappear *after* some keyword arguments, it is processed *before* the\nkeyword arguments (and the ``**expression`` argument, if any -- see\nbelow).  So:\n\n   >>> def f(a, b):\n   ...  print(a, b)\n   ...\n   >>> f(b=1, *(2,))\n   2 1\n   >>> f(a=1, *(2,))\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in ?\n   TypeError: f() got multiple values for keyword argument \'a\'\n   >>> f(1, *(2,))\n   1 2\n\nIt is unusual for both keyword arguments and the ``*expression``\nsyntax to be used in the same call, so in practice this confusion does\nnot arise.\n\nIf the syntax ``**expression`` appears in the function call,\n``expression`` must evaluate to a mapping, the contents of which are\ntreated as additional keyword arguments.  In the case of a keyword\nappearing in both ``expression`` and as an explicit keyword argument,\na ``TypeError`` exception is raised.\n\nFormal parameters using the syntax ``*identifier`` or ``**identifier``\ncannot be used as positional argument slots or as keyword argument\nnames.\n\nA call always returns some value, possibly ``None``, unless it raises\nan exception.  How this value is computed depends on the type of the\ncallable object.\n\nIf it is---\n\na user-defined function:\n   The code block for the function is executed, passing it the\n   argument list.  The first thing the code block will do is bind the\n   formal parameters to the arguments; this is described in section\n   *Function definitions*.  When the code block executes a ``return``\n   statement, this specifies the return value of the function call.\n\na built-in function or method:\n   The result is up to the interpreter; see *Built-in Functions* for\n   the descriptions of built-in functions and methods.\n\na class object:\n   A new instance of that class is returned.\n\na class instance method:\n   The corresponding user-defined function is called, with an argument\n   list that is one longer than the argument list of the call: the\n   instance becomes the first argument.\n\na class instance:\n   The class must define a ``__call__()`` method; the effect is then\n   the same as if that method was called.\n',
  'class': '\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= [decorators] "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [parameter_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing.  Classes without an inheritance\nlist inherit, by default, from the base class ``object``; hence,\n\n   class Foo:\n       pass\n\nis equivalent to\n\n   class Foo(object):\n       pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.)  When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n   @f1(arg)\n   @f2\n   class Foo: pass\n\nis equivalent to\n\n   class Foo: pass\n   Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators.  The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances.  Instance attributes\ncan be set in a method with ``self.name = value``.  Both class and\ninstance attributes are accessible through the notation\n"``self.name``", and an instance attribute hides a class attribute\nwith the same name when accessed in this way.  Class attributes can be\nused as defaults for instance attributes, but using mutable values\nthere can lead to unexpected results.  *Descriptors* can be used to\ncreate instance variables with different implementation details.\n\nSee also:\n\n   **PEP 3115** - Metaclasses in Python 3 **PEP 3129** - Class\n   Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless there\n    is a ``finally`` clause which happens to raise another exception.\n    That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of an\n    exception or the execution of a ``return``, ``continue``, or\n    ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n    body is transformed into the function\'s ``__doc__`` attribute and\n    therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n    body is transformed into the namespace\'s ``__doc__`` item and\n    therefore the class\'s *docstring*.\n',
  'comparisons': '\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation.  Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n   comparison    ::= or_expr ( comp_operator or_expr )*\n   comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n                     | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects.  The objects need not have the same type.\nIf both are numbers, they are converted to a common type.  Otherwise,\nthe ``==`` and ``!=`` operators *always* consider objects of different\ntypes to be unequal, while the ``<``, ``>``, ``>=`` and ``<=``\noperators raise a ``TypeError`` when comparing objects of different\ntypes that do not implement these operators for the given pair of\ntypes.  You can control comparison behavior of objects of non-built-in\ntypes by defining rich comparison methods like ``__gt__()``, described\nin section *Basic customization*.\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* The values ``float(\'NaN\')`` and ``Decimal(\'NaN\')`` are special. The\n  are identical to themselves, ``x is x`` but are not equal to\n  themselves, ``x != x``.  Additionally, comparing any value to a\n  not-a-number value will return ``False``.  For example, both ``3 <\n  float(\'NaN\')`` and ``float(\'NaN\') < 3`` will return ``False``.\n\n* Bytes objects are compared lexicographically using the numeric\n  values of their elements.\n\n* Strings are compared lexicographically using the numeric equivalents\n  (the result of the built-in function ``ord()``) of their characters.\n  [3] String and bytes object can\'t be compared!\n\n* Tuples and lists are compared lexicographically using comparison of\n  corresponding elements.  This means that to compare equal, each\n  element must compare equal and the two sequences must be of the same\n  type and have the same length.\n\n  If not equal, the sequences are ordered the same as their first\n  differing elements.  For example, ``[1,2,x] <= [1,2,y]`` has the\n  same value as ``x <= y``.  If the corresponding element does not\n  exist, the shorter sequence is ordered first (for example, ``[1,2] <\n  [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if they have the\n  same ``(key, value)`` pairs. Order comparisons ``(\'<\', \'<=\', \'>=\',\n  \'>\')`` raise ``TypeError``.\n\n* Sets and frozensets define comparison operators to mean subset and\n  superset tests.  Those relations do not define total orderings (the\n  two sets ``{1,2}`` and {2,3} are not equal, nor subsets of one\n  another, nor supersets of one another).  Accordingly, sets are not\n  appropriate arguments for functions which depend on total ordering.\n  For example, ``min()``, ``max()``, and ``sorted()`` produce\n  undefined results given a list of sets as inputs.\n\n* Most other objects of built-in types compare unequal unless they are\n  the same object; the choice whether one object is considered smaller\n  or larger than another one is made arbitrarily but consistently\n  within one execution of a program.\n\nComparison of objects of the differing types depends on whether either\nof the types provide explicit support for the comparison.  Most\nnumeric types can be compared with one another, but comparisons of\n``float`` and ``Decimal`` are not supported to avoid the inevitable\nconfusion arising from representation issues such as ``float(\'1.1\')``\nbeing inexactly represented and therefore not exactly equal to\n``Decimal(\'1.1\')`` which is.  When cross-type comparison is not\nsupported, the comparison method returns ``NotImplemented``.  This can\ncreate the illusion of non-transitivity between supported cross-type\ncomparisons and unsupported comparisons.  For example, ``Decimal(2) ==\n2`` and ``2 == float(2)`` but ``Decimal(2) != float(2)``.\n\nThe operators ``in`` and ``not in`` test for membership.  ``x in s``\nevaluates to true if *x* is a member of *s*, and false otherwise.  ``x\nnot in s`` returns the negation of ``x in s``.  All built-in sequences\nand set types support this as well as dictionary, for which ``in``\ntests whether a the dictionary has a given key. For container types\nsuch as list, tuple, set, frozenset, dict, or collections.deque, the\nexpression ``x in y`` is equivalent to ``any(x is e or x == e for e in\ny)``.\n\nFor the string and bytes types, ``x in y`` is true if and only if *x*\nis a substring of *y*.  An equivalent test is ``y.find(x) != -1``.\nEmpty strings are always considered to be a substring of any other\nstring, so ``"" in "abc"`` will return ``True``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``.  If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception.  (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object.  ``x is\nnot y`` yields the inverse truth value. [4]\n',
- 'compound': '\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way.  In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs.  ``try`` specifies exception handlers and/or\ncleanup code for a group of statements, while the ``with`` statement\nallows the execution of initialization and finalization code around a\nblock of code.  Function and class definitions are also syntactically\ncompound statements.\n\nCompound statements consist of one or more \'clauses.\'  A clause\nconsists of a header and a \'suite.\'  The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon.  A suite is a group of statements controlled by a\nclause.  A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines.  Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n   if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print()`` calls are executed:\n\n   if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n   compound_stmt ::= if_stmt\n                     | while_stmt\n                     | for_stmt\n                     | try_stmt\n                     | with_stmt\n                     | funcdef\n                     | classdef\n   suite         ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n   statement     ::= stmt_list NEWLINE | compound_stmt\n   stmt_list     ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``.  Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n``expression_list``.  The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted.  When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a ``StopIteration``\nexception), the suite in the ``else`` clause, if present, is executed,\nand the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop.  Hint: the built-in function ``range()`` returns an\niterator of integers suitable to emulate the effect of Pascal\'s ``for\ni := a to b do``; e.g., ``list(range(3))`` returns the list ``[0, 1,\n2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n  (this can only occur for mutable sequences, i.e. lists).  An\n  internal counter is used to keep track of which item is used next,\n  and this is incremented on each iteration.  When this counter has\n  reached the length of the sequence the loop terminates.  This means\n  that if the suite deletes the current (or a previous) item from the\n  sequence, the next item will be skipped (since it gets the index of\n  the current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression ["as" target]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started.  This search inspects the except\nclauses in turn until one is found that matches the exception.  An\nexpression-less except clause, if present, must be last; it matches\nany exception.  For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception.  An object is\ncompatible with an exception if it is the class or a base class of the\nexception object or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the ``as`` keyword in that except clause,\nif present, and the except clause\'s suite is executed.  All except\nclauses must have an executable block.  When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using ``as target``, it is cleared\nat the end of the except clause.  This is as if\n\n   except E as N:\n       foo\n\nwas translated to\n\n   except E as N:\n       try:\n           foo\n       finally:\n           del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause.  Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the ``sys`` module and can be access via\n``sys.exc_info()``. ``sys.exc_info()`` returns a 3-tuple consisting of\nthe exception class, the exception instance and a traceback object\n(see section *The standard type hierarchy*) identifying the point in\nthe program where the exception occurred.  ``sys.exc_info()`` values\nare restored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler.  The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses.  If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted.  If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is set as the context of the new exception.  The exception\ninformation is not available to the program during execution of the\n``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n   is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n   value from ``__enter__()`` is assigned to it.\n\n   Note: The ``with`` statement guarantees that if the ``__enter__()``\n     method returns without an error, then ``__exit__()`` will always\n     be called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked.  If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to ``__exit__()``. Otherwise,\n   three ``None`` arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the ``__exit__()`` method was false, the exception is\n   reraised.  If the return value was true, the exception is\n   suppressed, and execution continues with the statement following\n   the ``with`` statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from ``__exit__()`` is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   funcdef        ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      (  "*" [parameter] ("," defparameter)*\n                      [, "**" parameter]\n                      | "**" parameter\n                      | defparameter [","] )\n   parameter      ::= identifier [":" expression]\n   defparameter   ::= parameter ["=" expression]\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis equivalent to\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nWhen one or more parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters up until the "``*``" must also have a default value ---\nthis is a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call.  This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended.  A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple.  If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after "``*``" or "``*identifier``" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "``: expression``"\nfollowing the parameter name.  Any parameter may have an annotation\neven those of the form ``*identifier`` or ``**identifier``.  Functions\nmay have "return" annotation of the form "``-> expression``" after the\nparameter list.  These annotations can be any valid Python expression\nand are evaluated when the function definition is executed.\nAnnotations may be evaluated in a different order than they appear in\nthe source code.  The presence of annotations does not change the\nsemantics of a function.  The annotation values are available as\nvalues of a dictionary keyed by the parameters\' names in the\n``__annotations__`` attribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda forms,\ndescribed in section *Lambdas*.  Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form.  The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects.  A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around.  Free variables used in the\nnested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= [decorators] "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [parameter_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing.  Classes without an inheritance\nlist inherit, by default, from the base class ``object``; hence,\n\n   class Foo:\n       pass\n\nis equivalent to\n\n   class Foo(object):\n       pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.)  When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n   @f1(arg)\n   @f2\n   class Foo: pass\n\nis equivalent to\n\n   class Foo: pass\n   Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators.  The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances.  Instance attributes\ncan be set in a method with ``self.name = value``.  Both class and\ninstance attributes are accessible through the notation\n"``self.name``", and an instance attribute hides a class attribute\nwith the same name when accessed in this way.  Class attributes can be\nused as defaults for instance attributes, but using mutable values\nthere can lead to unexpected results.  *Descriptors* can be used to\ncreate instance variables with different implementation details.\n\nSee also:\n\n   **PEP 3115** - Metaclasses in Python 3 **PEP 3129** - Class\n   Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless there\n    is a ``finally`` clause which happens to raise another exception.\n    That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of an\n    exception or the execution of a ``return``, ``continue``, or\n    ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n    body is transformed into the function\'s ``__doc__`` attribute and\n    therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n    body is transformed into the namespace\'s ``__doc__`` item and\n    therefore the class\'s *docstring*.\n',
+ 'compound': '\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way.  In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs.  ``try`` specifies exception handlers and/or\ncleanup code for a group of statements, while the ``with`` statement\nallows the execution of initialization and finalization code around a\nblock of code.  Function and class definitions are also syntactically\ncompound statements.\n\nCompound statements consist of one or more \'clauses.\'  A clause\nconsists of a header and a \'suite.\'  The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon.  A suite is a group of statements controlled by a\nclause.  A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines.  Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n   if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print()`` calls are executed:\n\n   if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n   compound_stmt ::= if_stmt\n                     | while_stmt\n                     | for_stmt\n                     | try_stmt\n                     | with_stmt\n                     | funcdef\n                     | classdef\n   suite         ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n   statement     ::= stmt_list NEWLINE | compound_stmt\n   stmt_list     ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``.  Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n``expression_list``.  The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted.  When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a ``StopIteration``\nexception), the suite in the ``else`` clause, if present, is executed,\nand the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop.  Hint: the built-in function ``range()`` returns an\niterator of integers suitable to emulate the effect of Pascal\'s ``for\ni := a to b do``; e.g., ``list(range(3))`` returns the list ``[0, 1,\n2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n  (this can only occur for mutable sequences, i.e. lists).  An\n  internal counter is used to keep track of which item is used next,\n  and this is incremented on each iteration.  When this counter has\n  reached the length of the sequence the loop terminates.  This means\n  that if the suite deletes the current (or a previous) item from the\n  sequence, the next item will be skipped (since it gets the index of\n  the current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression ["as" target]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started.  This search inspects the except\nclauses in turn until one is found that matches the exception.  An\nexpression-less except clause, if present, must be last; it matches\nany exception.  For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception.  An object is\ncompatible with an exception if it is the class or a base class of the\nexception object or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the ``as`` keyword in that except clause,\nif present, and the except clause\'s suite is executed.  All except\nclauses must have an executable block.  When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using ``as target``, it is cleared\nat the end of the except clause.  This is as if\n\n   except E as N:\n       foo\n\nwas translated to\n\n   except E as N:\n       try:\n           foo\n       finally:\n           del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause.  Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the ``sys`` module and can be access via\n``sys.exc_info()``. ``sys.exc_info()`` returns a 3-tuple consisting of\nthe exception class, the exception instance and a traceback object\n(see section *The standard type hierarchy*) identifying the point in\nthe program where the exception occurred.  ``sys.exc_info()`` values\nare restored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler.  The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses.  If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted.  If there is a saved exception or ``break`` statement, it is\nre-raised at the end of the ``finally`` clause. If the ``finally``\nclause raises another exception the saved exception is set as the\ncontext of the new exception; if the ``finally`` clause executes a\n``return`` statement, the saved exception is discarded:\n\n   def f():\n       try:\n           1/0\n       finally:\n           return 42\n\n   >>> f()\n   42\n\nThe exception information is not available to the program during\nexecution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n   is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n   value from ``__enter__()`` is assigned to it.\n\n   Note: The ``with`` statement guarantees that if the ``__enter__()``\n     method returns without an error, then ``__exit__()`` will always\n     be called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked.  If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to ``__exit__()``. Otherwise,\n   three ``None`` arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the ``__exit__()`` method was false, the exception is\n   reraised.  If the return value was true, the exception is\n   suppressed, and execution continues with the statement following\n   the ``with`` statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from ``__exit__()`` is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   funcdef        ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      (  "*" [parameter] ("," defparameter)*\n                      [, "**" parameter]\n                      | "**" parameter\n                      | defparameter [","] )\n   parameter      ::= identifier [":" expression]\n   defparameter   ::= parameter ["=" expression]\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis equivalent to\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nWhen one or more parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters up until the "``*``" must also have a default value ---\nthis is a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call.  This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended.  A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple.  If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after "``*``" or "``*identifier``" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "``: expression``"\nfollowing the parameter name.  Any parameter may have an annotation\neven those of the form ``*identifier`` or ``**identifier``.  Functions\nmay have "return" annotation of the form "``-> expression``" after the\nparameter list.  These annotations can be any valid Python expression\nand are evaluated when the function definition is executed.\nAnnotations may be evaluated in a different order than they appear in\nthe source code.  The presence of annotations does not change the\nsemantics of a function.  The annotation values are available as\nvalues of a dictionary keyed by the parameters\' names in the\n``__annotations__`` attribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda forms,\ndescribed in section *Lambdas*.  Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form.  The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects.  A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around.  Free variables used in the\nnested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n\nSee also:\n\n   **PEP 3107** - Function Annotations\n      The original specification for function annotations.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n   classdef    ::= [decorators] "class" classname [inheritance] ":" suite\n   inheritance ::= "(" [parameter_list] ")"\n   classname   ::= identifier\n\nA class definition is an executable statement.  The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing.  Classes without an inheritance\nlist inherit, by default, from the base class ``object``; hence,\n\n   class Foo:\n       pass\n\nis equivalent to\n\n   class Foo(object):\n       pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.)  When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n   @f1(arg)\n   @f2\n   class Foo: pass\n\nis equivalent to\n\n   class Foo: pass\n   Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators.  The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances.  Instance attributes\ncan be set in a method with ``self.name = value``.  Both class and\ninstance attributes are accessible through the notation\n"``self.name``", and an instance attribute hides a class attribute\nwith the same name when accessed in this way.  Class attributes can be\nused as defaults for instance attributes, but using mutable values\nthere can lead to unexpected results.  *Descriptors* can be used to\ncreate instance variables with different implementation details.\n\nSee also:\n\n   **PEP 3115** - Metaclasses in Python 3 **PEP 3129** - Class\n   Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless there\n    is a ``finally`` clause which happens to raise another exception.\n    That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of an\n    exception or the execution of a ``return``, ``continue``, or\n    ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n    body is transformed into the function\'s ``__doc__`` attribute and\n    therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n    body is transformed into the namespace\'s ``__doc__`` item and\n    therefore the class\'s *docstring*.\n',
  'context-managers': '\nWith Statement Context Managers\n*******************************\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code.  Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The ``with``\n   statement will bind this method\'s return value to the target(s)\n   specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be ``None``.\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that ``__exit__()`` methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n',
  'continue': '\nThe ``continue`` statement\n**************************\n\n   continue_stmt ::= "continue"\n\n``continue`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition or\n``finally`` clause within that loop.  It continues with the next cycle\nof the nearest enclosing loop.\n\nWhen ``continue`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nstarting the next loop cycle.\n',
  'conversions': '\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," this means\nthat the operator implementation for built-in types works that way:\n\n* If either argument is a complex number, the other is converted to\n  complex;\n\n* otherwise, if either argument is a floating point number, the other\n  is converted to floating point;\n\n* otherwise, both must be integers and no conversion is necessary.\n\nSome additional rules apply for certain operators (e.g., a string left\nargument to the \'%\' operator).  Extensions must define their own\nconversion behavior.\n',
- 'customization': '\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  ``__new__()`` is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of ``__new__()`` should be the new object instance (usually\n   an instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s ``__new__()`` method using\n   ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n   arguments and then modifying the newly-created instance as\n   necessary before returning it.\n\n   If ``__new__()`` returns an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will be invoked like\n   ``__init__(self[, ...])``, where *self* is the new instance and the\n   remaining arguments are the same as were passed to ``__new__()``.\n\n   If ``__new__()`` does not return an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will not be invoked.\n\n   ``__new__()`` is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called when the instance is created.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   ``__init__()`` method, the derived class\'s ``__init__()`` method,\n   if any, must explicitly call it to ensure proper initialization of\n   the base class part of the instance; for example:\n   ``BaseClass.__init__(self, [args...])``.  As a special constraint\n   on constructors, no value may be returned; doing so will cause a\n   ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a ``__del__()`` method,\n   the derived class\'s ``__del__()`` method, if any, must explicitly\n   call it to ensure proper deletion of the base class part of the\n   instance.  Note that it is possible (though not recommended!) for\n   the ``__del__()`` method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   ``__del__()`` methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n     decrements the reference count for ``x`` by one, and the latter\n     is only called when ``x``\'s reference count reaches zero.  Some\n     common situations that may prevent the reference count of an\n     object from going to zero include: circular references between\n     objects (e.g., a doubly-linked list or a tree data structure with\n     parent and child pointers); a reference to the object on the\n     stack frame of a function that caught an exception (the traceback\n     stored in ``sys.exc_info()[2]`` keeps the stack frame alive); or\n     a reference to the object on the stack frame that raised an\n     unhandled exception in interactive mode (the traceback stored in\n     ``sys.last_traceback`` keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the latter two situations can be resolved by storing ``None`` in\n     ``sys.last_traceback``. Circular references which are garbage are\n     detected when the option cycle detector is enabled (it\'s on by\n     default), but can only be cleaned up if there are no Python-\n     level ``__del__()`` methods involved. Refer to the documentation\n     for the ``gc`` module for more information about how\n     ``__del__()`` methods are handled by the cycle detector,\n     particularly the description of the ``garbage`` value.\n\n   Warning: Due to the precarious circumstances under which ``__del__()``\n     methods are invoked, exceptions that occur during their execution\n     are ignored, and a warning is printed to ``sys.stderr`` instead.\n     Also, when ``__del__()`` is invoked in response to a module being\n     deleted (e.g., when execution of the program is done), other\n     globals referenced by the ``__del__()`` method may already have\n     been deleted or in the process of being torn down (e.g. the\n     import machinery shutting down).  For this reason, ``__del__()``\n     methods should do the absolute minimum needed to maintain\n     external invariants.  Starting with version 1.5, Python\n     guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n   Called by the ``repr()`` built-in function to compute the\n   "official" string representation of an object.  If at all possible,\n   this should look like a valid Python expression that could be used\n   to recreate an object with the same value (given an appropriate\n   environment).  If this is not possible, a string of the form\n   ``<...some useful description...>`` should be returned. The return\n   value must be a string object. If a class defines ``__repr__()``\n   but not ``__str__()``, then ``__repr__()`` is also used when an\n   "informal" string representation of instances of that class is\n   required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by the ``str()`` built-in function and by the ``print()``\n   function to compute the "informal" string representation of an\n   object.  This differs from ``__repr__()`` in that it does not have\n   to be a valid Python expression: a more convenient or concise\n   representation may be used instead. The return value must be a\n   string object.\n\nobject.__bytes__(self)\n\n   Called by ``bytes()`` to compute a byte-string representation of an\n   object. This should return a ``bytes`` object.\n\nobject.__format__(self, format_spec)\n\n   Called by the ``format()`` built-in function (and by extension, the\n   ``format()`` method of class ``str``) to produce a "formatted"\n   string representation of an object. The ``format_spec`` argument is\n   a string that contains a description of the formatting options\n   desired. The interpretation of the ``format_spec`` argument is up\n   to the type implementing ``__format__()``, however most classes\n   will either delegate formatting to one of the built-in types, or\n   use a similar formatting option syntax.\n\n   See *Format Specification Mini-Language* for a description of the\n   standard formatting syntax.\n\n   The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   These are the so-called "rich comparison" methods. The\n   correspondence between operator symbols and method names is as\n   follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n   ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` calls\n   ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls\n   ``x.__ge__(y)``.\n\n   A rich comparison method may return the singleton\n   ``NotImplemented`` if it does not implement the operation for a\n   given pair of arguments. By convention, ``False`` and ``True`` are\n   returned for a successful comparison. However, these methods can\n   return any value, so if the comparison operator is used in a\n   Boolean context (e.g., in the condition of an ``if`` statement),\n   Python will call ``bool()`` on the value to determine if the result\n   is true or false.\n\n   There are no implied relationships among the comparison operators.\n   The truth of ``x==y`` does not imply that ``x!=y`` is false.\n   Accordingly, when defining ``__eq__()``, one should also define\n   ``__ne__()`` so that the operators will behave as expected.  See\n   the paragraph on ``__hash__()`` for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n   other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n   reflection, and ``__eq__()`` and ``__ne__()`` are their own\n   reflection.\n\n   Arguments to rich comparison methods are never coerced.\n\n   To automatically generate ordering operations from a single root\n   operation, see ``functools.total_ordering()``.\n\nobject.__hash__(self)\n\n   Called by built-in function ``hash()`` and for operations on\n   members of hashed collections including ``set``, ``frozenset``, and\n   ``dict``.  ``__hash__()`` should return an integer.  The only\n   required property is that objects which compare equal have the same\n   hash value; it is advised to somehow mix together (e.g. using\n   exclusive or) the hash values for the components of the object that\n   also play a part in comparison of objects.\n\n   If a class does not define an ``__eq__()`` method it should not\n   define a ``__hash__()`` operation either; if it defines\n   ``__eq__()`` but not ``__hash__()``, its instances will not be\n   usable as items in hashable collections.  If a class defines\n   mutable objects and implements an ``__eq__()`` method, it should\n   not implement ``__hash__()``, since the implementation of hashable\n   collections requires that a key\'s hash value is immutable (if the\n   object\'s hash value changes, it will be in the wrong hash bucket).\n\n   User-defined classes have ``__eq__()`` and ``__hash__()`` methods\n   by default; with them, all objects compare unequal (except with\n   themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n   Classes which inherit a ``__hash__()`` method from a parent class\n   but change the meaning of ``__eq__()`` such that the hash value\n   returned is no longer appropriate (e.g. by switching to a value-\n   based concept of equality instead of the default identity based\n   equality) can explicitly flag themselves as being unhashable by\n   setting ``__hash__ = None`` in the class definition. Doing so means\n   that not only will instances of the class raise an appropriate\n   ``TypeError`` when a program attempts to retrieve their hash value,\n   but they will also be correctly identified as unhashable when\n   checking ``isinstance(obj, collections.Hashable)`` (unlike classes\n   which define their own ``__hash__()`` to explicitly raise\n   ``TypeError``).\n\n   If a class that overrides ``__eq__()`` needs to retain the\n   implementation of ``__hash__()`` from a parent class, the\n   interpreter must be told this explicitly by setting ``__hash__ =\n   <ParentClass>.__hash__``. Otherwise the inheritance of\n   ``__hash__()`` will be blocked, just as if ``__hash__`` had been\n   explicitly set to ``None``.\n\n   See also the *-R* command-line option.\n\nobject.__bool__(self)\n\n   Called to implement truth value testing and the built-in operation\n   ``bool()``; should return ``False`` or ``True``.  When this method\n   is not defined, ``__len__()`` is called, if it is defined, and the\n   object is considered true if its result is nonzero.  If a class\n   defines neither ``__len__()`` nor ``__bool__()``, all its instances\n   are considered true.\n',
- 'debugger': '\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs.  It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame.  It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible -- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source.  The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> pdb.run(\'mymodule.test()\')\n   > <string>(0)?()\n   (Pdb) continue\n   > <string>(1)?()\n   (Pdb) continue\n   NameError: \'spam\'\n   > <string>(1)?()\n   (Pdb)\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n   python3 -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally.  After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program.  Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 3.2: ``pdb.py`` now accepts a ``-c`` option that\nexecutes commands as if given in a ``.pdbrc`` file, see *Debugger\nCommands*.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n   import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger.  You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``continue`` command.\n\nThe typical usage to inspect a crashed program is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> mymodule.test()\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in ?\n     File "./mymodule.py", line 4, in test\n       test2()\n     File "./mymodule.py", line 3, in test2\n       print(spam)\n   NameError: spam\n   >>> pdb.pm()\n   > ./mymodule.py(3)test2()\n   -> print(spam)\n   (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement, globals=None, locals=None)\n\n   Execute the *statement* (given as a string or a code object) under\n   debugger control.  The debugger prompt appears before any code is\n   executed; you can set breakpoints and type ``continue``, or you can\n   step through the statement using ``step`` or ``next`` (all these\n   commands are explained below).  The optional *globals* and *locals*\n   arguments specify the environment in which the code is executed; by\n   default the dictionary of the module ``__main__`` is used.  (See\n   the explanation of the built-in ``exec()`` or ``eval()``\n   functions.)\n\npdb.runeval(expression, globals=None, locals=None)\n\n   Evaluate the *expression* (given as a string or a code object)\n   under debugger control.  When ``runeval()`` returns, it returns the\n   value of the expression.  Otherwise this function is similar to\n   ``run()``.\n\npdb.runcall(function, *args, **kwds)\n\n   Call the *function* (a function or method object, not a string)\n   with the given arguments.  When ``runcall()`` returns, it returns\n   whatever the function call returned.  The debugger prompt appears\n   as soon as the function is entered.\n\npdb.set_trace()\n\n   Enter the debugger at the calling stack frame.  This is useful to\n   hard-code a breakpoint at a given point in a program, even if the\n   code is not otherwise being debugged (e.g. when an assertion\n   fails).\n\npdb.post_mortem(traceback=None)\n\n   Enter post-mortem debugging of the given *traceback* object.  If no\n   *traceback* is given, it uses the one of the exception that is\n   currently being handled (an exception must be being handled if the\n   default is to be used).\n\npdb.pm()\n\n   Enter post-mortem debugging of the traceback found in\n   ``sys.last_traceback``.\n\nThe ``run*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname.  If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None, nosigint=False)\n\n   ``Pdb`` is the debugger class.\n\n   The *completekey*, *stdin* and *stdout* arguments are passed to the\n   underlying ``cmd.Cmd`` class; see the description there.\n\n   The *skip* argument, if given, must be an iterable of glob-style\n   module name patterns.  The debugger will not step into frames that\n   originate in a module that matches one of these patterns. [1]\n\n   By default, Pdb sets a handler for the SIGINT signal (which is sent\n   when the user presses Ctrl-C on the console) when you give a\n   ``continue`` command. This allows you to break into the debugger\n   again by pressing Ctrl-C.  If you want Pdb not to touch the SIGINT\n   handler, set *nosigint* tot true.\n\n   Example call to enable tracing with *skip*:\n\n      import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n   New in version 3.1: The *skip* argument.\n\n   New in version 3.2: The *nosigint* argument.  Previously, a SIGINT\n   handler was never set by Pdb.\n\n   run(statement, globals=None, locals=None)\n   runeval(expression, globals=None, locals=None)\n   runcall(function, *args, **kwds)\n   set_trace()\n\n      See the documentation for the functions explained above.\n\n\nDebugger Commands\n=================\n\nThe commands recognized by the debugger are listed below.  Most\ncommands can be abbreviated to one or two letters as indicated; e.g.\n``h(elp)`` means that either ``h`` or ``help`` can be used to enter\nthe help command (but not ``he`` or ``hel``, nor ``H`` or ``Help`` or\n``HELP``).  Arguments to commands must be separated by whitespace\n(spaces or tabs).  Optional arguments are enclosed in square brackets\n(``[]``) in the command syntax; the square brackets must not be typed.\nAlternatives in the command syntax are separated by a vertical bar\n(``|``).\n\nEntering a blank line repeats the last command entered.  Exception: if\nthe last command was a ``list`` command, the next 11 lines are listed.\n\nCommands that the debugger doesn\'t recognize are assumed to be Python\nstatements and are executed in the context of the program being\ndebugged.  Python statements can also be prefixed with an exclamation\npoint (``!``).  This is a powerful way to inspect the program being\ndebugged; it is even possible to change a variable or call a function.\nWhen an exception occurs in such a statement, the exception name is\nprinted but the debugger\'s state is not changed.\n\nThe debugger supports *aliases*.  Aliases can have parameters which\nallows one a certain level of adaptability to the context under\nexamination.\n\nMultiple commands may be entered on a single line, separated by\n``;;``.  (A single ``;`` is not used as it is the separator for\nmultiple commands in a line that is passed to the Python parser.)  No\nintelligence is applied to separating the commands; the input is split\nat the first ``;;`` pair, even if it is in the middle of a quoted\nstring.\n\nIf a file ``.pdbrc`` exists in the user\'s home directory or in the\ncurrent directory, it is read in and executed as if it had been typed\nat the debugger prompt.  This is particularly useful for aliases.  If\nboth files exist, the one in the home directory is read first and\naliases defined there can be overridden by the local file.\n\nChanged in version 3.2: ``.pdbrc`` can now contain commands that\ncontinue debugging, such as ``continue`` or ``next``.  Previously,\nthese commands had no effect.\n\nh(elp) [command]\n\n   Without argument, print the list of available commands.  With a\n   *command* as argument, print help about that command.  ``help pdb``\n   displays the full documentation (the docstring of the ``pdb``\n   module).  Since the *command* argument must be an identifier,\n   ``help exec`` must be entered to get help on the ``!`` command.\n\nw(here)\n\n   Print a stack trace, with the most recent frame at the bottom.  An\n   arrow indicates the current frame, which determines the context of\n   most commands.\n\nd(own) [count]\n\n   Move the current frame *count* (default one) levels down in the\n   stack trace (to a newer frame).\n\nu(p) [count]\n\n   Move the current frame *count* (default one) levels up in the stack\n   trace (to an older frame).\n\nb(reak) [([filename:]lineno | function) [, condition]]\n\n   With a *lineno* argument, set a break there in the current file.\n   With a *function* argument, set a break at the first executable\n   statement within that function.  The line number may be prefixed\n   with a filename and a colon, to specify a breakpoint in another\n   file (probably one that hasn\'t been loaded yet).  The file is\n   searched on ``sys.path``.  Note that each breakpoint is assigned a\n   number to which all the other breakpoint commands refer.\n\n   If a second argument is present, it is an expression which must\n   evaluate to true before the breakpoint is honored.\n\n   Without argument, list all breaks, including for each breakpoint,\n   the number of times that breakpoint has been hit, the current\n   ignore count, and the associated condition if any.\n\ntbreak [([filename:]lineno | function) [, condition]]\n\n   Temporary breakpoint, which is removed automatically when it is\n   first hit. The arguments are the same as for ``break``.\n\ncl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n\n   With a *filename:lineno* argument, clear all the breakpoints at\n   this line. With a space separated list of breakpoint numbers, clear\n   those breakpoints. Without argument, clear all breaks (but first\n   ask confirmation).\n\ndisable [bpnumber [bpnumber ...]]\n\n   Disable the breakpoints given as a space separated list of\n   breakpoint numbers.  Disabling a breakpoint means it cannot cause\n   the program to stop execution, but unlike clearing a breakpoint, it\n   remains in the list of breakpoints and can be (re-)enabled.\n\nenable [bpnumber [bpnumber ...]]\n\n   Enable the breakpoints specified.\n\nignore bpnumber [count]\n\n   Set the ignore count for the given breakpoint number.  If count is\n   omitted, the ignore count is set to 0.  A breakpoint becomes active\n   when the ignore count is zero.  When non-zero, the count is\n   decremented each time the breakpoint is reached and the breakpoint\n   is not disabled and any associated condition evaluates to true.\n\ncondition bpnumber [condition]\n\n   Set a new *condition* for the breakpoint, an expression which must\n   evaluate to true before the breakpoint is honored.  If *condition*\n   is absent, any existing condition is removed; i.e., the breakpoint\n   is made unconditional.\n\ncommands [bpnumber]\n\n   Specify a list of commands for breakpoint number *bpnumber*.  The\n   commands themselves appear on the following lines.  Type a line\n   containing just ``end`` to terminate the commands. An example:\n\n      (Pdb) commands 1\n      (com) print some_variable\n      (com) end\n      (Pdb)\n\n   To remove all commands from a breakpoint, type commands and follow\n   it immediately with ``end``; that is, give no commands.\n\n   With no *bpnumber* argument, commands refers to the last breakpoint\n   set.\n\n   You can use breakpoint commands to start your program up again.\n   Simply use the continue command, or step, or any other command that\n   resumes execution.\n\n   Specifying any command resuming execution (currently continue,\n   step, next, return, jump, quit and their abbreviations) terminates\n   the command list (as if that command was immediately followed by\n   end). This is because any time you resume execution (even with a\n   simple next or step), you may encounter another breakpoint--which\n   could have its own command list, leading to ambiguities about which\n   list to execute.\n\n   If you use the \'silent\' command in the command list, the usual\n   message about stopping at a breakpoint is not printed.  This may be\n   desirable for breakpoints that are to print a specific message and\n   then continue.  If none of the other commands print anything, you\n   see no sign that the breakpoint was reached.\n\ns(tep)\n\n   Execute the current line, stop at the first possible occasion\n   (either in a function that is called or on the next line in the\n   current function).\n\nn(ext)\n\n   Continue execution until the next line in the current function is\n   reached or it returns.  (The difference between ``next`` and\n   ``step`` is that ``step`` stops inside a called function, while\n   ``next`` executes called functions at (nearly) full speed, only\n   stopping at the next line in the current function.)\n\nunt(il) [lineno]\n\n   Without argument, continue execution until the line with a number\n   greater than the current one is reached.\n\n   With a line number, continue execution until a line with a number\n   greater or equal to that is reached.  In both cases, also stop when\n   the current frame returns.\n\n   Changed in version 3.2: Allow giving an explicit line number.\n\nr(eturn)\n\n   Continue execution until the current function returns.\n\nc(ont(inue))\n\n   Continue execution, only stop when a breakpoint is encountered.\n\nj(ump) lineno\n\n   Set the next line that will be executed.  Only available in the\n   bottom-most frame.  This lets you jump back and execute code again,\n   or jump forward to skip code that you don\'t want to run.\n\n   It should be noted that not all jumps are allowed -- for instance\n   it is not possible to jump into the middle of a ``for`` loop or out\n   of a ``finally`` clause.\n\nl(ist) [first[, last]]\n\n   List source code for the current file.  Without arguments, list 11\n   lines around the current line or continue the previous listing.\n   With ``.`` as argument, list 11 lines around the current line.\n   With one argument, list 11 lines around at that line.  With two\n   arguments, list the given range; if the second argument is less\n   than the first, it is interpreted as a count.\n\n   The current line in the current frame is indicated by ``->``.  If\n   an exception is being debugged, the line where the exception was\n   originally raised or propagated is indicated by ``>>``, if it\n   differs from the current line.\n\n   New in version 3.2: The ``>>`` marker.\n\nll | longlist\n\n   List all source code for the current function or frame.\n   Interesting lines are marked as for ``list``.\n\n   New in version 3.2.\n\na(rgs)\n\n   Print the argument list of the current function.\n\np(rint) expression\n\n   Evaluate the *expression* in the current context and print its\n   value.\n\npp expression\n\n   Like the ``print`` command, except the value of the expression is\n   pretty-printed using the ``pprint`` module.\n\nwhatis expression\n\n   Print the type of the *expression*.\n\nsource expression\n\n   Try to get source code for the given object and display it.\n\n   New in version 3.2.\n\ndisplay [expression]\n\n   Display the value of the expression if it changed, each time\n   execution stops in the current frame.\n\n   Without expression, list all display expressions for the current\n   frame.\n\n   New in version 3.2.\n\nundisplay [expression]\n\n   Do not display the expression any more in the current frame.\n   Without expression, clear all display expressions for the current\n   frame.\n\n   New in version 3.2.\n\ninteract\n\n   Start an interative interpreter (using the ``code`` module) whose\n   global namespace contains all the (global and local) names found in\n   the current scope.\n\n   New in version 3.2.\n\nalias [name [command]]\n\n   Create an alias called *name* that executes *command*.  The command\n   must *not* be enclosed in quotes.  Replaceable parameters can be\n   indicated by ``%1``, ``%2``, and so on, while ``%*`` is replaced by\n   all the parameters. If no command is given, the current alias for\n   *name* is shown. If no arguments are given, all aliases are listed.\n\n   Aliases may be nested and can contain anything that can be legally\n   typed at the pdb prompt.  Note that internal pdb commands *can* be\n   overridden by aliases.  Such a command is then hidden until the\n   alias is removed.  Aliasing is recursively applied to the first\n   word of the command line; all other words in the line are left\n   alone.\n\n   As an example, here are two useful aliases (especially when placed\n   in the ``.pdbrc`` file):\n\n      # Print instance variables (usage "pi classInst")\n      alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])\n      # Print instance variables in self\n      alias ps pi self\n\nunalias name\n\n   Delete the specified alias.\n\n! statement\n\n   Execute the (one-line) *statement* in the context of the current\n   stack frame. The exclamation point can be omitted unless the first\n   word of the statement resembles a debugger command.  To set a\n   global variable, you can prefix the assignment command with a\n   ``global`` statement on the same line, e.g.:\n\n      (Pdb) global list_options; list_options = [\'-l\']\n      (Pdb)\n\nrun [args ...]\nrestart [args ...]\n\n   Restart the debugged Python program.  If an argument is supplied,\n   it is split with ``shlex`` and the result is used as the new\n   ``sys.argv``. History, breakpoints, actions and debugger options\n   are preserved. ``restart`` is an alias for ``run``.\n\nq(uit)\n\n   Quit from the debugger.  The program being executed is aborted.\n\n-[ Footnotes ]-\n\n[1] Whether a frame is considered to originate in a certain module is\n    determined by the ``__name__`` in the frame globals.\n',
- 'del': '\nThe ``del`` statement\n*********************\n\n   del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather than spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a ``global``\nstatement in the same code block.  If the name is unbound, a\n``NameError`` exception will be raised.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n\nChanged in version 3.2.\n',
+ 'customization': '\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  ``__new__()`` is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of ``__new__()`` should be the new object instance (usually\n   an instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s ``__new__()`` method using\n   ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n   arguments and then modifying the newly-created instance as\n   necessary before returning it.\n\n   If ``__new__()`` returns an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will be invoked like\n   ``__init__(self[, ...])``, where *self* is the new instance and the\n   remaining arguments are the same as were passed to ``__new__()``.\n\n   If ``__new__()`` does not return an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will not be invoked.\n\n   ``__new__()`` is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called when the instance is created.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   ``__init__()`` method, the derived class\'s ``__init__()`` method,\n   if any, must explicitly call it to ensure proper initialization of\n   the base class part of the instance; for example:\n   ``BaseClass.__init__(self, [args...])``.  As a special constraint\n   on constructors, no value may be returned; doing so will cause a\n   ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a ``__del__()`` method,\n   the derived class\'s ``__del__()`` method, if any, must explicitly\n   call it to ensure proper deletion of the base class part of the\n   instance.  Note that it is possible (though not recommended!) for\n   the ``__del__()`` method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   ``__del__()`` methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n     decrements the reference count for ``x`` by one, and the latter\n     is only called when ``x``\'s reference count reaches zero.  Some\n     common situations that may prevent the reference count of an\n     object from going to zero include: circular references between\n     objects (e.g., a doubly-linked list or a tree data structure with\n     parent and child pointers); a reference to the object on the\n     stack frame of a function that caught an exception (the traceback\n     stored in ``sys.exc_info()[2]`` keeps the stack frame alive); or\n     a reference to the object on the stack frame that raised an\n     unhandled exception in interactive mode (the traceback stored in\n     ``sys.last_traceback`` keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the latter two situations can be resolved by storing ``None`` in\n     ``sys.last_traceback``. Circular references which are garbage are\n     detected when the option cycle detector is enabled (it\'s on by\n     default), but can only be cleaned up if there are no Python-\n     level ``__del__()`` methods involved. Refer to the documentation\n     for the ``gc`` module for more information about how\n     ``__del__()`` methods are handled by the cycle detector,\n     particularly the description of the ``garbage`` value.\n\n   Warning: Due to the precarious circumstances under which ``__del__()``\n     methods are invoked, exceptions that occur during their execution\n     are ignored, and a warning is printed to ``sys.stderr`` instead.\n     Also, when ``__del__()`` is invoked in response to a module being\n     deleted (e.g., when execution of the program is done), other\n     globals referenced by the ``__del__()`` method may already have\n     been deleted or in the process of being torn down (e.g. the\n     import machinery shutting down).  For this reason, ``__del__()``\n     methods should do the absolute minimum needed to maintain\n     external invariants.  Starting with version 1.5, Python\n     guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n   Called by the ``repr()`` built-in function to compute the\n   "official" string representation of an object.  If at all possible,\n   this should look like a valid Python expression that could be used\n   to recreate an object with the same value (given an appropriate\n   environment).  If this is not possible, a string of the form\n   ``<...some useful description...>`` should be returned. The return\n   value must be a string object. If a class defines ``__repr__()``\n   but not ``__str__()``, then ``__repr__()`` is also used when an\n   "informal" string representation of instances of that class is\n   required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by the ``str()`` built-in function and by the ``print()``\n   function to compute the "informal" string representation of an\n   object.  This differs from ``__repr__()`` in that it does not have\n   to be a valid Python expression: a more convenient or concise\n   representation may be used instead. The return value must be a\n   string object.\n\nobject.__bytes__(self)\n\n   Called by ``bytes()`` to compute a byte-string representation of an\n   object. This should return a ``bytes`` object.\n\nobject.__format__(self, format_spec)\n\n   Called by the ``format()`` built-in function (and by extension, the\n   ``format()`` method of class ``str``) to produce a "formatted"\n   string representation of an object. The ``format_spec`` argument is\n   a string that contains a description of the formatting options\n   desired. The interpretation of the ``format_spec`` argument is up\n   to the type implementing ``__format__()``, however most classes\n   will either delegate formatting to one of the built-in types, or\n   use a similar formatting option syntax.\n\n   See *Format Specification Mini-Language* for a description of the\n   standard formatting syntax.\n\n   The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   These are the so-called "rich comparison" methods. The\n   correspondence between operator symbols and method names is as\n   follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n   ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` calls\n   ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls\n   ``x.__ge__(y)``.\n\n   A rich comparison method may return the singleton\n   ``NotImplemented`` if it does not implement the operation for a\n   given pair of arguments. By convention, ``False`` and ``True`` are\n   returned for a successful comparison. However, these methods can\n   return any value, so if the comparison operator is used in a\n   Boolean context (e.g., in the condition of an ``if`` statement),\n   Python will call ``bool()`` on the value to determine if the result\n   is true or false.\n\n   There are no implied relationships among the comparison operators.\n   The truth of ``x==y`` does not imply that ``x!=y`` is false.\n   Accordingly, when defining ``__eq__()``, one should also define\n   ``__ne__()`` so that the operators will behave as expected.  See\n   the paragraph on ``__hash__()`` for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n   other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n   reflection, and ``__eq__()`` and ``__ne__()`` are their own\n   reflection.\n\n   Arguments to rich comparison methods are never coerced.\n\n   To automatically generate ordering operations from a single root\n   operation, see ``functools.total_ordering()``.\n\nobject.__hash__(self)\n\n   Called by built-in function ``hash()`` and for operations on\n   members of hashed collections including ``set``, ``frozenset``, and\n   ``dict``.  ``__hash__()`` should return an integer.  The only\n   required property is that objects which compare equal have the same\n   hash value; it is advised to somehow mix together (e.g. using\n   exclusive or) the hash values for the components of the object that\n   also play a part in comparison of objects.\n\n   If a class does not define an ``__eq__()`` method it should not\n   define a ``__hash__()`` operation either; if it defines\n   ``__eq__()`` but not ``__hash__()``, its instances will not be\n   usable as items in hashable collections.  If a class defines\n   mutable objects and implements an ``__eq__()`` method, it should\n   not implement ``__hash__()``, since the implementation of hashable\n   collections requires that a key\'s hash value is immutable (if the\n   object\'s hash value changes, it will be in the wrong hash bucket).\n\n   User-defined classes have ``__eq__()`` and ``__hash__()`` methods\n   by default; with them, all objects compare unequal (except with\n   themselves) and ``x.__hash__()`` returns an appropriate value such\n   that ``x == y`` implies both that ``x is y`` and ``hash(x) ==\n   hash(y)``.\n\n   Classes which inherit a ``__hash__()`` method from a parent class\n   but change the meaning of ``__eq__()`` such that the hash value\n   returned is no longer appropriate (e.g. by switching to a value-\n   based concept of equality instead of the default identity based\n   equality) can explicitly flag themselves as being unhashable by\n   setting ``__hash__ = None`` in the class definition. Doing so means\n   that not only will instances of the class raise an appropriate\n   ``TypeError`` when a program attempts to retrieve their hash value,\n   but they will also be correctly identified as unhashable when\n   checking ``isinstance(obj, collections.Hashable)`` (unlike classes\n   which define their own ``__hash__()`` to explicitly raise\n   ``TypeError``).\n\n   If a class that overrides ``__eq__()`` needs to retain the\n   implementation of ``__hash__()`` from a parent class, the\n   interpreter must be told this explicitly by setting ``__hash__ =\n   <ParentClass>.__hash__``. Otherwise the inheritance of\n   ``__hash__()`` will be blocked, just as if ``__hash__`` had been\n   explicitly set to ``None``.\n\n   Note: By default, the ``__hash__()`` values of str, bytes and datetime\n     objects are "salted" with an unpredictable random value.\n     Although they remain constant within an individual Python\n     process, they are not predictable between repeated invocations of\n     Python.This is intended to provide protection against a denial-\n     of-service caused by carefully-chosen inputs that exploit the\n     worst case performance of a dict insertion, O(n^2) complexity.\n     See http://www.ocert.org/advisories/ocert-2011-003.html for\n     details.Changing hash values affects the iteration order of\n     dicts, sets and other mappings.  Python has never made guarantees\n     about this ordering (and it typically varies between 32-bit and\n     64-bit builds).See also ``PYTHONHASHSEED``.\n\n   Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n   Called to implement truth value testing and the built-in operation\n   ``bool()``; should return ``False`` or ``True``.  When this method\n   is not defined, ``__len__()`` is called, if it is defined, and the\n   object is considered true if its result is nonzero.  If a class\n   defines neither ``__len__()`` nor ``__bool__()``, all its instances\n   are considered true.\n',
+ 'debugger': '\n``pdb`` --- The Python Debugger\n*******************************\n\nThe module ``pdb`` defines an interactive source code debugger for\nPython programs.  It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame.  It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible -- it is actually defined as the class\n``Pdb``. This is currently undocumented but easily understood by\nreading the source.  The extension interface uses the modules ``bdb``\nand ``cmd``.\n\nThe debugger\'s prompt is ``(Pdb)``. Typical usage to run a program\nunder control of the debugger is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> pdb.run(\'mymodule.test()\')\n   > <string>(0)?()\n   (Pdb) continue\n   > <string>(1)?()\n   (Pdb) continue\n   NameError: \'spam\'\n   > <string>(1)?()\n   (Pdb)\n\nChanged in version 3.3: Tab-completion via the ``readline`` module is\navailable for commands and command arguments, e.g. the current global\nand local names are offered as arguments of the ``print`` command.\n\n``pdb.py`` can also be invoked as a script to debug other scripts.\nFor example:\n\n   python3 -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally.  After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program.  Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 3.2: ``pdb.py`` now accepts a ``-c`` option that\nexecutes commands as if given in a ``.pdbrc`` file, see *Debugger\nCommands*.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n   import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger.  You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the ``continue`` command.\n\nThe typical usage to inspect a crashed program is:\n\n   >>> import pdb\n   >>> import mymodule\n   >>> mymodule.test()\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in ?\n     File "./mymodule.py", line 4, in test\n       test2()\n     File "./mymodule.py", line 3, in test2\n       print(spam)\n   NameError: spam\n   >>> pdb.pm()\n   > ./mymodule.py(3)test2()\n   -> print(spam)\n   (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement, globals=None, locals=None)\n\n   Execute the *statement* (given as a string or a code object) under\n   debugger control.  The debugger prompt appears before any code is\n   executed; you can set breakpoints and type ``continue``, or you can\n   step through the statement using ``step`` or ``next`` (all these\n   commands are explained below).  The optional *globals* and *locals*\n   arguments specify the environment in which the code is executed; by\n   default the dictionary of the module ``__main__`` is used.  (See\n   the explanation of the built-in ``exec()`` or ``eval()``\n   functions.)\n\npdb.runeval(expression, globals=None, locals=None)\n\n   Evaluate the *expression* (given as a string or a code object)\n   under debugger control.  When ``runeval()`` returns, it returns the\n   value of the expression.  Otherwise this function is similar to\n   ``run()``.\n\npdb.runcall(function, *args, **kwds)\n\n   Call the *function* (a function or method object, not a string)\n   with the given arguments.  When ``runcall()`` returns, it returns\n   whatever the function call returned.  The debugger prompt appears\n   as soon as the function is entered.\n\npdb.set_trace()\n\n   Enter the debugger at the calling stack frame.  This is useful to\n   hard-code a breakpoint at a given point in a program, even if the\n   code is not otherwise being debugged (e.g. when an assertion\n   fails).\n\npdb.post_mortem(traceback=None)\n\n   Enter post-mortem debugging of the given *traceback* object.  If no\n   *traceback* is given, it uses the one of the exception that is\n   currently being handled (an exception must be being handled if the\n   default is to be used).\n\npdb.pm()\n\n   Enter post-mortem debugging of the traceback found in\n   ``sys.last_traceback``.\n\nThe ``run*`` functions and ``set_trace()`` are aliases for\ninstantiating the ``Pdb`` class and calling the method of the same\nname.  If you want to access further features, you have to do this\nyourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None, nosigint=False)\n\n   ``Pdb`` is the debugger class.\n\n   The *completekey*, *stdin* and *stdout* arguments are passed to the\n   underlying ``cmd.Cmd`` class; see the description there.\n\n   The *skip* argument, if given, must be an iterable of glob-style\n   module name patterns.  The debugger will not step into frames that\n   originate in a module that matches one of these patterns. [1]\n\n   By default, Pdb sets a handler for the SIGINT signal (which is sent\n   when the user presses Ctrl-C on the console) when you give a\n   ``continue`` command. This allows you to break into the debugger\n   again by pressing Ctrl-C.  If you want Pdb not to touch the SIGINT\n   handler, set *nosigint* tot true.\n\n   Example call to enable tracing with *skip*:\n\n      import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n   New in version 3.1: The *skip* argument.\n\n   New in version 3.2: The *nosigint* argument.  Previously, a SIGINT\n   handler was never set by Pdb.\n\n   run(statement, globals=None, locals=None)\n   runeval(expression, globals=None, locals=None)\n   runcall(function, *args, **kwds)\n   set_trace()\n\n      See the documentation for the functions explained above.\n\n\nDebugger Commands\n=================\n\nThe commands recognized by the debugger are listed below.  Most\ncommands can be abbreviated to one or two letters as indicated; e.g.\n``h(elp)`` means that either ``h`` or ``help`` can be used to enter\nthe help command (but not ``he`` or ``hel``, nor ``H`` or ``Help`` or\n``HELP``).  Arguments to commands must be separated by whitespace\n(spaces or tabs).  Optional arguments are enclosed in square brackets\n(``[]``) in the command syntax; the square brackets must not be typed.\nAlternatives in the command syntax are separated by a vertical bar\n(``|``).\n\nEntering a blank line repeats the last command entered.  Exception: if\nthe last command was a ``list`` command, the next 11 lines are listed.\n\nCommands that the debugger doesn\'t recognize are assumed to be Python\nstatements and are executed in the context of the program being\ndebugged.  Python statements can also be prefixed with an exclamation\npoint (``!``).  This is a powerful way to inspect the program being\ndebugged; it is even possible to change a variable or call a function.\nWhen an exception occurs in such a statement, the exception name is\nprinted but the debugger\'s state is not changed.\n\nThe debugger supports *aliases*.  Aliases can have parameters which\nallows one a certain level of adaptability to the context under\nexamination.\n\nMultiple commands may be entered on a single line, separated by\n``;;``.  (A single ``;`` is not used as it is the separator for\nmultiple commands in a line that is passed to the Python parser.)  No\nintelligence is applied to separating the commands; the input is split\nat the first ``;;`` pair, even if it is in the middle of a quoted\nstring.\n\nIf a file ``.pdbrc`` exists in the user\'s home directory or in the\ncurrent directory, it is read in and executed as if it had been typed\nat the debugger prompt.  This is particularly useful for aliases.  If\nboth files exist, the one in the home directory is read first and\naliases defined there can be overridden by the local file.\n\nChanged in version 3.2: ``.pdbrc`` can now contain commands that\ncontinue debugging, such as ``continue`` or ``next``.  Previously,\nthese commands had no effect.\n\nh(elp) [command]\n\n   Without argument, print the list of available commands.  With a\n   *command* as argument, print help about that command.  ``help pdb``\n   displays the full documentation (the docstring of the ``pdb``\n   module).  Since the *command* argument must be an identifier,\n   ``help exec`` must be entered to get help on the ``!`` command.\n\nw(here)\n\n   Print a stack trace, with the most recent frame at the bottom.  An\n   arrow indicates the current frame, which determines the context of\n   most commands.\n\nd(own) [count]\n\n   Move the current frame *count* (default one) levels down in the\n   stack trace (to a newer frame).\n\nu(p) [count]\n\n   Move the current frame *count* (default one) levels up in the stack\n   trace (to an older frame).\n\nb(reak) [([filename:]lineno | function) [, condition]]\n\n   With a *lineno* argument, set a break there in the current file.\n   With a *function* argument, set a break at the first executable\n   statement within that function.  The line number may be prefixed\n   with a filename and a colon, to specify a breakpoint in another\n   file (probably one that hasn\'t been loaded yet).  The file is\n   searched on ``sys.path``.  Note that each breakpoint is assigned a\n   number to which all the other breakpoint commands refer.\n\n   If a second argument is present, it is an expression which must\n   evaluate to true before the breakpoint is honored.\n\n   Without argument, list all breaks, including for each breakpoint,\n   the number of times that breakpoint has been hit, the current\n   ignore count, and the associated condition if any.\n\ntbreak [([filename:]lineno | function) [, condition]]\n\n   Temporary breakpoint, which is removed automatically when it is\n   first hit. The arguments are the same as for ``break``.\n\ncl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n\n   With a *filename:lineno* argument, clear all the breakpoints at\n   this line. With a space separated list of breakpoint numbers, clear\n   those breakpoints. Without argument, clear all breaks (but first\n   ask confirmation).\n\ndisable [bpnumber [bpnumber ...]]\n\n   Disable the breakpoints given as a space separated list of\n   breakpoint numbers.  Disabling a breakpoint means it cannot cause\n   the program to stop execution, but unlike clearing a breakpoint, it\n   remains in the list of breakpoints and can be (re-)enabled.\n\nenable [bpnumber [bpnumber ...]]\n\n   Enable the breakpoints specified.\n\nignore bpnumber [count]\n\n   Set the ignore count for the given breakpoint number.  If count is\n   omitted, the ignore count is set to 0.  A breakpoint becomes active\n   when the ignore count is zero.  When non-zero, the count is\n   decremented each time the breakpoint is reached and the breakpoint\n   is not disabled and any associated condition evaluates to true.\n\ncondition bpnumber [condition]\n\n   Set a new *condition* for the breakpoint, an expression which must\n   evaluate to true before the breakpoint is honored.  If *condition*\n   is absent, any existing condition is removed; i.e., the breakpoint\n   is made unconditional.\n\ncommands [bpnumber]\n\n   Specify a list of commands for breakpoint number *bpnumber*.  The\n   commands themselves appear on the following lines.  Type a line\n   containing just ``end`` to terminate the commands. An example:\n\n      (Pdb) commands 1\n      (com) print some_variable\n      (com) end\n      (Pdb)\n\n   To remove all commands from a breakpoint, type commands and follow\n   it immediately with ``end``; that is, give no commands.\n\n   With no *bpnumber* argument, commands refers to the last breakpoint\n   set.\n\n   You can use breakpoint commands to start your program up again.\n   Simply use the continue command, or step, or any other command that\n   resumes execution.\n\n   Specifying any command resuming execution (currently continue,\n   step, next, return, jump, quit and their abbreviations) terminates\n   the command list (as if that command was immediately followed by\n   end). This is because any time you resume execution (even with a\n   simple next or step), you may encounter another breakpoint--which\n   could have its own command list, leading to ambiguities about which\n   list to execute.\n\n   If you use the \'silent\' command in the command list, the usual\n   message about stopping at a breakpoint is not printed.  This may be\n   desirable for breakpoints that are to print a specific message and\n   then continue.  If none of the other commands print anything, you\n   see no sign that the breakpoint was reached.\n\ns(tep)\n\n   Execute the current line, stop at the first possible occasion\n   (either in a function that is called or on the next line in the\n   current function).\n\nn(ext)\n\n   Continue execution until the next line in the current function is\n   reached or it returns.  (The difference between ``next`` and\n   ``step`` is that ``step`` stops inside a called function, while\n   ``next`` executes called functions at (nearly) full speed, only\n   stopping at the next line in the current function.)\n\nunt(il) [lineno]\n\n   Without argument, continue execution until the line with a number\n   greater than the current one is reached.\n\n   With a line number, continue execution until a line with a number\n   greater or equal to that is reached.  In both cases, also stop when\n   the current frame returns.\n\n   Changed in version 3.2: Allow giving an explicit line number.\n\nr(eturn)\n\n   Continue execution until the current function returns.\n\nc(ont(inue))\n\n   Continue execution, only stop when a breakpoint is encountered.\n\nj(ump) lineno\n\n   Set the next line that will be executed.  Only available in the\n   bottom-most frame.  This lets you jump back and execute code again,\n   or jump forward to skip code that you don\'t want to run.\n\n   It should be noted that not all jumps are allowed -- for instance\n   it is not possible to jump into the middle of a ``for`` loop or out\n   of a ``finally`` clause.\n\nl(ist) [first[, last]]\n\n   List source code for the current file.  Without arguments, list 11\n   lines around the current line or continue the previous listing.\n   With ``.`` as argument, list 11 lines around the current line.\n   With one argument, list 11 lines around at that line.  With two\n   arguments, list the given range; if the second argument is less\n   than the first, it is interpreted as a count.\n\n   The current line in the current frame is indicated by ``->``.  If\n   an exception is being debugged, the line where the exception was\n   originally raised or propagated is indicated by ``>>``, if it\n   differs from the current line.\n\n   New in version 3.2: The ``>>`` marker.\n\nll | longlist\n\n   List all source code for the current function or frame.\n   Interesting lines are marked as for ``list``.\n\n   New in version 3.2.\n\na(rgs)\n\n   Print the argument list of the current function.\n\np(rint) expression\n\n   Evaluate the *expression* in the current context and print its\n   value.\n\npp expression\n\n   Like the ``print`` command, except the value of the expression is\n   pretty-printed using the ``pprint`` module.\n\nwhatis expression\n\n   Print the type of the *expression*.\n\nsource expression\n\n   Try to get source code for the given object and display it.\n\n   New in version 3.2.\n\ndisplay [expression]\n\n   Display the value of the expression if it changed, each time\n   execution stops in the current frame.\n\n   Without expression, list all display expressions for the current\n   frame.\n\n   New in version 3.2.\n\nundisplay [expression]\n\n   Do not display the expression any more in the current frame.\n   Without expression, clear all display expressions for the current\n   frame.\n\n   New in version 3.2.\n\ninteract\n\n   Start an interative interpreter (using the ``code`` module) whose\n   global namespace contains all the (global and local) names found in\n   the current scope.\n\n   New in version 3.2.\n\nalias [name [command]]\n\n   Create an alias called *name* that executes *command*.  The command\n   must *not* be enclosed in quotes.  Replaceable parameters can be\n   indicated by ``%1``, ``%2``, and so on, while ``%*`` is replaced by\n   all the parameters. If no command is given, the current alias for\n   *name* is shown. If no arguments are given, all aliases are listed.\n\n   Aliases may be nested and can contain anything that can be legally\n   typed at the pdb prompt.  Note that internal pdb commands *can* be\n   overridden by aliases.  Such a command is then hidden until the\n   alias is removed.  Aliasing is recursively applied to the first\n   word of the command line; all other words in the line are left\n   alone.\n\n   As an example, here are two useful aliases (especially when placed\n   in the ``.pdbrc`` file):\n\n      # Print instance variables (usage "pi classInst")\n      alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])\n      # Print instance variables in self\n      alias ps pi self\n\nunalias name\n\n   Delete the specified alias.\n\n! statement\n\n   Execute the (one-line) *statement* in the context of the current\n   stack frame. The exclamation point can be omitted unless the first\n   word of the statement resembles a debugger command.  To set a\n   global variable, you can prefix the assignment command with a\n   ``global`` statement on the same line, e.g.:\n\n      (Pdb) global list_options; list_options = [\'-l\']\n      (Pdb)\n\nrun [args ...]\nrestart [args ...]\n\n   Restart the debugged Python program.  If an argument is supplied,\n   it is split with ``shlex`` and the result is used as the new\n   ``sys.argv``. History, breakpoints, actions and debugger options\n   are preserved. ``restart`` is an alias for ``run``.\n\nq(uit)\n\n   Quit from the debugger.  The program being executed is aborted.\n\n-[ Footnotes ]-\n\n[1] Whether a frame is considered to originate in a certain module is\n    determined by the ``__name__`` in the frame globals.\n',
+ 'del': '\nThe ``del`` statement\n*********************\n\n   del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather than spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a ``global``\nstatement in the same code block.  If the name is unbound, a\n``NameError`` exception will be raised.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n\nChanged in version 3.2: Previously it was illegal to delete a name\nfrom the local namespace if it occurs as a free variable in a nested\nblock.\n',
  'dict': '\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n   dict_display       ::= "{" [key_datum_list | dict_comprehension] "}"\n   key_datum_list     ::= key_datum ("," key_datum)* [","]\n   key_datum          ::= expression ":" expression\n   dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum.  This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*.  (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.)  Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n',
  'dynamic-features': '\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name.  An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nThe ``eval()`` and ``exec()`` functions do not have access to the full\nenvironment for resolving names.  Names may be resolved in the local\nand global namespaces of the caller.  Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace.  [1]\nThe ``exec()`` and ``eval()`` functions have optional arguments to\noverride the global and local namespace.  If only one namespace is\nspecified, it is used for both.\n',
  'else': '\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n',
@@ -33,14 +34,14 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss
  'exprlists': '\nExpression lists\n****************\n\n   expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple.  The\nlength of the tuple is the number of expressions in the list.  The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases.  A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n',
  'floating': '\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n   floatnumber   ::= pointfloat | exponentfloat\n   pointfloat    ::= [intpart] fraction | intpart "."\n   exponentfloat ::= (intpart | pointfloat) exponent\n   intpart       ::= digit+\n   fraction      ::= "." digit+\n   exponent      ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts are always interpreted using\nradix 10. For example, ``077e010`` is legal, and denotes the same\nnumber as ``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n   3.14    10.    .001    1e100    3.14e-10    0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n',
  'for': '\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n   for_stmt ::= "for" target_list "in" expression_list ":" suite\n                ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject.  An iterator is created for the result of the\n``expression_list``.  The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices.  Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted.  When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a ``StopIteration``\nexception), the suite in the ``else`` clause, if present, is executed,\nand the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop.  Hint: the built-in function ``range()`` returns an\niterator of integers suitable to emulate the effect of Pascal\'s ``for\ni := a to b do``; e.g., ``list(range(3))`` returns the list ``[0, 1,\n2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n  (this can only occur for mutable sequences, i.e. lists).  An\n  internal counter is used to keep track of which item is used next,\n  and this is incremented on each iteration.  When this counter has\n  reached the length of the sequence the loop terminates.  This means\n  that if the suite deletes the current (or a previous) item from the\n  sequence, the next item will be skipped (since it gets the index of\n  the current item which has already been treated).  Likewise, if the\n  suite inserts an item in the sequence before the current item, the\n  current item will be treated again the next time through the loop.\n  This can lead to nasty bugs that can be avoided by making a\n  temporary copy using a slice of the whole sequence, e.g.,\n\n     for x in a[:]:\n         if x < 0: a.remove(x)\n',
- 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output.  If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n      replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n      field_name        ::= arg_name ("." attribute_name | "[" element_index "]")*\n      arg_name          ::= [identifier | integer]\n      attribute_name    ::= identifier\n      element_index     ::= integer | index_string\n      index_string      ::= <any source character except "]"> +\n      conversion        ::= "r" | "s" | "a"\n      format_spec       ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a  *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``.  These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword.  If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument.  If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 3.1: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n   "First, thou shalt count to {0}" # References first positional argument\n   "Bring me a {}"                  # Implicitly references the first positional argument\n   "From {} to {}"                  # Same as "From {0} to {1}"\n   "My quest is {name}"             # References keyword argument \'name\'\n   "Weight in tons {0.weight}"      # \'weight\' attribute of first positional arg\n   "Units destroyed: {players[0]}"  # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself.  However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting.  By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nThree conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, ``\'!r\'`` which calls ``repr()`` and ``\'!a\'``\nwhich calls ``ascii()``.\n\nSome examples:\n\n   "Harold\'s a clever {0!s}"        # Calls str() on the argument first\n   "Bring out the holy {name!r}"    # Calls repr() on the argument first\n   "More {!a}"                      # Calls ascii() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on.  Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed.  The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*).  They can also be passed directly to the\nbuilt-in ``format()`` function.  Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n   format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n   fill        ::= <a character other than \'}\'>\n   align       ::= "<" | ">" | "=" | "^"\n   sign        ::= "+" | "-" | " "\n   width       ::= integer\n   precision   ::= integer\n   type        ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'.  The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options.  If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'<\'``   | Forces the field to be left-aligned within the available   |\n   |           | space (this is the default for most objects).              |\n   +-----------+------------------------------------------------------------+\n   | ``\'>\'``   | Forces the field to be right-aligned within the available  |\n   |           | space (this is the default for numbers).                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'=\'``   | Forces the padding to be placed after the sign (if any)    |\n   |           | but before the digits.  This is used for printing fields   |\n   |           | in the form \'+000000120\'. This alignment option is only    |\n   |           | valid for numeric types.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'^\'``   | Forces the field to be centered within the available       |\n   |           | space.                                                     |\n   +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'+\'``   | indicates that a sign should be used for both positive as  |\n   |           | well as negative numbers.                                  |\n   +-----------+------------------------------------------------------------+\n   | ``\'-\'``   | indicates that a sign should be used only for negative     |\n   |           | numbers (this is the default behavior).                    |\n   +-----------+------------------------------------------------------------+\n   | space     | indicates that a leading space should be used on positive  |\n   |           | numbers, and a minus sign on negative numbers.             |\n   +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option causes the "alternate form" to be used for the\nconversion.  The alternate form is defined differently for different\ntypes.  This option is only valid for integer, float, complex and\nDecimal types. For integers, when binary, octal, or hexadecimal output\nis used, this option adds the prefix respective ``\'0b\'``, ``\'0o\'``, or\n``\'0x\'`` to the output value. For floats, complex and Decimal the\nalternate form causes the result of the conversion to always contain a\ndecimal-point character, even if no digits follow it. Normally, a\ndecimal-point character appears in the result of these conversions\nonly if a digit follows it. In addition, for ``\'g\'`` and ``\'G\'``\nconversions, trailing zeros are not removed from the result.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 3.1: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width.  If not\nspecified, then the field width will be determined by the content.\n\nIf the *width* field is preceded by a zero (``\'0\'``) character, this\nenables zero-padding.  This is equivalent to an *alignment* type of\n``\'=\'`` and a *fill* character of ``\'0\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'s\'``   | String format. This is the default type for strings and    |\n   |           | may be omitted.                                            |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'s\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'b\'``   | Binary format. Outputs the number in base 2.               |\n   +-----------+------------------------------------------------------------+\n   | ``\'c\'``   | Character. Converts the integer to the corresponding       |\n   |           | unicode character before printing.                         |\n   +-----------+------------------------------------------------------------+\n   | ``\'d\'``   | Decimal Integer. Outputs the number in base 10.            |\n   +-----------+------------------------------------------------------------+\n   | ``\'o\'``   | Octal format. Outputs the number in base 8.                |\n   +-----------+------------------------------------------------------------+\n   | ``\'x\'``   | Hex format. Outputs the number in base 16, using lower-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'X\'``   | Hex format. Outputs the number in base 16, using upper-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'d\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'d\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'e\'``   | Exponent notation. Prints the number in scientific         |\n   |           | notation using the letter \'e\' to indicate the exponent.    |\n   +-----------+------------------------------------------------------------+\n   | ``\'E\'``   | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n   |           | case \'E\' as the separator character.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'f\'``   | Fixed point. Displays the number as a fixed-point number.  |\n   +-----------+------------------------------------------------------------+\n   | ``\'F\'``   | Fixed point. Same as ``\'f\'``, but converts ``nan`` to      |\n   |           | ``NAN`` and ``inf`` to ``INF``.                            |\n   +-----------+------------------------------------------------------------+\n   | ``\'g\'``   | General format.  For a given precision ``p >= 1``, this    |\n   |           | rounds the number to ``p`` significant digits and then     |\n   |           | formats the result in either fixed-point format or in      |\n   |           | scientific notation, depending on its magnitude.  The      |\n   |           | precise rules are as follows: suppose that the result      |\n   |           | formatted with presentation type ``\'e\'`` and precision     |\n   |           | ``p-1`` would have exponent ``exp``.  Then if ``-4 <= exp  |\n   |           | < p``, the number is formatted with presentation type      |\n   |           | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number   |\n   |           | is formatted with presentation type ``\'e\'`` and precision  |\n   |           | ``p-1``. In both cases insignificant trailing zeros are    |\n   |           | removed from the significand, and the decimal point is     |\n   |           | also removed if there are no remaining digits following    |\n   |           | it.  Positive and negative infinity, positive and negative |\n   |           | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n   |           | ``-0`` and ``nan`` respectively, regardless of the         |\n   |           | precision.  A precision of ``0`` is treated as equivalent  |\n   |           | to a precision of ``1``.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'G\'``   | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n   |           | if the number gets too large. The representations of       |\n   |           | infinity and NaN are uppercased, too.                      |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'g\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | ``\'%\'``   | Percentage. Multiplies the number by 100 and displays in   |\n   |           | fixed (``\'f\'``) format, followed by a percent sign.        |\n   +-----------+------------------------------------------------------------+\n   | None      | Similar to ``\'g\'``, except with at least one digit past    |\n   |           | the decimal point and a default precision of 12. This is   |\n   |           | intended to match ``str()``, except you can add the other  |\n   |           | format modifiers.                                          |\n   +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n   >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n   \'a, b, c\'\n   >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\')  # 3.1+ only\n   \'a, b, c\'\n   >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n   \'c, b, a\'\n   >>> \'{2}, {1}, {0}\'.format(*\'abc\')      # unpacking argument sequence\n   \'c, b, a\'\n   >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\')   # arguments\' indices can be repeated\n   \'abracadabra\'\n\nAccessing arguments by name:\n\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n   \'Coordinates: 37.24N, -115.81W\'\n   >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n   \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n   >>> c = 3-5j\n   >>> (\'The complex number {0} is formed from the real part {0.real} \'\n   ...  \'and the imaginary part {0.imag}.\').format(c)\n   \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n   >>> class Point:\n   ...     def __init__(self, x, y):\n   ...         self.x, self.y = x, y\n   ...     def __str__(self):\n   ...         return \'Point({self.x}, {self.y})\'.format(self=self)\n   ...\n   >>> str(Point(4, 2))\n   \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n   >>> coord = (3, 5)\n   >>> \'X: {0[0]};  Y: {0[1]}\'.format(coord)\n   \'X: 3;  Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n   >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n   "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n   >>> \'{:<30}\'.format(\'left aligned\')\n   \'left aligned                  \'\n   >>> \'{:>30}\'.format(\'right aligned\')\n   \'                 right aligned\'\n   >>> \'{:^30}\'.format(\'centered\')\n   \'           centered           \'\n   >>> \'{:*^30}\'.format(\'centered\')  # use \'*\' as a fill char\n   \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n   >>> \'{:+f}; {:+f}\'.format(3.14, -3.14)  # show it always\n   \'+3.140000; -3.140000\'\n   >>> \'{: f}; {: f}\'.format(3.14, -3.14)  # show a space for positive numbers\n   \' 3.140000; -3.140000\'\n   >>> \'{:-f}; {:-f}\'.format(3.14, -3.14)  # show only the minus -- same as \'{:f}; {:f}\'\n   \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n   >>> # format also supports binary numbers\n   >>> "int: {0:d};  hex: {0:x};  oct: {0:o};  bin: {0:b}".format(42)\n   \'int: 42;  hex: 2a;  oct: 52;  bin: 101010\'\n   >>> # with 0x, 0o, or 0b as prefix:\n   >>> "int: {0:d};  hex: {0:#x};  oct: {0:#o};  bin: {0:#b}".format(42)\n   \'int: 42;  hex: 0x2a;  oct: 0o52;  bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n   >>> \'{:,}\'.format(1234567890)\n   \'1,234,567,890\'\n\nExpressing a percentage:\n\n   >>> points = 19\n   >>> total = 22\n   >>> \'Correct answers: {:.2%}\'.format(points/total)\n   \'Correct answers: 86.36%\'\n\nUsing type-specific formatting:\n\n   >>> import datetime\n   >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n   >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n   \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n   >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n   ...     \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n   ...\n   \'left<<<<<<<<<<<<\'\n   \'^^^^^center^^^^^\'\n   \'>>>>>>>>>>>right\'\n   >>>\n   >>> octets = [192, 168, 0, 1]\n   >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n   \'C0A80001\'\n   >>> int(_, 16)\n   3232235521\n   >>>\n   >>> width = 5\n   >>> for num in range(5,12):\n   ...     for base in \'dXob\':\n   ...         print(\'{0:{width}{base}}\'.format(num, base=base, width=width), end=\' \')\n   ...     print()\n   ...\n       5     5     5   101\n       6     6     6   110\n       7     7     7   111\n       8     8    10  1000\n       9     9    11  1001\n      10     A    12  1010\n      11     B    13  1011\n',
- 'function': '\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   funcdef        ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      (  "*" [parameter] ("," defparameter)*\n                      [, "**" parameter]\n                      | "**" parameter\n                      | defparameter [","] )\n   parameter      ::= identifier [":" expression]\n   defparameter   ::= parameter ["=" expression]\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis equivalent to\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nWhen one or more parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters up until the "``*``" must also have a default value ---\nthis is a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call.  This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended.  A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple.  If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after "``*``" or "``*identifier``" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "``: expression``"\nfollowing the parameter name.  Any parameter may have an annotation\neven those of the form ``*identifier`` or ``**identifier``.  Functions\nmay have "return" annotation of the form "``-> expression``" after the\nparameter list.  These annotations can be any valid Python expression\nand are evaluated when the function definition is executed.\nAnnotations may be evaluated in a different order than they appear in\nthe source code.  The presence of annotations does not change the\nsemantics of a function.  The annotation values are available as\nvalues of a dictionary keyed by the parameters\' names in the\n``__annotations__`` attribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda forms,\ndescribed in section *Lambdas*.  Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form.  The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects.  A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around.  Free variables used in the\nnested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n',
+ 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output.  If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n      replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n      field_name        ::= arg_name ("." attribute_name | "[" element_index "]")*\n      arg_name          ::= [identifier | integer]\n      attribute_name    ::= identifier\n      element_index     ::= integer | index_string\n      index_string      ::= <any source character except "]"> +\n      conversion        ::= "r" | "s" | "a"\n      format_spec       ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a  *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``.  These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword.  If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument.  If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 3.1: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n   "First, thou shalt count to {0}" # References first positional argument\n   "Bring me a {}"                  # Implicitly references the first positional argument\n   "From {} to {}"                  # Same as "From {0} to {1}"\n   "My quest is {name}"             # References keyword argument \'name\'\n   "Weight in tons {0.weight}"      # \'weight\' attribute of first positional arg\n   "Units destroyed: {players[0]}"  # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself.  However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting.  By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nThree conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, ``\'!r\'`` which calls ``repr()`` and ``\'!a\'``\nwhich calls ``ascii()``.\n\nSome examples:\n\n   "Harold\'s a clever {0!s}"        # Calls str() on the argument first\n   "Bring out the holy {name!r}"    # Calls repr() on the argument first\n   "More {!a}"                      # Calls ascii() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on.  Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed.  The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*).  They can also be passed directly to the\nbuilt-in ``format()`` function.  Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n   format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n   fill        ::= <a character other than \'{\' or \'}\'>\n   align       ::= "<" | ">" | "=" | "^"\n   sign        ::= "+" | "-" | " "\n   width       ::= integer\n   precision   ::= integer\n   type        ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'.  The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options.  If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'<\'``   | Forces the field to be left-aligned within the available   |\n   |           | space (this is the default for most objects).              |\n   +-----------+------------------------------------------------------------+\n   | ``\'>\'``   | Forces the field to be right-aligned within the available  |\n   |           | space (this is the default for numbers).                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'=\'``   | Forces the padding to be placed after the sign (if any)    |\n   |           | but before the digits.  This is used for printing fields   |\n   |           | in the form \'+000000120\'. This alignment option is only    |\n   |           | valid for numeric types.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'^\'``   | Forces the field to be centered within the available       |\n   |           | space.                                                     |\n   +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n   +-----------+------------------------------------------------------------+\n   | Option    | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'+\'``   | indicates that a sign should be used for both positive as  |\n   |           | well as negative numbers.                                  |\n   +-----------+------------------------------------------------------------+\n   | ``\'-\'``   | indicates that a sign should be used only for negative     |\n   |           | numbers (this is the default behavior).                    |\n   +-----------+------------------------------------------------------------+\n   | space     | indicates that a leading space should be used on positive  |\n   |           | numbers, and a minus sign on negative numbers.             |\n   +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option causes the "alternate form" to be used for the\nconversion.  The alternate form is defined differently for different\ntypes.  This option is only valid for integer, float, complex and\nDecimal types. For integers, when binary, octal, or hexadecimal output\nis used, this option adds the prefix respective ``\'0b\'``, ``\'0o\'``, or\n``\'0x\'`` to the output value. For floats, complex and Decimal the\nalternate form causes the result of the conversion to always contain a\ndecimal-point character, even if no digits follow it. Normally, a\ndecimal-point character appears in the result of these conversions\nonly if a digit follows it. In addition, for ``\'g\'`` and ``\'G\'``\nconversions, trailing zeros are not removed from the result.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 3.1: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width.  If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero (``\'0\'``) character enables\nsign-aware zero-padding for numeric types.  This is equivalent to a\n*fill* character of ``\'0\'`` with an *alignment* type of ``\'=\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'s\'``   | String format. This is the default type for strings and    |\n   |           | may be omitted.                                            |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'s\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'b\'``   | Binary format. Outputs the number in base 2.               |\n   +-----------+------------------------------------------------------------+\n   | ``\'c\'``   | Character. Converts the integer to the corresponding       |\n   |           | unicode character before printing.                         |\n   +-----------+------------------------------------------------------------+\n   | ``\'d\'``   | Decimal Integer. Outputs the number in base 10.            |\n   +-----------+------------------------------------------------------------+\n   | ``\'o\'``   | Octal format. Outputs the number in base 8.                |\n   +-----------+------------------------------------------------------------+\n   | ``\'x\'``   | Hex format. Outputs the number in base 16, using lower-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'X\'``   | Hex format. Outputs the number in base 16, using upper-    |\n   |           | case letters for the digits above 9.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'d\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | None      | The same as ``\'d\'``.                                       |\n   +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n   +-----------+------------------------------------------------------------+\n   | Type      | Meaning                                                    |\n   +===========+============================================================+\n   | ``\'e\'``   | Exponent notation. Prints the number in scientific         |\n   |           | notation using the letter \'e\' to indicate the exponent.    |\n   +-----------+------------------------------------------------------------+\n   | ``\'E\'``   | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n   |           | case \'E\' as the separator character.                       |\n   +-----------+------------------------------------------------------------+\n   | ``\'f\'``   | Fixed point. Displays the number as a fixed-point number.  |\n   +-----------+------------------------------------------------------------+\n   | ``\'F\'``   | Fixed point. Same as ``\'f\'``, but converts ``nan`` to      |\n   |           | ``NAN`` and ``inf`` to ``INF``.                            |\n   +-----------+------------------------------------------------------------+\n   | ``\'g\'``   | General format.  For a given precision ``p >= 1``, this    |\n   |           | rounds the number to ``p`` significant digits and then     |\n   |           | formats the result in either fixed-point format or in      |\n   |           | scientific notation, depending on its magnitude.  The      |\n   |           | precise rules are as follows: suppose that the result      |\n   |           | formatted with presentation type ``\'e\'`` and precision     |\n   |           | ``p-1`` would have exponent ``exp``.  Then if ``-4 <= exp  |\n   |           | < p``, the number is formatted with presentation type      |\n   |           | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number   |\n   |           | is formatted with presentation type ``\'e\'`` and precision  |\n   |           | ``p-1``. In both cases insignificant trailing zeros are    |\n   |           | removed from the significand, and the decimal point is     |\n   |           | also removed if there are no remaining digits following    |\n   |           | it.  Positive and negative infinity, positive and negative |\n   |           | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n   |           | ``-0`` and ``nan`` respectively, regardless of the         |\n   |           | precision.  A precision of ``0`` is treated as equivalent  |\n   |           | to a precision of ``1``.                                   |\n   +-----------+------------------------------------------------------------+\n   | ``\'G\'``   | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n   |           | if the number gets too large. The representations of       |\n   |           | infinity and NaN are uppercased, too.                      |\n   +-----------+------------------------------------------------------------+\n   | ``\'n\'``   | Number. This is the same as ``\'g\'``, except that it uses   |\n   |           | the current locale setting to insert the appropriate       |\n   |           | number separator characters.                               |\n   +-----------+------------------------------------------------------------+\n   | ``\'%\'``   | Percentage. Multiplies the number by 100 and displays in   |\n   |           | fixed (``\'f\'``) format, followed by a percent sign.        |\n   +-----------+------------------------------------------------------------+\n   | None      | Similar to ``\'g\'``, except with at least one digit past    |\n   |           | the decimal point and a default precision of 12. This is   |\n   |           | intended to match ``str()``, except you can add the other  |\n   |           | format modifiers.                                          |\n   +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n   >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n   \'a, b, c\'\n   >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\')  # 3.1+ only\n   \'a, b, c\'\n   >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n   \'c, b, a\'\n   >>> \'{2}, {1}, {0}\'.format(*\'abc\')      # unpacking argument sequence\n   \'c, b, a\'\n   >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\')   # arguments\' indices can be repeated\n   \'abracadabra\'\n\nAccessing arguments by name:\n\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n   \'Coordinates: 37.24N, -115.81W\'\n   >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n   >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n   \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n   >>> c = 3-5j\n   >>> (\'The complex number {0} is formed from the real part {0.real} \'\n   ...  \'and the imaginary part {0.imag}.\').format(c)\n   \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n   >>> class Point:\n   ...     def __init__(self, x, y):\n   ...         self.x, self.y = x, y\n   ...     def __str__(self):\n   ...         return \'Point({self.x}, {self.y})\'.format(self=self)\n   ...\n   >>> str(Point(4, 2))\n   \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n   >>> coord = (3, 5)\n   >>> \'X: {0[0]};  Y: {0[1]}\'.format(coord)\n   \'X: 3;  Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n   >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n   "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n   >>> \'{:<30}\'.format(\'left aligned\')\n   \'left aligned                  \'\n   >>> \'{:>30}\'.format(\'right aligned\')\n   \'                 right aligned\'\n   >>> \'{:^30}\'.format(\'centered\')\n   \'           centered           \'\n   >>> \'{:*^30}\'.format(\'centered\')  # use \'*\' as a fill char\n   \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n   >>> \'{:+f}; {:+f}\'.format(3.14, -3.14)  # show it always\n   \'+3.140000; -3.140000\'\n   >>> \'{: f}; {: f}\'.format(3.14, -3.14)  # show a space for positive numbers\n   \' 3.140000; -3.140000\'\n   >>> \'{:-f}; {:-f}\'.format(3.14, -3.14)  # show only the minus -- same as \'{:f}; {:f}\'\n   \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n   >>> # format also supports binary numbers\n   >>> "int: {0:d};  hex: {0:x};  oct: {0:o};  bin: {0:b}".format(42)\n   \'int: 42;  hex: 2a;  oct: 52;  bin: 101010\'\n   >>> # with 0x, 0o, or 0b as prefix:\n   >>> "int: {0:d};  hex: {0:#x};  oct: {0:#o};  bin: {0:#b}".format(42)\n   \'int: 42;  hex: 0x2a;  oct: 0o52;  bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n   >>> \'{:,}\'.format(1234567890)\n   \'1,234,567,890\'\n\nExpressing a percentage:\n\n   >>> points = 19\n   >>> total = 22\n   >>> \'Correct answers: {:.2%}\'.format(points/total)\n   \'Correct answers: 86.36%\'\n\nUsing type-specific formatting:\n\n   >>> import datetime\n   >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n   >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n   \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n   >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n   ...     \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n   ...\n   \'left<<<<<<<<<<<<\'\n   \'^^^^^center^^^^^\'\n   \'>>>>>>>>>>>right\'\n   >>>\n   >>> octets = [192, 168, 0, 1]\n   >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n   \'C0A80001\'\n   >>> int(_, 16)\n   3232235521\n   >>>\n   >>> width = 5\n   >>> for num in range(5,12):\n   ...     for base in \'dXob\':\n   ...         print(\'{0:{width}{base}}\'.format(num, base=base, width=width), end=\' \')\n   ...     print()\n   ...\n       5     5     5   101\n       6     6     6   110\n       7     7     7   111\n       8     8    10  1000\n       9     9    11  1001\n      10     A    12  1010\n      11     B    13  1011\n',
+ 'function': '\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n   funcdef        ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n   decorators     ::= decorator+\n   decorator      ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n   dotted_name    ::= identifier ("." identifier)*\n   parameter_list ::= (defparameter ",")*\n                      (  "*" [parameter] ("," defparameter)*\n                      [, "**" parameter]\n                      | "**" parameter\n                      | defparameter [","] )\n   parameter      ::= identifier [":" expression]\n   defparameter   ::= parameter ["=" expression]\n   funcname       ::= identifier\n\nA function definition is an executable statement.  Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function).  This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition.  The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object.  Multiple decorators are applied in\nnested fashion. For example, the following code\n\n   @f1(arg)\n   @f2\n   def func(): pass\n\nis equivalent to\n\n   def func(): pass\n   func = f1(arg)(f2(func))\n\nWhen one or more parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted.  If a parameter has a default value, all following\nparameters up until the "``*``" must also have a default value ---\nthis is a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call.  This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended.  A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n   def whats_on_the_telly(penguin=None):\n       if penguin is None:\n           penguin = []\n       penguin.append("property of the zoo")\n       return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values.  If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple.  If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after "``*``" or "``*identifier``" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "``: expression``"\nfollowing the parameter name.  Any parameter may have an annotation\neven those of the form ``*identifier`` or ``**identifier``.  Functions\nmay have "return" annotation of the form "``-> expression``" after the\nparameter list.  These annotations can be any valid Python expression\nand are evaluated when the function definition is executed.\nAnnotations may be evaluated in a different order than they appear in\nthe source code.  The presence of annotations does not change the\nsemantics of a function.  The annotation values are available as\nvalues of a dictionary keyed by the parameters\' names in the\n``__annotations__`` attribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions.  This uses lambda forms,\ndescribed in section *Lambdas*.  Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form.  The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects.  A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around.  Free variables used in the\nnested function can access the local variables of the function\ncontaining the def.  See section *Naming and binding* for details.\n\nSee also:\n\n   **PEP 3107** - Function Annotations\n      The original specification for function annotations.\n',
  'global': '\nThe ``global`` statement\n************************\n\n   global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block.  It means that the listed identifiers are to be\ninterpreted as globals.  It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in a string\nor code object supplied to the built-in ``exec()`` function does not\naffect the code block *containing* the function call, and code\ncontained in such a string is unaffected by ``global`` statements in\nthe code containing the function call.  The same applies to the\n``eval()`` and ``compile()`` functions.\n',
  'id-classes': '\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n   Not imported by ``from module import *``.  The special identifier\n   ``_`` is used in the interactive interpreter to store the result of\n   the last evaluation; it is stored in the ``builtins`` module.  When\n   not in interactive mode, ``_`` has no special meaning and is not\n   defined. See section *The import statement*.\n\n   Note: The name ``_`` is often used in conjunction with\n     internationalization; refer to the documentation for the\n     ``gettext`` module for more information on this convention.\n\n``__*__``\n   System-defined names. These names are defined by the interpreter\n   and its implementation (including the standard library).  Current\n   system names are discussed in the *Special method names* section\n   and elsewhere.  More will likely be defined in future versions of\n   Python.  *Any* use of ``__*__`` names, in any context, that does\n   not follow explicitly documented use, is subject to breakage\n   without warning.\n\n``__*``\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
  'identifiers': '\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions.\n\nThe syntax of identifiers in Python is based on the Unicode standard\nannex UAX-31, with elaboration and changes as defined below; see also\n**PEP 3131** for further details.\n\nWithin the ASCII range (U+0001..U+007F), the valid characters for\nidentifiers are the same as in Python 2.x: the uppercase and lowercase\nletters ``A`` through ``Z``, the underscore ``_`` and, except for the\nfirst character, the digits ``0`` through ``9``.\n\nPython 3.0 introduces additional characters from outside the ASCII\nrange (see **PEP 3131**).  For these characters, the classification\nuses the version of the Unicode Character Database as included in the\n``unicodedata`` module.\n\nIdentifiers are unlimited in length.  Case is significant.\n\n   identifier   ::= xid_start xid_continue*\n   id_start     ::= <all characters in general categories Lu, Ll, Lt, Lm, Lo, Nl, the underscore, and characters with the Other_ID_Start property>\n   id_continue  ::= <all characters in id_start, plus characters in the categories Mn, Mc, Nd, Pc and others with the Other_ID_Continue property>\n   xid_start    ::= <all characters in id_start whose NFKC normalization is in "id_start xid_continue*">\n   xid_continue ::= <all characters in id_continue whose NFKC normalization is in "id_continue*">\n\nThe Unicode category codes mentioned above stand for:\n\n* *Lu* - uppercase letters\n\n* *Ll* - lowercase letters\n\n* *Lt* - titlecase letters\n\n* *Lm* - modifier letters\n\n* *Lo* - other letters\n\n* *Nl* - letter numbers\n\n* *Mn* - nonspacing marks\n\n* *Mc* - spacing combining marks\n\n* *Nd* - decimal numbers\n\n* *Pc* - connector punctuations\n\n* *Other_ID_Start* - explicit list of characters in PropList.txt to\n  support backwards compatibility\n\n* *Other_ID_Continue* - likewise\n\nAll identifiers are converted into the normal form NFKC while parsing;\ncomparison of identifiers is based on NFKC.\n\nA non-normative HTML file listing all valid identifier characters for\nUnicode 4.1 can be found at http://www.dcl.hpi.uni-\npotsdam.de/home/loewis/table-3131.html.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers.  They must\nbe spelled exactly as written here:\n\n   False      class      finally    is         return\n   None       continue   for        lambda     try\n   True       def        from       nonlocal   while\n   and        del        global     not        with\n   as         elif       if         or         yield\n   assert     else       import     pass\n   break      except     in         raise\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings.  These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n   Not imported by ``from module import *``.  The special identifier\n   ``_`` is used in the interactive interpreter to store the result of\n   the last evaluation; it is stored in the ``builtins`` module.  When\n   not in interactive mode, ``_`` has no special meaning and is not\n   defined. See section *The import statement*.\n\n   Note: The name ``_`` is often used in conjunction with\n     internationalization; refer to the documentation for the\n     ``gettext`` module for more information on this convention.\n\n``__*__``\n   System-defined names. These names are defined by the interpreter\n   and its implementation (including the standard library).  Current\n   system names are discussed in the *Special method names* section\n   and elsewhere.  More will likely be defined in future versions of\n   Python.  *Any* use of ``__*__`` names, in any context, that does\n   not follow explicitly documented use, is subject to breakage\n   without warning.\n\n``__*``\n   Class-private names.  Names in this category, when used within the\n   context of a class definition, are re-written to use a mangled form\n   to help avoid name clashes between "private" attributes of base and\n   derived classes. See section *Identifiers (Names)*.\n',
  'if': '\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n   if_stmt ::= "if" expression ":" suite\n               ( "elif" expression ":" suite )*\n               ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n',
  'imaginary': '\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n   imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range.  To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., ``(3+4j)``.  Some examples of imaginary literals:\n\n   3.14j   10.j    10j     .001j   1e100j  3.14e-10j\n',
- 'import': '\nThe ``import`` statement\n************************\n\n   import_stmt     ::= "import" module ["as" name] ( "," module ["as" name] )*\n                   | "from" relative_module "import" identifier ["as" name]\n                   ( "," identifier ["as" name] )*\n                   | "from" relative_module "import" "(" identifier ["as" name]\n                   ( "," identifier ["as" name] )* [","] ")"\n                   | "from" module "import" "*"\n   module          ::= (identifier ".")* identifier\n   relative_module ::= "."* module | "."+\n   name            ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly. For a reference\nimplementation of step (1), see the ``importlib`` module.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import unless ``None`` is found in ``sys.modules``, in which case\n``ImportError`` is raised.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any.  If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound.  As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname".  If a name is not\nfound, ``ImportError`` is raised.  If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement.\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module.  The names given in ``__all__`` are all considered public\nand are required to exist.  If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope.  The\nwild card form of import --- ``import *`` --- is only allowed at the\nmodule level. Attempting to use it in class or function definitions\nwill raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python.  The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language.  It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n   future_statement ::= "from" "__future__" "import" feature ["as" name]\n                        ("," feature ["as" name])*\n                        | "from" "__future__" "import" "(" feature ["as" name]\n                        ("," feature ["as" name])* [","] ")"\n   feature          ::= identifier\n   name             ::= identifier\n\nA future statement must appear near the top of the module.  The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 3.0 are ``absolute_import``,\n``division``, ``generators``, ``unicode_literals``,\n``print_function``, ``nested_scopes`` and ``with_statement``.  They\nare all redundant because they are always enabled, and only kept for\nbackwards compatibility.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code.  It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently.  Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n   import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by calls to the built-in functions ``exec()`` and\n``compile()`` that occur in a module ``M`` containing a future\nstatement will, by default, use the new syntax or semantics associated\nwith the future statement.  This can be controlled by optional\narguments to ``compile()`` --- see the documentation of that function\nfor details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session.  If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n   **PEP 236** - Back to the __future__\n      The original proposal for the __future__ mechanism.\n',
+ 'import': '\nThe ``import`` statement\n************************\n\n   import_stmt     ::= "import" module ["as" name] ( "," module ["as" name] )*\n                   | "from" relative_module "import" identifier ["as" name]\n                   ( "," identifier ["as" name] )*\n                   | "from" relative_module "import" "(" identifier ["as" name]\n                   ( "," identifier ["as" name] )* [","] ")"\n                   | "from" module "import" "*"\n   module          ::= (identifier ".")* identifier\n   relative_module ::= "."* module | "."+\n   name            ::= identifier\n\nThe basic import statement (no ``from`` clause) is executed in two\nsteps:\n\n1. find a module, loading and initializing it if necessary\n\n2. define a name or names in the local namespace for the scope where\n   the ``import`` statement occurs.\n\nWhen the statement contains multiple clauses (separated by commas) the\ntwo steps are carried out separately for each clause, just as though\nthe clauses had been separated out into individiual import statements.\n\nThe details of the first step, finding and loading modules is\ndescribed in greater detail in the section on the *import system*,\nwhich also describes the various types of packages and modules that\ncan be imported, as well as all the hooks that can be used to\ncustomize the import system. Note that failures in this step may\nindicate either that the module could not be located, *or* that an\nerror occurred while initializing the module, which includes execution\nof the module\'s code.\n\nIf the requested module is retrieved successfully, it will be made\navailable in the local namespace in one of three ways:\n\n* If the module name is followed by ``as``, then the name following\n  ``as`` is bound directly to the imported module.\n\n* If no other name is specified, and the module being imported is a\n  top level module, the module\'s name is bound in the local namespace\n  as a reference to the imported module\n\n* If the module being imported is *not* a top level module, then the\n  name of the top level package that contains the module is bound in\n  the local namespace as a reference to the top level package. The\n  imported module must be accessed using its full qualified name\n  rather than directly\n\nThe ``from`` form uses a slightly more complex process:\n\n1. find the module specified in the ``from`` clause loading and\n   initializing it if necessary;\n\n2. for each of the identifiers specified in the ``import`` clauses:\n\n   1. check if the imported module has an attribute by that name\n\n   2. if not, attempt to import a submodule with that name and then\n      check the imported module again for that attribute\n\n   3. if the attribute is not found, ``ImportError`` is raised.\n\n   4. otherwise, a reference to that value is bound in the local\n      namespace, using the name in the ``as`` clause if it is present,\n      otherwise using the attribute name\n\nExamples:\n\n   import foo                 # foo imported and bound locally\n   import foo.bar.baz         # foo.bar.baz imported, foo bound locally\n   import foo.bar.baz as fbb  # foo.bar.baz imported and bound as fbb\n   from foo.bar import baz    # foo.bar.baz imported and bound as baz\n   from foo import attr       # foo imported and foo.attr bound as attr\n\nIf the list of identifiers is replaced by a star (``\'*\'``), all public\nnames defined in the module are bound in the local namespace for the\nscope where the ``import`` statement occurs.\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module.  The names given in ``__all__`` are all considered public\nand are required to exist.  If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``).  ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope.\nAttempting to use it in class or function definitions will raise a\n``SyntaxError``.\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module.  The names given in ``__all__`` are all considered public\nand are required to exist.  If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``).  ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope.  The\nwild card form of import --- ``import *`` --- is only allowed at the\nmodule level. Attempting to use it in class or function definitions\nwill raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python.  The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language.  It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n   future_statement ::= "from" "__future__" "import" feature ["as" name]\n                        ("," feature ["as" name])*\n                        | "from" "__future__" "import" "(" feature ["as" name]\n                        ("," feature ["as" name])* [","] ")"\n   feature          ::= identifier\n   name             ::= identifier\n\nA future statement must appear near the top of the module.  The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 3.0 are ``absolute_import``,\n``division``, ``generators``, ``unicode_literals``,\n``print_function``, ``nested_scopes`` and ``with_statement``.  They\nare all redundant because they are always enabled, and only kept for\nbackwards compatibility.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code.  It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently.  Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n   import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by calls to the built-in functions ``exec()`` and\n``compile()`` that occur in a module ``M`` containing a future\nstatement will, by default, use the new syntax or semantics associated\nwith the future statement.  This can be controlled by optional\narguments to ``compile()`` --- see the documentation of that function\nfor details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session.  If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n   **PEP 236** - Back to the __future__\n      The original proposal for the __future__ mechanism.\n',
  'in': '\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation.  Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n   comparison    ::= or_expr ( comp_operator or_expr )*\n   comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n                     | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects.  The objects need not have the same type.\nIf both are numbers, they are converted to a common type.  Otherwise,\nthe ``==`` and ``!=`` operators *always* consider objects of different\ntypes to be unequal, while the ``<``, ``>``, ``>=`` and ``<=``\noperators raise a ``TypeError`` when comparing objects of different\ntypes that do not implement these operators for the given pair of\ntypes.  You can control comparison behavior of objects of non-built-in\ntypes by defining rich comparison methods like ``__gt__()``, described\nin section *Basic customization*.\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* The values ``float(\'NaN\')`` and ``Decimal(\'NaN\')`` are special. The\n  are identical to themselves, ``x is x`` but are not equal to\n  themselves, ``x != x``.  Additionally, comparing any value to a\n  not-a-number value will return ``False``.  For example, both ``3 <\n  float(\'NaN\')`` and ``float(\'NaN\') < 3`` will return ``False``.\n\n* Bytes objects are compared lexicographically using the numeric\n  values of their elements.\n\n* Strings are compared lexicographically using the numeric equivalents\n  (the result of the built-in function ``ord()``) of their characters.\n  [3] String and bytes object can\'t be compared!\n\n* Tuples and lists are compared lexicographically using comparison of\n  corresponding elements.  This means that to compare equal, each\n  element must compare equal and the two sequences must be of the same\n  type and have the same length.\n\n  If not equal, the sequences are ordered the same as their first\n  differing elements.  For example, ``[1,2,x] <= [1,2,y]`` has the\n  same value as ``x <= y``.  If the corresponding element does not\n  exist, the shorter sequence is ordered first (for example, ``[1,2] <\n  [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if they have the\n  same ``(key, value)`` pairs. Order comparisons ``(\'<\', \'<=\', \'>=\',\n  \'>\')`` raise ``TypeError``.\n\n* Sets and frozensets define comparison operators to mean subset and\n  superset tests.  Those relations do not define total orderings (the\n  two sets ``{1,2}`` and {2,3} are not equal, nor subsets of one\n  another, nor supersets of one another).  Accordingly, sets are not\n  appropriate arguments for functions which depend on total ordering.\n  For example, ``min()``, ``max()``, and ``sorted()`` produce\n  undefined results given a list of sets as inputs.\n\n* Most other objects of built-in types compare unequal unless they are\n  the same object; the choice whether one object is considered smaller\n  or larger than another one is made arbitrarily but consistently\n  within one execution of a program.\n\nComparison of objects of the differing types depends on whether either\nof the types provide explicit support for the comparison.  Most\nnumeric types can be compared with one another, but comparisons of\n``float`` and ``Decimal`` are not supported to avoid the inevitable\nconfusion arising from representation issues such as ``float(\'1.1\')``\nbeing inexactly represented and therefore not exactly equal to\n``Decimal(\'1.1\')`` which is.  When cross-type comparison is not\nsupported, the comparison method returns ``NotImplemented``.  This can\ncreate the illusion of non-transitivity between supported cross-type\ncomparisons and unsupported comparisons.  For example, ``Decimal(2) ==\n2`` and ``2 == float(2)`` but ``Decimal(2) != float(2)``.\n\nThe operators ``in`` and ``not in`` test for membership.  ``x in s``\nevaluates to true if *x* is a member of *s*, and false otherwise.  ``x\nnot in s`` returns the negation of ``x in s``.  All built-in sequences\nand set types support this as well as dictionary, for which ``in``\ntests whether a the dictionary has a given key. For container types\nsuch as list, tuple, set, frozenset, dict, or collections.deque, the\nexpression ``x in y`` is equivalent to ``any(x is e or x == e for e in\ny)``.\n\nFor the string and bytes types, ``x in y`` is true if and only if *x*\nis a substring of *y*.  An equivalent test is ``y.find(x) != -1``.\nEmpty strings are always considered to be a substring of any other\nstring, so ``"" in "abc"`` will return ``True``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``.  If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception.  (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object.  ``x is\nnot y`` yields the inverse truth value. [4]\n',
  'integers': '\nInteger literals\n****************\n\nInteger literals are described by the following lexical definitions:\n\n   integer        ::= decimalinteger | octinteger | hexinteger | bininteger\n   decimalinteger ::= nonzerodigit digit* | "0"+\n   nonzerodigit   ::= "1"..."9"\n   digit          ::= "0"..."9"\n   octinteger     ::= "0" ("o" | "O") octdigit+\n   hexinteger     ::= "0" ("x" | "X") hexdigit+\n   bininteger     ::= "0" ("b" | "B") bindigit+\n   octdigit       ::= "0"..."7"\n   hexdigit       ::= digit | "a"..."f" | "A"..."F"\n   bindigit       ::= "0" | "1"\n\nThere is no limit for the length of integer literals apart from what\ncan be stored in available memory.\n\nNote that leading zeros in a non-zero decimal number are not allowed.\nThis is for disambiguation with C-style octal literals, which Python\nused before version 3.0.\n\nSome examples of integer literals:\n\n   7     2147483647                        0o177    0b100110111\n   3     79228162514264337593543950336     0o377    0x100000000\n         79228162514264337593543950336              0xdeadbeef\n',
  'lambda': '\nLambdas\n*******\n\n   lambda_form        ::= "lambda" [parameter_list]: expression\n   lambda_form_nocond ::= "lambda" [parameter_list]: expression_nocond\n\nLambda forms (lambda expressions) have the same syntactic position as\nexpressions.  They are a shorthand to create anonymous functions; the\nexpression ``lambda arguments: expression`` yields a function object.\nThe unnamed object behaves like a function object defined with\n\n   def <lambda>(arguments):\n       return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda forms cannot contain\nstatements or annotations.\n',
@@ -49,30 +50,30 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss
  'nonlocal': '\nThe ``nonlocal`` statement\n**************************\n\n   nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n\nThe ``nonlocal`` statement causes the listed identifiers to refer to\npreviously bound variables in the nearest enclosing scope.  This is\nimportant because the default behavior for binding is to search the\nlocal namespace first.  The statement allows encapsulated code to\nrebind variables outside of the local scope besides the global\n(module) scope.\n\nNames listed in a ``nonlocal`` statement, unlike to those listed in a\n``global`` statement, must refer to pre-existing bindings in an\nenclosing scope (the scope in which a new binding should be created\ncannot be determined unambiguously).\n\nNames listed in a ``nonlocal`` statement must not collide with pre-\nexisting bindings in the local scope.\n\nSee also:\n\n   **PEP 3104** - Access to Names in Outer Scopes\n      The specification for the ``nonlocal`` statement.\n',
  'numbers': "\nNumeric literals\n****************\n\nThere are three types of numeric literals: integers, floating point\nnumbers, and imaginary numbers.  There are no complex literals\n(complex numbers can be formed by adding a real number and an\nimaginary number).\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator '``-``' and\nthe literal ``1``.\n",
  'numeric-types': "\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n   ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n   ``|``).  For instance, to evaluate the expression ``x + y``, where\n   *x* is an instance of a class that has an ``__add__()`` method,\n   ``x.__add__(y)`` is called.  The ``__divmod__()`` method should be\n   the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n   should not be related to ``__truediv__()``.  Note that\n   ``__pow__()`` should be defined to accept an optional third\n   argument if the ternary version of the built-in ``pow()`` function\n   is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return ``NotImplemented``.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n   ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n   ``|``) with reflected (swapped) operands. These functions are only\n   called if the left operand does not support the corresponding\n   operation and the operands are of different types. [2]  For\n   instance, to evaluate the expression ``x - y``, where *y* is an\n   instance of a class that has an ``__rsub__()`` method,\n   ``y.__rsub__(x)`` is called if ``x.__sub__(y)`` returns\n   *NotImplemented*.\n\n   Note that ternary ``pow()`` will not try calling ``__rpow__()``\n   (the coercion rules would become too complicated).\n\n   Note: If the right operand's type is a subclass of the left operand's\n     type and that subclass provides the reflected method for the\n     operation, this method will be called before the left operand's\n     non-reflected method.  This behavior allows subclasses to\n     override their ancestors' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n   ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``).  These methods\n   should attempt to do the operation in-place (modifying *self*) and\n   return the result (which could be, but does not have to be,\n   *self*).  If a specific method is not defined, the augmented\n   assignment falls back to the normal methods.  For instance, to\n   execute the statement ``x += y``, where *x* is an instance of a\n   class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n   called.  If *x* is an instance of a class that does not define a\n   ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n   considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations (``-``, ``+``,\n   ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n   Called to implement the built-in functions ``complex()``,\n   ``int()``, ``float()`` and ``round()``.  Should return a value of\n   the appropriate type.\n\nobject.__index__(self)\n\n   Called to implement ``operator.index()``.  Also called whenever\n   Python needs an integer object (such as in slicing, or in the\n   built-in ``bin()``, ``hex()`` and ``oct()`` functions). Must return\n   an integer.\n",
- 'objects': '\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data.  All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value.  An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory.  The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype.  The ``type()`` function returns an object\'s type (which is an\nobject itself).  The *value* of some objects can change.  Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed.  So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected.  An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references.  See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows.  It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects.  The\n\'``try``...``finally``\' statement and the \'``with``\' statement provide\nconvenient ways to do this.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries.  The references are part of a container\'s value.  In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied.  So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior.  Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed.  E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n',
+ 'objects': '\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data.  All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value.  An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory.  The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity.\n\n**CPython implementation detail:** For CPython, ``id(x)`` is the\nmemory address where ``x`` is stored.\n\nAn object\'s type determines the operations that the object supports\n(e.g., "does it have a length?") and also defines the possible values\nfor objects of that type.  The ``type()`` function returns an object\'s\ntype (which is an object itself).  Like its identity, an object\'s\n*type* is also unchangeable. [1]\n\nThe *value* of some objects can change.  Objects whose value can\nchange are said to be *mutable*; objects whose value is unchangeable\nonce they are created are called *immutable*. (The value of an\nimmutable container object that contains a reference to a mutable\nobject can change when the latter\'s value is changed; however the\ncontainer is still considered immutable, because the collection of\nobjects it contains cannot be changed.  So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected.  An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references.  See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows.  It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects.  The\n\'``try``...``finally``\' statement and the \'``with``\' statement provide\nconvenient ways to do this.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries.  The references are part of a container\'s value.  In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied.  So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior.  Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed.  E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n',
  'operator-summary': '\nSummary\n*******\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding).  Operators in the same box have the same precedence.  Unless\nthe syntax is explicitly given, operators are binary.  Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator                                        | Description                           |\n+=================================================+=======================================+\n| ``lambda``                                      | Lambda expression                     |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else``                              | Conditional expression                |\n+-------------------------------------------------+---------------------------------------+\n| ``or``                                          | Boolean OR                            |\n+-------------------------------------------------+---------------------------------------+\n| ``and``                                         | Boolean AND                           |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` *x*                                     | Boolean NOT                           |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not`` ``in``, ``is``, ``is not``,     | Comparisons, including membership     |\n| ``<``, ``<=``, ``>``, ``>=``, ``!=``, ``==``    | tests and identity tests,             |\n+-------------------------------------------------+---------------------------------------+\n| ``|``                                           | Bitwise OR                            |\n+-------------------------------------------------+---------------------------------------+\n| ``^``                                           | Bitwise XOR                           |\n+-------------------------------------------------+---------------------------------------+\n| ``&``                                           | Bitwise AND                           |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>``                                  | Shifts                                |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-``                                    | Addition and subtraction              |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%``                     | Multiplication, division, remainder   |\n|                                                 | [5]                                   |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x``                          | Positive, negative, bitwise NOT       |\n+-------------------------------------------------+---------------------------------------+\n| ``**``                                          | Exponentiation [6]                    |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``,               | Subscription, slicing, call,          |\n| ``x(arguments...)``, ``x.attribute``            | attribute reference                   |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``,     | Binding or tuple display, list        |\n| ``{key:datum...}``, ``{expressions...}``        | display, dictionary display, set      |\n|                                                 | display                               |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n    may not be true numerically due to roundoff.  For example, and\n    assuming a platform on which a Python float is an IEEE 754 double-\n    precision number, in order that ``-1e-100 % 1e100`` have the same\n    sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n    which is numerically exactly equal to ``1e100``.  The function\n    ``math.fmod()`` returns a result whose sign matches the sign of\n    the first argument instead, and so returns ``-1e-100`` in this\n    case. Which approach is more appropriate depends on the\n    application.\n\n[2] If x is very close to an exact integer multiple of y, it\'s\n    possible for ``x//y`` to be one larger than ``(x-x%y)//y`` due to\n    rounding.  In such cases, Python returns the latter result, in\n    order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n    close to ``x``.\n\n[3] While comparisons between strings make sense at the byte level,\n    they may be counter-intuitive to users.  For example, the strings\n    ``"\\u00C7"`` and ``"\\u0327\\u0043"`` compare differently, even\n    though they both represent the same unicode character (LATIN\n    CAPITAL LETTER C WITH CEDILLA).  To compare strings in a human\n    recognizable way, compare using ``unicodedata.normalize()``.\n\n[4] Due to automatic garbage-collection, free lists, and the dynamic\n    nature of descriptors, you may notice seemingly unusual behaviour\n    in certain uses of the ``is`` operator, like those involving\n    comparisons between instance methods, or constants.  Check their\n    documentation for more info.\n\n[5] The ``%`` operator is also used for string formatting; the same\n    precedence applies.\n\n[6] The power operator ``**`` binds less tightly than an arithmetic or\n    bitwise unary operator on its right, that is, ``2**-1`` is\n    ``0.5``.\n',
  'pass': '\nThe ``pass`` statement\n**********************\n\n   pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n   def f(arg): pass    # a function that does nothing (yet)\n\n   class C: pass       # a class with no methods (yet)\n',
  'power': '\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right.  The\nsyntax is:\n\n   power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument.  The numeric arguments are\nfirst converted to a common type, and the result is of that type.\n\nFor int operands, the result has the same type as the operands unless\nthe second argument is negative; in that case, all arguments are\nconverted to float and a float result is delivered. For example,\n``10**2`` returns ``100``, but ``10**-2`` returns ``0.01``.\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``complex`` number. (In earlier versions it raised a\n``ValueError``.)\n',
  'raise': '\nThe ``raise`` statement\n***********************\n\n   raise_stmt ::= "raise" [expression ["from" expression]]\n\nIf no expressions are present, ``raise`` re-raises the last exception\nthat was active in the current scope.  If no exception is active in\nthe current scope, a ``RuntimeError`` exception is raised indicating\nthat this is an error.\n\nOtherwise, ``raise`` evaluates the first expression as the exception\nobject.  It must be either a subclass or an instance of\n``BaseException``. If it is a class, the exception instance will be\nobtained when needed by instantiating the class with no arguments.\n\nThe *type* of the exception is the exception instance\'s class, the\n*value* is the instance itself.\n\nA traceback object is normally created automatically when an exception\nis raised and attached to it as the ``__traceback__`` attribute, which\nis writable. You can create an exception and set your own traceback in\none step using the ``with_traceback()`` exception method (which\nreturns the same exception instance, with its traceback set to its\nargument), like so:\n\n   raise Exception("foo occurred").with_traceback(tracebackobj)\n\nThe ``from`` clause is used for exception chaining: if given, the\nsecond *expression* must be another exception class or instance, which\nwill then be attached to the raised exception as the ``__cause__``\nattribute (which is writable).  If the raised exception is not\nhandled, both exceptions will be printed:\n\n   >>> try:\n   ...     print(1 / 0)\n   ... except Exception as exc:\n   ...     raise RuntimeError("Something bad happened") from exc\n   ...\n   Traceback (most recent call last):\n     File "<stdin>", line 2, in <module>\n   ZeroDivisionError: int division or modulo by zero\n\n   The above exception was the direct cause of the following exception:\n\n   Traceback (most recent call last):\n     File "<stdin>", line 4, in <module>\n   RuntimeError: Something bad happened\n\nA similar mechanism works implicitly if an exception is raised inside\nan exception handler: the previous exception is then attached as the\nnew exception\'s ``__context__`` attribute:\n\n   >>> try:\n   ...     print(1 / 0)\n   ... except:\n   ...     raise RuntimeError("Something bad happened")\n   ...\n   Traceback (most recent call last):\n     File "<stdin>", line 2, in <module>\n   ZeroDivisionError: int division or modulo by zero\n\n   During handling of the above exception, another exception occurred:\n\n   Traceback (most recent call last):\n     File "<stdin>", line 4, in <module>\n   RuntimeError: Something bad happened\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information about handling exceptions is in section\n*The try statement*.\n',
- 'return': '\nThe ``return`` statement\n************************\n\n   return_stmt ::= "return" [expression_list]\n\n``return`` may only occur syntactically nested in a function\ndefinition, not within a nested class definition.\n\nIf an expression list is present, it is evaluated, else ``None`` is\nsubstituted.\n\n``return`` leaves the current function call with the expression list\n(or ``None``) as return value.\n\nWhen ``return`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nleaving the function.\n\nIn a generator function, the ``return`` statement is not allowed to\ninclude an ``expression_list``.  In that context, a bare ``return``\nindicates that the generator is done and will cause ``StopIteration``\nto be raised.\n',
+ 'return': '\nThe ``return`` statement\n************************\n\n   return_stmt ::= "return" [expression_list]\n\n``return`` may only occur syntactically nested in a function\ndefinition, not within a nested class definition.\n\nIf an expression list is present, it is evaluated, else ``None`` is\nsubstituted.\n\n``return`` leaves the current function call with the expression list\n(or ``None``) as return value.\n\nWhen ``return`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nleaving the function.\n\nIn a generator function, the ``return`` statement indicates that the\ngenerator is done and will cause ``StopIteration`` to be raised. The\nreturned value (if any) is used as an argument to construct\n``StopIteration`` and becomes the ``StopIteration.value`` attribute.\n',
  'sequence-types': "\nEmulating container types\n*************************\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well.  The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items.  It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``get()``,\n``clear()``, ``setdefault()``, ``pop()``, ``popitem()``, ``copy()``,\nand ``update()`` behaving similar to those for Python's standard\ndictionary objects.  The ``collections`` module provides a\n``MutableMapping`` abstract base class to help create those methods\nfrom a base set of ``__getitem__()``, ``__setitem__()``,\n``__delitem__()``, and ``keys()``. Mutable sequences should provide\nmethods ``append()``, ``count()``, ``index()``, ``extend()``,\n``insert()``, ``pop()``, ``remove()``, ``reverse()`` and ``sort()``,\nlike Python standard list objects.  Finally, sequence types should\nimplement addition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods ``__add__()``, ``__radd__()``,\n``__iadd__()``, ``__mul__()``, ``__rmul__()`` and ``__imul__()``\ndescribed below; they should not define other numerical operators.  It\nis recommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should search the mapping's keys; for\nsequences, it should search through the values.  It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``keys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n   Called to implement the built-in function ``len()``.  Should return\n   the length of the object, an integer ``>=`` 0.  Also, an object\n   that doesn't define a ``__bool__()`` method and whose ``__len__()``\n   method returns zero is considered to be false in a Boolean context.\n\nNote: Slicing is done exclusively with the following three methods.  A\n  call like\n\n     a[1:2] = b\n\n  is translated to\n\n     a[slice(1, 2, None)] = b\n\n  and so forth.  Missing slice items are always filled in with\n  ``None``.\n\nobject.__getitem__(self, key)\n\n   Called to implement evaluation of ``self[key]``. For sequence\n   types, the accepted keys should be integers and slice objects.\n   Note that the special interpretation of negative indexes (if the\n   class wishes to emulate a sequence type) is up to the\n   ``__getitem__()`` method. If *key* is of an inappropriate type,\n   ``TypeError`` may be raised; if of a value outside the set of\n   indexes for the sequence (after any special interpretation of\n   negative values), ``IndexError`` should be raised. For mapping\n   types, if *key* is missing (not in the container), ``KeyError``\n   should be raised.\n\n   Note: ``for`` loops expect that an ``IndexError`` will be raised for\n     illegal indexes to allow proper detection of the end of the\n     sequence.\n\nobject.__setitem__(self, key, value)\n\n   Called to implement assignment to ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support changes to the values for keys, or if new keys\n   can be added, or for sequences if elements can be replaced.  The\n   same exceptions should be raised for improper *key* values as for\n   the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n   Called to implement deletion of ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support removal of keys, or for sequences if elements\n   can be removed from the sequence.  The same exceptions should be\n   raised for improper *key* values as for the ``__getitem__()``\n   method.\n\nobject.__iter__(self)\n\n   This method is called when an iterator is required for a container.\n   This method should return a new iterator object that can iterate\n   over all the objects in the container.  For mappings, it should\n   iterate over the keys of the container, and should also be made\n   available as the method ``keys()``.\n\n   Iterator objects also need to implement this method; they are\n   required to return themselves.  For more information on iterator\n   objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n   Called (if present) by the ``reversed()`` built-in to implement\n   reverse iteration.  It should return a new iterator object that\n   iterates over all the objects in the container in reverse order.\n\n   If the ``__reversed__()`` method is not provided, the\n   ``reversed()`` built-in will fall back to using the sequence\n   protocol (``__len__()`` and ``__getitem__()``).  Objects that\n   support the sequence protocol should only provide\n   ``__reversed__()`` if they can provide an implementation that is\n   more efficient than the one provided by ``reversed()``.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence.  However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n   Called to implement membership test operators.  Should return true\n   if *item* is in *self*, false otherwise.  For mapping objects, this\n   should consider the keys of the mapping rather than the values or\n   the key-item pairs.\n\n   For objects that don't define ``__contains__()``, the membership\n   test first tries iteration via ``__iter__()``, then the old\n   sequence iteration protocol via ``__getitem__()``, see *this\n   section in the language reference*.\n",
  'shifting': '\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n   shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept integers as arguments.  They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as division by ``pow(2,n)``.  A\nleft shift by *n* bits is defined as multiplication with ``pow(2,n)``.\n\nNote: In the current implementation, the right-hand operand is required to\n  be at most ``sys.maxsize``.  If the right-hand operand is larger\n  than ``sys.maxsize`` an ``OverflowError`` exception is raised.\n',
  'slicings': '\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list).  Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements.  The syntax for a\nslicing:\n\n   slicing      ::= primary "[" slice_list "]"\n   slice_list   ::= slice_item ("," slice_item)* [","]\n   slice_item   ::= expression | proper_slice\n   proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" [stride] ]\n   lower_bound  ::= expression\n   upper_bound  ::= expression\n   stride       ::= expression\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing.  Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice).\n\nThe semantics for a slicing are as follows.  The primary must evaluate\nto a mapping object, and it is indexed (using the same\n``__getitem__()`` method as normal subscription) with a key that is\nconstructed from the slice list, as follows.  If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key.  The conversion of a slice item that is an\nexpression is that expression.  The conversion of a proper slice is a\nslice object (see section *The standard type hierarchy*) whose\n``start``, ``stop`` and ``step`` attributes are the values of the\nexpressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n',
- 'specialattrs': '\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant.  Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n   A dictionary or other mapping object used to store an object\'s\n   (writable) attributes.\n\ninstance.__class__\n\n   The class to which a class instance belongs.\n\nclass.__bases__\n\n   The tuple of base classes of a class object.\n\nclass.__name__\n\n   The name of the class or type.\n\nclass.__mro__\n\n   This attribute is a tuple of classes that are considered when\n   looking for base classes during method resolution.\n\nclass.mro()\n\n   This method can be overridden by a metaclass to customize the\n   method resolution order for its instances.  It is called at class\n   instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n   Each class keeps a list of weak references to its immediate\n   subclasses.  This method returns a list of all those references\n   still alive. Example:\n\n      >>> int.__subclasses__()\n      [<class \'bool\'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n    the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n    ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can\'t tell the type of the\n    operands.\n\n[4] Cased characters are those with general category property being\n    one of "Lu" (Letter, uppercase), "Ll" (Letter, lowercase), or "Lt"\n    (Letter, titlecase).\n\n[5] To format only a tuple you should therefore provide a singleton\n    tuple whose only element is the tuple to be formatted.\n',
- 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators.  For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``type(x).__getitem__(x,\ni)``.  Except where mentioned, attempts to execute an operation raise\nan exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled.  For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense.  (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  ``__new__()`` is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of ``__new__()`` should be the new object instance (usually\n   an instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s ``__new__()`` method using\n   ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n   arguments and then modifying the newly-created instance as\n   necessary before returning it.\n\n   If ``__new__()`` returns an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will be invoked like\n   ``__init__(self[, ...])``, where *self* is the new instance and the\n   remaining arguments are the same as were passed to ``__new__()``.\n\n   If ``__new__()`` does not return an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will not be invoked.\n\n   ``__new__()`` is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called when the instance is created.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   ``__init__()`` method, the derived class\'s ``__init__()`` method,\n   if any, must explicitly call it to ensure proper initialization of\n   the base class part of the instance; for example:\n   ``BaseClass.__init__(self, [args...])``.  As a special constraint\n   on constructors, no value may be returned; doing so will cause a\n   ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a ``__del__()`` method,\n   the derived class\'s ``__del__()`` method, if any, must explicitly\n   call it to ensure proper deletion of the base class part of the\n   instance.  Note that it is possible (though not recommended!) for\n   the ``__del__()`` method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   ``__del__()`` methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n     decrements the reference count for ``x`` by one, and the latter\n     is only called when ``x``\'s reference count reaches zero.  Some\n     common situations that may prevent the reference count of an\n     object from going to zero include: circular references between\n     objects (e.g., a doubly-linked list or a tree data structure with\n     parent and child pointers); a reference to the object on the\n     stack frame of a function that caught an exception (the traceback\n     stored in ``sys.exc_info()[2]`` keeps the stack frame alive); or\n     a reference to the object on the stack frame that raised an\n     unhandled exception in interactive mode (the traceback stored in\n     ``sys.last_traceback`` keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the latter two situations can be resolved by storing ``None`` in\n     ``sys.last_traceback``. Circular references which are garbage are\n     detected when the option cycle detector is enabled (it\'s on by\n     default), but can only be cleaned up if there are no Python-\n     level ``__del__()`` methods involved. Refer to the documentation\n     for the ``gc`` module for more information about how\n     ``__del__()`` methods are handled by the cycle detector,\n     particularly the description of the ``garbage`` value.\n\n   Warning: Due to the precarious circumstances under which ``__del__()``\n     methods are invoked, exceptions that occur during their execution\n     are ignored, and a warning is printed to ``sys.stderr`` instead.\n     Also, when ``__del__()`` is invoked in response to a module being\n     deleted (e.g., when execution of the program is done), other\n     globals referenced by the ``__del__()`` method may already have\n     been deleted or in the process of being torn down (e.g. the\n     import machinery shutting down).  For this reason, ``__del__()``\n     methods should do the absolute minimum needed to maintain\n     external invariants.  Starting with version 1.5, Python\n     guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n   Called by the ``repr()`` built-in function to compute the\n   "official" string representation of an object.  If at all possible,\n   this should look like a valid Python expression that could be used\n   to recreate an object with the same value (given an appropriate\n   environment).  If this is not possible, a string of the form\n   ``<...some useful description...>`` should be returned. The return\n   value must be a string object. If a class defines ``__repr__()``\n   but not ``__str__()``, then ``__repr__()`` is also used when an\n   "informal" string representation of instances of that class is\n   required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by the ``str()`` built-in function and by the ``print()``\n   function to compute the "informal" string representation of an\n   object.  This differs from ``__repr__()`` in that it does not have\n   to be a valid Python expression: a more convenient or concise\n   representation may be used instead. The return value must be a\n   string object.\n\nobject.__bytes__(self)\n\n   Called by ``bytes()`` to compute a byte-string representation of an\n   object. This should return a ``bytes`` object.\n\nobject.__format__(self, format_spec)\n\n   Called by the ``format()`` built-in function (and by extension, the\n   ``format()`` method of class ``str``) to produce a "formatted"\n   string representation of an object. The ``format_spec`` argument is\n   a string that contains a description of the formatting options\n   desired. The interpretation of the ``format_spec`` argument is up\n   to the type implementing ``__format__()``, however most classes\n   will either delegate formatting to one of the built-in types, or\n   use a similar formatting option syntax.\n\n   See *Format Specification Mini-Language* for a description of the\n   standard formatting syntax.\n\n   The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   These are the so-called "rich comparison" methods. The\n   correspondence between operator symbols and method names is as\n   follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n   ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` calls\n   ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls\n   ``x.__ge__(y)``.\n\n   A rich comparison method may return the singleton\n   ``NotImplemented`` if it does not implement the operation for a\n   given pair of arguments. By convention, ``False`` and ``True`` are\n   returned for a successful comparison. However, these methods can\n   return any value, so if the comparison operator is used in a\n   Boolean context (e.g., in the condition of an ``if`` statement),\n   Python will call ``bool()`` on the value to determine if the result\n   is true or false.\n\n   There are no implied relationships among the comparison operators.\n   The truth of ``x==y`` does not imply that ``x!=y`` is false.\n   Accordingly, when defining ``__eq__()``, one should also define\n   ``__ne__()`` so that the operators will behave as expected.  See\n   the paragraph on ``__hash__()`` for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n   other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n   reflection, and ``__eq__()`` and ``__ne__()`` are their own\n   reflection.\n\n   Arguments to rich comparison methods are never coerced.\n\n   To automatically generate ordering operations from a single root\n   operation, see ``functools.total_ordering()``.\n\nobject.__hash__(self)\n\n   Called by built-in function ``hash()`` and for operations on\n   members of hashed collections including ``set``, ``frozenset``, and\n   ``dict``.  ``__hash__()`` should return an integer.  The only\n   required property is that objects which compare equal have the same\n   hash value; it is advised to somehow mix together (e.g. using\n   exclusive or) the hash values for the components of the object that\n   also play a part in comparison of objects.\n\n   If a class does not define an ``__eq__()`` method it should not\n   define a ``__hash__()`` operation either; if it defines\n   ``__eq__()`` but not ``__hash__()``, its instances will not be\n   usable as items in hashable collections.  If a class defines\n   mutable objects and implements an ``__eq__()`` method, it should\n   not implement ``__hash__()``, since the implementation of hashable\n   collections requires that a key\'s hash value is immutable (if the\n   object\'s hash value changes, it will be in the wrong hash bucket).\n\n   User-defined classes have ``__eq__()`` and ``__hash__()`` methods\n   by default; with them, all objects compare unequal (except with\n   themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n   Classes which inherit a ``__hash__()`` method from a parent class\n   but change the meaning of ``__eq__()`` such that the hash value\n   returned is no longer appropriate (e.g. by switching to a value-\n   based concept of equality instead of the default identity based\n   equality) can explicitly flag themselves as being unhashable by\n   setting ``__hash__ = None`` in the class definition. Doing so means\n   that not only will instances of the class raise an appropriate\n   ``TypeError`` when a program attempts to retrieve their hash value,\n   but they will also be correctly identified as unhashable when\n   checking ``isinstance(obj, collections.Hashable)`` (unlike classes\n   which define their own ``__hash__()`` to explicitly raise\n   ``TypeError``).\n\n   If a class that overrides ``__eq__()`` needs to retain the\n   implementation of ``__hash__()`` from a parent class, the\n   interpreter must be told this explicitly by setting ``__hash__ =\n   <ParentClass>.__hash__``. Otherwise the inheritance of\n   ``__hash__()`` will be blocked, just as if ``__hash__`` had been\n   explicitly set to ``None``.\n\n   See also the *-R* command-line option.\n\nobject.__bool__(self)\n\n   Called to implement truth value testing and the built-in operation\n   ``bool()``; should return ``False`` or ``True``.  When this method\n   is not defined, ``__len__()`` is called, if it is defined, and the\n   object is considered true if its result is nonzero.  If a class\n   defines neither ``__len__()`` nor ``__bool__()``, all its instances\n   are considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control over attribute access.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary). *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should call the base class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\nobject.__dir__(self)\n\n   Called when ``dir()`` is called on the object.  A list must be\n   returned.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to an object instance, ``a.x`` is transformed into the\n   call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a class, ``A.x`` is transformed into the call:\n   ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage.  This wastes space for objects having very few instance\nvariables.  The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable.  Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   class, *__slots__* reserves space for the declared variables and\n   prevents the automatic creation of *__dict__* and *__weakref__* for\n   each instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``int``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using ``type()``. A class\ndefinition is read into a separate namespace and the value of class\nname is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if a callable ``metaclass`` keyword\nargument is passed after the bases in the class definition, the\ncallable given will be called instead of ``type()``.  If other keyword\narguments are passed, they will also be passed to the metaclass.  This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n  role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties.  This example adds a new\nelement to the class dictionary before creating the class:\n\n   class metacls(type):\n       def __new__(mcs, name, bases, dict):\n           dict[\'foo\'] = \'metacls was here\'\n           return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\nIf the metaclass has a ``__prepare__()`` attribute (usually\nimplemented as a class or static method), it is called before the\nclass body is evaluated with the name of the class and a tuple of its\nbases for arguments.  It should return an object that supports the\nmapping interface that will be used to store the namespace of the\nclass.  The default is a plain dictionary.  This could be used, for\nexample, to keep track of the order that class attributes are declared\nin by returning an ordered dictionary.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If the ``metaclass`` keyword argument is passed with the bases, it\n  is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n  used.\n\n* Otherwise, the default metaclass (``type``) is used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n``collections.OrderedDict`` to remember the order that class members\nwere defined:\n\n   class OrderedClass(type):\n\n        @classmethod\n        def __prepare__(metacls, name, bases, **kwds):\n           return collections.OrderedDict()\n\n        def __new__(cls, name, bases, classdict):\n           result = type.__new__(cls, name, bases, dict(classdict))\n           result.members = tuple(classdict)\n           return result\n\n   class A(metaclass=OrderedClass):\n       def one(self): pass\n       def two(self): pass\n       def three(self): pass\n       def four(self): pass\n\n   >>> A.members\n   (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s ``__prepare__()`` method which returns an\nempty ``collections.OrderedDict``.  That mapping records the methods\nand attributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s ``__new__()`` method gets\ninvoked.  That method builds the new type and it saves the ordered\ndictionary keys in an attribute called ``members``.\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n   Return true if *instance* should be considered a (direct or\n   indirect) instance of *class*. If defined, called to implement\n   ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n   Return true if *subclass* should be considered a (direct or\n   indirect) subclass of *class*.  If defined, called to implement\n   ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass.  They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n   **PEP 3119** - Introducing Abstract Base Classes\n      Includes the specification for customizing ``isinstance()`` and\n      ``issubclass()`` behavior through ``__instancecheck__()`` and\n      ``__subclasscheck__()``, with motivation for this functionality\n      in the context of adding Abstract Base Classes (see the ``abc``\n      module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n   Called when the instance is "called" as a function; if this method\n   is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n   ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well.  The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items.  It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``get()``,\n``clear()``, ``setdefault()``, ``pop()``, ``popitem()``, ``copy()``,\nand ``update()`` behaving similar to those for Python\'s standard\ndictionary objects.  The ``collections`` module provides a\n``MutableMapping`` abstract base class to help create those methods\nfrom a base set of ``__getitem__()``, ``__setitem__()``,\n``__delitem__()``, and ``keys()``. Mutable sequences should provide\nmethods ``append()``, ``count()``, ``index()``, ``extend()``,\n``insert()``, ``pop()``, ``remove()``, ``reverse()`` and ``sort()``,\nlike Python standard list objects.  Finally, sequence types should\nimplement addition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods ``__add__()``, ``__radd__()``,\n``__iadd__()``, ``__mul__()``, ``__rmul__()`` and ``__imul__()``\ndescribed below; they should not define other numerical operators.  It\nis recommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should search the mapping\'s keys; for\nsequences, it should search through the values.  It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``keys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n   Called to implement the built-in function ``len()``.  Should return\n   the length of the object, an integer ``>=`` 0.  Also, an object\n   that doesn\'t define a ``__bool__()`` method and whose ``__len__()``\n   method returns zero is considered to be false in a Boolean context.\n\nNote: Slicing is done exclusively with the following three methods.  A\n  call like\n\n     a[1:2] = b\n\n  is translated to\n\n     a[slice(1, 2, None)] = b\n\n  and so forth.  Missing slice items are always filled in with\n  ``None``.\n\nobject.__getitem__(self, key)\n\n   Called to implement evaluation of ``self[key]``. For sequence\n   types, the accepted keys should be integers and slice objects.\n   Note that the special interpretation of negative indexes (if the\n   class wishes to emulate a sequence type) is up to the\n   ``__getitem__()`` method. If *key* is of an inappropriate type,\n   ``TypeError`` may be raised; if of a value outside the set of\n   indexes for the sequence (after any special interpretation of\n   negative values), ``IndexError`` should be raised. For mapping\n   types, if *key* is missing (not in the container), ``KeyError``\n   should be raised.\n\n   Note: ``for`` loops expect that an ``IndexError`` will be raised for\n     illegal indexes to allow proper detection of the end of the\n     sequence.\n\nobject.__setitem__(self, key, value)\n\n   Called to implement assignment to ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support changes to the values for keys, or if new keys\n   can be added, or for sequences if elements can be replaced.  The\n   same exceptions should be raised for improper *key* values as for\n   the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n   Called to implement deletion of ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support removal of keys, or for sequences if elements\n   can be removed from the sequence.  The same exceptions should be\n   raised for improper *key* values as for the ``__getitem__()``\n   method.\n\nobject.__iter__(self)\n\n   This method is called when an iterator is required for a container.\n   This method should return a new iterator object that can iterate\n   over all the objects in the container.  For mappings, it should\n   iterate over the keys of the container, and should also be made\n   available as the method ``keys()``.\n\n   Iterator objects also need to implement this method; they are\n   required to return themselves.  For more information on iterator\n   objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n   Called (if present) by the ``reversed()`` built-in to implement\n   reverse iteration.  It should return a new iterator object that\n   iterates over all the objects in the container in reverse order.\n\n   If the ``__reversed__()`` method is not provided, the\n   ``reversed()`` built-in will fall back to using the sequence\n   protocol (``__len__()`` and ``__getitem__()``).  Objects that\n   support the sequence protocol should only provide\n   ``__reversed__()`` if they can provide an implementation that is\n   more efficient than the one provided by ``reversed()``.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence.  However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n   Called to implement membership test operators.  Should return true\n   if *item* is in *self*, false otherwise.  For mapping objects, this\n   should consider the keys of the mapping rather than the values or\n   the key-item pairs.\n\n   For objects that don\'t define ``__contains__()``, the membership\n   test first tries iteration via ``__iter__()``, then the old\n   sequence iteration protocol via ``__getitem__()``, see *this\n   section in the language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n   ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n   ``|``).  For instance, to evaluate the expression ``x + y``, where\n   *x* is an instance of a class that has an ``__add__()`` method,\n   ``x.__add__(y)`` is called.  The ``__divmod__()`` method should be\n   the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n   should not be related to ``__truediv__()``.  Note that\n   ``__pow__()`` should be defined to accept an optional third\n   argument if the ternary version of the built-in ``pow()`` function\n   is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return ``NotImplemented``.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n   ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n   ``|``) with reflected (swapped) operands. These functions are only\n   called if the left operand does not support the corresponding\n   operation and the operands are of different types. [2]  For\n   instance, to evaluate the expression ``x - y``, where *y* is an\n   instance of a class that has an ``__rsub__()`` method,\n   ``y.__rsub__(x)`` is called if ``x.__sub__(y)`` returns\n   *NotImplemented*.\n\n   Note that ternary ``pow()`` will not try calling ``__rpow__()``\n   (the coercion rules would become too complicated).\n\n   Note: If the right operand\'s type is a subclass of the left operand\'s\n     type and that subclass provides the reflected method for the\n     operation, this method will be called before the left operand\'s\n     non-reflected method.  This behavior allows subclasses to\n     override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n   ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``).  These methods\n   should attempt to do the operation in-place (modifying *self*) and\n   return the result (which could be, but does not have to be,\n   *self*).  If a specific method is not defined, the augmented\n   assignment falls back to the normal methods.  For instance, to\n   execute the statement ``x += y``, where *x* is an instance of a\n   class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n   called.  If *x* is an instance of a class that does not define a\n   ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n   considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations (``-``, ``+``,\n   ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n   Called to implement the built-in functions ``complex()``,\n   ``int()``, ``float()`` and ``round()``.  Should return a value of\n   the appropriate type.\n\nobject.__index__(self)\n\n   Called to implement ``operator.index()``.  Also called whenever\n   Python needs an integer object (such as in slicing, or in the\n   built-in ``bin()``, ``hex()`` and ``oct()`` functions). Must return\n   an integer.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code.  Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The ``with``\n   statement will bind this method\'s return value to the target(s)\n   specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be ``None``.\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that ``__exit__()`` methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary.  That behaviour is the reason why\nthe following code raises an exception:\n\n   >>> class C:\n   ...     pass\n   ...\n   >>> c = C()\n   >>> c.__len__ = lambda: 5\n   >>> len(c)\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n   >>> 1 .__hash__() == hash(1)\n   True\n   >>> int.__hash__() == hash(int)\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n   >>> type(1).__hash__(1) == hash(1)\n   True\n   >>> type(int).__hash__(int) == hash(int)\n   True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n   >>> class Meta(type):\n   ...    def __getattribute__(*args):\n   ...       print("Metaclass getattribute invoked")\n   ...       return type.__getattribute__(*args)\n   ...\n   >>> class C(object, metaclass=Meta):\n   ...     def __len__(self):\n   ...         return 10\n   ...     def __getattribute__(*args):\n   ...         print("Class getattribute invoked")\n   ...         return object.__getattribute__(*args)\n   ...\n   >>> c = C()\n   >>> c.__len__()                 # Explicit lookup via instance\n   Class getattribute invoked\n   10\n   >>> type(c).__len__(c)          # Explicit lookup via type\n   Metaclass getattribute invoked\n   10\n   >>> len(c)                      # Implicit lookup\n   10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n    certain controlled conditions. It generally isn\'t a good idea\n    though, since it can lead to some very strange behaviour if it is\n    handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n    reflected method (such as ``__add__()``) fails the operation is\n    not supported, which is why the reflected method is not called.\n',
- 'string-methods': '\nString Methods\n**************\n\nString objects support the methods listed below.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n   Return a copy of the string with its first character capitalized\n   and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n   Return an encoded version of the string as a bytes object. Default\n   encoding is ``\'utf-8\'``. *errors* may be given to set a different\n   error handling scheme. The default for *errors* is ``\'strict\'``,\n   meaning that encoding errors raise a ``UnicodeError``. Other\n   possible values are ``\'ignore\'``, ``\'replace\'``,\n   ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and any other name\n   registered via ``codecs.register_error()``, see section *Codec Base\n   Classes*. For a list of possible encodings, see section *Standard\n   Encodings*.\n\n   Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return ``True`` if the string ends with the specified *suffix*,\n   otherwise return ``False``.  *suffix* can also be a tuple of\n   suffixes to look for.  With optional *start*, test beginning at\n   that position.  With optional *end*, stop comparing at that\n   position.\n\nstr.expandtabs([tabsize])\n\n   Return a copy of the string where all tab characters are replaced\n   by zero or more spaces, depending on the current column and the\n   given tab size.  The column number is reset to zero after each\n   newline occurring in the string. If *tabsize* is not given, a tab\n   size of ``8`` characters is assumed.  This doesn\'t understand other\n   non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found, such that *sub* is contained in the slice ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` if *sub* is not found.\n\n   Note: The ``find()`` method should be used only if you need to know the\n     position of *sub*.  To check if *sub* is a substring or not, use\n     the ``in`` operator:\n\n        >>> \'Py\' in \'Python\'\n        True\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces ``{}``.  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n   Similar to ``str.format(**mapping)``, except that ``mapping`` is\n   used directly and not copied to a ``dict`` .  This is useful if for\n   example ``mapping`` is a dict subclass:\n\n   >>> class Default(dict):\n   ...     def __missing__(self, key):\n   ...         return key\n   ...\n   >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n   \'Guido was born in country\'\n\n   New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n   Like ``find()``, but raise ``ValueError`` when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.  A character\n   ``c`` is alphanumeric if one of the following returns ``True``:\n   ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n   ``c.isnumeric()``.\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.  Alphabetic\n   characters are those characters defined in the Unicode character\n   database as "Letter", i.e., those with general category property\n   being one of "Lm", "Lt", "Lu", "Ll", or "Lo".  Note that this is\n   different from the "Alphabetic" property defined in the Unicode\n   Standard.\n\nstr.isdecimal()\n\n   Return true if all characters in the string are decimal characters\n   and there is at least one character, false otherwise. Decimal\n   characters are those from general category "Nd". This category\n   includes digit characters, and all characters that can be used to\n   form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.  Digits include decimal\n   characters and digits that need special handling, such as the\n   compatibility superscript digits.  Formally, a digit is a character\n   that has the property value Numeric_Type=Digit or\n   Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n   Return true if the string is a valid identifier according to the\n   language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n   Return true if all cased characters [4] in the string are lowercase\n   and there is at least one cased character, false otherwise.\n\nstr.isnumeric()\n\n   Return true if all characters in the string are numeric characters,\n   and there is at least one character, false otherwise. Numeric\n   characters include digit characters, and all characters that have\n   the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n   ONE FIFTH.  Formally, numeric characters are those with the\n   property value Numeric_Type=Digit, Numeric_Type=Decimal or\n   Numeric_Type=Numeric.\n\nstr.isprintable()\n\n   Return true if all characters in the string are printable or the\n   string is empty, false otherwise.  Nonprintable characters are\n   those characters defined in the Unicode character database as\n   "Other" or "Separator", excepting the ASCII space (0x20) which is\n   considered printable.  (Note that printable characters in this\n   context are those which should not be escaped when ``repr()`` is\n   invoked on a string.  It has no bearing on the handling of strings\n   written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.  Whitespace\n   characters  are those characters defined in the Unicode character\n   database as "Other" or "Separator" and those with bidirectional\n   property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\nstr.isupper()\n\n   Return true if all cased characters [4] in the string are uppercase\n   and there is at least one cased character, false otherwise.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in the\n   *iterable* *iterable*.  A ``TypeError`` will be raised if there are\n   any non-string values in *iterable*, including ``bytes`` objects.\n   The separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space).  The original string is returned if *width* is less than or\n   equal to ``len(s)``.\n\nstr.lower()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to lowercase.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.lstrip()\n   \'spacious   \'\n   >>> \'www.example.com\'.lstrip(\'cmowz.\')\n   \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n   This static method returns a translation table usable for\n   ``str.translate()``.\n\n   If there is only one argument, it must be a dictionary mapping\n   Unicode ordinals (integers) or characters (strings of length 1) to\n   Unicode ordinals, strings (of arbitrary lengths) or None.\n   Character keys will then be converted to ordinals.\n\n   If there are two arguments, they must be strings of equal length,\n   and in the resulting dictionary, each character in x will be mapped\n   to the character at the same position in y.  If there is a third\n   argument, it must be a string, whose characters will be mapped to\n   None in the result.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n   is not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space). The original string is returned if *width* is less than or\n   equal to ``len(s)``.\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   ``None``, any whitespace string is a separator.  Except for\n   splitting from the right, ``rsplit()`` behaves like ``split()``\n   which is described in detail below.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.rstrip()\n   \'   spacious\'\n   >>> \'mississippi\'.rstrip(\'ipz\')\n   \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most ``maxsplit+1``\n   elements).  If *maxsplit* is not specified, then there is no limit\n   on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``).  The *sep*\n   argument may consist of multiple characters (for example,\n   ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n   an empty string with a specified separator returns ``[\'\']``.\n\n   If *sep* is not specified or is ``None``, a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a ``None`` separator returns\n   ``[]``.\n\n   For example, ``\' 1  2   3  \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n   and ``\'  1  2   3  \'.split(None, 1)`` returns ``[\'1\', \'2   3  \']``.\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries.  Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n   Return ``True`` if string starts with the *prefix*, otherwise\n   return ``False``. *prefix* can also be a tuple of prefixes to look\n   for.  With optional *start*, test string beginning at that\n   position.  With optional *end*, stop comparing string at that\n   position.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or ``None``, the\n   *chars* argument defaults to removing whitespace. The *chars*\n   argument is not a prefix or suffix; rather, all combinations of its\n   values are stripped:\n\n   >>> \'   spacious   \'.strip()\n   \'spacious\'\n   >>> \'www.example.com\'.strip(\'cmowz.\')\n   \'example\'\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa.\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n              return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n                            lambda mo: mo.group(0)[0].upper() +\n                                       mo.group(0)[1:].lower(),\n                            s)\n\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n   Return a copy of the *s* where all characters have been mapped\n   through the *map* which must be a dictionary of Unicode ordinals\n   (integers) to Unicode ordinals, strings or ``None``.  Unmapped\n   characters are left untouched. Characters mapped to ``None`` are\n   deleted.\n\n   You can use ``str.maketrans()`` to create a translation map from\n   character-to-character mappings in different formats.\n\n   Note: An even more flexible approach is to create a custom character\n     mapping codec using the ``codecs`` module (see\n     ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to uppercase.  Note that ``str.upper().isupper()`` might\n   be ``False`` if ``s`` contains uncased characters or if the Unicode\n   category of the resulting character(s) is not "Lu" (Letter,\n   uppercase), but e.g. "Lt" (Letter, titlecase).\n\nstr.zfill(width)\n\n   Return the numeric string left filled with zeros in a string of\n   length *width*.  A sign prefix is handled correctly.  The original\n   string is returned if *width* is less than or equal to ``len(s)``.\n',
- 'strings': '\nString and Bytes literals\n*************************\n\nString literals are described by the following lexical definitions:\n\n   stringliteral   ::= [stringprefix](shortstring | longstring)\n   stringprefix    ::= "r" | "R"\n   shortstring     ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n   longstring      ::= "\'\'\'" longstringitem* "\'\'\'" | \'"""\' longstringitem* \'"""\'\n   shortstringitem ::= shortstringchar | stringescapeseq\n   longstringitem  ::= longstringchar | stringescapeseq\n   shortstringchar ::= <any source character except "\\" or newline or the quote>\n   longstringchar  ::= <any source character except "\\">\n   stringescapeseq ::= "\\" <any source character>\n\n   bytesliteral   ::= bytesprefix(shortbytes | longbytes)\n   bytesprefix    ::= "b" | "B" | "br" | "Br" | "bR" | "BR"\n   shortbytes     ::= "\'" shortbytesitem* "\'" | \'"\' shortbytesitem* \'"\'\n   longbytes      ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' longbytesitem* \'"""\'\n   shortbytesitem ::= shortbyteschar | bytesescapeseq\n   longbytesitem  ::= longbyteschar | bytesescapeseq\n   shortbyteschar ::= <any ASCII character except "\\" or newline or the quote>\n   longbyteschar  ::= <any ASCII character except "\\">\n   bytesescapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the ``stringprefix`` or\n``bytesprefix`` and the rest of the literal. The source character set\nis defined by the encoding declaration; it is UTF-8 if no encoding\ndeclaration is given in the source file; see section *Encoding\ndeclarations*.\n\nIn plain English: Both types of literals can be enclosed in matching\nsingle quotes (``\'``) or double quotes (``"``).  They can also be\nenclosed in matching groups of three single or double quotes (these\nare generally referred to as *triple-quoted strings*).  The backslash\n(``\\``) character is used to escape characters that otherwise have a\nspecial meaning, such as newline, backslash itself, or the quote\ncharacter.\n\nBytes literals are always prefixed with ``\'b\'`` or ``\'B\'``; they\nproduce an instance of the ``bytes`` type instead of the ``str`` type.\nThey may only contain ASCII characters; bytes with a numeric value of\n128 or greater must be expressed with escapes.\n\nBoth string and bytes literals may optionally be prefixed with a\nletter ``\'r\'`` or ``\'R\'``; such strings are called *raw strings* and\ntreat backslashes as literal characters.  As a result, in string\nliterals, ``\'\\U\'`` and ``\'\\u\'`` escapes in raw strings are not treated\nspecially.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string.  (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C.  The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| ``\\newline``      | Backslash and newline ignored     |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\\``            | Backslash (``\\``)                 |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\'``            | Single quote (``\'``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\"``            | Double quote (``"``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\a``            | ASCII Bell (BEL)                  |         |\n+-------------------+-----------------------------------+---------+\n| ``\\b``            | ASCII Backspace (BS)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\f``            | ASCII Formfeed (FF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\n``            | ASCII Linefeed (LF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\r``            | ASCII Carriage Return (CR)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\t``            | ASCII Horizontal Tab (TAB)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\v``            | ASCII Vertical Tab (VT)           |         |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo``          | Character with octal value *ooo*  | (1,3)   |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh``          | Character with hex value *hh*     | (2,3)   |\n+-------------------+-----------------------------------+---------+\n\nEscape sequences only recognized in string literals are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| ``\\N{name}``      | Character named *name* in the     |         |\n|                   | Unicode database                  |         |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx``        | Character with 16-bit hex value   | (4)     |\n|                   | *xxxx*                            |         |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx``    | Character with 32-bit hex value   | (5)     |\n|                   | *xxxxxxxx*                        |         |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. As in Standard C, up to three octal digits are accepted.\n\n2. Unlike in Standard C, exactly two hex digits are required.\n\n3. In a bytes literal, hexadecimal and octal escapes denote the byte\n   with the given value. In a string literal, these escapes denote a\n   Unicode character with the given value.\n\n4. Individual code units which form parts of a surrogate pair can be\n   encoded using this escape sequence.  Exactly four hex digits are\n   required.\n\n5. Any Unicode character can be encoded this way, but characters\n   outside the Basic Multilingual Plane (BMP) will be encoded using a\n   surrogate pair if Python is compiled to use 16-bit code units (the\n   default).  Exactly eight hex digits are required.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*.  (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.)  It is also\nimportant to note that the escape sequences only recognized in string\nliterals fall into the category of unrecognized escapes for bytes\nliterals.\n\nEven in a raw string, string quotes can be escaped with a backslash,\nbut the backslash remains in the string; for example, ``r"\\""`` is a\nvalid string literal consisting of two characters: a backslash and a\ndouble quote; ``r"\\"`` is not a valid string literal (even a raw\nstring cannot end in an odd number of backslashes).  Specifically, *a\nraw string cannot end in a single backslash* (since the backslash\nwould escape the following quote character).  Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n',
+ 'specialattrs': '\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant.  Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n   A dictionary or other mapping object used to store an object\'s\n   (writable) attributes.\n\ninstance.__class__\n\n   The class to which a class instance belongs.\n\nclass.__bases__\n\n   The tuple of base classes of a class object.\n\nclass.__name__\n\n   The name of the class or type.\n\nclass.__qualname__\n\n   The *qualified name* of the class or type.\n\n   New in version 3.3.\n\nclass.__mro__\n\n   This attribute is a tuple of classes that are considered when\n   looking for base classes during method resolution.\n\nclass.mro()\n\n   This method can be overridden by a metaclass to customize the\n   method resolution order for its instances.  It is called at class\n   instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n   Each class keeps a list of weak references to its immediate\n   subclasses.  This method returns a list of all those references\n   still alive. Example:\n\n      >>> int.__subclasses__()\n      [<class \'bool\'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n    the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n    ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can\'t tell the type of the\n    operands.\n\n[4] Cased characters are those with general category property being\n    one of "Lu" (Letter, uppercase), "Ll" (Letter, lowercase), or "Lt"\n    (Letter, titlecase).\n\n[5] To format only a tuple you should therefore provide a singleton\n    tuple whose only element is the tuple to be formatted.\n',
+ 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators.  For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``type(x).__getitem__(x,\ni)``.  Except where mentioned, attempts to execute an operation raise\nan exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled.  For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense.  (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n   Called to create a new instance of class *cls*.  ``__new__()`` is a\n   static method (special-cased so you need not declare it as such)\n   that takes the class of which an instance was requested as its\n   first argument.  The remaining arguments are those passed to the\n   object constructor expression (the call to the class).  The return\n   value of ``__new__()`` should be the new object instance (usually\n   an instance of *cls*).\n\n   Typical implementations create a new instance of the class by\n   invoking the superclass\'s ``__new__()`` method using\n   ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n   arguments and then modifying the newly-created instance as\n   necessary before returning it.\n\n   If ``__new__()`` returns an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will be invoked like\n   ``__init__(self[, ...])``, where *self* is the new instance and the\n   remaining arguments are the same as were passed to ``__new__()``.\n\n   If ``__new__()`` does not return an instance of *cls*, then the new\n   instance\'s ``__init__()`` method will not be invoked.\n\n   ``__new__()`` is intended mainly to allow subclasses of immutable\n   types (like int, str, or tuple) to customize instance creation.  It\n   is also commonly overridden in custom metaclasses in order to\n   customize class creation.\n\nobject.__init__(self[, ...])\n\n   Called when the instance is created.  The arguments are those\n   passed to the class constructor expression.  If a base class has an\n   ``__init__()`` method, the derived class\'s ``__init__()`` method,\n   if any, must explicitly call it to ensure proper initialization of\n   the base class part of the instance; for example:\n   ``BaseClass.__init__(self, [args...])``.  As a special constraint\n   on constructors, no value may be returned; doing so will cause a\n   ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n   Called when the instance is about to be destroyed.  This is also\n   called a destructor.  If a base class has a ``__del__()`` method,\n   the derived class\'s ``__del__()`` method, if any, must explicitly\n   call it to ensure proper deletion of the base class part of the\n   instance.  Note that it is possible (though not recommended!) for\n   the ``__del__()`` method to postpone destruction of the instance by\n   creating a new reference to it.  It may then be called at a later\n   time when this new reference is deleted.  It is not guaranteed that\n   ``__del__()`` methods are called for objects that still exist when\n   the interpreter exits.\n\n   Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n     decrements the reference count for ``x`` by one, and the latter\n     is only called when ``x``\'s reference count reaches zero.  Some\n     common situations that may prevent the reference count of an\n     object from going to zero include: circular references between\n     objects (e.g., a doubly-linked list or a tree data structure with\n     parent and child pointers); a reference to the object on the\n     stack frame of a function that caught an exception (the traceback\n     stored in ``sys.exc_info()[2]`` keeps the stack frame alive); or\n     a reference to the object on the stack frame that raised an\n     unhandled exception in interactive mode (the traceback stored in\n     ``sys.last_traceback`` keeps the stack frame alive).  The first\n     situation can only be remedied by explicitly breaking the cycles;\n     the latter two situations can be resolved by storing ``None`` in\n     ``sys.last_traceback``. Circular references which are garbage are\n     detected when the option cycle detector is enabled (it\'s on by\n     default), but can only be cleaned up if there are no Python-\n     level ``__del__()`` methods involved. Refer to the documentation\n     for the ``gc`` module for more information about how\n     ``__del__()`` methods are handled by the cycle detector,\n     particularly the description of the ``garbage`` value.\n\n   Warning: Due to the precarious circumstances under which ``__del__()``\n     methods are invoked, exceptions that occur during their execution\n     are ignored, and a warning is printed to ``sys.stderr`` instead.\n     Also, when ``__del__()`` is invoked in response to a module being\n     deleted (e.g., when execution of the program is done), other\n     globals referenced by the ``__del__()`` method may already have\n     been deleted or in the process of being torn down (e.g. the\n     import machinery shutting down).  For this reason, ``__del__()``\n     methods should do the absolute minimum needed to maintain\n     external invariants.  Starting with version 1.5, Python\n     guarantees that globals whose name begins with a single\n     underscore are deleted from their module before other globals are\n     deleted; if no other references to such globals exist, this may\n     help in assuring that imported modules are still available at the\n     time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n   Called by the ``repr()`` built-in function to compute the\n   "official" string representation of an object.  If at all possible,\n   this should look like a valid Python expression that could be used\n   to recreate an object with the same value (given an appropriate\n   environment).  If this is not possible, a string of the form\n   ``<...some useful description...>`` should be returned. The return\n   value must be a string object. If a class defines ``__repr__()``\n   but not ``__str__()``, then ``__repr__()`` is also used when an\n   "informal" string representation of instances of that class is\n   required.\n\n   This is typically used for debugging, so it is important that the\n   representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n   Called by the ``str()`` built-in function and by the ``print()``\n   function to compute the "informal" string representation of an\n   object.  This differs from ``__repr__()`` in that it does not have\n   to be a valid Python expression: a more convenient or concise\n   representation may be used instead. The return value must be a\n   string object.\n\nobject.__bytes__(self)\n\n   Called by ``bytes()`` to compute a byte-string representation of an\n   object. This should return a ``bytes`` object.\n\nobject.__format__(self, format_spec)\n\n   Called by the ``format()`` built-in function (and by extension, the\n   ``format()`` method of class ``str``) to produce a "formatted"\n   string representation of an object. The ``format_spec`` argument is\n   a string that contains a description of the formatting options\n   desired. The interpretation of the ``format_spec`` argument is up\n   to the type implementing ``__format__()``, however most classes\n   will either delegate formatting to one of the built-in types, or\n   use a similar formatting option syntax.\n\n   See *Format Specification Mini-Language* for a description of the\n   standard formatting syntax.\n\n   The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n   These are the so-called "rich comparison" methods. The\n   correspondence between operator symbols and method names is as\n   follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n   ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` calls\n   ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls\n   ``x.__ge__(y)``.\n\n   A rich comparison method may return the singleton\n   ``NotImplemented`` if it does not implement the operation for a\n   given pair of arguments. By convention, ``False`` and ``True`` are\n   returned for a successful comparison. However, these methods can\n   return any value, so if the comparison operator is used in a\n   Boolean context (e.g., in the condition of an ``if`` statement),\n   Python will call ``bool()`` on the value to determine if the result\n   is true or false.\n\n   There are no implied relationships among the comparison operators.\n   The truth of ``x==y`` does not imply that ``x!=y`` is false.\n   Accordingly, when defining ``__eq__()``, one should also define\n   ``__ne__()`` so that the operators will behave as expected.  See\n   the paragraph on ``__hash__()`` for some important notes on\n   creating *hashable* objects which support custom comparison\n   operations and are usable as dictionary keys.\n\n   There are no swapped-argument versions of these methods (to be used\n   when the left argument does not support the operation but the right\n   argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n   other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n   reflection, and ``__eq__()`` and ``__ne__()`` are their own\n   reflection.\n\n   Arguments to rich comparison methods are never coerced.\n\n   To automatically generate ordering operations from a single root\n   operation, see ``functools.total_ordering()``.\n\nobject.__hash__(self)\n\n   Called by built-in function ``hash()`` and for operations on\n   members of hashed collections including ``set``, ``frozenset``, and\n   ``dict``.  ``__hash__()`` should return an integer.  The only\n   required property is that objects which compare equal have the same\n   hash value; it is advised to somehow mix together (e.g. using\n   exclusive or) the hash values for the components of the object that\n   also play a part in comparison of objects.\n\n   If a class does not define an ``__eq__()`` method it should not\n   define a ``__hash__()`` operation either; if it defines\n   ``__eq__()`` but not ``__hash__()``, its instances will not be\n   usable as items in hashable collections.  If a class defines\n   mutable objects and implements an ``__eq__()`` method, it should\n   not implement ``__hash__()``, since the implementation of hashable\n   collections requires that a key\'s hash value is immutable (if the\n   object\'s hash value changes, it will be in the wrong hash bucket).\n\n   User-defined classes have ``__eq__()`` and ``__hash__()`` methods\n   by default; with them, all objects compare unequal (except with\n   themselves) and ``x.__hash__()`` returns an appropriate value such\n   that ``x == y`` implies both that ``x is y`` and ``hash(x) ==\n   hash(y)``.\n\n   Classes which inherit a ``__hash__()`` method from a parent class\n   but change the meaning of ``__eq__()`` such that the hash value\n   returned is no longer appropriate (e.g. by switching to a value-\n   based concept of equality instead of the default identity based\n   equality) can explicitly flag themselves as being unhashable by\n   setting ``__hash__ = None`` in the class definition. Doing so means\n   that not only will instances of the class raise an appropriate\n   ``TypeError`` when a program attempts to retrieve their hash value,\n   but they will also be correctly identified as unhashable when\n   checking ``isinstance(obj, collections.Hashable)`` (unlike classes\n   which define their own ``__hash__()`` to explicitly raise\n   ``TypeError``).\n\n   If a class that overrides ``__eq__()`` needs to retain the\n   implementation of ``__hash__()`` from a parent class, the\n   interpreter must be told this explicitly by setting ``__hash__ =\n   <ParentClass>.__hash__``. Otherwise the inheritance of\n   ``__hash__()`` will be blocked, just as if ``__hash__`` had been\n   explicitly set to ``None``.\n\n   Note: By default, the ``__hash__()`` values of str, bytes and datetime\n     objects are "salted" with an unpredictable random value.\n     Although they remain constant within an individual Python\n     process, they are not predictable between repeated invocations of\n     Python.This is intended to provide protection against a denial-\n     of-service caused by carefully-chosen inputs that exploit the\n     worst case performance of a dict insertion, O(n^2) complexity.\n     See http://www.ocert.org/advisories/ocert-2011-003.html for\n     details.Changing hash values affects the iteration order of\n     dicts, sets and other mappings.  Python has never made guarantees\n     about this ordering (and it typically varies between 32-bit and\n     64-bit builds).See also ``PYTHONHASHSEED``.\n\n   Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n   Called to implement truth value testing and the built-in operation\n   ``bool()``; should return ``False`` or ``True``.  When this method\n   is not defined, ``__len__()`` is called, if it is defined, and the\n   object is considered true if its result is nonzero.  If a class\n   defines neither ``__len__()`` nor ``__bool__()``, all its instances\n   are considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n   Called when an attribute lookup has not found the attribute in the\n   usual places (i.e. it is not an instance attribute nor is it found\n   in the class tree for ``self``).  ``name`` is the attribute name.\n   This method should return the (computed) attribute value or raise\n   an ``AttributeError`` exception.\n\n   Note that if the attribute is found through the normal mechanism,\n   ``__getattr__()`` is not called.  (This is an intentional asymmetry\n   between ``__getattr__()`` and ``__setattr__()``.) This is done both\n   for efficiency reasons and because otherwise ``__getattr__()``\n   would have no way to access other attributes of the instance.  Note\n   that at least for instance variables, you can fake total control by\n   not inserting any values in the instance attribute dictionary (but\n   instead inserting them in another object).  See the\n   ``__getattribute__()`` method below for a way to actually get total\n   control over attribute access.\n\nobject.__getattribute__(self, name)\n\n   Called unconditionally to implement attribute accesses for\n   instances of the class. If the class also defines\n   ``__getattr__()``, the latter will not be called unless\n   ``__getattribute__()`` either calls it explicitly or raises an\n   ``AttributeError``. This method should return the (computed)\n   attribute value or raise an ``AttributeError`` exception. In order\n   to avoid infinite recursion in this method, its implementation\n   should always call the base class method with the same name to\n   access any attributes it needs, for example,\n   ``object.__getattribute__(self, name)``.\n\n   Note: This method may still be bypassed when looking up special methods\n     as the result of implicit invocation via language syntax or\n     built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n   Called when an attribute assignment is attempted.  This is called\n   instead of the normal mechanism (i.e. store the value in the\n   instance dictionary). *name* is the attribute name, *value* is the\n   value to be assigned to it.\n\n   If ``__setattr__()`` wants to assign to an instance attribute, it\n   should call the base class method with the same name, for example,\n   ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n   Like ``__setattr__()`` but for attribute deletion instead of\n   assignment.  This should only be implemented if ``del obj.name`` is\n   meaningful for the object.\n\nobject.__dir__(self)\n\n   Called when ``dir()`` is called on the object. A sequence must be\n   returned. ``dir()`` converts the returned sequence to a list and\n   sorts it.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents).  In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n   Called to get the attribute of the owner class (class attribute\n   access) or of an instance of that class (instance attribute\n   access). *owner* is always the owner class, while *instance* is the\n   instance that the attribute was accessed through, or ``None`` when\n   the attribute is accessed through the *owner*.  This method should\n   return the (computed) attribute value or raise an\n   ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n   Called to set the attribute on an instance *instance* of the owner\n   class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n   Called to delete the attribute on an instance *instance* of the\n   owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol:  ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead.  Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n   The simplest and least common call is when user code directly\n   invokes a descriptor method:    ``x.__get__(a)``.\n\nInstance Binding\n   If binding to an object instance, ``a.x`` is transformed into the\n   call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n   If binding to a class, ``A.x`` is transformed into the call:\n   ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n   If ``a`` is an instance of ``super``, then the binding ``super(B,\n   obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n   ``A`` immediately preceding ``B`` and then invokes the descriptor\n   with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined.  A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary.  If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor.  Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method.  Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary.  In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors.  Accordingly, instances can\nredefine and override methods.  This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage.  This wastes space for objects having very few instance\nvariables.  The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable.  Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n   This class variable can be assigned a string, iterable, or sequence\n   of strings with variable names used by instances.  If defined in a\n   class, *__slots__* reserves space for the declared variables and\n   prevents the automatic creation of *__dict__* and *__weakref__* for\n   each instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n  attribute of that class will always be accessible, so a *__slots__*\n  definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n  variables not listed in the *__slots__* definition.  Attempts to\n  assign to an unlisted variable name raises ``AttributeError``. If\n  dynamic assignment of new variables is desired, then add\n  ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n  declaration.\n\n* Without a *__weakref__* variable for each instance, classes defining\n  *__slots__* do not support weak references to its instances. If weak\n  reference support is needed, then add ``\'__weakref__\'`` to the\n  sequence of strings in the *__slots__* declaration.\n\n* *__slots__* are implemented at the class level by creating\n  descriptors (*Implementing Descriptors*) for each variable name.  As\n  a result, class attributes cannot be used to set default values for\n  instance variables defined by *__slots__*; otherwise, the class\n  attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n  where it is defined.  As a result, subclasses will have a *__dict__*\n  unless they also define *__slots__* (which must only contain names\n  of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n  variable defined by the base class slot is inaccessible (except by\n  retrieving its descriptor directly from the base class). This\n  renders the meaning of the program undefined.  In the future, a\n  check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n  "variable-length" built-in types such as ``int``, ``str`` and\n  ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n  also be used; however, in the future, special meaning may be\n  assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n  *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using ``type()``. The class body\nis executed in a new namespace and the class name is bound locally to\nthe result of ``type(name, bases, namespace)``.\n\nThe class creation process can be customised by passing the\n``metaclass`` keyword argument in the class definition line, or by\ninheriting from an existing class that included such an argument. In\nthe following example, both ``MyClass`` and ``MySubclass`` are\ninstances of ``Meta``:\n\n   class Meta(type):\n       pass\n\n   class MyClass(metaclass=Meta):\n       pass\n\n   class MySubclass(MyClass):\n       pass\n\nAny other keyword arguments that are specified in the class definition\nare passed through to all metaclass operations described below.\n\nWhen a class definition is executed, the following steps occur:\n\n* the appropriate metaclass is determined\n\n* the class namespace is prepared\n\n* the class body is executed\n\n* the class object is created\n\n\nDetermining the appropriate metaclass\n-------------------------------------\n\nThe appropriate metaclass for a class definition is determined as\nfollows:\n\n* if no bases and no explicit metaclass are given, then ``type()`` is\n  used\n\n* if an explicit metaclass is given and it is *not* an instance of\n  ``type()``, then it is used directly as the metaclass\n\n* if an instance of ``type()`` is given as the explicit metaclass, or\n  bases are defined, then the most derived metaclass is used\n\nThe most derived metaclass is selected from the explicitly specified\nmetaclass (if any) and the metaclasses (i.e. ``type(cls)``) of all\nspecified base classes. The most derived metaclass is one which is a\nsubtype of *all* of these candidate metaclasses. If none of the\ncandidate metaclasses meets that criterion, then the class definition\nwill fail with ``TypeError``.\n\n\nPreparing the class namespace\n-----------------------------\n\nOnce the appropriate metaclass has been identified, then the class\nnamespace is prepared. If the metaclass has a ``__prepare__``\nattribute, it is called as ``namespace = metaclass.__prepare__(name,\nbases, **kwds)`` (where the additional keyword arguments, if any, come\nfrom the class definition).\n\nIf the metaclass has no ``__prepare__`` attribute, then the class\nnamespace is initialised as an empty ``dict()`` instance.\n\nSee also:\n\n   **PEP 3115** - Metaclasses in Python 3000\n      Introduced the ``__prepare__`` namespace hook\n\n\nExecuting the class body\n------------------------\n\nThe class body is executed (approximately) as ``exec(body, globals(),\nnamespace)``. The key difference from a normal call to ``exec()`` is\nthat lexical scoping allows the class body (including any methods) to\nreference names from the current and outer scopes when the class\ndefinition occurs inside a function.\n\nHowever, even when the class definition occurs inside the function,\nmethods defined inside the class still cannot see names defined at the\nclass scope. Class variables must be accessed through the first\nparameter of instance or class methods, and cannot be accessed at all\nfrom static methods.\n\n\nCreating the class object\n-------------------------\n\nOnce the class namespace has been populated by executing the class\nbody, the class object is created by calling ``metaclass(name, bases,\nnamespace, **kwds)`` (the additional keywords passed here are the same\nas those passed to ``__prepare__``).\n\nThis class object is the one that will be referenced by the zero-\nargument form of ``super()``. ``__class__`` is an implicit closure\nreference created by the compiler if any methods in a class body refer\nto either ``__class__`` or ``super``. This allows the zero argument\nform of ``super()`` to correctly identify the class being defined\nbased on lexical scoping, while the class or instance that was used to\nmake the current call is identified based on the first argument passed\nto the method.\n\nAfter the class object is created, it is passed to the class\ndecorators included in the class definition (if any) and the resulting\nobject is bound in the local namespace as the defined class.\n\nSee also:\n\n   **PEP 3135** - New super\n      Describes the implicit ``__class__`` closure reference\n\n\nMetaclass example\n-----------------\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored include logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n``collections.OrderedDict`` to remember the order that class members\nwere defined:\n\n   class OrderedClass(type):\n\n        @classmethod\n        def __prepare__(metacls, name, bases, **kwds):\n           return collections.OrderedDict()\n\n        def __new__(cls, name, bases, namespace, **kwds):\n           result = type.__new__(cls, name, bases, dict(namespace))\n           result.members = tuple(namespace)\n           return result\n\n   class A(metaclass=OrderedClass):\n       def one(self): pass\n       def two(self): pass\n       def three(self): pass\n       def four(self): pass\n\n   >>> A.members\n   (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s ``__prepare__()`` method which returns an\nempty ``collections.OrderedDict``.  That mapping records the methods\nand attributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s ``__new__()`` method gets\ninvoked.  That method builds the new type and it saves the ordered\ndictionary keys in an attribute called ``members``.\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n   Return true if *instance* should be considered a (direct or\n   indirect) instance of *class*. If defined, called to implement\n   ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n   Return true if *subclass* should be considered a (direct or\n   indirect) subclass of *class*.  If defined, called to implement\n   ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass.  They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n   **PEP 3119** - Introducing Abstract Base Classes\n      Includes the specification for customizing ``isinstance()`` and\n      ``issubclass()`` behavior through ``__instancecheck__()`` and\n      ``__subclasscheck__()``, with motivation for this functionality\n      in the context of adding Abstract Base Classes (see the ``abc``\n      module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n   Called when the instance is "called" as a function; if this method\n   is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n   ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well.  The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items.  It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``get()``,\n``clear()``, ``setdefault()``, ``pop()``, ``popitem()``, ``copy()``,\nand ``update()`` behaving similar to those for Python\'s standard\ndictionary objects.  The ``collections`` module provides a\n``MutableMapping`` abstract base class to help create those methods\nfrom a base set of ``__getitem__()``, ``__setitem__()``,\n``__delitem__()``, and ``keys()``. Mutable sequences should provide\nmethods ``append()``, ``count()``, ``index()``, ``extend()``,\n``insert()``, ``pop()``, ``remove()``, ``reverse()`` and ``sort()``,\nlike Python standard list objects.  Finally, sequence types should\nimplement addition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods ``__add__()``, ``__radd__()``,\n``__iadd__()``, ``__mul__()``, ``__rmul__()`` and ``__imul__()``\ndescribed below; they should not define other numerical operators.  It\nis recommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should search the mapping\'s keys; for\nsequences, it should search through the values.  It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``keys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n   Called to implement the built-in function ``len()``.  Should return\n   the length of the object, an integer ``>=`` 0.  Also, an object\n   that doesn\'t define a ``__bool__()`` method and whose ``__len__()``\n   method returns zero is considered to be false in a Boolean context.\n\nNote: Slicing is done exclusively with the following three methods.  A\n  call like\n\n     a[1:2] = b\n\n  is translated to\n\n     a[slice(1, 2, None)] = b\n\n  and so forth.  Missing slice items are always filled in with\n  ``None``.\n\nobject.__getitem__(self, key)\n\n   Called to implement evaluation of ``self[key]``. For sequence\n   types, the accepted keys should be integers and slice objects.\n   Note that the special interpretation of negative indexes (if the\n   class wishes to emulate a sequence type) is up to the\n   ``__getitem__()`` method. If *key* is of an inappropriate type,\n   ``TypeError`` may be raised; if of a value outside the set of\n   indexes for the sequence (after any special interpretation of\n   negative values), ``IndexError`` should be raised. For mapping\n   types, if *key* is missing (not in the container), ``KeyError``\n   should be raised.\n\n   Note: ``for`` loops expect that an ``IndexError`` will be raised for\n     illegal indexes to allow proper detection of the end of the\n     sequence.\n\nobject.__setitem__(self, key, value)\n\n   Called to implement assignment to ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support changes to the values for keys, or if new keys\n   can be added, or for sequences if elements can be replaced.  The\n   same exceptions should be raised for improper *key* values as for\n   the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n   Called to implement deletion of ``self[key]``.  Same note as for\n   ``__getitem__()``.  This should only be implemented for mappings if\n   the objects support removal of keys, or for sequences if elements\n   can be removed from the sequence.  The same exceptions should be\n   raised for improper *key* values as for the ``__getitem__()``\n   method.\n\nobject.__iter__(self)\n\n   This method is called when an iterator is required for a container.\n   This method should return a new iterator object that can iterate\n   over all the objects in the container.  For mappings, it should\n   iterate over the keys of the container, and should also be made\n   available as the method ``keys()``.\n\n   Iterator objects also need to implement this method; they are\n   required to return themselves.  For more information on iterator\n   objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n   Called (if present) by the ``reversed()`` built-in to implement\n   reverse iteration.  It should return a new iterator object that\n   iterates over all the objects in the container in reverse order.\n\n   If the ``__reversed__()`` method is not provided, the\n   ``reversed()`` built-in will fall back to using the sequence\n   protocol (``__len__()`` and ``__getitem__()``).  Objects that\n   support the sequence protocol should only provide\n   ``__reversed__()`` if they can provide an implementation that is\n   more efficient than the one provided by ``reversed()``.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence.  However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n   Called to implement membership test operators.  Should return true\n   if *item* is in *self*, false otherwise.  For mapping objects, this\n   should consider the keys of the mapping rather than the values or\n   the key-item pairs.\n\n   For objects that don\'t define ``__contains__()``, the membership\n   test first tries iteration via ``__iter__()``, then the old\n   sequence iteration protocol via ``__getitem__()``, see *this\n   section in the language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n   ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n   ``|``).  For instance, to evaluate the expression ``x + y``, where\n   *x* is an instance of a class that has an ``__add__()`` method,\n   ``x.__add__(y)`` is called.  The ``__divmod__()`` method should be\n   the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n   should not be related to ``__truediv__()``.  Note that\n   ``__pow__()`` should be defined to accept an optional third\n   argument if the ternary version of the built-in ``pow()`` function\n   is to be supported.\n\n   If one of those methods does not support the operation with the\n   supplied arguments, it should return ``NotImplemented``.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n   These methods are called to implement the binary arithmetic\n   operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n   ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n   ``|``) with reflected (swapped) operands. These functions are only\n   called if the left operand does not support the corresponding\n   operation and the operands are of different types. [2]  For\n   instance, to evaluate the expression ``x - y``, where *y* is an\n   instance of a class that has an ``__rsub__()`` method,\n   ``y.__rsub__(x)`` is called if ``x.__sub__(y)`` returns\n   *NotImplemented*.\n\n   Note that ternary ``pow()`` will not try calling ``__rpow__()``\n   (the coercion rules would become too complicated).\n\n   Note: If the right operand\'s type is a subclass of the left operand\'s\n     type and that subclass provides the reflected method for the\n     operation, this method will be called before the left operand\'s\n     non-reflected method.  This behavior allows subclasses to\n     override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n   These methods are called to implement the augmented arithmetic\n   assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n   ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``).  These methods\n   should attempt to do the operation in-place (modifying *self*) and\n   return the result (which could be, but does not have to be,\n   *self*).  If a specific method is not defined, the augmented\n   assignment falls back to the normal methods.  For instance, to\n   execute the statement ``x += y``, where *x* is an instance of a\n   class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n   called.  If *x* is an instance of a class that does not define a\n   ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n   considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n   Called to implement the unary arithmetic operations (``-``, ``+``,\n   ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n   Called to implement the built-in functions ``complex()``,\n   ``int()``, ``float()`` and ``round()``.  Should return a value of\n   the appropriate type.\n\nobject.__index__(self)\n\n   Called to implement ``operator.index()``.  Also called whenever\n   Python needs an integer object (such as in slicing, or in the\n   built-in ``bin()``, ``hex()`` and ``oct()`` functions). Must return\n   an integer.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code.  Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n   Enter the runtime context related to this object. The ``with``\n   statement will bind this method\'s return value to the target(s)\n   specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n   Exit the runtime context related to this object. The parameters\n   describe the exception that caused the context to be exited. If the\n   context was exited without an exception, all three arguments will\n   be ``None``.\n\n   If an exception is supplied, and the method wishes to suppress the\n   exception (i.e., prevent it from being propagated), it should\n   return a true value. Otherwise, the exception will be processed\n   normally upon exit from this method.\n\n   Note that ``__exit__()`` methods should not reraise the passed-in\n   exception; this is the caller\'s responsibility.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary.  That behaviour is the reason why\nthe following code raises an exception:\n\n   >>> class C:\n   ...     pass\n   ...\n   >>> c = C()\n   >>> c.__len__ = lambda: 5\n   >>> len(c)\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n   >>> 1 .__hash__() == hash(1)\n   True\n   >>> int.__hash__() == hash(int)\n   Traceback (most recent call last):\n     File "<stdin>", line 1, in <module>\n   TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n   >>> type(1).__hash__(1) == hash(1)\n   True\n   >>> type(int).__hash__(int) == hash(int)\n   True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n   >>> class Meta(type):\n   ...    def __getattribute__(*args):\n   ...       print("Metaclass getattribute invoked")\n   ...       return type.__getattribute__(*args)\n   ...\n   >>> class C(object, metaclass=Meta):\n   ...     def __len__(self):\n   ...         return 10\n   ...     def __getattribute__(*args):\n   ...         print("Class getattribute invoked")\n   ...         return object.__getattribute__(*args)\n   ...\n   >>> c = C()\n   >>> c.__len__()                 # Explicit lookup via instance\n   Class getattribute invoked\n   10\n   >>> type(c).__len__(c)          # Explicit lookup via type\n   Metaclass getattribute invoked\n   10\n   >>> len(c)                      # Implicit lookup\n   10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n    certain controlled conditions. It generally isn\'t a good idea\n    though, since it can lead to some very strange behaviour if it is\n    handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n    reflected method (such as ``__add__()``) fails the operation is\n    not supported, which is why the reflected method is not called.\n',
+ 'string-methods': '\nString Methods\n**************\n\nStrings implement all of the *common* sequence operations, along with\nthe additional methods described below.\n\nStrings also support two styles of string formatting, one providing a\nlarge degree of flexibility and customization (see ``str.format()``,\n*Format String Syntax* and *String Formatting*) and the other based on\nC ``printf`` style formatting that handles a narrower range of types\nand is slightly harder to use correctly, but is often faster for the\ncases it can handle (*printf-style String Formatting*).\n\nThe *Text Processing Services* section of the standard library covers\na number of other modules that provide various text related utilities\n(including regular expression support in the ``re`` module).\n\nstr.capitalize()\n\n   Return a copy of the string with its first character capitalized\n   and the rest lowercased.\n\nstr.casefold()\n\n   Return a casefolded copy of the string. Casefolded strings may be\n   used for caseless matching.\n\n   Casefolding is similar to lowercasing but more aggressive because\n   it is intended to remove all case distinctions in a string. For\n   example, the German lowercase letter ``\'\xc3\x9f\'`` is equivalent to\n   ``"ss"``. Since it is already lowercase, ``lower()`` would do\n   nothing to ``\'\xc3\x9f\'``; ``casefold()`` converts it to ``"ss"``.\n\n   The casefolding algorithm is described in section 3.13 of the\n   Unicode Standard.\n\n   New in version 3.3.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n   Return an encoded version of the string as a bytes object. Default\n   encoding is ``\'utf-8\'``. *errors* may be given to set a different\n   error handling scheme. The default for *errors* is ``\'strict\'``,\n   meaning that encoding errors raise a ``UnicodeError``. Other\n   possible values are ``\'ignore\'``, ``\'replace\'``,\n   ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and any other name\n   registered via ``codecs.register_error()``, see section *Codec Base\n   Classes*. For a list of possible encodings, see section *Standard\n   Encodings*.\n\n   Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return ``True`` if the string ends with the specified *suffix*,\n   otherwise return ``False``.  *suffix* can also be a tuple of\n   suffixes to look for.  With optional *start*, test beginning at\n   that position.  With optional *end*, stop comparing at that\n   position.\n\nstr.expandtabs([tabsize])\n\n   Return a copy of the string where all tab characters are replaced\n   by zero or more spaces, depending on the current column and the\n   given tab size.  The column number is reset to zero after each\n   newline occurring in the string. If *tabsize* is not given, a tab\n   size of ``8`` characters is assumed.  This doesn\'t understand other\n   non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found, such that *sub* is contained in the slice ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` if *sub* is not found.\n\n   Note: The ``find()`` method should be used only if you need to know the\n     position of *sub*.  To check if *sub* is a substring or not, use\n     the ``in`` operator:\n\n        >>> \'Py\' in \'Python\'\n        True\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces ``{}``.  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n   Similar to ``str.format(**mapping)``, except that ``mapping`` is\n   used directly and not copied to a ``dict`` .  This is useful if for\n   example ``mapping`` is a dict subclass:\n\n   >>> class Default(dict):\n   ...     def __missing__(self, key):\n   ...         return key\n   ...\n   >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n   \'Guido was born in country\'\n\n   New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n   Like ``find()``, but raise ``ValueError`` when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.  A character\n   ``c`` is alphanumeric if one of the following returns ``True``:\n   ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n   ``c.isnumeric()``.\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.  Alphabetic\n   characters are those characters defined in the Unicode character\n   database as "Letter", i.e., those with general category property\n   being one of "Lm", "Lt", "Lu", "Ll", or "Lo".  Note that this is\n   different from the "Alphabetic" property defined in the Unicode\n   Standard.\n\nstr.isdecimal()\n\n   Return true if all characters in the string are decimal characters\n   and there is at least one character, false otherwise. Decimal\n   characters are those from general category "Nd". This category\n   includes digit characters, and all characters that can be used to\n   form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.  Digits include decimal\n   characters and digits that need special handling, such as the\n   compatibility superscript digits.  Formally, a digit is a character\n   that has the property value Numeric_Type=Digit or\n   Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n   Return true if the string is a valid identifier according to the\n   language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n   Return true if all cased characters [4] in the string are lowercase\n   and there is at least one cased character, false otherwise.\n\nstr.isnumeric()\n\n   Return true if all characters in the string are numeric characters,\n   and there is at least one character, false otherwise. Numeric\n   characters include digit characters, and all characters that have\n   the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n   ONE FIFTH.  Formally, numeric characters are those with the\n   property value Numeric_Type=Digit, Numeric_Type=Decimal or\n   Numeric_Type=Numeric.\n\nstr.isprintable()\n\n   Return true if all characters in the string are printable or the\n   string is empty, false otherwise.  Nonprintable characters are\n   those characters defined in the Unicode character database as\n   "Other" or "Separator", excepting the ASCII space (0x20) which is\n   considered printable.  (Note that printable characters in this\n   context are those which should not be escaped when ``repr()`` is\n   invoked on a string.  It has no bearing on the handling of strings\n   written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.  Whitespace\n   characters  are those characters defined in the Unicode character\n   database as "Other" or "Separator" and those with bidirectional\n   property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\nstr.isupper()\n\n   Return true if all cased characters [4] in the string are uppercase\n   and there is at least one cased character, false otherwise.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in the\n   *iterable* *iterable*.  A ``TypeError`` will be raised if there are\n   any non-string values in *iterable*, including ``bytes`` objects.\n   The separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space).  The original string is returned if *width* is less than or\n   equal to ``len(s)``.\n\nstr.lower()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to lowercase.\n\n   The lowercasing algorithm used is described in section 3.13 of the\n   Unicode Standard.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.lstrip()\n   \'spacious   \'\n   >>> \'www.example.com\'.lstrip(\'cmowz.\')\n   \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n   This static method returns a translation table usable for\n   ``str.translate()``.\n\n   If there is only one argument, it must be a dictionary mapping\n   Unicode ordinals (integers) or characters (strings of length 1) to\n   Unicode ordinals, strings (of arbitrary lengths) or None.\n   Character keys will then be converted to ordinals.\n\n   If there are two arguments, they must be strings of equal length,\n   and in the resulting dictionary, each character in x will be mapped\n   to the character at the same position in y.  If there is a third\n   argument, it must be a string, whose characters will be mapped to\n   None in the result.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n   is not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space). The original string is returned if *width* is less than or\n   equal to ``len(s)``.\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\nstr.rsplit(sep=None, maxsplit=-1)\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   ``None``, any whitespace string is a separator.  Except for\n   splitting from the right, ``rsplit()`` behaves like ``split()``\n   which is described in detail below.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.rstrip()\n   \'   spacious\'\n   >>> \'mississippi\'.rstrip(\'ipz\')\n   \'mississ\'\n\nstr.split(sep=None, maxsplit=-1)\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most ``maxsplit+1``\n   elements).  If *maxsplit* is not specified or ``-1``, then there is\n   no limit on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``).  The *sep*\n   argument may consist of multiple characters (for example,\n   ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n   an empty string with a specified separator returns ``[\'\']``.\n\n   If *sep* is not specified or is ``None``, a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a ``None`` separator returns\n   ``[]``.\n\n   For example, ``\' 1  2   3  \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n   and ``\'  1  2   3  \'.split(None, 1)`` returns ``[\'1\', \'2   3  \']``.\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries. This method uses the *universal newlines* approach to\n   splitting lines. Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\n   For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n   ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n   ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n   \'kl\\r\\n\']``.\n\n   Unlike ``split()`` when a delimiter string *sep* is given, this\n   method returns an empty list for the empty string, and a terminal\n   line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n   Return ``True`` if string starts with the *prefix*, otherwise\n   return ``False``. *prefix* can also be a tuple of prefixes to look\n   for.  With optional *start*, test string beginning at that\n   position.  With optional *end*, stop comparing string at that\n   position.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or ``None``, the\n   *chars* argument defaults to removing whitespace. The *chars*\n   argument is not a prefix or suffix; rather, all combinations of its\n   values are stripped:\n\n   >>> \'   spacious   \'.strip()\n   \'spacious\'\n   >>> \'www.example.com\'.strip(\'cmowz.\')\n   \'example\'\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa. Note that it is not necessarily true that\n   ``s.swapcase().swapcase() == s``.\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n              return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n                            lambda mo: mo.group(0)[0].upper() +\n                                       mo.group(0)[1:].lower(),\n                            s)\n\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n   Return a copy of the *s* where all characters have been mapped\n   through the *map* which must be a dictionary of Unicode ordinals\n   (integers) to Unicode ordinals, strings or ``None``.  Unmapped\n   characters are left untouched. Characters mapped to ``None`` are\n   deleted.\n\n   You can use ``str.maketrans()`` to create a translation map from\n   character-to-character mappings in different formats.\n\n   Note: An even more flexible approach is to create a custom character\n     mapping codec using the ``codecs`` module (see\n     ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to uppercase.  Note that ``str.upper().isupper()`` might\n   be ``False`` if ``s`` contains uncased characters or if the Unicode\n   category of the resulting character(s) is not "Lu" (Letter,\n   uppercase), but e.g. "Lt" (Letter, titlecase).\n\n   The uppercasing algorithm used is described in section 3.13 of the\n   Unicode Standard.\n\nstr.zfill(width)\n\n   Return the numeric string left filled with zeros in a string of\n   length *width*.  A sign prefix is handled correctly.  The original\n   string is returned if *width* is less than or equal to ``len(s)``.\n',
+ 'strings': '\nString and Bytes literals\n*************************\n\nString literals are described by the following lexical definitions:\n\n   stringliteral   ::= [stringprefix](shortstring | longstring)\n   stringprefix    ::= "r" | "u" | "R" | "U"\n   shortstring     ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n   longstring      ::= "\'\'\'" longstringitem* "\'\'\'" | \'"""\' longstringitem* \'"""\'\n   shortstringitem ::= shortstringchar | stringescapeseq\n   longstringitem  ::= longstringchar | stringescapeseq\n   shortstringchar ::= <any source character except "\\" or newline or the quote>\n   longstringchar  ::= <any source character except "\\">\n   stringescapeseq ::= "\\" <any source character>\n\n   bytesliteral   ::= bytesprefix(shortbytes | longbytes)\n   bytesprefix    ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB"\n   shortbytes     ::= "\'" shortbytesitem* "\'" | \'"\' shortbytesitem* \'"\'\n   longbytes      ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' longbytesitem* \'"""\'\n   shortbytesitem ::= shortbyteschar | bytesescapeseq\n   longbytesitem  ::= longbyteschar | bytesescapeseq\n   shortbyteschar ::= <any ASCII character except "\\" or newline or the quote>\n   longbyteschar  ::= <any ASCII character except "\\">\n   bytesescapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the ``stringprefix`` or\n``bytesprefix`` and the rest of the literal. The source character set\nis defined by the encoding declaration; it is UTF-8 if no encoding\ndeclaration is given in the source file; see section *Encoding\ndeclarations*.\n\nIn plain English: Both types of literals can be enclosed in matching\nsingle quotes (``\'``) or double quotes (``"``).  They can also be\nenclosed in matching groups of three single or double quotes (these\nare generally referred to as *triple-quoted strings*).  The backslash\n(``\\``) character is used to escape characters that otherwise have a\nspecial meaning, such as newline, backslash itself, or the quote\ncharacter.\n\nBytes literals are always prefixed with ``\'b\'`` or ``\'B\'``; they\nproduce an instance of the ``bytes`` type instead of the ``str`` type.\nThey may only contain ASCII characters; bytes with a numeric value of\n128 or greater must be expressed with escapes.\n\nAs of Python 3.3 it is possible again to prefix unicode strings with a\n``u`` prefix to simplify maintenance of dual 2.x and 3.x codebases.\n\nBoth string and bytes literals may optionally be prefixed with a\nletter ``\'r\'`` or ``\'R\'``; such strings are called *raw strings* and\ntreat backslashes as literal characters.  As a result, in string\nliterals, ``\'\\U\'`` and ``\'\\u\'`` escapes in raw strings are not treated\nspecially. Given that Python 2.x\'s raw unicode literals behave\ndifferently than Python 3.x\'s the ``\'ur\'`` syntax is not supported.\n\n   New in version 3.3: The ``\'rb\'`` prefix of raw bytes literals has\n   been added as a synonym of ``\'br\'``.\n\n   New in version 3.3: Support for the unicode legacy literal\n   (``u\'value\'``) was reintroduced to simplify the maintenance of dual\n   Python 2.x and 3.x codebases. See **PEP 414** for more information.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string.  (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C.  The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| ``\\newline``      | Backslash and newline ignored     |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\\``            | Backslash (``\\``)                 |         |\n+-------------------+-----------------------------------+---------+\n| ``\\\'``            | Single quote (``\'``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\"``            | Double quote (``"``)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\a``            | ASCII Bell (BEL)                  |         |\n+-------------------+-----------------------------------+---------+\n| ``\\b``            | ASCII Backspace (BS)              |         |\n+-------------------+-----------------------------------+---------+\n| ``\\f``            | ASCII Formfeed (FF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\n``            | ASCII Linefeed (LF)               |         |\n+-------------------+-----------------------------------+---------+\n| ``\\r``            | ASCII Carriage Return (CR)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\t``            | ASCII Horizontal Tab (TAB)        |         |\n+-------------------+-----------------------------------+---------+\n| ``\\v``            | ASCII Vertical Tab (VT)           |         |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo``          | Character with octal value *ooo*  | (1,3)   |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh``          | Character with hex value *hh*     | (2,3)   |\n+-------------------+-----------------------------------+---------+\n\nEscape sequences only recognized in string literals are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence   | Meaning                           | Notes   |\n+===================+===================================+=========+\n| ``\\N{name}``      | Character named *name* in the     | (4)     |\n|                   | Unicode database                  |         |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx``        | Character with 16-bit hex value   | (5)     |\n|                   | *xxxx*                            |         |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx``    | Character with 32-bit hex value   | (6)     |\n|                   | *xxxxxxxx*                        |         |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. As in Standard C, up to three octal digits are accepted.\n\n2. Unlike in Standard C, exactly two hex digits are required.\n\n3. In a bytes literal, hexadecimal and octal escapes denote the byte\n   with the given value. In a string literal, these escapes denote a\n   Unicode character with the given value.\n\n4. Changed in version 3.3: Support for name aliases [1] has been\n   added.\n\n5. Individual code units which form parts of a surrogate pair can be\n   encoded using this escape sequence.  Exactly four hex digits are\n   required.\n\n6. Any Unicode character can be encoded this way, but characters\n   outside the Basic Multilingual Plane (BMP) will be encoded using a\n   surrogate pair if Python is compiled to use 16-bit code units (the\n   default).  Exactly eight hex digits are required.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*.  (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.)  It is also\nimportant to note that the escape sequences only recognized in string\nliterals fall into the category of unrecognized escapes for bytes\nliterals.\n\nEven in a raw string, string quotes can be escaped with a backslash,\nbut the backslash remains in the string; for example, ``r"\\""`` is a\nvalid string literal consisting of two characters: a backslash and a\ndouble quote; ``r"\\"`` is not a valid string literal (even a raw\nstring cannot end in an odd number of backslashes).  Specifically, *a\nraw string cannot end in a single backslash* (since the backslash\nwould escape the following quote character).  Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n',
  'subscriptions': '\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n   subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object that supports subscription,\ne.g. a list or dictionary.  User-defined objects can support\nsubscription by defining a ``__getitem__()`` method.\n\nFor built-in objects, there are two types of objects that support\nsubscription:\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey.  (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to\nan integer or a slice (as discussed in the following section).\n\nThe formal syntax makes no special provision for negative indices in\nsequences; however, built-in sequences all provide a ``__getitem__()``\nmethod that interprets negative indices by adding the length of the\nsequence to the index (so that ``x[-1]`` selects the last item of\n``x``).  The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero). Since the support\nfor negative indices and slicing occurs in the object\'s\n``__getitem__()`` method, subclasses overriding this method will need\nto explicitly add that support.\n\nA string\'s items are characters.  A character is not a separate data\ntype but a string of exactly one character.\n',
  'truth': "\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0.0``, ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n  ``__bool__()`` or ``__len__()`` method, when that method returns the\n  integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n",
- 'try': '\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression ["as" target]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started.  This search inspects the except\nclauses in turn until one is found that matches the exception.  An\nexpression-less except clause, if present, must be last; it matches\nany exception.  For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception.  An object is\ncompatible with an exception if it is the class or a base class of the\nexception object or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the ``as`` keyword in that except clause,\nif present, and the except clause\'s suite is executed.  All except\nclauses must have an executable block.  When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using ``as target``, it is cleared\nat the end of the except clause.  This is as if\n\n   except E as N:\n       foo\n\nwas translated to\n\n   except E as N:\n       try:\n           foo\n       finally:\n           del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause.  Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the ``sys`` module and can be access via\n``sys.exc_info()``. ``sys.exc_info()`` returns a 3-tuple consisting of\nthe exception class, the exception instance and a traceback object\n(see section *The standard type hierarchy*) identifying the point in\nthe program where the exception occurred.  ``sys.exc_info()`` values\nare restored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler.  The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses.  If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted.  If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is set as the context of the new exception.  The exception\ninformation is not available to the program during execution of the\n``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n',
- 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python.  Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types.  Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\'  These are attributes that provide access to the\nimplementation and are not intended for general use.  Their definition\nmay change in the future.\n\nNone\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name ``None``.\n   It is used to signify the absence of a value in many situations,\n   e.g., it is returned from functions that don\'t explicitly return\n   anything. Its truth value is false.\n\nNotImplemented\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name\n   ``NotImplemented``. Numeric methods and rich comparison methods may\n   return this value if they do not implement the operation for the\n   operands provided.  (The interpreter will then try the reflected\n   operation, or some other fallback, depending on the operator.)  Its\n   truth value is true.\n\nEllipsis\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the literal ``...`` or the\n   built-in name ``Ellipsis``.  Its truth value is true.\n\n``numbers.Number``\n   These are created by numeric literals and returned as results by\n   arithmetic operators and arithmetic built-in functions.  Numeric\n   objects are immutable; once created their value never changes.\n   Python numbers are of course strongly related to mathematical\n   numbers, but subject to the limitations of numerical representation\n   in computers.\n\n   Python distinguishes between integers, floating point numbers, and\n   complex numbers:\n\n   ``numbers.Integral``\n      These represent elements from the mathematical set of integers\n      (positive and negative).\n\n      There are two types of integers:\n\n      Integers (``int``)\n\n         These represent numbers in an unlimited range, subject to\n         available (virtual) memory only.  For the purpose of shift\n         and mask operations, a binary representation is assumed, and\n         negative numbers are represented in a variant of 2\'s\n         complement which gives the illusion of an infinite string of\n         sign bits extending to the left.\n\n      Booleans (``bool``)\n         These represent the truth values False and True.  The two\n         objects representing the values False and True are the only\n         Boolean objects. The Boolean type is a subtype of the integer\n         type, and Boolean values behave like the values 0 and 1,\n         respectively, in almost all contexts, the exception being\n         that when converted to a string, the strings ``"False"`` or\n         ``"True"`` are returned, respectively.\n\n      The rules for integer representation are intended to give the\n      most meaningful interpretation of shift and mask operations\n      involving negative integers.\n\n   ``numbers.Real`` (``float``)\n      These represent machine-level double precision floating point\n      numbers. You are at the mercy of the underlying machine\n      architecture (and C or Java implementation) for the accepted\n      range and handling of overflow. Python does not support single-\n      precision floating point numbers; the savings in processor and\n      memory usage that are usually the reason for using these is\n      dwarfed by the overhead of using objects in Python, so there is\n      no reason to complicate the language with two kinds of floating\n      point numbers.\n\n   ``numbers.Complex`` (``complex``)\n      These represent complex numbers as a pair of machine-level\n      double precision floating point numbers.  The same caveats apply\n      as for floating point numbers. The real and imaginary parts of a\n      complex number ``z`` can be retrieved through the read-only\n      attributes ``z.real`` and ``z.imag``.\n\nSequences\n   These represent finite ordered sets indexed by non-negative\n   numbers. The built-in function ``len()`` returns the number of\n   items of a sequence. When the length of a sequence is *n*, the\n   index set contains the numbers 0, 1, ..., *n*-1.  Item *i* of\n   sequence *a* is selected by ``a[i]``.\n\n   Sequences also support slicing: ``a[i:j]`` selects all items with\n   index *k* such that *i* ``<=`` *k* ``<`` *j*.  When used as an\n   expression, a slice is a sequence of the same type.  This implies\n   that the index set is renumbered so that it starts at 0.\n\n   Some sequences also support "extended slicing" with a third "step"\n   parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n   where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n   *j*.\n\n   Sequences are distinguished according to their mutability:\n\n   Immutable sequences\n      An object of an immutable sequence type cannot change once it is\n      created.  (If the object contains references to other objects,\n      these other objects may be mutable and may be changed; however,\n      the collection of objects directly referenced by an immutable\n      object cannot change.)\n\n      The following types are immutable sequences:\n\n      Strings\n         The items of a string object are Unicode code units.  A\n         Unicode code unit is represented by a string object of one\n         item and can hold either a 16-bit or 32-bit value\n         representing a Unicode ordinal (the maximum value for the\n         ordinal is given in ``sys.maxunicode``, and depends on how\n         Python is configured at compile time).  Surrogate pairs may\n         be present in the Unicode object, and will be reported as two\n         separate items.  The built-in functions ``chr()`` and\n         ``ord()`` convert between code units and nonnegative integers\n         representing the Unicode ordinals as defined in the Unicode\n         Standard 3.0. Conversion from and to other encodings are\n         possible through the string method ``encode()``.\n\n      Tuples\n         The items of a tuple are arbitrary Python objects. Tuples of\n         two or more items are formed by comma-separated lists of\n         expressions.  A tuple of one item (a \'singleton\') can be\n         formed by affixing a comma to an expression (an expression by\n         itself does not create a tuple, since parentheses must be\n         usable for grouping of expressions).  An empty tuple can be\n         formed by an empty pair of parentheses.\n\n      Bytes\n         A bytes object is an immutable array.  The items are 8-bit\n         bytes, represented by integers in the range 0 <= x < 256.\n         Bytes literals (like ``b\'abc\'`` and the built-in function\n         ``bytes()`` can be used to construct bytes objects.  Also,\n         bytes objects can be decoded to strings via the ``decode()``\n         method.\n\n   Mutable sequences\n      Mutable sequences can be changed after they are created.  The\n      subscription and slicing notations can be used as the target of\n      assignment and ``del`` (delete) statements.\n\n      There are currently two intrinsic mutable sequence types:\n\n      Lists\n         The items of a list are arbitrary Python objects.  Lists are\n         formed by placing a comma-separated list of expressions in\n         square brackets. (Note that there are no special cases needed\n         to form lists of length 0 or 1.)\n\n      Byte Arrays\n         A bytearray object is a mutable array. They are created by\n         the built-in ``bytearray()`` constructor.  Aside from being\n         mutable (and hence unhashable), byte arrays otherwise provide\n         the same interface and functionality as immutable bytes\n         objects.\n\n      The extension module ``array`` provides an additional example of\n      a mutable sequence type, as does the ``collections`` module.\n\nSet types\n   These represent unordered, finite sets of unique, immutable\n   objects. As such, they cannot be indexed by any subscript. However,\n   they can be iterated over, and the built-in function ``len()``\n   returns the number of items in a set. Common uses for sets are fast\n   membership testing, removing duplicates from a sequence, and\n   computing mathematical operations such as intersection, union,\n   difference, and symmetric difference.\n\n   For set elements, the same immutability rules apply as for\n   dictionary keys. Note that numeric types obey the normal rules for\n   numeric comparison: if two numbers compare equal (e.g., ``1`` and\n   ``1.0``), only one of them can be contained in a set.\n\n   There are currently two intrinsic set types:\n\n   Sets\n      These represent a mutable set. They are created by the built-in\n      ``set()`` constructor and can be modified afterwards by several\n      methods, such as ``add()``.\n\n   Frozen sets\n      These represent an immutable set.  They are created by the\n      built-in ``frozenset()`` constructor.  As a frozenset is\n      immutable and *hashable*, it can be used again as an element of\n      another set, or as a dictionary key.\n\nMappings\n   These represent finite sets of objects indexed by arbitrary index\n   sets. The subscript notation ``a[k]`` selects the item indexed by\n   ``k`` from the mapping ``a``; this can be used in expressions and\n   as the target of assignments or ``del`` statements. The built-in\n   function ``len()`` returns the number of items in a mapping.\n\n   There is currently a single intrinsic mapping type:\n\n   Dictionaries\n      These represent finite sets of objects indexed by nearly\n      arbitrary values.  The only types of values not acceptable as\n      keys are values containing lists or dictionaries or other\n      mutable types that are compared by value rather than by object\n      identity, the reason being that the efficient implementation of\n      dictionaries requires a key\'s hash value to remain constant.\n      Numeric types used for keys obey the normal rules for numeric\n      comparison: if two numbers compare equal (e.g., ``1`` and\n      ``1.0``) then they can be used interchangeably to index the same\n      dictionary entry.\n\n      Dictionaries are mutable; they can be created by the ``{...}``\n      notation (see section *Dictionary displays*).\n\n      The extension modules ``dbm.ndbm`` and ``dbm.gnu`` provide\n      additional examples of mapping types, as does the\n      ``collections`` module.\n\nCallable types\n   These are the types to which the function call operation (see\n   section *Calls*) can be applied:\n\n   User-defined functions\n      A user-defined function object is created by a function\n      definition (see section *Function definitions*).  It should be\n      called with an argument list containing the same number of items\n      as the function\'s formal parameter list.\n\n      Special attributes:\n\n      +---------------------------+---------------------------------+-------------+\n      | Attribute                 | Meaning                         |             |\n      +===========================+=================================+=============+\n      | ``__doc__``               | The function\'s documentation    | Writable    |\n      |                           | string, or ``None`` if          |             |\n      |                           | unavailable                     |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__name__``              | The function\'s name             | Writable    |\n      +---------------------------+---------------------------------+-------------+\n      | ``__module__``            | The name of the module the      | Writable    |\n      |                           | function was defined in, or     |             |\n      |                           | ``None`` if unavailable.        |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__defaults__``          | A tuple containing default      | Writable    |\n      |                           | argument values for those       |             |\n      |                           | arguments that have defaults,   |             |\n      |                           | or ``None`` if no arguments     |             |\n      |                           | have a default value            |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__code__``              | The code object representing    | Writable    |\n      |                           | the compiled function body.     |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__globals__``           | A reference to the dictionary   | Read-only   |\n      |                           | that holds the function\'s       |             |\n      |                           | global variables --- the global |             |\n      |                           | namespace of the module in      |             |\n      |                           | which the function was defined. |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__dict__``              | The namespace supporting        | Writable    |\n      |                           | arbitrary function attributes.  |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__closure__``           | ``None`` or a tuple of cells    | Read-only   |\n      |                           | that contain bindings for the   |             |\n      |                           | function\'s free variables.      |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__annotations__``       | A dict containing annotations   | Writable    |\n      |                           | of parameters.  The keys of the |             |\n      |                           | dict are the parameter names,   |             |\n      |                           | or ``\'return\'`` for the return  |             |\n      |                           | annotation, if provided.        |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__kwdefaults__``        | A dict containing defaults for  | Writable    |\n      |                           | keyword-only parameters.        |             |\n      +---------------------------+---------------------------------+-------------+\n\n      Most of the attributes labelled "Writable" check the type of the\n      assigned value.\n\n      Function objects also support getting and setting arbitrary\n      attributes, which can be used, for example, to attach metadata\n      to functions.  Regular attribute dot-notation is used to get and\n      set such attributes. *Note that the current implementation only\n      supports function attributes on user-defined functions. Function\n      attributes on built-in functions may be supported in the\n      future.*\n\n      Additional information about a function\'s definition can be\n      retrieved from its code object; see the description of internal\n      types below.\n\n   Instance methods\n      An instance method object combines a class, a class instance and\n      any callable object (normally a user-defined function).\n\n      Special read-only attributes: ``__self__`` is the class instance\n      object, ``__func__`` is the function object; ``__doc__`` is the\n      method\'s documentation (same as ``__func__.__doc__``);\n      ``__name__`` is the method name (same as ``__func__.__name__``);\n      ``__module__`` is the name of the module the method was defined\n      in, or ``None`` if unavailable.\n\n      Methods also support accessing (but not setting) the arbitrary\n      function attributes on the underlying function object.\n\n      User-defined method objects may be created when getting an\n      attribute of a class (perhaps via an instance of that class), if\n      that attribute is a user-defined function object or a class\n      method object.\n\n      When an instance method object is created by retrieving a user-\n      defined function object from a class via one of its instances,\n      its ``__self__`` attribute is the instance, and the method\n      object is said to be bound.  The new method\'s ``__func__``\n      attribute is the original function object.\n\n      When a user-defined method object is created by retrieving\n      another method object from a class or instance, the behaviour is\n      the same as for a function object, except that the ``__func__``\n      attribute of the new instance is not the original method object\n      but its ``__func__`` attribute.\n\n      When an instance method object is created by retrieving a class\n      method object from a class or instance, its ``__self__``\n      attribute is the class itself, and its ``__func__`` attribute is\n      the function object underlying the class method.\n\n      When an instance method object is called, the underlying\n      function (``__func__``) is called, inserting the class instance\n      (``__self__``) in front of the argument list.  For instance,\n      when ``C`` is a class which contains a definition for a function\n      ``f()``, and ``x`` is an instance of ``C``, calling ``x.f(1)``\n      is equivalent to calling ``C.f(x, 1)``.\n\n      When an instance method object is derived from a class method\n      object, the "class instance" stored in ``__self__`` will\n      actually be the class itself, so that calling either ``x.f(1)``\n      or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n      the underlying function.\n\n      Note that the transformation from function object to instance\n      method object happens each time the attribute is retrieved from\n      the instance.  In some cases, a fruitful optimization is to\n      assign the attribute to a local variable and call that local\n      variable. Also notice that this transformation only happens for\n      user-defined functions; other callable objects (and all non-\n      callable objects) are retrieved without transformation.  It is\n      also important to note that user-defined functions which are\n      attributes of a class instance are not converted to bound\n      methods; this *only* happens when the function is an attribute\n      of the class.\n\n   Generator functions\n      A function or method which uses the ``yield`` statement (see\n      section *The yield statement*) is called a *generator function*.\n      Such a function, when called, always returns an iterator object\n      which can be used to execute the body of the function:  calling\n      the iterator\'s ``__next__()`` method will cause the function to\n      execute until it provides a value using the ``yield`` statement.\n      When the function executes a ``return`` statement or falls off\n      the end, a ``StopIteration`` exception is raised and the\n      iterator will have reached the end of the set of values to be\n      returned.\n\n   Built-in functions\n      A built-in function object is a wrapper around a C function.\n      Examples of built-in functions are ``len()`` and ``math.sin()``\n      (``math`` is a standard built-in module). The number and type of\n      the arguments are determined by the C function. Special read-\n      only attributes: ``__doc__`` is the function\'s documentation\n      string, or ``None`` if unavailable; ``__name__`` is the\n      function\'s name; ``__self__`` is set to ``None`` (but see the\n      next item); ``__module__`` is the name of the module the\n      function was defined in or ``None`` if unavailable.\n\n   Built-in methods\n      This is really a different disguise of a built-in function, this\n      time containing an object passed to the C function as an\n      implicit extra argument.  An example of a built-in method is\n      ``alist.append()``, assuming *alist* is a list object. In this\n      case, the special read-only attribute ``__self__`` is set to the\n      object denoted by *alist*.\n\n   Classes\n      Classes are callable.  These objects normally act as factories\n      for new instances of themselves, but variations are possible for\n      class types that override ``__new__()``.  The arguments of the\n      call are passed to ``__new__()`` and, in the typical case, to\n      ``__init__()`` to initialize the new instance.\n\n   Class Instances\n      Instances of arbitrary classes can be made callable by defining\n      a ``__call__()`` method in their class.\n\nModules\n   Modules are imported by the ``import`` statement (see section *The\n   import statement*). A module object has a namespace implemented by\n   a dictionary object (this is the dictionary referenced by the\n   __globals__ attribute of functions defined in the module).\n   Attribute references are translated to lookups in this dictionary,\n   e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n   does not contain the code object used to initialize the module\n   (since it isn\'t needed once the initialization is done).\n\n   Attribute assignment updates the module\'s namespace dictionary,\n   e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n   Special read-only attribute: ``__dict__`` is the module\'s namespace\n   as a dictionary object.\n\n   **CPython implementation detail:** Because of the way CPython\n   clears module dictionaries, the module dictionary will be cleared\n   when the module falls out of scope even if the dictionary still has\n   live references.  To avoid this, copy the dictionary or keep the\n   module around while using its dictionary directly.\n\n   Predefined (writable) attributes: ``__name__`` is the module\'s\n   name; ``__doc__`` is the module\'s documentation string, or ``None``\n   if unavailable; ``__file__`` is the pathname of the file from which\n   the module was loaded, if it was loaded from a file. The\n   ``__file__`` attribute is not present for C modules that are\n   statically linked into the interpreter; for extension modules\n   loaded dynamically from a shared library, it is the pathname of the\n   shared library file.\n\nCustom classes\n   Custom class types are typically created by class definitions (see\n   section *Class definitions*).  A class has a namespace implemented\n   by a dictionary object. Class attribute references are translated\n   to lookups in this dictionary, e.g., ``C.x`` is translated to\n   ``C.__dict__["x"]`` (although there are a number of hooks which\n   allow for other means of locating attributes). When the attribute\n   name is not found there, the attribute search continues in the base\n   classes. This search of the base classes uses the C3 method\n   resolution order which behaves correctly even in the presence of\n   \'diamond\' inheritance structures where there are multiple\n   inheritance paths leading back to a common ancestor. Additional\n   details on the C3 MRO used by Python can be found in the\n   documentation accompanying the 2.3 release at\n   http://www.python.org/download/releases/2.3/mro/.\n\n   When a class attribute reference (for class ``C``, say) would yield\n   a class method object, it is transformed into an instance method\n   object whose ``__self__`` attributes is ``C``.  When it would yield\n   a static method object, it is transformed into the object wrapped\n   by the static method object. See section *Implementing Descriptors*\n   for another way in which attributes retrieved from a class may\n   differ from those actually contained in its ``__dict__``.\n\n   Class attribute assignments update the class\'s dictionary, never\n   the dictionary of a base class.\n\n   A class object can be called (see above) to yield a class instance\n   (see below).\n\n   Special attributes: ``__name__`` is the class name; ``__module__``\n   is the module name in which the class was defined; ``__dict__`` is\n   the dictionary containing the class\'s namespace; ``__bases__`` is a\n   tuple (possibly empty or a singleton) containing the base classes,\n   in the order of their occurrence in the base class list;\n   ``__doc__`` is the class\'s documentation string, or None if\n   undefined.\n\nClass instances\n   A class instance is created by calling a class object (see above).\n   A class instance has a namespace implemented as a dictionary which\n   is the first place in which attribute references are searched.\n   When an attribute is not found there, and the instance\'s class has\n   an attribute by that name, the search continues with the class\n   attributes.  If a class attribute is found that is a user-defined\n   function object, it is transformed into an instance method object\n   whose ``__self__`` attribute is the instance.  Static method and\n   class method objects are also transformed; see above under\n   "Classes".  See section *Implementing Descriptors* for another way\n   in which attributes of a class retrieved via its instances may\n   differ from the objects actually stored in the class\'s\n   ``__dict__``.  If no class attribute is found, and the object\'s\n   class has a ``__getattr__()`` method, that is called to satisfy the\n   lookup.\n\n   Attribute assignments and deletions update the instance\'s\n   dictionary, never a class\'s dictionary.  If the class has a\n   ``__setattr__()`` or ``__delattr__()`` method, this is called\n   instead of updating the instance dictionary directly.\n\n   Class instances can pretend to be numbers, sequences, or mappings\n   if they have methods with certain special names.  See section\n   *Special method names*.\n\n   Special attributes: ``__dict__`` is the attribute dictionary;\n   ``__class__`` is the instance\'s class.\n\nI/O objects (also known as file objects)\n   A *file object* represents an open file.  Various shortcuts are\n   available to create file objects: the ``open()`` built-in function,\n   and also ``os.popen()``, ``os.fdopen()``, and the ``makefile()``\n   method of socket objects (and perhaps by other functions or methods\n   provided by extension modules).\n\n   The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are\n   initialized to file objects corresponding to the interpreter\'s\n   standard input, output and error streams; they are all open in text\n   mode and therefore follow the interface defined by the\n   ``io.TextIOBase`` abstract class.\n\nInternal types\n   A few types used internally by the interpreter are exposed to the\n   user. Their definitions may change with future versions of the\n   interpreter, but they are mentioned here for completeness.\n\n   Code objects\n      Code objects represent *byte-compiled* executable Python code,\n      or *bytecode*. The difference between a code object and a\n      function object is that the function object contains an explicit\n      reference to the function\'s globals (the module in which it was\n      defined), while a code object contains no context; also the\n      default argument values are stored in the function object, not\n      in the code object (because they represent values calculated at\n      run-time).  Unlike function objects, code objects are immutable\n      and contain no references (directly or indirectly) to mutable\n      objects.\n\n      Special read-only attributes: ``co_name`` gives the function\n      name; ``co_argcount`` is the number of positional arguments\n      (including arguments with default values); ``co_nlocals`` is the\n      number of local variables used by the function (including\n      arguments); ``co_varnames`` is a tuple containing the names of\n      the local variables (starting with the argument names);\n      ``co_cellvars`` is a tuple containing the names of local\n      variables that are referenced by nested functions;\n      ``co_freevars`` is a tuple containing the names of free\n      variables; ``co_code`` is a string representing the sequence of\n      bytecode instructions; ``co_consts`` is a tuple containing the\n      literals used by the bytecode; ``co_names`` is a tuple\n      containing the names used by the bytecode; ``co_filename`` is\n      the filename from which the code was compiled;\n      ``co_firstlineno`` is the first line number of the function;\n      ``co_lnotab`` is a string encoding the mapping from bytecode\n      offsets to line numbers (for details see the source code of the\n      interpreter); ``co_stacksize`` is the required stack size\n      (including local variables); ``co_flags`` is an integer encoding\n      a number of flags for the interpreter.\n\n      The following flag bits are defined for ``co_flags``: bit\n      ``0x04`` is set if the function uses the ``*arguments`` syntax\n      to accept an arbitrary number of positional arguments; bit\n      ``0x08`` is set if the function uses the ``**keywords`` syntax\n      to accept arbitrary keyword arguments; bit ``0x20`` is set if\n      the function is a generator.\n\n      Future feature declarations (``from __future__ import\n      division``) also use bits in ``co_flags`` to indicate whether a\n      code object was compiled with a particular feature enabled: bit\n      ``0x2000`` is set if the function was compiled with future\n      division enabled; bits ``0x10`` and ``0x1000`` were used in\n      earlier versions of Python.\n\n      Other bits in ``co_flags`` are reserved for internal use.\n\n      If a code object represents a function, the first item in\n      ``co_consts`` is the documentation string of the function, or\n      ``None`` if undefined.\n\n   Frame objects\n      Frame objects represent execution frames.  They may occur in\n      traceback objects (see below).\n\n      Special read-only attributes: ``f_back`` is to the previous\n      stack frame (towards the caller), or ``None`` if this is the\n      bottom stack frame; ``f_code`` is the code object being executed\n      in this frame; ``f_locals`` is the dictionary used to look up\n      local variables; ``f_globals`` is used for global variables;\n      ``f_builtins`` is used for built-in (intrinsic) names;\n      ``f_lasti`` gives the precise instruction (this is an index into\n      the bytecode string of the code object).\n\n      Special writable attributes: ``f_trace``, if not ``None``, is a\n      function called at the start of each source code line (this is\n      used by the debugger); ``f_lineno`` is the current line number\n      of the frame --- writing to this from within a trace function\n      jumps to the given line (only for the bottom-most frame).  A\n      debugger can implement a Jump command (aka Set Next Statement)\n      by writing to f_lineno.\n\n   Traceback objects\n      Traceback objects represent a stack trace of an exception.  A\n      traceback object is created when an exception occurs.  When the\n      search for an exception handler unwinds the execution stack, at\n      each unwound level a traceback object is inserted in front of\n      the current traceback.  When an exception handler is entered,\n      the stack trace is made available to the program. (See section\n      *The try statement*.) It is accessible as the third item of the\n      tuple returned by ``sys.exc_info()``. When the program contains\n      no suitable handler, the stack trace is written (nicely\n      formatted) to the standard error stream; if the interpreter is\n      interactive, it is also made available to the user as\n      ``sys.last_traceback``.\n\n      Special read-only attributes: ``tb_next`` is the next level in\n      the stack trace (towards the frame where the exception\n      occurred), or ``None`` if there is no next level; ``tb_frame``\n      points to the execution frame of the current level;\n      ``tb_lineno`` gives the line number where the exception\n      occurred; ``tb_lasti`` indicates the precise instruction.  The\n      line number and last instruction in the traceback may differ\n      from the line number of its frame object if the exception\n      occurred in a ``try`` statement with no matching except clause\n      or with a finally clause.\n\n   Slice objects\n      Slice objects are used to represent slices for ``__getitem__()``\n      methods.  They are also created by the built-in ``slice()``\n      function.\n\n      Special read-only attributes: ``start`` is the lower bound;\n      ``stop`` is the upper bound; ``step`` is the step value; each is\n      ``None`` if omitted. These attributes can have any type.\n\n      Slice objects support one method:\n\n      slice.indices(self, length)\n\n         This method takes a single integer argument *length* and\n         computes information about the slice that the slice object\n         would describe if applied to a sequence of *length* items.\n         It returns a tuple of three integers; respectively these are\n         the *start* and *stop* indices and the *step* or stride\n         length of the slice. Missing or out-of-bounds indices are\n         handled in a manner consistent with regular slices.\n\n   Static method objects\n      Static method objects provide a way of defeating the\n      transformation of function objects to method objects described\n      above. A static method object is a wrapper around any other\n      object, usually a user-defined method object. When a static\n      method object is retrieved from a class or a class instance, the\n      object actually returned is the wrapped object, which is not\n      subject to any further transformation. Static method objects are\n      not themselves callable, although the objects they wrap usually\n      are. Static method objects are created by the built-in\n      ``staticmethod()`` constructor.\n\n   Class method objects\n      A class method object, like a static method object, is a wrapper\n      around another object that alters the way in which that object\n      is retrieved from classes and class instances. The behaviour of\n      class method objects upon such retrieval is described above,\n      under "User-defined methods". Class method objects are created\n      by the built-in ``classmethod()`` constructor.\n',
+ 'try': '\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n   try_stmt  ::= try1_stmt | try2_stmt\n   try1_stmt ::= "try" ":" suite\n                 ("except" [expression ["as" target]] ":" suite)+\n                 ["else" ":" suite]\n                 ["finally" ":" suite]\n   try2_stmt ::= "try" ":" suite\n                 "finally" ":" suite\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started.  This search inspects the except\nclauses in turn until one is found that matches the exception.  An\nexpression-less except clause, if present, must be last; it matches\nany exception.  For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception.  An object is\ncompatible with an exception if it is the class or a base class of the\nexception object or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the ``as`` keyword in that except clause,\nif present, and the except clause\'s suite is executed.  All except\nclauses must have an executable block.  When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using ``as target``, it is cleared\nat the end of the except clause.  This is as if\n\n   except E as N:\n       foo\n\nwas translated to\n\n   except E as N:\n       try:\n           foo\n       finally:\n           del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause.  Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the ``sys`` module and can be access via\n``sys.exc_info()``. ``sys.exc_info()`` returns a 3-tuple consisting of\nthe exception class, the exception instance and a traceback object\n(see section *The standard type hierarchy*) identifying the point in\nthe program where the exception occurred.  ``sys.exc_info()`` values\nare restored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler.  The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses.  If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted.  If there is a saved exception or ``break`` statement, it is\nre-raised at the end of the ``finally`` clause. If the ``finally``\nclause raises another exception the saved exception is set as the\ncontext of the new exception; if the ``finally`` clause executes a\n``return`` statement, the saved exception is discarded:\n\n   def f():\n       try:\n           1/0\n       finally:\n           return 42\n\n   >>> f()\n   42\n\nThe exception information is not available to the program during\nexecution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n',
+ 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python.  Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types.  Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\'  These are attributes that provide access to the\nimplementation and are not intended for general use.  Their definition\nmay change in the future.\n\nNone\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name ``None``.\n   It is used to signify the absence of a value in many situations,\n   e.g., it is returned from functions that don\'t explicitly return\n   anything. Its truth value is false.\n\nNotImplemented\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the built-in name\n   ``NotImplemented``. Numeric methods and rich comparison methods may\n   return this value if they do not implement the operation for the\n   operands provided.  (The interpreter will then try the reflected\n   operation, or some other fallback, depending on the operator.)  Its\n   truth value is true.\n\nEllipsis\n   This type has a single value.  There is a single object with this\n   value. This object is accessed through the literal ``...`` or the\n   built-in name ``Ellipsis``.  Its truth value is true.\n\n``numbers.Number``\n   These are created by numeric literals and returned as results by\n   arithmetic operators and arithmetic built-in functions.  Numeric\n   objects are immutable; once created their value never changes.\n   Python numbers are of course strongly related to mathematical\n   numbers, but subject to the limitations of numerical representation\n   in computers.\n\n   Python distinguishes between integers, floating point numbers, and\n   complex numbers:\n\n   ``numbers.Integral``\n      These represent elements from the mathematical set of integers\n      (positive and negative).\n\n      There are two types of integers:\n\n      Integers (``int``)\n\n         These represent numbers in an unlimited range, subject to\n         available (virtual) memory only.  For the purpose of shift\n         and mask operations, a binary representation is assumed, and\n         negative numbers are represented in a variant of 2\'s\n         complement which gives the illusion of an infinite string of\n         sign bits extending to the left.\n\n      Booleans (``bool``)\n         These represent the truth values False and True.  The two\n         objects representing the values False and True are the only\n         Boolean objects. The Boolean type is a subtype of the integer\n         type, and Boolean values behave like the values 0 and 1,\n         respectively, in almost all contexts, the exception being\n         that when converted to a string, the strings ``"False"`` or\n         ``"True"`` are returned, respectively.\n\n      The rules for integer representation are intended to give the\n      most meaningful interpretation of shift and mask operations\n      involving negative integers.\n\n   ``numbers.Real`` (``float``)\n      These represent machine-level double precision floating point\n      numbers. You are at the mercy of the underlying machine\n      architecture (and C or Java implementation) for the accepted\n      range and handling of overflow. Python does not support single-\n      precision floating point numbers; the savings in processor and\n      memory usage that are usually the reason for using these is\n      dwarfed by the overhead of using objects in Python, so there is\n      no reason to complicate the language with two kinds of floating\n      point numbers.\n\n   ``numbers.Complex`` (``complex``)\n      These represent complex numbers as a pair of machine-level\n      double precision floating point numbers.  The same caveats apply\n      as for floating point numbers. The real and imaginary parts of a\n      complex number ``z`` can be retrieved through the read-only\n      attributes ``z.real`` and ``z.imag``.\n\nSequences\n   These represent finite ordered sets indexed by non-negative\n   numbers. The built-in function ``len()`` returns the number of\n   items of a sequence. When the length of a sequence is *n*, the\n   index set contains the numbers 0, 1, ..., *n*-1.  Item *i* of\n   sequence *a* is selected by ``a[i]``.\n\n   Sequences also support slicing: ``a[i:j]`` selects all items with\n   index *k* such that *i* ``<=`` *k* ``<`` *j*.  When used as an\n   expression, a slice is a sequence of the same type.  This implies\n   that the index set is renumbered so that it starts at 0.\n\n   Some sequences also support "extended slicing" with a third "step"\n   parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n   where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n   *j*.\n\n   Sequences are distinguished according to their mutability:\n\n   Immutable sequences\n      An object of an immutable sequence type cannot change once it is\n      created.  (If the object contains references to other objects,\n      these other objects may be mutable and may be changed; however,\n      the collection of objects directly referenced by an immutable\n      object cannot change.)\n\n      The following types are immutable sequences:\n\n      Strings\n         A string is a sequence of values that represent Unicode\n         codepoints. All the codepoints in range ``U+0000 - U+10FFFF``\n         can be represented in a string.  Python doesn\'t have a\n         ``chr`` type, and every character in the string is\n         represented as a string object with length ``1``.  The built-\n         in function ``ord()`` converts a character to its codepoint\n         (as an integer); ``chr()`` converts an integer in range ``0 -\n         10FFFF`` to the corresponding character. ``str.encode()`` can\n         be used to convert a ``str`` to ``bytes`` using the given\n         encoding, and ``bytes.decode()`` can be used to achieve the\n         opposite.\n\n      Tuples\n         The items of a tuple are arbitrary Python objects. Tuples of\n         two or more items are formed by comma-separated lists of\n         expressions.  A tuple of one item (a \'singleton\') can be\n         formed by affixing a comma to an expression (an expression by\n         itself does not create a tuple, since parentheses must be\n         usable for grouping of expressions).  An empty tuple can be\n         formed by an empty pair of parentheses.\n\n      Bytes\n         A bytes object is an immutable array.  The items are 8-bit\n         bytes, represented by integers in the range 0 <= x < 256.\n         Bytes literals (like ``b\'abc\'`` and the built-in function\n         ``bytes()`` can be used to construct bytes objects.  Also,\n         bytes objects can be decoded to strings via the ``decode()``\n         method.\n\n   Mutable sequences\n      Mutable sequences can be changed after they are created.  The\n      subscription and slicing notations can be used as the target of\n      assignment and ``del`` (delete) statements.\n\n      There are currently two intrinsic mutable sequence types:\n\n      Lists\n         The items of a list are arbitrary Python objects.  Lists are\n         formed by placing a comma-separated list of expressions in\n         square brackets. (Note that there are no special cases needed\n         to form lists of length 0 or 1.)\n\n      Byte Arrays\n         A bytearray object is a mutable array. They are created by\n         the built-in ``bytearray()`` constructor.  Aside from being\n         mutable (and hence unhashable), byte arrays otherwise provide\n         the same interface and functionality as immutable bytes\n         objects.\n\n      The extension module ``array`` provides an additional example of\n      a mutable sequence type, as does the ``collections`` module.\n\nSet types\n   These represent unordered, finite sets of unique, immutable\n   objects. As such, they cannot be indexed by any subscript. However,\n   they can be iterated over, and the built-in function ``len()``\n   returns the number of items in a set. Common uses for sets are fast\n   membership testing, removing duplicates from a sequence, and\n   computing mathematical operations such as intersection, union,\n   difference, and symmetric difference.\n\n   For set elements, the same immutability rules apply as for\n   dictionary keys. Note that numeric types obey the normal rules for\n   numeric comparison: if two numbers compare equal (e.g., ``1`` and\n   ``1.0``), only one of them can be contained in a set.\n\n   There are currently two intrinsic set types:\n\n   Sets\n      These represent a mutable set. They are created by the built-in\n      ``set()`` constructor and can be modified afterwards by several\n      methods, such as ``add()``.\n\n   Frozen sets\n      These represent an immutable set.  They are created by the\n      built-in ``frozenset()`` constructor.  As a frozenset is\n      immutable and *hashable*, it can be used again as an element of\n      another set, or as a dictionary key.\n\nMappings\n   These represent finite sets of objects indexed by arbitrary index\n   sets. The subscript notation ``a[k]`` selects the item indexed by\n   ``k`` from the mapping ``a``; this can be used in expressions and\n   as the target of assignments or ``del`` statements. The built-in\n   function ``len()`` returns the number of items in a mapping.\n\n   There is currently a single intrinsic mapping type:\n\n   Dictionaries\n      These represent finite sets of objects indexed by nearly\n      arbitrary values.  The only types of values not acceptable as\n      keys are values containing lists or dictionaries or other\n      mutable types that are compared by value rather than by object\n      identity, the reason being that the efficient implementation of\n      dictionaries requires a key\'s hash value to remain constant.\n      Numeric types used for keys obey the normal rules for numeric\n      comparison: if two numbers compare equal (e.g., ``1`` and\n      ``1.0``) then they can be used interchangeably to index the same\n      dictionary entry.\n\n      Dictionaries are mutable; they can be created by the ``{...}``\n      notation (see section *Dictionary displays*).\n\n      The extension modules ``dbm.ndbm`` and ``dbm.gnu`` provide\n      additional examples of mapping types, as does the\n      ``collections`` module.\n\nCallable types\n   These are the types to which the function call operation (see\n   section *Calls*) can be applied:\n\n   User-defined functions\n      A user-defined function object is created by a function\n      definition (see section *Function definitions*).  It should be\n      called with an argument list containing the same number of items\n      as the function\'s formal parameter list.\n\n      Special attributes:\n\n      +---------------------------+---------------------------------+-------------+\n      | Attribute                 | Meaning                         |             |\n      +===========================+=================================+=============+\n      | ``__doc__``               | The function\'s documentation    | Writable    |\n      |                           | string, or ``None`` if          |             |\n      |                           | unavailable                     |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__name__``              | The function\'s name             | Writable    |\n      +---------------------------+---------------------------------+-------------+\n      | ``__qualname__``          | The function\'s *qualified name* | Writable    |\n      |                           | New in version 3.3.             |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__module__``            | The name of the module the      | Writable    |\n      |                           | function was defined in, or     |             |\n      |                           | ``None`` if unavailable.        |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__defaults__``          | A tuple containing default      | Writable    |\n      |                           | argument values for those       |             |\n      |                           | arguments that have defaults,   |             |\n      |                           | or ``None`` if no arguments     |             |\n      |                           | have a default value            |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__code__``              | The code object representing    | Writable    |\n      |                           | the compiled function body.     |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__globals__``           | A reference to the dictionary   | Read-only   |\n      |                           | that holds the function\'s       |             |\n      |                           | global variables --- the global |             |\n      |                           | namespace of the module in      |             |\n      |                           | which the function was defined. |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__dict__``              | The namespace supporting        | Writable    |\n      |                           | arbitrary function attributes.  |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__closure__``           | ``None`` or a tuple of cells    | Read-only   |\n      |                           | that contain bindings for the   |             |\n      |                           | function\'s free variables.      |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__annotations__``       | A dict containing annotations   | Writable    |\n      |                           | of parameters.  The keys of the |             |\n      |                           | dict are the parameter names,   |             |\n      |                           | or ``\'return\'`` for the return  |             |\n      |                           | annotation, if provided.        |             |\n      +---------------------------+---------------------------------+-------------+\n      | ``__kwdefaults__``        | A dict containing defaults for  | Writable    |\n      |                           | keyword-only parameters.        |             |\n      +---------------------------+---------------------------------+-------------+\n\n      Most of the attributes labelled "Writable" check the type of the\n      assigned value.\n\n      Function objects also support getting and setting arbitrary\n      attributes, which can be used, for example, to attach metadata\n      to functions.  Regular attribute dot-notation is used to get and\n      set such attributes. *Note that the current implementation only\n      supports function attributes on user-defined functions. Function\n      attributes on built-in functions may be supported in the\n      future.*\n\n      Additional information about a function\'s definition can be\n      retrieved from its code object; see the description of internal\n      types below.\n\n   Instance methods\n      An instance method object combines a class, a class instance and\n      any callable object (normally a user-defined function).\n\n      Special read-only attributes: ``__self__`` is the class instance\n      object, ``__func__`` is the function object; ``__doc__`` is the\n      method\'s documentation (same as ``__func__.__doc__``);\n      ``__name__`` is the method name (same as ``__func__.__name__``);\n      ``__module__`` is the name of the module the method was defined\n      in, or ``None`` if unavailable.\n\n      Methods also support accessing (but not setting) the arbitrary\n      function attributes on the underlying function object.\n\n      User-defined method objects may be created when getting an\n      attribute of a class (perhaps via an instance of that class), if\n      that attribute is a user-defined function object or a class\n      method object.\n\n      When an instance method object is created by retrieving a user-\n      defined function object from a class via one of its instances,\n      its ``__self__`` attribute is the instance, and the method\n      object is said to be bound.  The new method\'s ``__func__``\n      attribute is the original function object.\n\n      When a user-defined method object is created by retrieving\n      another method object from a class or instance, the behaviour is\n      the same as for a function object, except that the ``__func__``\n      attribute of the new instance is not the original method object\n      but its ``__func__`` attribute.\n\n      When an instance method object is created by retrieving a class\n      method object from a class or instance, its ``__self__``\n      attribute is the class itself, and its ``__func__`` attribute is\n      the function object underlying the class method.\n\n      When an instance method object is called, the underlying\n      function (``__func__``) is called, inserting the class instance\n      (``__self__``) in front of the argument list.  For instance,\n      when ``C`` is a class which contains a definition for a function\n      ``f()``, and ``x`` is an instance of ``C``, calling ``x.f(1)``\n      is equivalent to calling ``C.f(x, 1)``.\n\n      When an instance method object is derived from a class method\n      object, the "class instance" stored in ``__self__`` will\n      actually be the class itself, so that calling either ``x.f(1)``\n      or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n      the underlying function.\n\n      Note that the transformation from function object to instance\n      method object happens each time the attribute is retrieved from\n      the instance.  In some cases, a fruitful optimization is to\n      assign the attribute to a local variable and call that local\n      variable. Also notice that this transformation only happens for\n      user-defined functions; other callable objects (and all non-\n      callable objects) are retrieved without transformation.  It is\n      also important to note that user-defined functions which are\n      attributes of a class instance are not converted to bound\n      methods; this *only* happens when the function is an attribute\n      of the class.\n\n   Generator functions\n      A function or method which uses the ``yield`` statement (see\n      section *The yield statement*) is called a *generator function*.\n      Such a function, when called, always returns an iterator object\n      which can be used to execute the body of the function:  calling\n      the iterator\'s ``__next__()`` method will cause the function to\n      execute until it provides a value using the ``yield`` statement.\n      When the function executes a ``return`` statement or falls off\n      the end, a ``StopIteration`` exception is raised and the\n      iterator will have reached the end of the set of values to be\n      returned.\n\n   Built-in functions\n      A built-in function object is a wrapper around a C function.\n      Examples of built-in functions are ``len()`` and ``math.sin()``\n      (``math`` is a standard built-in module). The number and type of\n      the arguments are determined by the C function. Special read-\n      only attributes: ``__doc__`` is the function\'s documentation\n      string, or ``None`` if unavailable; ``__name__`` is the\n      function\'s name; ``__self__`` is set to ``None`` (but see the\n      next item); ``__module__`` is the name of the module the\n      function was defined in or ``None`` if unavailable.\n\n   Built-in methods\n      This is really a different disguise of a built-in function, this\n      time containing an object passed to the C function as an\n      implicit extra argument.  An example of a built-in method is\n      ``alist.append()``, assuming *alist* is a list object. In this\n      case, the special read-only attribute ``__self__`` is set to the\n      object denoted by *alist*.\n\n   Classes\n      Classes are callable.  These objects normally act as factories\n      for new instances of themselves, but variations are possible for\n      class types that override ``__new__()``.  The arguments of the\n      call are passed to ``__new__()`` and, in the typical case, to\n      ``__init__()`` to initialize the new instance.\n\n   Class Instances\n      Instances of arbitrary classes can be made callable by defining\n      a ``__call__()`` method in their class.\n\nModules\n   Modules are a basic organizational unit of Python code, and are\n   created by the *import system* as invoked either by the ``import``\n   statement (see ``import``), or by calling functions such as\n   ``importlib.import_module()`` and built-in ``__import__()``.  A\n   module object has a namespace implemented by a dictionary object\n   (this is the dictionary referenced by the ``__globals__`` attribute\n   of functions defined in the module).  Attribute references are\n   translated to lookups in this dictionary, e.g., ``m.x`` is\n   equivalent to ``m.__dict__["x"]``. A module object does not contain\n   the code object used to initialize the module (since it isn\'t\n   needed once the initialization is done).\n\n   Attribute assignment updates the module\'s namespace dictionary,\n   e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n   Special read-only attribute: ``__dict__`` is the module\'s namespace\n   as a dictionary object.\n\n   **CPython implementation detail:** Because of the way CPython\n   clears module dictionaries, the module dictionary will be cleared\n   when the module falls out of scope even if the dictionary still has\n   live references.  To avoid this, copy the dictionary or keep the\n   module around while using its dictionary directly.\n\n   Predefined (writable) attributes: ``__name__`` is the module\'s\n   name; ``__doc__`` is the module\'s documentation string, or ``None``\n   if unavailable; ``__file__`` is the pathname of the file from which\n   the module was loaded, if it was loaded from a file. The\n   ``__file__`` attribute may be missing for certain types of modules,\n   such as C modules that are statically linked into the interpreter;\n   for extension modules loaded dynamically from a shared library, it\n   is the pathname of the shared library file.\n\nCustom classes\n   Custom class types are typically created by class definitions (see\n   section *Class definitions*).  A class has a namespace implemented\n   by a dictionary object. Class attribute references are translated\n   to lookups in this dictionary, e.g., ``C.x`` is translated to\n   ``C.__dict__["x"]`` (although there are a number of hooks which\n   allow for other means of locating attributes). When the attribute\n   name is not found there, the attribute search continues in the base\n   classes. This search of the base classes uses the C3 method\n   resolution order which behaves correctly even in the presence of\n   \'diamond\' inheritance structures where there are multiple\n   inheritance paths leading back to a common ancestor. Additional\n   details on the C3 MRO used by Python can be found in the\n   documentation accompanying the 2.3 release at\n   http://www.python.org/download/releases/2.3/mro/.\n\n   When a class attribute reference (for class ``C``, say) would yield\n   a class method object, it is transformed into an instance method\n   object whose ``__self__`` attributes is ``C``.  When it would yield\n   a static method object, it is transformed into the object wrapped\n   by the static method object. See section *Implementing Descriptors*\n   for another way in which attributes retrieved from a class may\n   differ from those actually contained in its ``__dict__``.\n\n   Class attribute assignments update the class\'s dictionary, never\n   the dictionary of a base class.\n\n   A class object can be called (see above) to yield a class instance\n   (see below).\n\n   Special attributes: ``__name__`` is the class name; ``__module__``\n   is the module name in which the class was defined; ``__dict__`` is\n   the dictionary containing the class\'s namespace; ``__bases__`` is a\n   tuple (possibly empty or a singleton) containing the base classes,\n   in the order of their occurrence in the base class list;\n   ``__doc__`` is the class\'s documentation string, or None if\n   undefined.\n\nClass instances\n   A class instance is created by calling a class object (see above).\n   A class instance has a namespace implemented as a dictionary which\n   is the first place in which attribute references are searched.\n   When an attribute is not found there, and the instance\'s class has\n   an attribute by that name, the search continues with the class\n   attributes.  If a class attribute is found that is a user-defined\n   function object, it is transformed into an instance method object\n   whose ``__self__`` attribute is the instance.  Static method and\n   class method objects are also transformed; see above under\n   "Classes".  See section *Implementing Descriptors* for another way\n   in which attributes of a class retrieved via its instances may\n   differ from the objects actually stored in the class\'s\n   ``__dict__``.  If no class attribute is found, and the object\'s\n   class has a ``__getattr__()`` method, that is called to satisfy the\n   lookup.\n\n   Attribute assignments and deletions update the instance\'s\n   dictionary, never a class\'s dictionary.  If the class has a\n   ``__setattr__()`` or ``__delattr__()`` method, this is called\n   instead of updating the instance dictionary directly.\n\n   Class instances can pretend to be numbers, sequences, or mappings\n   if they have methods with certain special names.  See section\n   *Special method names*.\n\n   Special attributes: ``__dict__`` is the attribute dictionary;\n   ``__class__`` is the instance\'s class.\n\nI/O objects (also known as file objects)\n   A *file object* represents an open file.  Various shortcuts are\n   available to create file objects: the ``open()`` built-in function,\n   and also ``os.popen()``, ``os.fdopen()``, and the ``makefile()``\n   method of socket objects (and perhaps by other functions or methods\n   provided by extension modules).\n\n   The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are\n   initialized to file objects corresponding to the interpreter\'s\n   standard input, output and error streams; they are all open in text\n   mode and therefore follow the interface defined by the\n   ``io.TextIOBase`` abstract class.\n\nInternal types\n   A few types used internally by the interpreter are exposed to the\n   user. Their definitions may change with future versions of the\n   interpreter, but they are mentioned here for completeness.\n\n   Code objects\n      Code objects represent *byte-compiled* executable Python code,\n      or *bytecode*. The difference between a code object and a\n      function object is that the function object contains an explicit\n      reference to the function\'s globals (the module in which it was\n      defined), while a code object contains no context; also the\n      default argument values are stored in the function object, not\n      in the code object (because they represent values calculated at\n      run-time).  Unlike function objects, code objects are immutable\n      and contain no references (directly or indirectly) to mutable\n      objects.\n\n      Special read-only attributes: ``co_name`` gives the function\n      name; ``co_argcount`` is the number of positional arguments\n      (including arguments with default values); ``co_nlocals`` is the\n      number of local variables used by the function (including\n      arguments); ``co_varnames`` is a tuple containing the names of\n      the local variables (starting with the argument names);\n      ``co_cellvars`` is a tuple containing the names of local\n      variables that are referenced by nested functions;\n      ``co_freevars`` is a tuple containing the names of free\n      variables; ``co_code`` is a string representing the sequence of\n      bytecode instructions; ``co_consts`` is a tuple containing the\n      literals used by the bytecode; ``co_names`` is a tuple\n      containing the names used by the bytecode; ``co_filename`` is\n      the filename from which the code was compiled;\n      ``co_firstlineno`` is the first line number of the function;\n      ``co_lnotab`` is a string encoding the mapping from bytecode\n      offsets to line numbers (for details see the source code of the\n      interpreter); ``co_stacksize`` is the required stack size\n      (including local variables); ``co_flags`` is an integer encoding\n      a number of flags for the interpreter.\n\n      The following flag bits are defined for ``co_flags``: bit\n      ``0x04`` is set if the function uses the ``*arguments`` syntax\n      to accept an arbitrary number of positional arguments; bit\n      ``0x08`` is set if the function uses the ``**keywords`` syntax\n      to accept arbitrary keyword arguments; bit ``0x20`` is set if\n      the function is a generator.\n\n      Future feature declarations (``from __future__ import\n      division``) also use bits in ``co_flags`` to indicate whether a\n      code object was compiled with a particular feature enabled: bit\n      ``0x2000`` is set if the function was compiled with future\n      division enabled; bits ``0x10`` and ``0x1000`` were used in\n      earlier versions of Python.\n\n      Other bits in ``co_flags`` are reserved for internal use.\n\n      If a code object represents a function, the first item in\n      ``co_consts`` is the documentation string of the function, or\n      ``None`` if undefined.\n\n   Frame objects\n      Frame objects represent execution frames.  They may occur in\n      traceback objects (see below).\n\n      Special read-only attributes: ``f_back`` is to the previous\n      stack frame (towards the caller), or ``None`` if this is the\n      bottom stack frame; ``f_code`` is the code object being executed\n      in this frame; ``f_locals`` is the dictionary used to look up\n      local variables; ``f_globals`` is used for global variables;\n      ``f_builtins`` is used for built-in (intrinsic) names;\n      ``f_lasti`` gives the precise instruction (this is an index into\n      the bytecode string of the code object).\n\n      Special writable attributes: ``f_trace``, if not ``None``, is a\n      function called at the start of each source code line (this is\n      used by the debugger); ``f_lineno`` is the current line number\n      of the frame --- writing to this from within a trace function\n      jumps to the given line (only for the bottom-most frame).  A\n      debugger can implement a Jump command (aka Set Next Statement)\n      by writing to f_lineno.\n\n   Traceback objects\n      Traceback objects represent a stack trace of an exception.  A\n      traceback object is created when an exception occurs.  When the\n      search for an exception handler unwinds the execution stack, at\n      each unwound level a traceback object is inserted in front of\n      the current traceback.  When an exception handler is entered,\n      the stack trace is made available to the program. (See section\n      *The try statement*.) It is accessible as the third item of the\n      tuple returned by ``sys.exc_info()``. When the program contains\n      no suitable handler, the stack trace is written (nicely\n      formatted) to the standard error stream; if the interpreter is\n      interactive, it is also made available to the user as\n      ``sys.last_traceback``.\n\n      Special read-only attributes: ``tb_next`` is the next level in\n      the stack trace (towards the frame where the exception\n      occurred), or ``None`` if there is no next level; ``tb_frame``\n      points to the execution frame of the current level;\n      ``tb_lineno`` gives the line number where the exception\n      occurred; ``tb_lasti`` indicates the precise instruction.  The\n      line number and last instruction in the traceback may differ\n      from the line number of its frame object if the exception\n      occurred in a ``try`` statement with no matching except clause\n      or with a finally clause.\n\n   Slice objects\n      Slice objects are used to represent slices for ``__getitem__()``\n      methods.  They are also created by the built-in ``slice()``\n      function.\n\n      Special read-only attributes: ``start`` is the lower bound;\n      ``stop`` is the upper bound; ``step`` is the step value; each is\n      ``None`` if omitted. These attributes can have any type.\n\n      Slice objects support one method:\n\n      slice.indices(self, length)\n\n         This method takes a single integer argument *length* and\n         computes information about the slice that the slice object\n         would describe if applied to a sequence of *length* items.\n         It returns a tuple of three integers; respectively these are\n         the *start* and *stop* indices and the *step* or stride\n         length of the slice. Missing or out-of-bounds indices are\n         handled in a manner consistent with regular slices.\n\n   Static method objects\n      Static method objects provide a way of defeating the\n      transformation of function objects to method objects described\n      above. A static method object is a wrapper around any other\n      object, usually a user-defined method object. When a static\n      method object is retrieved from a class or a class instance, the\n      object actually returned is the wrapped object, which is not\n      subject to any further transformation. Static method objects are\n      not themselves callable, although the objects they wrap usually\n      are. Static method objects are created by the built-in\n      ``staticmethod()`` constructor.\n\n   Class method objects\n      A class method object, like a static method object, is a wrapper\n      around another object that alters the way in which that object\n      is retrieved from classes and class instances. The behaviour of\n      class method objects upon such retrieval is described above,\n      under "User-defined methods". Class method objects are created\n      by the built-in ``classmethod()`` constructor.\n',
  'typesfunctions': '\nFunctions\n*********\n\nFunction objects are created by function definitions.  The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions.  Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n',
- 'typesmapping': '\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects.  There is currently only one standard\nmapping type, the *dictionary*.  (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values.  Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys.  Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry.  (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n   Return a new dictionary initialized from an optional positional\n   argument or from a set of keyword arguments.  If no arguments are\n   given, return a new empty dictionary.  If the positional argument\n   *arg* is a mapping object, return a dictionary mapping the same\n   keys to the same values as does the mapping object.  Otherwise the\n   positional argument must be a sequence, a container that supports\n   iteration, or an iterator object.  The elements of the argument\n   must each also be of one of those kinds, and each must in turn\n   contain exactly two objects.  The first is used as a key in the new\n   dictionary, and the second as the key\'s value.  If a given key is\n   seen more than once, the last value associated with it is retained\n   in the new dictionary.\n\n   If keyword arguments are given, the keywords themselves with their\n   associated values are added as items to the dictionary.  If a key\n   is specified both in the positional argument and as a keyword\n   argument, the value associated with the keyword is retained in the\n   dictionary.  For example, these all return a dictionary equal to\n   ``{"one": 1, "two": 2}``:\n\n   * ``dict(one=1, two=2)``\n\n   * ``dict({\'one\': 1, \'two\': 2})``\n\n   * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n   * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n   The first example only works for keys that are valid Python\n   identifiers; the others work with any valid keys.\n\n   These are the operations that dictionaries support (and therefore,\n   custom mapping types should support too):\n\n   len(d)\n\n      Return the number of items in the dictionary *d*.\n\n   d[key]\n\n      Return the item of *d* with key *key*.  Raises a ``KeyError`` if\n      *key* is not in the map.\n\n      If a subclass of dict defines a method ``__missing__()``, if the\n      key *key* is not present, the ``d[key]`` operation calls that\n      method with the key *key* as argument.  The ``d[key]`` operation\n      then returns or raises whatever is returned or raised by the\n      ``__missing__(key)`` call if the key is not present. No other\n      operations or methods invoke ``__missing__()``. If\n      ``__missing__()`` is not defined, ``KeyError`` is raised.\n      ``__missing__()`` must be a method; it cannot be an instance\n      variable:\n\n         >>> class Counter(dict):\n         ...     def __missing__(self, key):\n         ...         return 0\n         >>> c = Counter()\n         >>> c[\'red\']\n         0\n         >>> c[\'red\'] += 1\n         >>> c[\'red\']\n         1\n\n      See ``collections.Counter`` for a complete implementation\n      including other methods helpful for accumulating and managing\n      tallies.\n\n   d[key] = value\n\n      Set ``d[key]`` to *value*.\n\n   del d[key]\n\n      Remove ``d[key]`` from *d*.  Raises a ``KeyError`` if *key* is\n      not in the map.\n\n   key in d\n\n      Return ``True`` if *d* has a key *key*, else ``False``.\n\n   key not in d\n\n      Equivalent to ``not key in d``.\n\n   iter(d)\n\n      Return an iterator over the keys of the dictionary.  This is a\n      shortcut for ``iter(d.keys())``.\n\n   clear()\n\n      Remove all items from the dictionary.\n\n   copy()\n\n      Return a shallow copy of the dictionary.\n\n   classmethod fromkeys(seq[, value])\n\n      Create a new dictionary with keys from *seq* and values set to\n      *value*.\n\n      ``fromkeys()`` is a class method that returns a new dictionary.\n      *value* defaults to ``None``.\n\n   get(key[, default])\n\n      Return the value for *key* if *key* is in the dictionary, else\n      *default*. If *default* is not given, it defaults to ``None``,\n      so that this method never raises a ``KeyError``.\n\n   items()\n\n      Return a new view of the dictionary\'s items (``(key, value)``\n      pairs).  See below for documentation of view objects.\n\n   keys()\n\n      Return a new view of the dictionary\'s keys.  See below for\n      documentation of view objects.\n\n   pop(key[, default])\n\n      If *key* is in the dictionary, remove it and return its value,\n      else return *default*.  If *default* is not given and *key* is\n      not in the dictionary, a ``KeyError`` is raised.\n\n   popitem()\n\n      Remove and return an arbitrary ``(key, value)`` pair from the\n      dictionary.\n\n      ``popitem()`` is useful to destructively iterate over a\n      dictionary, as often used in set algorithms.  If the dictionary\n      is empty, calling ``popitem()`` raises a ``KeyError``.\n\n   setdefault(key[, default])\n\n      If *key* is in the dictionary, return its value.  If not, insert\n      *key* with a value of *default* and return *default*.  *default*\n      defaults to ``None``.\n\n   update([other])\n\n      Update the dictionary with the key/value pairs from *other*,\n      overwriting existing keys.  Return ``None``.\n\n      ``update()`` accepts either another dictionary object or an\n      iterable of key/value pairs (as tuples or other iterables of\n      length two).  If keyword arguments are specified, the dictionary\n      is then updated with those key/value pairs: ``d.update(red=1,\n      blue=2)``.\n\n   values()\n\n      Return a new view of the dictionary\'s values.  See below for\n      documentation of view objects.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.keys()``, ``dict.values()`` and\n``dict.items()`` are *view objects*.  They provide a dynamic view on\nthe dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n   Return the number of entries in the dictionary.\n\niter(dictview)\n\n   Return an iterator over the keys, values or items (represented as\n   tuples of ``(key, value)``) in the dictionary.\n\n   Keys and values are iterated over in an arbitrary order which is\n   non-random, varies across Python implementations, and depends on\n   the dictionary\'s history of insertions and deletions. If keys,\n   values and items views are iterated over with no intervening\n   modifications to the dictionary, the order of items will directly\n   correspond.  This allows the creation of ``(value, key)`` pairs\n   using ``zip()``: ``pairs = zip(d.values(), d.keys())``.  Another\n   way to create the same list is ``pairs = [(v, k) for (k, v) in\n   d.items()]``.\n\n   Iterating views while adding or deleting entries in the dictionary\n   may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n   Return ``True`` if *x* is in the underlying dictionary\'s keys,\n   values or items (in the latter case, *x* should be a ``(key,\n   value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that ``(key, value)`` pairs are unique\nand hashable, then the items view is also set-like.  (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class ``collections.Set`` are available (for example, ``==``,\n``<``, or ``^``).\n\nAn example of dictionary view usage:\n\n   >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n   >>> keys = dishes.keys()\n   >>> values = dishes.values()\n\n   >>> # iteration\n   >>> n = 0\n   >>> for val in values:\n   ...     n += val\n   >>> print(n)\n   504\n\n   >>> # keys and values are iterated over in the same order\n   >>> list(keys)\n   [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n   >>> list(values)\n   [2, 1, 1, 500]\n\n   >>> # view objects are dynamic and reflect dict changes\n   >>> del dishes[\'eggs\']\n   >>> del dishes[\'sausage\']\n   >>> list(keys)\n   [\'spam\', \'bacon\']\n\n   >>> # set operations\n   >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n   {\'bacon\'}\n   >>> keys ^ {\'sausage\', \'juice\'}\n   {\'juice\', \'sausage\', \'bacon\', \'spam\'}\n',
+ 'typesmapping': '\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects.  There is currently only one standard\nmapping type, the *dictionary*.  (For other containers see the built-\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values.  Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys.  Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry.  (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n   Return a new dictionary initialized from an optional positional\n   argument or from a set of keyword arguments.  If no arguments are\n   given, return a new empty dictionary.  If the positional argument\n   *arg* is a mapping object, return a dictionary mapping the same\n   keys to the same values as does the mapping object.  Otherwise the\n   positional argument must be a sequence, a container that supports\n   iteration, or an iterator object.  The elements of the argument\n   must each also be of one of those kinds, and each must in turn\n   contain exactly two objects.  The first is used as a key in the new\n   dictionary, and the second as the key\'s value.  If a given key is\n   seen more than once, the last value associated with it is retained\n   in the new dictionary.\n\n   If keyword arguments are given, the keywords themselves with their\n   associated values are added as items to the dictionary.  If a key\n   is specified both in the positional argument and as a keyword\n   argument, the value associated with the keyword is retained in the\n   dictionary.  For example, these all return a dictionary equal to\n   ``{"one": 1, "two": 2}``:\n\n   * ``dict(one=1, two=2)``\n\n   * ``dict({\'one\': 1, \'two\': 2})``\n\n   * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n   * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n   The first example only works for keys that are valid Python\n   identifiers; the others work with any valid keys.\n\n   These are the operations that dictionaries support (and therefore,\n   custom mapping types should support too):\n\n   len(d)\n\n      Return the number of items in the dictionary *d*.\n\n   d[key]\n\n      Return the item of *d* with key *key*.  Raises a ``KeyError`` if\n      *key* is not in the map.\n\n      If a subclass of dict defines a method ``__missing__()``, if the\n      key *key* is not present, the ``d[key]`` operation calls that\n      method with the key *key* as argument.  The ``d[key]`` operation\n      then returns or raises whatever is returned or raised by the\n      ``__missing__(key)`` call if the key is not present. No other\n      operations or methods invoke ``__missing__()``. If\n      ``__missing__()`` is not defined, ``KeyError`` is raised.\n      ``__missing__()`` must be a method; it cannot be an instance\n      variable:\n\n         >>> class Counter(dict):\n         ...     def __missing__(self, key):\n         ...         return 0\n         >>> c = Counter()\n         >>> c[\'red\']\n         0\n         >>> c[\'red\'] += 1\n         >>> c[\'red\']\n         1\n\n      See ``collections.Counter`` for a complete implementation\n      including other methods helpful for accumulating and managing\n      tallies.\n\n   d[key] = value\n\n      Set ``d[key]`` to *value*.\n\n   del d[key]\n\n      Remove ``d[key]`` from *d*.  Raises a ``KeyError`` if *key* is\n      not in the map.\n\n   key in d\n\n      Return ``True`` if *d* has a key *key*, else ``False``.\n\n   key not in d\n\n      Equivalent to ``not key in d``.\n\n   iter(d)\n\n      Return an iterator over the keys of the dictionary.  This is a\n      shortcut for ``iter(d.keys())``.\n\n   clear()\n\n      Remove all items from the dictionary.\n\n   copy()\n\n      Return a shallow copy of the dictionary.\n\n   classmethod fromkeys(seq[, value])\n\n      Create a new dictionary with keys from *seq* and values set to\n      *value*.\n\n      ``fromkeys()`` is a class method that returns a new dictionary.\n      *value* defaults to ``None``.\n\n   get(key[, default])\n\n      Return the value for *key* if *key* is in the dictionary, else\n      *default*. If *default* is not given, it defaults to ``None``,\n      so that this method never raises a ``KeyError``.\n\n   items()\n\n      Return a new view of the dictionary\'s items (``(key, value)``\n      pairs). See the *documentation of view objects*.\n\n   keys()\n\n      Return a new view of the dictionary\'s keys.  See the\n      *documentation of view objects*.\n\n   pop(key[, default])\n\n      If *key* is in the dictionary, remove it and return its value,\n      else return *default*.  If *default* is not given and *key* is\n      not in the dictionary, a ``KeyError`` is raised.\n\n   popitem()\n\n      Remove and return an arbitrary ``(key, value)`` pair from the\n      dictionary.\n\n      ``popitem()`` is useful to destructively iterate over a\n      dictionary, as often used in set algorithms.  If the dictionary\n      is empty, calling ``popitem()`` raises a ``KeyError``.\n\n   setdefault(key[, default])\n\n      If *key* is in the dictionary, return its value.  If not, insert\n      *key* with a value of *default* and return *default*.  *default*\n      defaults to ``None``.\n\n   update([other])\n\n      Update the dictionary with the key/value pairs from *other*,\n      overwriting existing keys.  Return ``None``.\n\n      ``update()`` accepts either another dictionary object or an\n      iterable of key/value pairs (as tuples or other iterables of\n      length two).  If keyword arguments are specified, the dictionary\n      is then updated with those key/value pairs: ``d.update(red=1,\n      blue=2)``.\n\n   values()\n\n      Return a new view of the dictionary\'s values.  See the\n      *documentation of view objects*.\n\nSee also:\n\n   ``types.MappingProxyType`` can be used to create a read-only view\n   of a ``dict``.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.keys()``, ``dict.values()`` and\n``dict.items()`` are *view objects*.  They provide a dynamic view on\nthe dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n   Return the number of entries in the dictionary.\n\niter(dictview)\n\n   Return an iterator over the keys, values or items (represented as\n   tuples of ``(key, value)``) in the dictionary.\n\n   Keys and values are iterated over in an arbitrary order which is\n   non-random, varies across Python implementations, and depends on\n   the dictionary\'s history of insertions and deletions. If keys,\n   values and items views are iterated over with no intervening\n   modifications to the dictionary, the order of items will directly\n   correspond.  This allows the creation of ``(value, key)`` pairs\n   using ``zip()``: ``pairs = zip(d.values(), d.keys())``.  Another\n   way to create the same list is ``pairs = [(v, k) for (k, v) in\n   d.items()]``.\n\n   Iterating views while adding or deleting entries in the dictionary\n   may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n   Return ``True`` if *x* is in the underlying dictionary\'s keys,\n   values or items (in the latter case, *x* should be a ``(key,\n   value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that ``(key, value)`` pairs are unique\nand hashable, then the items view is also set-like.  (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class ``collections.abc.Set`` are available (for example, ``==``,\n``<``, or ``^``).\n\nAn example of dictionary view usage:\n\n   >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n   >>> keys = dishes.keys()\n   >>> values = dishes.values()\n\n   >>> # iteration\n   >>> n = 0\n   >>> for val in values:\n   ...     n += val\n   >>> print(n)\n   504\n\n   >>> # keys and values are iterated over in the same order\n   >>> list(keys)\n   [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n   >>> list(values)\n   [2, 1, 1, 500]\n\n   >>> # view objects are dynamic and reflect dict changes\n   >>> del dishes[\'eggs\']\n   >>> del dishes[\'sausage\']\n   >>> list(keys)\n   [\'spam\', \'bacon\']\n\n   >>> # set operations\n   >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n   {\'bacon\'}\n   >>> keys ^ {\'sausage\', \'juice\'}\n   {\'juice\', \'sausage\', \'bacon\', \'spam\'}\n',
  'typesmethods': "\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods.  Built-in methods are described\nwith the types that support them.\n\nIf you access a method (a function defined in a class namespace)\nthrough an instance, you get a special object: a *bound method* (also\ncalled *instance method*) object. When called, it will add the\n``self`` argument to the argument list.  Bound methods have two\nspecial read-only attributes: ``m.__self__`` is the object on which\nthe method operates, and ``m.__func__`` is the function implementing\nthe method.  Calling ``m(arg-1, arg-2, ..., arg-n)`` is completely\nequivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ...,\narg-n)``.\n\nLike function objects, bound method objects support getting arbitrary\nattributes.  However, since method attributes are actually stored on\nthe underlying function object (``meth.__func__``), setting method\nattributes on bound methods is disallowed.  Attempting to set a method\nattribute results in a ``TypeError`` being raised.  In order to set a\nmethod attribute, you need to explicitly set it on the underlying\nfunction object:\n\n   class C:\n       def method(self):\n           pass\n\n   c = C()\n   c.method.__func__.whoami = 'my name is c'\n\nSee *The standard type hierarchy* for more information.\n",
  'typesmodules': "\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to.  (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special attribute of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``).  Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ``<module\n'sys' (built-in)>``.  If loaded from a file, they are written as\n``<module 'os' from '/usr/local/lib/pythonX.Y/os.pyc'>``.\n",
- 'typesseq': '\nSequence Types --- ``str``, ``bytes``, ``bytearray``, ``list``, ``tuple``, ``range``\n************************************************************************************\n\nThere are six sequence types: strings, byte sequences (``bytes``\nobjects), byte arrays (``bytearray`` objects), lists, tuples, and\nrange objects.  For other containers see the built in ``dict`` and\n``set`` classes, and the ``collections`` module.\n\nStrings contain Unicode characters.  Their literals are written in\nsingle or double quotes: ``\'xyzzy\'``, ``"frobozz"``.  See *String and\nBytes literals* for more about string literals.  In addition to the\nfunctionality described here, there are also string-specific methods\ndescribed in the *String Methods* section.\n\nBytes and bytearray objects contain single bytes -- the former is\nimmutable while the latter is a mutable sequence.  Bytes objects can\nbe constructed the constructor, ``bytes()``, and from literals; use a\n``b`` prefix with normal string syntax: ``b\'xyzzy\'``.  To construct\nbyte arrays, use the ``bytearray()`` function.\n\nWhile string objects are sequences of characters (represented by\nstrings of length 1), bytes and bytearray objects are sequences of\n*integers* (between 0 and 255), representing the ASCII value of single\nbytes.  That means that for a bytes or bytearray object *b*, ``b[0]``\nwill be an integer, while ``b[0:1]`` will be a bytes or bytearray\nobject of length 1.  The representation of bytes objects uses the\nliteral format (``b\'...\'``) since it is generally more useful than\ne.g. ``bytes([50, 19, 100])``.  You can always convert a bytes object\ninto a list of integers using ``list(b)``.\n\nAlso, while in previous Python versions, byte strings and Unicode\nstrings could be exchanged for each other rather freely (barring\nencoding issues), strings and bytes are now completely separate\nconcepts.  There\'s no implicit en-/decoding if you pass an object of\nthe wrong type.  A string always compares unequal to a bytes or\nbytearray object.\n\nLists are constructed with square brackets, separating items with\ncommas: ``[a, b, c]``.  Tuples are constructed by the comma operator\n(not within square brackets), with or without enclosing parentheses,\nbut an empty tuple must have the enclosing parentheses, such as ``a,\nb, c`` or ``()``.  A single item tuple must have a trailing comma,\nsuch as ``(d,)``.\n\nObjects of type range are created using the ``range()`` function.\nThey don\'t support concatenation or repetition, and using ``min()`` or\n``max()`` on them is inefficient.\n\nMost sequence types support the following operations.  The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations.  The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority).  In the table,\n*s* and *t* are sequences of the same type; *n*, *i*, *j* and *k* are\nintegers.\n\n+--------------------+----------------------------------+------------+\n| Operation          | Result                           | Notes      |\n+====================+==================================+============+\n| ``x in s``         | ``True`` if an item of *s* is    | (1)        |\n|                    | equal to *x*, else ``False``     |            |\n+--------------------+----------------------------------+------------+\n| ``x not in s``     | ``False`` if an item of *s* is   | (1)        |\n|                    | equal to *x*, else ``True``      |            |\n+--------------------+----------------------------------+------------+\n| ``s + t``          | the concatenation of *s* and *t* | (6)        |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s``   | *n* shallow copies of *s*        | (2)        |\n|                    | concatenated                     |            |\n+--------------------+----------------------------------+------------+\n| ``s[i]``           | *i*th item of *s*, origin 0      | (3)        |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]``         | slice of *s* from *i* to *j*     | (3)(4)     |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]``       | slice of *s* from *i* to *j*     | (3)(5)     |\n|                    | with step *k*                    |            |\n+--------------------+----------------------------------+------------+\n| ``len(s)``         | length of *s*                    |            |\n+--------------------+----------------------------------+------------+\n| ``min(s)``         | smallest item of *s*             |            |\n+--------------------+----------------------------------+------------+\n| ``max(s)``         | largest item of *s*              |            |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)``     | index of the first occurence of  |            |\n|                    | *i* in *s*                       |            |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)``     | total number of occurences of    |            |\n|                    | *i* in *s*                       |            |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons.  In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements.  This means that to compare equal, every element must\ncompare equal and the two sequences must be of the same type and have\nthe same length.  (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string object, the ``in`` and ``not in`` operations\n   act like a substring test.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n   empty sequence of the same type as *s*).  Note also that the copies\n   are shallow; nested structures are not copied.  This often haunts\n   new Python programmers; consider:\n\n   >>> lists = [[]] * 3\n   >>> lists\n   [[], [], []]\n   >>> lists[0].append(3)\n   >>> lists\n   [[3], [3], [3]]\n\n   What has happened is that ``[[]]`` is a one-element list containing\n   an empty list, so all three elements of ``[[]] * 3`` are (pointers\n   to) this single empty list.  Modifying any of the elements of\n   ``lists`` modifies this single list. You can create a list of\n   different lists this way:\n\n   >>> lists = [[] for i in range(3)]\n   >>> lists[0].append(3)\n   >>> lists[1].append(5)\n   >>> lists[2].append(7)\n   >>> lists\n   [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n   string: ``len(s) + i`` or ``len(s) + j`` is substituted.  But note\n   that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n   items with index *k* such that ``i <= k < j``.  If *i* or *j* is\n   greater than ``len(s)``, use ``len(s)``.  If *i* is omitted or\n   ``None``, use ``0``.  If *j* is omitted or ``None``, use\n   ``len(s)``.  If *i* is greater than or equal to *j*, the slice is\n   empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n   sequence of items with index  ``x = i + n*k`` such that ``0 <= n <\n   (j-i)/k``.  In other words, the indices are ``i``, ``i+k``,\n   ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n   never including *j*).  If *i* or *j* is greater than ``len(s)``,\n   use ``len(s)``.  If *i* or *j* are omitted or ``None``, they become\n   "end" values (which end depends on the sign of *k*).  Note, *k*\n   cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. Concatenating immutable strings always results in a new object.\n   This means that building up a string by repeated concatenation will\n   have a quadratic runtime cost in the total string length.  To get a\n   linear runtime cost, you must switch to one of the alternatives\n   below:\n\n   * if concatenating ``str`` objects, you can build a list and use\n     ``str.join()`` at the end;\n\n   * if concatenating ``bytes`` objects, you can similarly use\n     ``bytes.join()``, or you can do in-place concatenation with a\n     ``bytearray`` object.  ``bytearray`` objects are mutable and have\n     an efficient overallocation mechanism.\n\n\nString Methods\n==============\n\nString objects support the methods listed below.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n   Return a copy of the string with its first character capitalized\n   and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n   Return centered in a string of length *width*. Padding is done\n   using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n   Return the number of non-overlapping occurrences of substring *sub*\n   in the range [*start*, *end*].  Optional arguments *start* and\n   *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n   Return an encoded version of the string as a bytes object. Default\n   encoding is ``\'utf-8\'``. *errors* may be given to set a different\n   error handling scheme. The default for *errors* is ``\'strict\'``,\n   meaning that encoding errors raise a ``UnicodeError``. Other\n   possible values are ``\'ignore\'``, ``\'replace\'``,\n   ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and any other name\n   registered via ``codecs.register_error()``, see section *Codec Base\n   Classes*. For a list of possible encodings, see section *Standard\n   Encodings*.\n\n   Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n   Return ``True`` if the string ends with the specified *suffix*,\n   otherwise return ``False``.  *suffix* can also be a tuple of\n   suffixes to look for.  With optional *start*, test beginning at\n   that position.  With optional *end*, stop comparing at that\n   position.\n\nstr.expandtabs([tabsize])\n\n   Return a copy of the string where all tab characters are replaced\n   by zero or more spaces, depending on the current column and the\n   given tab size.  The column number is reset to zero after each\n   newline occurring in the string. If *tabsize* is not given, a tab\n   size of ``8`` characters is assumed.  This doesn\'t understand other\n   non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n   Return the lowest index in the string where substring *sub* is\n   found, such that *sub* is contained in the slice ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` if *sub* is not found.\n\n   Note: The ``find()`` method should be used only if you need to know the\n     position of *sub*.  To check if *sub* is a substring or not, use\n     the ``in`` operator:\n\n        >>> \'Py\' in \'Python\'\n        True\n\nstr.format(*args, **kwargs)\n\n   Perform a string formatting operation.  The string on which this\n   method is called can contain literal text or replacement fields\n   delimited by braces ``{}``.  Each replacement field contains either\n   the numeric index of a positional argument, or the name of a\n   keyword argument.  Returns a copy of the string where each\n   replacement field is replaced with the string value of the\n   corresponding argument.\n\n   >>> "The sum of 1 + 2 is {0}".format(1+2)\n   \'The sum of 1 + 2 is 3\'\n\n   See *Format String Syntax* for a description of the various\n   formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n   Similar to ``str.format(**mapping)``, except that ``mapping`` is\n   used directly and not copied to a ``dict`` .  This is useful if for\n   example ``mapping`` is a dict subclass:\n\n   >>> class Default(dict):\n   ...     def __missing__(self, key):\n   ...         return key\n   ...\n   >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n   \'Guido was born in country\'\n\n   New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n   Like ``find()``, but raise ``ValueError`` when the substring is not\n   found.\n\nstr.isalnum()\n\n   Return true if all characters in the string are alphanumeric and\n   there is at least one character, false otherwise.  A character\n   ``c`` is alphanumeric if one of the following returns ``True``:\n   ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n   ``c.isnumeric()``.\n\nstr.isalpha()\n\n   Return true if all characters in the string are alphabetic and\n   there is at least one character, false otherwise.  Alphabetic\n   characters are those characters defined in the Unicode character\n   database as "Letter", i.e., those with general category property\n   being one of "Lm", "Lt", "Lu", "Ll", or "Lo".  Note that this is\n   different from the "Alphabetic" property defined in the Unicode\n   Standard.\n\nstr.isdecimal()\n\n   Return true if all characters in the string are decimal characters\n   and there is at least one character, false otherwise. Decimal\n   characters are those from general category "Nd". This category\n   includes digit characters, and all characters that can be used to\n   form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\nstr.isdigit()\n\n   Return true if all characters in the string are digits and there is\n   at least one character, false otherwise.  Digits include decimal\n   characters and digits that need special handling, such as the\n   compatibility superscript digits.  Formally, a digit is a character\n   that has the property value Numeric_Type=Digit or\n   Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n   Return true if the string is a valid identifier according to the\n   language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n   Return true if all cased characters [4] in the string are lowercase\n   and there is at least one cased character, false otherwise.\n\nstr.isnumeric()\n\n   Return true if all characters in the string are numeric characters,\n   and there is at least one character, false otherwise. Numeric\n   characters include digit characters, and all characters that have\n   the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n   ONE FIFTH.  Formally, numeric characters are those with the\n   property value Numeric_Type=Digit, Numeric_Type=Decimal or\n   Numeric_Type=Numeric.\n\nstr.isprintable()\n\n   Return true if all characters in the string are printable or the\n   string is empty, false otherwise.  Nonprintable characters are\n   those characters defined in the Unicode character database as\n   "Other" or "Separator", excepting the ASCII space (0x20) which is\n   considered printable.  (Note that printable characters in this\n   context are those which should not be escaped when ``repr()`` is\n   invoked on a string.  It has no bearing on the handling of strings\n   written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n   Return true if there are only whitespace characters in the string\n   and there is at least one character, false otherwise.  Whitespace\n   characters  are those characters defined in the Unicode character\n   database as "Other" or "Separator" and those with bidirectional\n   property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n   Return true if the string is a titlecased string and there is at\n   least one character, for example uppercase characters may only\n   follow uncased characters and lowercase characters only cased ones.\n   Return false otherwise.\n\nstr.isupper()\n\n   Return true if all cased characters [4] in the string are uppercase\n   and there is at least one cased character, false otherwise.\n\nstr.join(iterable)\n\n   Return a string which is the concatenation of the strings in the\n   *iterable* *iterable*.  A ``TypeError`` will be raised if there are\n   any non-string values in *iterable*, including ``bytes`` objects.\n   The separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n   Return the string left justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space).  The original string is returned if *width* is less than or\n   equal to ``len(s)``.\n\nstr.lower()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to lowercase.\n\nstr.lstrip([chars])\n\n   Return a copy of the string with leading characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a prefix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.lstrip()\n   \'spacious   \'\n   >>> \'www.example.com\'.lstrip(\'cmowz.\')\n   \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n   This static method returns a translation table usable for\n   ``str.translate()``.\n\n   If there is only one argument, it must be a dictionary mapping\n   Unicode ordinals (integers) or characters (strings of length 1) to\n   Unicode ordinals, strings (of arbitrary lengths) or None.\n   Character keys will then be converted to ordinals.\n\n   If there are two arguments, they must be strings of equal length,\n   and in the resulting dictionary, each character in x will be mapped\n   to the character at the same position in y.  If there is a third\n   argument, it must be a string, whose characters will be mapped to\n   None in the result.\n\nstr.partition(sep)\n\n   Split the string at the first occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing the string itself, followed by\n   two empty strings.\n\nstr.replace(old, new[, count])\n\n   Return a copy of the string with all occurrences of substring *old*\n   replaced by *new*.  If the optional argument *count* is given, only\n   the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n   Return the highest index in the string where substring *sub* is\n   found, such that *sub* is contained within ``s[start:end]``.\n   Optional arguments *start* and *end* are interpreted as in slice\n   notation.  Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n   Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n   is not found.\n\nstr.rjust(width[, fillchar])\n\n   Return the string right justified in a string of length *width*.\n   Padding is done using the specified *fillchar* (default is a\n   space). The original string is returned if *width* is less than or\n   equal to ``len(s)``.\n\nstr.rpartition(sep)\n\n   Split the string at the last occurrence of *sep*, and return a\n   3-tuple containing the part before the separator, the separator\n   itself, and the part after the separator.  If the separator is not\n   found, return a 3-tuple containing two empty strings, followed by\n   the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n   are done, the *rightmost* ones.  If *sep* is not specified or\n   ``None``, any whitespace string is a separator.  Except for\n   splitting from the right, ``rsplit()`` behaves like ``split()``\n   which is described in detail below.\n\nstr.rstrip([chars])\n\n   Return a copy of the string with trailing characters removed.  The\n   *chars* argument is a string specifying the set of characters to be\n   removed.  If omitted or ``None``, the *chars* argument defaults to\n   removing whitespace.  The *chars* argument is not a suffix; rather,\n   all combinations of its values are stripped:\n\n   >>> \'   spacious   \'.rstrip()\n   \'   spacious\'\n   >>> \'mississippi\'.rstrip(\'ipz\')\n   \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n   Return a list of the words in the string, using *sep* as the\n   delimiter string.  If *maxsplit* is given, at most *maxsplit*\n   splits are done (thus, the list will have at most ``maxsplit+1``\n   elements).  If *maxsplit* is not specified, then there is no limit\n   on the number of splits (all possible splits are made).\n\n   If *sep* is given, consecutive delimiters are not grouped together\n   and are deemed to delimit empty strings (for example,\n   ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``).  The *sep*\n   argument may consist of multiple characters (for example,\n   ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n   an empty string with a specified separator returns ``[\'\']``.\n\n   If *sep* is not specified or is ``None``, a different splitting\n   algorithm is applied: runs of consecutive whitespace are regarded\n   as a single separator, and the result will contain no empty strings\n   at the start or end if the string has leading or trailing\n   whitespace.  Consequently, splitting an empty string or a string\n   consisting of just whitespace with a ``None`` separator returns\n   ``[]``.\n\n   For example, ``\' 1  2   3  \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n   and ``\'  1  2   3  \'.split(None, 1)`` returns ``[\'1\', \'2   3  \']``.\n\nstr.splitlines([keepends])\n\n   Return a list of the lines in the string, breaking at line\n   boundaries.  Line breaks are not included in the resulting list\n   unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n   Return ``True`` if string starts with the *prefix*, otherwise\n   return ``False``. *prefix* can also be a tuple of prefixes to look\n   for.  With optional *start*, test string beginning at that\n   position.  With optional *end*, stop comparing string at that\n   position.\n\nstr.strip([chars])\n\n   Return a copy of the string with the leading and trailing\n   characters removed. The *chars* argument is a string specifying the\n   set of characters to be removed. If omitted or ``None``, the\n   *chars* argument defaults to removing whitespace. The *chars*\n   argument is not a prefix or suffix; rather, all combinations of its\n   values are stripped:\n\n   >>> \'   spacious   \'.strip()\n   \'spacious\'\n   >>> \'www.example.com\'.strip(\'cmowz.\')\n   \'example\'\n\nstr.swapcase()\n\n   Return a copy of the string with uppercase characters converted to\n   lowercase and vice versa.\n\nstr.title()\n\n   Return a titlecased version of the string where words start with an\n   uppercase character and the remaining characters are lowercase.\n\n   The algorithm uses a simple language-independent definition of a\n   word as groups of consecutive letters.  The definition works in\n   many contexts but it means that apostrophes in contractions and\n   possessives form word boundaries, which may not be the desired\n   result:\n\n      >>> "they\'re bill\'s friends from the UK".title()\n      "They\'Re Bill\'S Friends From The Uk"\n\n   A workaround for apostrophes can be constructed using regular\n   expressions:\n\n      >>> import re\n      >>> def titlecase(s):\n              return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n                            lambda mo: mo.group(0)[0].upper() +\n                                       mo.group(0)[1:].lower(),\n                            s)\n\n      >>> titlecase("they\'re bill\'s friends.")\n      "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n   Return a copy of the *s* where all characters have been mapped\n   through the *map* which must be a dictionary of Unicode ordinals\n   (integers) to Unicode ordinals, strings or ``None``.  Unmapped\n   characters are left untouched. Characters mapped to ``None`` are\n   deleted.\n\n   You can use ``str.maketrans()`` to create a translation map from\n   character-to-character mappings in different formats.\n\n   Note: An even more flexible approach is to create a custom character\n     mapping codec using the ``codecs`` module (see\n     ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n   Return a copy of the string with all the cased characters [4]\n   converted to uppercase.  Note that ``str.upper().isupper()`` might\n   be ``False`` if ``s`` contains uncased characters or if the Unicode\n   category of the resulting character(s) is not "Lu" (Letter,\n   uppercase), but e.g. "Lt" (Letter, titlecase).\n\nstr.zfill(width)\n\n   Return the numeric string left filled with zeros in a string of\n   length *width*.  A sign prefix is handled correctly.  The original\n   string is returned if *width* is less than or equal to ``len(s)``.\n\n\nOld String Formatting Operations\n================================\n\nNote: The formatting operations described here are obsolete and may go\n  away in future versions of Python.  Use the new *String Formatting*\n  in new code.\n\nString objects have one unique built-in operation: the ``%`` operator\n(modulo). This is also known as the string *formatting* or\n*interpolation* operator. Given ``format % values`` (where *format* is\na string), ``%`` conversion specifications in *format* are replaced\nwith zero or more elements of *values*. The effect is similar to the\nusing ``sprintf()`` in the C language.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [5]  Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n   characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n   conversion types.\n\n4. Minimum field width (optional).  If specified as an ``\'*\'``\n   (asterisk), the actual width is read from the next element of the\n   tuple in *values*, and the object to convert comes after the\n   minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n   precision.  If specified as ``\'*\'`` (an asterisk), the actual\n   precision is read from the next element of the tuple in *values*,\n   and the value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print(\'%(language)s has %(number)03d quote types.\' %\n...       {\'language\': "Python", "number": 2})\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag      | Meaning                                                               |\n+===========+=======================================================================+\n| ``\'#\'``   | The value conversion will use the "alternate form" (where defined     |\n|           | below).                                                               |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'``   | The conversion will be zero padded for numeric values.                |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'``   | The converted value is left adjusted (overrides the ``\'0\'``           |\n|           | conversion if both are given).                                        |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'``   | (a space) A blank should be left before a positive number (or empty   |\n|           | string) produced by a signed conversion.                              |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'``   | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion     |\n|           | (overrides a "space" flag).                                           |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion   | Meaning                                               | Notes   |\n+==============+=======================================================+=========+\n| ``\'d\'``      | Signed integer decimal.                               |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'``      | Signed integer decimal.                               |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'``      | Signed octal value.                                   | (1)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'``      | Obsolete type -- it is identical to ``\'d\'``.          | (7)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'``      | Signed hexadecimal (lowercase).                       | (2)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'``      | Signed hexadecimal (uppercase).                       | (2)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'``      | Floating point exponential format (lowercase).        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'``      | Floating point exponential format (uppercase).        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'``      | Floating point decimal format.                        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'``      | Floating point decimal format.                        | (3)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'``      | Floating point format. Uses lowercase exponential     | (4)     |\n|              | format if exponent is less than -4 or not less than   |         |\n|              | precision, decimal format otherwise.                  |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'``      | Floating point format. Uses uppercase exponential     | (4)     |\n|              | format if exponent is less than -4 or not less than   |         |\n|              | precision, decimal format otherwise.                  |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'``      | Single character (accepts integer or single character |         |\n|              | string).                                              |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'``      | String (converts any Python object using ``repr()``). | (5)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'``      | String (converts any Python object using ``str()``).  | (5)     |\n+--------------+-------------------------------------------------------+---------+\n| ``\'a\'``      | String (converts any Python object using              | (5)     |\n|              | ``ascii()``).                                         |         |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'``      | No argument is converted, results in a ``\'%\'``        |         |\n|              | character in the result.                              |         |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n   between left-hand padding and the formatting of the number if the\n   leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n   on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n   between left-hand padding and the formatting of the number if the\n   leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n   point, even if no digits follow it.\n\n   The precision determines the number of digits after the decimal\n   point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n   point, and trailing zeroes are not removed as they would otherwise\n   be.\n\n   The precision determines the number of significant digits before\n   and after the decimal point and defaults to 6.\n\n5. If precision is ``N``, the output is truncated to ``N`` characters.\n\n1. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 3.1: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nRange Type\n==========\n\nThe ``range`` type is an immutable sequence which is commonly used for\nlooping.  The advantage of the ``range`` type is that an ``range``\nobject will always take the same amount of memory, no matter the size\nof the range it represents.\n\nRange objects have relatively little behavior: they support indexing,\ncontains, iteration, the ``len()`` function, and the following\nmethods:\n\nrange.count(x)\n\n   Return the number of *i*\'s for which ``s[i] == x``.\n\n      New in version 3.2.\n\nrange.index(x)\n\n   Return the smallest *i* such that ``s[i] == x``.  Raises\n   ``ValueError`` when *x* is not in the range.\n\n      New in version 3.2.\n\n\nMutable Sequence Types\n======================\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object.  Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | same as ``s[len(s):len(s)] =     |                       |\n|                                | [x]``                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)``                | same as ``s[len(s):len(s)] = x`` | (2)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)``                 | return number of *i*\'s for which |                       |\n|                                | ``s[i] == x``                    |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])``       | return smallest *k* such that    | (3)                   |\n|                                | ``s[k] == x`` and ``i <= k < j`` |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | same as ``s[i:i] = [x]``         | (4)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | same as ``x = s[i]; del s[i];    | (5)                   |\n|                                | return x``                       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | same as ``del s[s.index(x)]``    | (3)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (6)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])``   | sort the items of *s* in place   | (6), (7), (8)         |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n   index is passed as the second or third parameter to the ``index()``\n   method, the sequence length is added, as for slice indices.  If it\n   is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n   ``insert()`` method, the sequence length is added, as for slice\n   indices.  If it is still negative, it is truncated to zero, as for\n   slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n   place for economy of space when sorting or reversing a large\n   sequence.  To remind you that they operate by side effect, they\n   don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n   comparisons.  Each must be specified as a keyword argument.\n\n   *key* specifies a function of one argument that is used to extract\n   a comparison key from each list element: ``key=str.lower``.  The\n   default value is ``None``. Use ``functools.cmp_to_key()`` to\n   convert an old-style *cmp* function to a *key* function.\n\n   *reverse* is a boolean value.  If set to ``True``, then the list\n   elements are sorted as if each comparison were reversed.\n\n   The ``sort()`` method is guaranteed to be stable.  A sort is stable\n   if it guarantees not to change the relative order of elements that\n   compare equal --- this is helpful for sorting in multiple passes\n   (for example, sort by department, then by salary grade).\n\n   **CPython implementation detail:** While a list is being sorted,\n   the effect of attempting to mutate, or even inspect, the list is\n   undefined.  The C implementation of Python makes the list appear\n   empty for the duration, and raises ``ValueError`` if it can detect\n   that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n\n\nBytes and Byte Array Methods\n============================\n\nBytes and bytearray objects, being "strings of bytes", have all\nmethods found on strings, with the exception of ``encode()``,\n``format()`` and ``isidentifier()``, which do not make sense with\nthese types.  For converting the objects to strings, they have a\n``decode()`` method.\n\nWherever one of these methods needs to interpret the bytes as\ncharacters (e.g. the ``is...()`` methods), the ASCII character set is\nassumed.\n\nNote: The methods on bytes and bytearray objects don\'t accept strings as\n  their arguments, just as the methods on strings don\'t accept bytes\n  as their arguments.  For example, you have to write\n\n     a = "abc"\n     b = a.replace("a", "f")\n\n  and\n\n     a = b"abc"\n     b = a.replace(b"a", b"f")\n\nbytes.decode(encoding="utf-8", errors="strict")\nbytearray.decode(encoding="utf-8", errors="strict")\n\n   Return a string decoded from the given bytes.  Default encoding is\n   ``\'utf-8\'``. *errors* may be given to set a different error\n   handling scheme.  The default for *errors* is ``\'strict\'``, meaning\n   that encoding errors raise a ``UnicodeError``.  Other possible\n   values are ``\'ignore\'``, ``\'replace\'`` and any other name\n   registered via ``codecs.register_error()``, see section *Codec Base\n   Classes*. For a list of possible encodings, see section *Standard\n   Encodings*.\n\n   Changed in version 3.1: Added support for keyword arguments.\n\nThe bytes and bytearray types have an additional class method:\n\nclassmethod bytes.fromhex(string)\nclassmethod bytearray.fromhex(string)\n\n   This ``bytes`` class method returns a bytes or bytearray object,\n   decoding the given string object.  The string must contain two\n   hexadecimal digits per byte, spaces are ignored.\n\n   >>> bytes.fromhex(\'f0 f1f2  \')\n   b\'\\xf0\\xf1\\xf2\'\n\nThe maketrans and translate methods differ in semantics from the\nversions available on strings:\n\nbytes.translate(table[, delete])\nbytearray.translate(table[, delete])\n\n   Return a copy of the bytes or bytearray object where all bytes\n   occurring in the optional argument *delete* are removed, and the\n   remaining bytes have been mapped through the given translation\n   table, which must be a bytes object of length 256.\n\n   You can use the ``bytes.maketrans()`` method to create a\n   translation table.\n\n   Set the *table* argument to ``None`` for translations that only\n   delete characters:\n\n      >>> b\'read this short text\'.translate(None, b\'aeiou\')\n      b\'rd ths shrt txt\'\n\nstatic bytes.maketrans(from, to)\nstatic bytearray.maketrans(from, to)\n\n   This static method returns a translation table usable for\n   ``bytes.translate()`` that will map each character in *from* into\n   the character at the same position in *to*; *from* and *to* must be\n   bytes objects and have the same length.\n\n   New in version 3.1.\n',
- 'typesseq-mutable': '\nMutable Sequence Types\n**********************\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object.  Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | same as ``s[len(s):len(s)] =     |                       |\n|                                | [x]``                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)``                | same as ``s[len(s):len(s)] = x`` | (2)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)``                 | return number of *i*\'s for which |                       |\n|                                | ``s[i] == x``                    |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])``       | return smallest *k* such that    | (3)                   |\n|                                | ``s[k] == x`` and ``i <= k < j`` |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | same as ``s[i:i] = [x]``         | (4)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | same as ``x = s[i]; del s[i];    | (5)                   |\n|                                | return x``                       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | same as ``del s[s.index(x)]``    | (3)                   |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (6)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])``   | sort the items of *s* in place   | (6), (7), (8)         |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n   index is passed as the second or third parameter to the ``index()``\n   method, the sequence length is added, as for slice indices.  If it\n   is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n   ``insert()`` method, the sequence length is added, as for slice\n   indices.  If it is still negative, it is truncated to zero, as for\n   slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n   place for economy of space when sorting or reversing a large\n   sequence.  To remind you that they operate by side effect, they\n   don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n   comparisons.  Each must be specified as a keyword argument.\n\n   *key* specifies a function of one argument that is used to extract\n   a comparison key from each list element: ``key=str.lower``.  The\n   default value is ``None``. Use ``functools.cmp_to_key()`` to\n   convert an old-style *cmp* function to a *key* function.\n\n   *reverse* is a boolean value.  If set to ``True``, then the list\n   elements are sorted as if each comparison were reversed.\n\n   The ``sort()`` method is guaranteed to be stable.  A sort is stable\n   if it guarantees not to change the relative order of elements that\n   compare equal --- this is helpful for sorting in multiple passes\n   (for example, sort by department, then by salary grade).\n\n   **CPython implementation detail:** While a list is being sorted,\n   the effect of attempting to mutate, or even inspect, the list is\n   undefined.  The C implementation of Python makes the list appear\n   empty for the duration, and raises ``ValueError`` if it can detect\n   that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n',
+ 'typesseq': '\nSequence Types --- ``list``, ``tuple``, ``range``\n*************************************************\n\nThere are three basic sequence types: lists, tuples, and range\nobjects. Additional sequence types tailored for processing of *binary\ndata* and *text strings* are described in dedicated sections.\n\n\nCommon Sequence Operations\n==========================\n\nThe operations in the following table are supported by most sequence\ntypes, both mutable and immutable. The ``collections.abc.Sequence``\nABC is provided to make it easier to correctly implement these\noperations on custom sequence types.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority).  In the table,\n*s* and *t* are sequences of the same type, *n*, *i*, *j* and *k* are\nintegers and *x* is an arbitrary object that meets any type and value\nrestrictions imposed by *s*.\n\nThe ``in`` and ``not in`` operations have the same priorities as the\ncomparison operations. The ``+`` (concatenation) and ``*``\n(repetition) operations have the same priority as the corresponding\nnumeric operations.\n\n+----------------------------+----------------------------------+------------+\n| Operation                  | Result                           | Notes      |\n+============================+==================================+============+\n| ``x in s``                 | ``True`` if an item of *s* is    | (1)        |\n|                            | equal to *x*, else ``False``     |            |\n+----------------------------+----------------------------------+------------+\n| ``x not in s``             | ``False`` if an item of *s* is   | (1)        |\n|                            | equal to *x*, else ``True``      |            |\n+----------------------------+----------------------------------+------------+\n| ``s + t``                  | the concatenation of *s* and *t* | (6)(7)     |\n+----------------------------+----------------------------------+------------+\n| ``s * n`` or ``n * s``     | *n* shallow copies of *s*        | (2)(7)     |\n|                            | concatenated                     |            |\n+----------------------------+----------------------------------+------------+\n| ``s[i]``                   | *i*th item of *s*, origin 0      | (3)        |\n+----------------------------+----------------------------------+------------+\n| ``s[i:j]``                 | slice of *s* from *i* to *j*     | (3)(4)     |\n+----------------------------+----------------------------------+------------+\n| ``s[i:j:k]``               | slice of *s* from *i* to *j*     | (3)(5)     |\n|                            | with step *k*                    |            |\n+----------------------------+----------------------------------+------------+\n| ``len(s)``                 | length of *s*                    |            |\n+----------------------------+----------------------------------+------------+\n| ``min(s)``                 | smallest item of *s*             |            |\n+----------------------------+----------------------------------+------------+\n| ``max(s)``                 | largest item of *s*              |            |\n+----------------------------+----------------------------------+------------+\n| ``s.index(x[, i[, j]])``   | index of the first occurence of  | (8)        |\n|                            | *x* in *s* (at or after index    |            |\n|                            | *i* and before index *j*)        |            |\n+----------------------------+----------------------------------+------------+\n| ``s.count(x)``             | total number of occurences of    |            |\n|                            | *x* in *s*                       |            |\n+----------------------------+----------------------------------+------------+\n\nSequences of the same type also support comparisons.  In particular,\ntuples and lists are compared lexicographically by comparing\ncorresponding elements. This means that to compare equal, every\nelement must compare equal and the two sequences must be of the same\ntype and have the same length.  (For full details see *Comparisons* in\nthe language reference.)\n\nNotes:\n\n1. While the ``in`` and ``not in`` operations are used only for simple\n   containment testing in the general case, some specialised sequences\n   (such as ``str``, ``bytes`` and ``bytearray``) also use them for\n   subsequence testing:\n\n      >>> "gg" in "eggs"\n      True\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n   empty sequence of the same type as *s*).  Note also that the copies\n   are shallow; nested structures are not copied.  This often haunts\n   new Python programmers; consider:\n\n      >>> lists = [[]] * 3\n      >>> lists\n      [[], [], []]\n      >>> lists[0].append(3)\n      >>> lists\n      [[3], [3], [3]]\n\n   What has happened is that ``[[]]`` is a one-element list containing\n   an empty list, so all three elements of ``[[]] * 3`` are (pointers\n   to) this single empty list.  Modifying any of the elements of\n   ``lists`` modifies this single list. You can create a list of\n   different lists this way:\n\n      >>> lists = [[] for i in range(3)]\n      >>> lists[0].append(3)\n      >>> lists[1].append(5)\n      >>> lists[2].append(7)\n      >>> lists\n      [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n   string: ``len(s) + i`` or ``len(s) + j`` is substituted.  But note\n   that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n   items with index *k* such that ``i <= k < j``.  If *i* or *j* is\n   greater than ``len(s)``, use ``len(s)``.  If *i* is omitted or\n   ``None``, use ``0``.  If *j* is omitted or ``None``, use\n   ``len(s)``.  If *i* is greater than or equal to *j*, the slice is\n   empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n   sequence of items with index  ``x = i + n*k`` such that ``0 <= n <\n   (j-i)/k``.  In other words, the indices are ``i``, ``i+k``,\n   ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n   never including *j*).  If *i* or *j* is greater than ``len(s)``,\n   use ``len(s)``.  If *i* or *j* are omitted or ``None``, they become\n   "end" values (which end depends on the sign of *k*).  Note, *k*\n   cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. Concatenating immutable sequences always results in a new object.\n   This means that building up a sequence by repeated concatenation\n   will have a quadratic runtime cost in the total sequence length.\n   To get a linear runtime cost, you must switch to one of the\n   alternatives below:\n\n   * if concatenating ``str`` objects, you can build a list and use\n     ``str.join()`` at the end or else write to a ``io.StringIO``\n     instance and retrieve its value when complete\n\n   * if concatenating ``bytes`` objects, you can similarly use\n     ``bytes.join()`` or ``io.BytesIO``, or you can do in-place\n     concatenation with a ``bytearray`` object.  ``bytearray`` objects\n     are mutable and have an efficient overallocation mechanism\n\n   * if concatenating ``tuple`` objects, extend a ``list`` instead\n\n   * for other types, investigate the relevant class documentation\n\n7. Some sequence types (such as ``range``) only support item sequences\n   that follow specific patterns, and hence don\'t support sequence\n   concatenation or repetition.\n\n8. ``index`` raises ``ValueError`` when *x* is not found in *s*. When\n   supported, the additional arguments to the index method allow\n   efficient searching of subsections of the sequence. Passing the\n   extra arguments is roughly equivalent to using ``s[i:j].index(x)``,\n   only without copying any data and with the returned index being\n   relative to the start of the sequence rather than the start of the\n   slice.\n\n\nImmutable Sequence Types\n========================\n\nThe only operation that immutable sequence types generally implement\nthat is not also implemented by mutable sequence types is support for\nthe ``hash()`` built-in.\n\nThis support allows immutable sequences, such as ``tuple`` instances,\nto be used as ``dict`` keys and stored in ``set`` and ``frozenset``\ninstances.\n\nAttempting to hash an immutable sequence that contains unhashable\nvalues will result in ``TypeError``.\n\n\nMutable Sequence Types\n======================\n\nThe operations in the following table are defined on mutable sequence\ntypes. The ``collections.abc.MutableSequence`` ABC is provided to make\nit easier to correctly implement these operations on custom sequence\ntypes.\n\nIn the table *s* is an instance of a mutable sequence type, *t* is any\niterable object and *x* is an arbitrary object that meets any type and\nvalue restrictions imposed by *s* (for example, ``bytearray`` only\naccepts integers that meet the value restriction ``0 <= x <= 255``).\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | appends *x* to the end of the    |                       |\n|                                | sequence (same as                |                       |\n|                                | ``s[len(s):len(s)] = [x]``)      |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.clear()``                  | removes all items from ``s``     | (5)                   |\n|                                | (same as ``del s[:]``)           |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.copy()``                   | creates a shallow copy of ``s``  | (5)                   |\n|                                | (same as ``s[:]``)               |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(t)``                | extends *s* with the contents of |                       |\n|                                | *t* (same as ``s[len(s):len(s)]  |                       |\n|                                | = t``)                           |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | inserts *x* into *s* at the      |                       |\n|                                | index given by *i* (same as      |                       |\n|                                | ``s[i:i] = [x]``)                |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | retrieves the item at *i* and    | (2)                   |\n|                                | also removes it from *s*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | remove the first item from *s*   | (3)                   |\n|                                | where ``s[i] == x``              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (4)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n3. ``remove`` raises ``ValueError`` when *x* is not found in *s*.\n\n4. The ``reverse()`` method modifies the sequence in place for economy\n   of space when reversing a large sequence.  To remind users that it\n   operates by side effect, it does not return the reversed sequence.\n\n5. ``clear()`` and ``copy()`` are included for consistency with the\n   interfaces of mutable containers that don\'t support slicing\n   operations (such as ``dict`` and ``set``)\n\n   New in version 3.3: ``clear()`` and ``copy()`` methods.\n\n\nLists\n=====\n\nLists are mutable sequences, typically used to store collections of\nhomogeneous items (where the precise degree of similarity will vary by\napplication).\n\nclass class list([iterable])\n\n   Lists may be constructed in several ways:\n\n   * Using a pair of square brackets to denote the empty list: ``[]``\n\n   * Using square brackets, separating items with commas: ``[a]``,\n     ``[a, b, c]``\n\n   * Using a list comprehension: ``[x for x in iterable]``\n\n   * Using the type constructor: ``list()`` or ``list(iterable)``\n\n   The constructor builds a list whose items are the same and in the\n   same order as *iterable*\'s items.  *iterable* may be either a\n   sequence, a container that supports iteration, or an iterator\n   object.  If *iterable* is already a list, a copy is made and\n   returned, similar to ``iterable[:]``. For example, ``list(\'abc\')``\n   returns ``[\'a\', \'b\', \'c\']`` and ``list( (1, 2, 3) )`` returns ``[1,\n   2, 3]``. If no argument is given, the constructor creates a new\n   empty list, ``[]``.\n\n   Many other operations also produce lists, including the\n   ``sorted()`` built-in.\n\n   Lists implement all of the *common* and *mutable* sequence\n   operations. Lists also provide the following additional method:\n\n   sort(*, key=None, reverse=None)\n\n      This method sorts the list in place, using only ``<``\n      comparisons between items. Exceptions are not suppressed - if\n      any comparison operations fail, the entire sort operation will\n      fail (and the list will likely be left in a partially modified\n      state).\n\n      *key* specifies a function of one argument that is used to\n      extract a comparison key from each list element (for example,\n      ``key=str.lower``). The key corresponding to each item in the\n      list is calculated once and then used for the entire sorting\n      process. The default value of ``None`` means that list items are\n      sorted directly without calculating a separate key value.\n\n      The ``functools.cmp_to_key()`` utility is available to convert a\n      2.x style *cmp* function to a *key* function.\n\n      *reverse* is a boolean value.  If set to ``True``, then the list\n      elements are sorted as if each comparison were reversed.\n\n      This method modifies the sequence in place for economy of space\n      when sorting a large sequence.  To remind users that it operates\n      by side effect, it does not return the sorted sequence (use\n      ``sorted()`` to explicitly request a new sorted list instance).\n\n      The ``sort()`` method is guaranteed to be stable.  A sort is\n      stable if it guarantees not to change the relative order of\n      elements that compare equal --- this is helpful for sorting in\n      multiple passes (for example, sort by department, then by salary\n      grade).\n\n      **CPython implementation detail:** While a list is being sorted,\n      the effect of attempting to mutate, or even inspect, the list is\n      undefined.  The C implementation of Python makes the list appear\n      empty for the duration, and raises ``ValueError`` if it can\n      detect that the list has been mutated during a sort.\n\n\nTuples\n======\n\nTuples are immutable sequences, typically used to store collections of\nheterogeneous data (such as the 2-tuples produced by the\n``enumerate()`` built-in). Tuples are also used for cases where an\nimmutable sequence of homogeneous data is needed (such as allowing\nstorage in a ``set`` or ``dict`` instance).\n\nclass class tuple([iterable])\n\n   Tuples may be constructed in a number of ways:\n\n   * Using a pair of parentheses to denote the empty tuple: ``()``\n\n   * Using a trailing comma for a singleton tuple: ``a,`` or ``(a,)``\n\n   * Separating items with commas: ``a, b, c`` or ``(a, b, c)``\n\n   * Using the ``tuple()`` built-in: ``tuple()`` or\n     ``tuple(iterable)``\n\n   The constructor builds a tuple whose items are the same and in the\n   same order as *iterable*\'s items.  *iterable* may be either a\n   sequence, a container that supports iteration, or an iterator\n   object.  If *iterable* is already a tuple, it is returned\n   unchanged. For example, ``tuple(\'abc\')`` returns ``(\'a\', \'b\',\n   \'c\')`` and ``tuple( [1, 2, 3] )`` returns ``(1, 2, 3)``. If no\n   argument is given, the constructor creates a new empty tuple,\n   ``()``.\n\n   Note that it is actually the comma which makes a tuple, not the\n   parentheses. The parentheses are optional, except in the empty\n   tuple case, or when they are needed to avoid syntactic ambiguity.\n   For example, ``f(a, b, c)`` is a function call with three\n   arguments, while ``f((a, b, c))`` is a function call with a 3-tuple\n   as the sole argument.\n\n   Tuples implement all of the *common* sequence operations.\n\nFor heterogeneous collections of data where access by name is clearer\nthan access by index, ``collections.namedtuple()`` may be a more\nappropriate choice than a simple tuple object.\n\n\nRanges\n======\n\nThe ``range`` type represents an immutable sequence of numbers and is\ncommonly used for looping a specific number of times in ``for`` loops.\n\nclass class range([start], stop[, step])\n\n   The arguments to the range constructor must be integers (either\n   built-in ``int`` or any object that implements the ``__index__``\n   special method).  If the *step* argument is omitted, it defaults to\n   ``1``. If the *start* argument is omitted, it defaults to ``0``. If\n   *step* is zero, ``ValueError`` is raised.\n\n   For a positive *step*, the contents of a range ``r`` are determined\n   by the formula ``r[i] = start + step*i`` where ``i >= 0`` and\n   ``r[i] < stop``.\n\n   For a negative *step*, the contents of the range are still\n   determined by the formula ``r[i] = start + step*i``, but the\n   constraints are ``i >= 0`` and ``r[i] > stop``.\n\n   A range object will be empty if ``r[0]`` does not meant the value\n   constraint. Ranges do support negative indices, but these are\n   interpreted as indexing from the end of the sequence determined by\n   the positive indices.\n\n   Ranges containing absolute values larger than ``sys.maxsize`` are\n   permitted but some features (such as ``len()``) may raise\n   ``OverflowError``.\n\n   Range examples:\n\n      >>> list(range(10))\n      [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n      >>> list(range(1, 11))\n      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n      >>> list(range(0, 30, 5))\n      [0, 5, 10, 15, 20, 25]\n      >>> list(range(0, 10, 3))\n      [0, 3, 6, 9]\n      >>> list(range(0, -10, -1))\n      [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n      >>> list(range(0))\n      []\n      >>> list(range(1, 0))\n      []\n\n   Ranges implement all of the *common* sequence operations except\n   concatenation and repetition (due to the fact that range objects\n   can only represent sequences that follow a strict pattern and\n   repetition and concatenation will usually violate that pattern).\n\nThe advantage of the ``range`` type over a regular ``list`` or\n``tuple`` is that a ``range`` object will always take the same (small)\namount of memory, no matter the size of the range it represents (as it\nonly stores the ``start``, ``stop`` and ``step`` values, calculating\nindividual items and subranges as needed).\n\nRange objects implement the ``collections.Sequence`` ABC, and provide\nfeatures such as containment tests, element index lookup, slicing and\nsupport for negative indices (see *Sequence Types --- list, tuple,\nrange*):\n\n>>> r = range(0, 20, 2)\n>>> r\nrange(0, 20, 2)\n>>> 11 in r\nFalse\n>>> 10 in r\nTrue\n>>> r.index(10)\n5\n>>> r[5]\n10\n>>> r[:5]\nrange(0, 10, 2)\n>>> r[-1]\n18\n\nTesting range objects for equality with ``==`` and ``!=`` compares\nthem as sequences.  That is, two range objects are considered equal if\nthey represent the same sequence of values.  (Note that two range\nobjects that compare equal might have different ``start``, ``stop``\nand ``step`` attributes, for example ``range(0) == range(2, 1, 3)`` or\n``range(0, 3, 2) == range(0, 4, 2)``.)\n\nChanged in version 3.2: Implement the Sequence ABC. Support slicing\nand negative indices. Test ``int`` objects for membership in constant\ntime instead of iterating through all items.\n\nChanged in version 3.3: Define \'==\' and \'!=\' to compare range objects\nbased on the sequence of values they define (instead of comparing\nbased on object identity).\n\nNew in version 3.3: The ``start``, ``stop`` and ``step`` attributes.\n',
+ 'typesseq-mutable': "\nMutable Sequence Types\n**********************\n\nThe operations in the following table are defined on mutable sequence\ntypes. The ``collections.abc.MutableSequence`` ABC is provided to make\nit easier to correctly implement these operations on custom sequence\ntypes.\n\nIn the table *s* is an instance of a mutable sequence type, *t* is any\niterable object and *x* is an arbitrary object that meets any type and\nvalue restrictions imposed by *s* (for example, ``bytearray`` only\naccepts integers that meet the value restriction ``0 <= x <= 255``).\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation                      | Result                           | Notes                 |\n+================================+==================================+=======================+\n| ``s[i] = x``                   | item *i* of *s* is replaced by   |                       |\n|                                | *x*                              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t``                 | slice of *s* from *i* to *j* is  |                       |\n|                                | replaced by the contents of the  |                       |\n|                                | iterable *t*                     |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]``                 | same as ``s[i:j] = []``          |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t``               | the elements of ``s[i:j:k]`` are | (1)                   |\n|                                | replaced by those of *t*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]``               | removes the elements of          |                       |\n|                                | ``s[i:j:k]`` from the list       |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)``                | appends *x* to the end of the    |                       |\n|                                | sequence (same as                |                       |\n|                                | ``s[len(s):len(s)] = [x]``)      |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.clear()``                  | removes all items from ``s``     | (5)                   |\n|                                | (same as ``del s[:]``)           |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.copy()``                   | creates a shallow copy of ``s``  | (5)                   |\n|                                | (same as ``s[:]``)               |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(t)``                | extends *s* with the contents of |                       |\n|                                | *t* (same as ``s[len(s):len(s)]  |                       |\n|                                | = t``)                           |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)``             | inserts *x* into *s* at the      |                       |\n|                                | index given by *i* (same as      |                       |\n|                                | ``s[i:i] = [x]``)                |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])``                 | retrieves the item at *i* and    | (2)                   |\n|                                | also removes it from *s*         |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)``                | remove the first item from *s*   | (3)                   |\n|                                | where ``s[i] == x``              |                       |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()``                | reverses the items of *s* in     | (4)                   |\n|                                | place                            |                       |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The optional argument *i* defaults to ``-1``, so that by default\n   the last item is removed and returned.\n\n3. ``remove`` raises ``ValueError`` when *x* is not found in *s*.\n\n4. The ``reverse()`` method modifies the sequence in place for economy\n   of space when reversing a large sequence.  To remind users that it\n   operates by side effect, it does not return the reversed sequence.\n\n5. ``clear()`` and ``copy()`` are included for consistency with the\n   interfaces of mutable containers that don't support slicing\n   operations (such as ``dict`` and ``set``)\n\n   New in version 3.3: ``clear()`` and ``copy()`` methods.\n",
  'unary': '\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n   u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\ninteger argument.  The bitwise inversion of ``x`` is defined as\n``-(x+1)``.  It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n',
  'while': '\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n   while_stmt ::= "while" expression ":" suite\n                  ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite.  A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n',
  'with': '\nThe ``with`` statement\n**********************\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n   with_stmt ::= "with" with_item ("," with_item)* ":" suite\n   with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n   is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n   value from ``__enter__()`` is assigned to it.\n\n   Note: The ``with`` statement guarantees that if the ``__enter__()``\n     method returns without an error, then ``__exit__()`` will always\n     be called. Thus, if an error occurs during the assignment to the\n     target list, it will be treated the same as an error occurring\n     within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked.  If an\n   exception caused the suite to be exited, its type, value, and\n   traceback are passed as arguments to ``__exit__()``. Otherwise,\n   three ``None`` arguments are supplied.\n\n   If the suite was exited due to an exception, and the return value\n   from the ``__exit__()`` method was false, the exception is\n   reraised.  If the return value was true, the exception is\n   suppressed, and execution continues with the statement following\n   the ``with`` statement.\n\n   If the suite was exited for any reason other than an exception, the\n   return value from ``__exit__()`` is ignored, and execution proceeds\n   at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n   with A() as a, B() as b:\n       suite\n\nis equivalent to\n\n   with A() as a:\n       with B() as b:\n           suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also:\n\n   **PEP 0343** - The "with" statement\n      The specification, background, and examples for the Python\n      ``with`` statement.\n',
- 'yield': '\nThe ``yield`` statement\n***********************\n\n   yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function. When a generator function is called, it returns an\niterator known as a generator iterator, or more commonly, a generator.\nThe body of the generator function is executed by calling the\n``next()`` function on the generator repeatedly until it raises an\nexception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of ``expression_list`` is returned to\n``next()``\'s caller.  By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nThe ``yield`` statement is allowed in the ``try`` clause of a ``try``\n...  ``finally`` construct.  If the generator is not resumed before it\nis finalized (by reaching a zero reference count or by being garbage\ncollected), the generator-iterator\'s ``close()`` method will be\ncalled, allowing any pending ``finally`` clauses to execute.\n\nSee also:\n\n   **PEP 0255** - Simple Generators\n      The proposal for adding generators and the ``yield`` statement\n      to Python.\n\n   **PEP 0342** - Coroutines via Enhanced Generators\n      The proposal that, among other generator enhancements, proposed\n      allowing ``yield`` to appear inside a ``try`` ... ``finally``\n      block.\n'}
+ 'yield': '\nThe ``yield`` statement\n***********************\n\n   yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator.  The body of the\ngenerator function is executed by calling the ``next()`` function on\nthe generator repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of ``expression_list`` is returned to\n``next()``\'s caller.  By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nThe ``yield`` statement is allowed in the ``try`` clause of a ``try``\n...  ``finally`` construct.  If the generator is not resumed before it\nis finalized (by reaching a zero reference count or by being garbage\ncollected), the generator-iterator\'s ``close()`` method will be\ncalled, allowing any pending ``finally`` clauses to execute.\n\nWhen ``yield from <expr>`` is used, it treats the supplied expression\nas a subiterator, producing values from it until the underlying\niterator is exhausted.\n\n   Changed in version 3.3: Added ``yield from <expr>`` to delegate\n   control flow to a subiterator\n\nFor full details of ``yield`` semantics, refer to the *Yield\nexpressions* section.\n\nSee also:\n\n   **PEP 0255** - Simple Generators\n      The proposal for adding generators and the ``yield`` statement\n      to Python.\n\n   **PEP 0342** - Coroutines via Enhanced Generators\n      The proposal to enhance the API and syntax of generators, making\n      them usable as simple coroutines.\n\n   **PEP 0380** - Syntax for Delegating to a Subgenerator\n      The proposal to introduce the ``yield_from`` syntax, making\n      delegation to sub-generators easy.\n'}
diff --git a/Lib/queue.py b/Lib/queue.py
index bee7ed4086..c3296fe138 100644
--- a/Lib/queue.py
+++ b/Lib/queue.py
@@ -1,49 +1,57 @@
-"""A multi-producer, multi-consumer queue."""
+'''A multi-producer, multi-consumer queue.'''
 
-from time import time as _time
 try:
-    import threading as _threading
+    import threading
 except ImportError:
-    import dummy_threading as _threading
+    import dummy_threading as threading
 from collections import deque
-import heapq
+from heapq import heappush, heappop
+try:
+    from time import monotonic as time
+except ImportError:
+    from time import time
 
 __all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue']
 
 class Empty(Exception):
-    "Exception raised by Queue.get(block=0)/get_nowait()."
+    'Exception raised by Queue.get(block=0)/get_nowait().'
     pass
 
 class Full(Exception):
-    "Exception raised by Queue.put(block=0)/put_nowait()."
+    'Exception raised by Queue.put(block=0)/put_nowait().'
     pass
 
 class Queue:
-    """Create a queue object with a given maximum size.
+    '''Create a queue object with a given maximum size.
 
     If maxsize is <= 0, the queue size is infinite.
-    """
+    '''
+
     def __init__(self, maxsize=0):
         self.maxsize = maxsize
         self._init(maxsize)
+
         # mutex must be held whenever the queue is mutating.  All methods
         # that acquire mutex must release it before returning.  mutex
         # is shared between the three conditions, so acquiring and
         # releasing the conditions also acquires and releases mutex.
-        self.mutex = _threading.Lock()
+        self.mutex = threading.Lock()
+
         # Notify not_empty whenever an item is added to the queue; a
         # thread waiting to get is notified then.
-        self.not_empty = _threading.Condition(self.mutex)
+        self.not_empty = threading.Condition(self.mutex)
+
         # Notify not_full whenever an item is removed from the queue;
         # a thread waiting to put is notified then.
-        self.not_full = _threading.Condition(self.mutex)
+        self.not_full = threading.Condition(self.mutex)
+
         # Notify all_tasks_done whenever the number of unfinished tasks
         # drops to zero; thread waiting to join() is notified to resume
-        self.all_tasks_done = _threading.Condition(self.mutex)
+        self.all_tasks_done = threading.Condition(self.mutex)
         self.unfinished_tasks = 0
 
     def task_done(self):
-        """Indicate that a formerly enqueued task is complete.
+        '''Indicate that a formerly enqueued task is complete.
 
         Used by Queue consumer threads.  For each get() used to fetch a task,
         a subsequent call to task_done() tells the queue that the processing
@@ -55,43 +63,35 @@ class Queue:
 
         Raises a ValueError if called more times than there were items
         placed in the queue.
-        """
-        self.all_tasks_done.acquire()
-        try:
+        '''
+        with self.all_tasks_done:
             unfinished = self.unfinished_tasks - 1
             if unfinished <= 0:
                 if unfinished < 0:
                     raise ValueError('task_done() called too many times')
                 self.all_tasks_done.notify_all()
             self.unfinished_tasks = unfinished
-        finally:
-            self.all_tasks_done.release()
 
     def join(self):
-        """Blocks until all items in the Queue have been gotten and processed.
+        '''Blocks until all items in the Queue have been gotten and processed.
 
         The count of unfinished tasks goes up whenever an item is added to the
         queue. The count goes down whenever a consumer thread calls task_done()
         to indicate the item was retrieved and all work on it is complete.
 
         When the count of unfinished tasks drops to zero, join() unblocks.
-        """
-        self.all_tasks_done.acquire()
-        try:
+        '''
+        with self.all_tasks_done:
             while self.unfinished_tasks:
                 self.all_tasks_done.wait()
-        finally:
-            self.all_tasks_done.release()
 
     def qsize(self):
-        """Return the approximate size of the queue (not reliable!)."""
-        self.mutex.acquire()
-        n = self._qsize()
-        self.mutex.release()
-        return n
+        '''Return the approximate size of the queue (not reliable!).'''
+        with self.mutex:
+            return self._qsize()
 
     def empty(self):
-        """Return True if the queue is empty, False otherwise (not reliable!).
+        '''Return True if the queue is empty, False otherwise (not reliable!).
 
         This method is likely to be removed at some point.  Use qsize() == 0
         as a direct substitute, but be aware that either approach risks a race
@@ -100,29 +100,23 @@ class Queue:
 
         To create code that needs to wait for all queued tasks to be
         completed, the preferred technique is to use the join() method.
-
-        """
-        self.mutex.acquire()
-        n = not self._qsize()
-        self.mutex.release()
-        return n
+        '''
+        with self.mutex:
+            return not self._qsize()
 
     def full(self):
-        """Return True if the queue is full, False otherwise (not reliable!).
+        '''Return True if the queue is full, False otherwise (not reliable!).
 
         This method is likely to be removed at some point.  Use qsize() >= n
         as a direct substitute, but be aware that either approach risks a race
         condition where a queue can shrink before the result of full() or
         qsize() can be used.
-
-        """
-        self.mutex.acquire()
-        n = 0 < self.maxsize <= self._qsize()
-        self.mutex.release()
-        return n
+        '''
+        with self.mutex:
+            return 0 < self.maxsize <= self._qsize()
 
     def put(self, item, block=True, timeout=None):
-        """Put an item into the queue.
+        '''Put an item into the queue.
 
         If optional args 'block' is true and 'timeout' is None (the default),
         block if necessary until a free slot is available. If 'timeout' is
@@ -131,9 +125,8 @@ class Queue:
         Otherwise ('block' is false), put an item on the queue if a free slot
         is immediately available, else raise the Full exception ('timeout'
         is ignored in that case).
-        """
-        self.not_full.acquire()
-        try:
+        '''
+        with self.not_full:
             if self.maxsize > 0:
                 if not block:
                     if self._qsize() >= self.maxsize:
@@ -144,28 +137,18 @@ class Queue:
                 elif timeout < 0:
                     raise ValueError("'timeout' must be a positive number")
                 else:
-                    endtime = _time() + timeout
+                    endtime = time() + timeout
                     while self._qsize() >= self.maxsize:
-                        remaining = endtime - _time()
+                        remaining = endtime - time()
                         if remaining <= 0.0:
                             raise Full
                         self.not_full.wait(remaining)
             self._put(item)
             self.unfinished_tasks += 1
             self.not_empty.notify()
-        finally:
-            self.not_full.release()
-
-    def put_nowait(self, item):
-        """Put an item into the queue without blocking.
-
-        Only enqueue the item if a free slot is immediately available.
-        Otherwise raise the Full exception.
-        """
-        return self.put(item, False)
 
     def get(self, block=True, timeout=None):
-        """Remove and return an item from the queue.
+        '''Remove and return an item from the queue.
 
         If optional args 'block' is true and 'timeout' is None (the default),
         block if necessary until an item is available. If 'timeout' is
@@ -174,9 +157,8 @@ class Queue:
         Otherwise ('block' is false), return an item if one is immediately
         available, else raise the Empty exception ('timeout' is ignored
         in that case).
-        """
-        self.not_empty.acquire()
-        try:
+        '''
+        with self.not_empty:
             if not block:
                 if not self._qsize():
                     raise Empty
@@ -186,25 +168,31 @@ class Queue:
             elif timeout < 0:
                 raise ValueError("'timeout' must be a positive number")
             else:
-                endtime = _time() + timeout
+                endtime = time() + timeout
                 while not self._qsize():
-                    remaining = endtime - _time()
+                    remaining = endtime - time()
                     if remaining <= 0.0:
                         raise Empty
                     self.not_empty.wait(remaining)
             item = self._get()
             self.not_full.notify()
             return item
-        finally:
-            self.not_empty.release()
+
+    def put_nowait(self, item):
+        '''Put an item into the queue without blocking.
+
+        Only enqueue the item if a free slot is immediately available.
+        Otherwise raise the Full exception.
+        '''
+        return self.put(item, block=False)
 
     def get_nowait(self):
-        """Remove and return an item from the queue without blocking.
+        '''Remove and return an item from the queue without blocking.
 
         Only get an item if one is immediately available. Otherwise
         raise the Empty exception.
-        """
-        return self.get(False)
+        '''
+        return self.get(block=False)
 
     # Override these methods to implement other queue organizations
     # (e.g. stack or priority queue).
@@ -214,7 +202,7 @@ class Queue:
     def _init(self, maxsize):
         self.queue = deque()
 
-    def _qsize(self, len=len):
+    def _qsize(self):
         return len(self.queue)
 
     # Put a new item in the queue
@@ -235,13 +223,13 @@ class PriorityQueue(Queue):
     def _init(self, maxsize):
         self.queue = []
 
-    def _qsize(self, len=len):
+    def _qsize(self):
         return len(self.queue)
 
-    def _put(self, item, heappush=heapq.heappush):
+    def _put(self, item):
         heappush(self.queue, item)
 
-    def _get(self, heappop=heapq.heappop):
+    def _get(self):
         return heappop(self.queue)
 
 
@@ -251,7 +239,7 @@ class LifoQueue(Queue):
     def _init(self, maxsize):
         self.queue = []
 
-    def _qsize(self, len=len):
+    def _qsize(self):
         return len(self.queue)
 
     def _put(self, item):
diff --git a/Lib/random.py b/Lib/random.py
index 91be5adf84..7d8d4f3a40 100644
--- a/Lib/random.py
+++ b/Lib/random.py
@@ -41,7 +41,7 @@ from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethod
 from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil
 from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin
 from os import urandom as _urandom
-from collections import Set as _Set, Sequence as _Sequence
+from collections.abc import Set as _Set, Sequence as _Sequence
 from hashlib import sha512 as _sha512
 
 __all__ = ["Random","seed","random","uniform","randint","choice","sample",
diff --git a/Lib/re.py b/Lib/re.py
index 85c5a57024..952b60f45b 100644
--- a/Lib/re.py
+++ b/Lib/re.py
@@ -215,7 +215,7 @@ def compile(pattern, flags=0):
 
 def purge():
     "Clear the regular expression caches"
-    _compile_typed.cache_clear()
+    _compile.cache_clear()
     _compile_repl.cache_clear()
 
 def template(pattern, flags=0):
@@ -223,12 +223,14 @@ def template(pattern, flags=0):
     return _compile(pattern, flags|T)
 
 _alphanum_str = frozenset(
-    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890")
+    "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890")
 _alphanum_bytes = frozenset(
-    b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890")
+    b"_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890")
 
 def escape(pattern):
-    "Escape all non-alphanumeric characters in pattern."
+    """
+    Escape all the characters in pattern except ASCII letters, numbers and '_'.
+    """
     if isinstance(pattern, str):
         alphanum = _alphanum_str
         s = list(pattern)
@@ -259,11 +261,8 @@ def escape(pattern):
 
 _pattern_type = type(sre_compile.compile("", 0))
 
+@functools.lru_cache(maxsize=512, typed=True)
 def _compile(pattern, flags):
-    return _compile_typed(type(pattern), pattern, flags)
-
-@functools.lru_cache(maxsize=500)
-def _compile_typed(text_bytes_type, pattern, flags):
     # internal: compile pattern
     if isinstance(pattern, _pattern_type):
         if flags:
@@ -274,7 +273,7 @@ def _compile_typed(text_bytes_type, pattern, flags):
         raise TypeError("first argument must be string or compiled pattern")
     return sre_compile.compile(pattern, flags)
 
-@functools.lru_cache(maxsize=500)
+@functools.lru_cache(maxsize=512)
 def _compile_repl(repl, pattern):
     # internal: compile replacement pattern
     return sre_parse.parse_template(repl, pattern)
diff --git a/Lib/runpy.py b/Lib/runpy.py
index a14a62e128..39c0e9f7dd 100644
--- a/Lib/runpy.py
+++ b/Lib/runpy.py
@@ -9,13 +9,12 @@ importers when locating support scripts as well as when importing modules.
 # Written by Nick Coghlan <ncoghlan at gmail.com>
 #    to implement PEP 338 (Executing Modules as Scripts)
 
+
+import os
 import sys
+import importlib.machinery # importlib first so we can test #15386 via -m
 import imp
-from pkgutil import read_code
-try:
-    from imp import get_loader
-except ImportError:
-    from pkgutil import get_loader
+from pkgutil import read_code, get_loader, get_importer
 
 __all__ = [
     "run_module", "run_path",
@@ -95,7 +94,7 @@ def _get_filename(loader, mod_name):
     for attr in ("get_filename", "_get_filename"):
         meth = getattr(loader, attr, None)
         if meth is not None:
-            return meth(mod_name)
+            return os.path.abspath(meth(mod_name))
     return None
 
 # Helper to get the loader, code and filename for a module
@@ -181,47 +180,23 @@ def run_module(mod_name, init_globals=None,
 def _get_main_module_details():
     # Helper that gives a nicer error message when attempting to
     # execute a zipfile or directory by invoking __main__.py
+    # Also moves the standard __main__ out of the way so that the
+    # preexisting __loader__ entry doesn't cause issues
     main_name = "__main__"
+    saved_main = sys.modules[main_name]
+    del sys.modules[main_name]
     try:
         return _get_module_details(main_name)
     except ImportError as exc:
         if main_name in str(exc):
             raise ImportError("can't find %r module in %r" %
-                              (main_name, sys.path[0]))
+                              (main_name, sys.path[0])) from exc
         raise
+    finally:
+        sys.modules[main_name] = saved_main
 
 
-# XXX (ncoghlan): Perhaps expose the C API function
-# as imp.get_importer instead of reimplementing it in Python?
-def _get_importer(path_name):
-    """Python version of PyImport_GetImporter C API function"""
-    cache = sys.path_importer_cache
-    try:
-        importer = cache[path_name]
-    except KeyError:
-        # Not yet cached. Flag as using the
-        # standard machinery until we finish
-        # checking the hooks
-        cache[path_name] = None
-        for hook in sys.path_hooks:
-            try:
-                importer = hook(path_name)
-                break
-            except ImportError:
-                pass
-        else:
-            # The following check looks a bit odd. The trick is that
-            # NullImporter raises ImportError if the supplied path is a
-            # *valid* directory entry (and hence able to be handled
-            # by the standard import machinery)
-            try:
-                importer = imp.NullImporter(path_name)
-            except ImportError:
-                return None
-        cache[path_name] = importer
-    return importer
-
-def _get_code_from_file(fname):
+def _get_code_from_file(run_name, fname):
     # Check for a compiled file first
     with open(fname, "rb") as f:
         code = read_code(f)
@@ -229,7 +204,10 @@ def _get_code_from_file(fname):
         # That didn't work, so try it as normal source code
         with open(fname, "rb") as f:
             code = compile(f.read(), fname, 'exec')
-    return code
+            loader = importlib.machinery.SourceFileLoader(run_name, fname)
+    else:
+        loader = importlib.machinery.SourcelessFileLoader(run_name, fname)
+    return code, loader
 
 def run_path(path_name, init_globals=None, run_name=None):
     """Execute code located at the specified filesystem location
@@ -244,13 +222,13 @@ def run_path(path_name, init_globals=None, run_name=None):
     if run_name is None:
         run_name = "<run_path>"
     pkg_name = run_name.rpartition(".")[0]
-    importer = _get_importer(path_name)
-    if isinstance(importer, imp.NullImporter):
+    importer = get_importer(path_name)
+    if isinstance(importer, (type(None), imp.NullImporter)):
         # Not a valid sys.path entry, so run the code directly
         # execfile() doesn't help as we want to allow compiled files
-        code = _get_code_from_file(path_name)
+        code, mod_loader = _get_code_from_file(run_name, path_name)
         return _run_module_code(code, init_globals, run_name, path_name,
-                                pkg_name=pkg_name)
+                                mod_loader, pkg_name)
     else:
         # Importer is defined for path, so add it to
         # the start of sys.path
@@ -262,13 +240,7 @@ def run_path(path_name, init_globals=None, run_name=None):
             # have no choice and we have to remove it even while we read the
             # code. If we don't do this, a __loader__ attribute in the
             # existing __main__ module may prevent location of the new module.
-            main_name = "__main__"
-            saved_main = sys.modules[main_name]
-            del sys.modules[main_name]
-            try:
-                mod_name, loader, code, fname = _get_main_module_details()
-            finally:
-                sys.modules[main_name] = saved_main
+            mod_name, loader, code, fname = _get_main_module_details()
             with _TempModule(run_name) as temp_module, \
                  _ModifiedArgv0(path_name):
                 mod_globals = temp_module.module.__dict__
diff --git a/Lib/sched.py b/Lib/sched.py
index a119892c3f..ccf8ce9074 100644
--- a/Lib/sched.py
+++ b/Lib/sched.py
@@ -13,12 +13,12 @@ also be used to integrate scheduling with STDWIN events; the delay
 function is allowed to modify the queue.  Time can be expressed as
 integers or floating point numbers, as long as it is consistent.
 
-Events are specified by tuples (time, priority, action, argument).
+Events are specified by tuples (time, priority, action, argument, kwargs).
 As in UNIX, lower priority numbers mean higher priority; in this
 way the queue can be maintained as a priority queue.  Execution of the
 event means calling the action function, passing it the argument
 sequence in "argument" (remember that in Python, multiple function
-arguments are be packed in a sequence).
+arguments are be packed in a sequence) and keyword parameters in "kwargs".
 The action function may be an instance method so it
 has another way to reference private data (besides global variables).
 """
@@ -28,12 +28,21 @@ has another way to reference private data (besides global variables).
 # XXX instead of having to define a module or class just to hold
 # XXX the global state of your particular time and delay functions.
 
+import time
 import heapq
 from collections import namedtuple
+try:
+    import threading
+except ImportError:
+    import dummy_threading as threading
+try:
+    from time import monotonic as _time
+except ImportError:
+    from time import time as _time
 
 __all__ = ["scheduler"]
 
-class Event(namedtuple('Event', 'time, priority, action, argument')):
+class Event(namedtuple('Event', 'time, priority, action, argument, kwargs')):
     def __eq__(s, o): return (s.time, s.priority) == (o.time, o.priority)
     def __ne__(s, o): return (s.time, s.priority) != (o.time, o.priority)
     def __lt__(s, o): return (s.time, s.priority) <  (o.time, o.priority)
@@ -41,33 +50,41 @@ class Event(namedtuple('Event', 'time, priority, action, argument')):
     def __gt__(s, o): return (s.time, s.priority) >  (o.time, o.priority)
     def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority)
 
+_sentinel = object()
+
 class scheduler:
-    def __init__(self, timefunc, delayfunc):
+
+    def __init__(self, timefunc=_time, delayfunc=time.sleep):
         """Initialize a new instance, passing the time and delay
         functions"""
         self._queue = []
+        self._lock = threading.RLock()
         self.timefunc = timefunc
         self.delayfunc = delayfunc
 
-    def enterabs(self, time, priority, action, argument):
+    def enterabs(self, time, priority, action, argument=(), kwargs=_sentinel):
         """Enter a new event in the queue at an absolute time.
 
         Returns an ID for the event which can be used to remove it,
         if necessary.
 
         """
-        event = Event(time, priority, action, argument)
-        heapq.heappush(self._queue, event)
-        return event # The ID
-
-    def enter(self, delay, priority, action, argument):
+        if kwargs is _sentinel:
+            kwargs = {}
+        with self._lock:
+            event = Event(time, priority, action, argument, kwargs)
+            heapq.heappush(self._queue, event)
+            return event # The ID
+
+    def enter(self, delay, priority, action, argument=(), kwargs=_sentinel):
         """A variant that specifies the time as a relative time.
 
         This is actually the more commonly used interface.
 
         """
-        time = self.timefunc() + delay
-        return self.enterabs(time, priority, action, argument)
+        with self._lock:
+            time = self.timefunc() + delay
+            return self.enterabs(time, priority, action, argument, kwargs)
 
     def cancel(self, event):
         """Remove an event from the queue.
@@ -76,15 +93,20 @@ class scheduler:
         If the event is not in the queue, this raises ValueError.
 
         """
-        self._queue.remove(event)
-        heapq.heapify(self._queue)
+        with self._lock:
+            self._queue.remove(event)
+            heapq.heapify(self._queue)
 
     def empty(self):
         """Check whether the queue is empty."""
-        return not self._queue
+        with self._lock:
+            return not self._queue
 
-    def run(self):
+    def run(self, blocking=True):
         """Execute events until the queue is empty.
+        If blocking is False executes the scheduled events due to
+        expire soonest (if any) and then return the deadline of the
+        next scheduled call in the scheduler.
 
         When there is a positive delay until the first event, the
         delay function is called and the event is left in the queue;
@@ -106,35 +128,41 @@ class scheduler:
         """
         # localize variable access to minimize overhead
         # and to improve thread safety
+        lock = self._lock
         q = self._queue
         delayfunc = self.delayfunc
         timefunc = self.timefunc
         pop = heapq.heappop
-        while q:
-            time, priority, action, argument = checked_event = q[0]
-            now = timefunc()
-            if now < time:
+        while True:
+            with lock:
+                if not q:
+                    break
+                time, priority, action, argument, kwargs = q[0]
+                now = timefunc()
+                if time > now:
+                    delay = True
+                else:
+                    delay = False
+                    pop(q)
+            if delay:
+                if not blocking:
+                    return time - now
                 delayfunc(time - now)
             else:
-                event = pop(q)
-                # Verify that the event was not removed or altered
-                # by another thread after we last looked at q[0].
-                if event is checked_event:
-                    action(*argument)
-                    delayfunc(0)   # Let other threads run
-                else:
-                    heapq.heappush(q, event)
+                action(*argument, **kwargs)
+                delayfunc(0)   # Let other threads run
 
     @property
     def queue(self):
         """An ordered list of upcoming events.
 
         Events are named tuples with fields for:
-            time, priority, action, arguments
+            time, priority, action, arguments, kwargs
 
         """
         # Use heapq to sort the queue rather than using 'sorted(self._queue)'.
         # With heapq, two events scheduled at the same time will show in
         # the actual order they would be retrieved.
-        events = self._queue[:]
-        return map(heapq.heappop, [events]*len(events))
+        with self._lock:
+            events = self._queue[:]
+            return map(heapq.heappop, [events]*len(events))
diff --git a/Lib/shlex.py b/Lib/shlex.py
index 3edd3db1ed..69f3b456af 100644
--- a/Lib/shlex.py
+++ b/Lib/shlex.py
@@ -6,13 +6,14 @@
 # Posix compliance, split(), string arguments, and
 # iterator interface by Gustavo Niemeyer, April 2003.
 
-import os.path
+import os
+import re
 import sys
 from collections import deque
 
 from io import StringIO
 
-__all__ = ["shlex", "split"]
+__all__ = ["shlex", "split", "quote"]
 
 class shlex:
     "A lexical analyzer class for simple shell-like syntaxes."
@@ -274,6 +275,21 @@ def split(s, comments=False, posix=True):
         lex.commenters = ''
     return list(lex)
 
+
+_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search
+
+def quote(s):
+    """Return a shell-escaped version of the string *s*."""
+    if not s:
+        return "''"
+    if _find_unsafe(s) is None:
+        return s
+
+    # use single quotes, and put single quotes into double quotes
+    # the string $'b is then quoted as '$'"'"'b'
+    return "'" + s.replace("'", "'\"'\"'") + "'"
+
+
 if __name__ == '__main__':
     if len(sys.argv) == 1:
         lexer = shlex()
diff --git a/Lib/shutil.py b/Lib/shutil.py
index ef29ae2303..a188408308 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -15,6 +15,7 @@ import tarfile
 
 try:
     import bz2
+    del bz2
     _BZ2_SUPPORTED = True
 except ImportError:
     _BZ2_SUPPORTED = False
@@ -34,7 +35,9 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
            "ExecError", "make_archive", "get_archive_formats",
            "register_archive_format", "unregister_archive_format",
            "get_unpack_formats", "register_unpack_format",
-           "unregister_unpack_format", "unpack_archive", "ignore_patterns"]
+           "unregister_unpack_format", "unpack_archive",
+           "ignore_patterns", "chown", "which"]
+           # disk_usage is added later, if available on the platform
 
 class Error(EnvironmentError):
     pass
@@ -79,8 +82,13 @@ def _samefile(src, dst):
     return (os.path.normcase(os.path.abspath(src)) ==
             os.path.normcase(os.path.abspath(dst)))
 
-def copyfile(src, dst):
-    """Copy data from src to dst"""
+def copyfile(src, dst, *, follow_symlinks=True):
+    """Copy data from src to dst.
+
+    If follow_symlinks is not set and src is a symbolic link, a new
+    symlink will be created instead of copying the file it points to.
+
+    """
     if _samefile(src, dst):
         raise Error("`%s` and `%s` are the same file" % (src, dst))
 
@@ -95,56 +103,146 @@ def copyfile(src, dst):
             if stat.S_ISFIFO(st.st_mode):
                 raise SpecialFileError("`%s` is a named pipe" % fn)
 
-    with open(src, 'rb') as fsrc:
-        with open(dst, 'wb') as fdst:
-            copyfileobj(fsrc, fdst)
+    if not follow_symlinks and os.path.islink(src):
+        os.symlink(os.readlink(src), dst)
+    else:
+        with open(src, 'rb') as fsrc:
+            with open(dst, 'wb') as fdst:
+                copyfileobj(fsrc, fdst)
+    return dst
+
+def copymode(src, dst, *, follow_symlinks=True):
+    """Copy mode bits from src to dst.
 
-def copymode(src, dst):
-    """Copy mode bits from src to dst"""
-    if hasattr(os, 'chmod'):
-        st = os.stat(src)
-        mode = stat.S_IMODE(st.st_mode)
-        os.chmod(dst, mode)
+    If follow_symlinks is not set, symlinks aren't followed if and only
+    if both `src` and `dst` are symlinks.  If `lchmod` isn't available
+    (e.g. Linux) this method does nothing.
 
-def copystat(src, dst):
-    """Copy all stat info (mode bits, atime, mtime, flags) from src to dst"""
-    st = os.stat(src)
+    """
+    if not follow_symlinks and os.path.islink(src) and os.path.islink(dst):
+        if hasattr(os, 'lchmod'):
+            stat_func, chmod_func = os.lstat, os.lchmod
+        else:
+            return
+    elif hasattr(os, 'chmod'):
+        stat_func, chmod_func = os.stat, os.chmod
+    else:
+        return
+
+    st = stat_func(src)
+    chmod_func(dst, stat.S_IMODE(st.st_mode))
+
+if hasattr(os, 'listxattr'):
+    def _copyxattr(src, dst, *, follow_symlinks=True):
+        """Copy extended filesystem attributes from `src` to `dst`.
+
+        Overwrite existing attributes.
+
+        If `follow_symlinks` is false, symlinks won't be followed.
+
+        """
+
+        try:
+            names = os.listxattr(src, follow_symlinks=follow_symlinks)
+        except OSError as e:
+            if e.errno not in (errno.ENOTSUP, errno.ENODATA):
+                raise
+            return
+        for name in names:
+            try:
+                value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
+                os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
+            except OSError as e:
+                if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
+                    raise
+else:
+    def _copyxattr(*args, **kwargs):
+        pass
+
+def copystat(src, dst, *, follow_symlinks=True):
+    """Copy all stat info (mode bits, atime, mtime, flags) from src to dst.
+
+    If the optional flag `follow_symlinks` is not set, symlinks aren't followed if and
+    only if both `src` and `dst` are symlinks.
+
+    """
+    def _nop(*args, ns=None, follow_symlinks=None):
+        pass
+
+    # follow symlinks (aka don't not follow symlinks)
+    follow = follow_symlinks or not (os.path.islink(src) and os.path.islink(dst))
+    if follow:
+        # use the real function if it exists
+        def lookup(name):
+            return getattr(os, name, _nop)
+    else:
+        # use the real function only if it exists
+        # *and* it supports follow_symlinks
+        def lookup(name):
+            fn = getattr(os, name, _nop)
+            if fn in os.supports_follow_symlinks:
+                return fn
+            return _nop
+
+    st = lookup("stat")(src, follow_symlinks=follow)
     mode = stat.S_IMODE(st.st_mode)
-    if hasattr(os, 'utime'):
-        os.utime(dst, (st.st_atime, st.st_mtime))
-    if hasattr(os, 'chmod'):
-        os.chmod(dst, mode)
-    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
+    lookup("utime")(dst, ns=(st.st_atime_ns, st.st_mtime_ns),
+        follow_symlinks=follow)
+    try:
+        lookup("chmod")(dst, mode, follow_symlinks=follow)
+    except NotImplementedError:
+        # if we got a NotImplementedError, it's because
+        #   * follow_symlinks=False,
+        #   * lchown() is unavailable, and
+        #   * either
+        #       * fchownat() is unvailable or
+        #       * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW.
+        #         (it returned ENOSUP.)
+        # therefore we're out of options--we simply cannot chown the
+        # symlink.  give up, suppress the error.
+        # (which is what shutil always did in this circumstance.)
+        pass
+    if hasattr(st, 'st_flags'):
         try:
-            os.chflags(dst, st.st_flags)
+            lookup("chflags")(dst, st.st_flags, follow_symlinks=follow)
         except OSError as why:
             for err in 'EOPNOTSUPP', 'ENOTSUP':
                 if hasattr(errno, err) and why.errno == getattr(errno, err):
                     break
             else:
                 raise
+    _copyxattr(src, dst, follow_symlinks=follow)
 
-def copy(src, dst):
-    """Copy data and mode bits ("cp src dst").
+def copy(src, dst, *, follow_symlinks=True):
+    """Copy data and mode bits ("cp src dst"). Return the file's destination.
 
     The destination may be a directory.
 
+    If follow_symlinks is false, symlinks won't be followed. This
+    resembles GNU's "cp -P src dst".
+
     """
     if os.path.isdir(dst):
         dst = os.path.join(dst, os.path.basename(src))
-    copyfile(src, dst)
-    copymode(src, dst)
+    copyfile(src, dst, follow_symlinks=follow_symlinks)
+    copymode(src, dst, follow_symlinks=follow_symlinks)
+    return dst
 
-def copy2(src, dst):
-    """Copy data and all stat info ("cp -p src dst").
+def copy2(src, dst, *, follow_symlinks=True):
+    """Copy data and all stat info ("cp -p src dst"). Return the file's
+    destination."
 
     The destination may be a directory.
 
+    If follow_symlinks is false, symlinks won't be followed. This
+    resembles GNU's "cp -P src dst".
+
     """
     if os.path.isdir(dst):
         dst = os.path.join(dst, os.path.basename(src))
-    copyfile(src, dst)
-    copystat(src, dst)
+    copyfile(src, dst, follow_symlinks=follow_symlinks)
+    copystat(src, dst, follow_symlinks=follow_symlinks)
+    return dst
 
 def ignore_patterns(*patterns):
     """Function that can be used as copytree() ignore parameter.
@@ -211,7 +309,11 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2,
             if os.path.islink(srcname):
                 linkto = os.readlink(srcname)
                 if symlinks:
+                    # We can't just leave it to `copy_function` because legacy
+                    # code with a custom `copy_function` may rely on copytree
+                    # doing the right thing.
                     os.symlink(linkto, dstname)
+                    copystat(srcname, dstname, follow_symlinks=not symlinks)
                 else:
                     # ignore dangling symlink if the flag is on
                     if not os.path.exists(linkto) and ignore_dangling_symlinks:
@@ -239,24 +341,10 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2,
             errors.append((src, dst, str(why)))
     if errors:
         raise Error(errors)
+    return dst
 
-def rmtree(path, ignore_errors=False, onerror=None):
-    """Recursively delete a directory tree.
-
-    If ignore_errors is set, errors are ignored; otherwise, if onerror
-    is set, it is called to handle the error with arguments (func,
-    path, exc_info) where func is os.listdir, os.remove, or os.rmdir;
-    path is the argument to that function that caused it to fail; and
-    exc_info is a tuple returned by sys.exc_info().  If ignore_errors
-    is false and onerror is None, an exception is raised.
-
-    """
-    if ignore_errors:
-        def onerror(*args):
-            pass
-    elif onerror is None:
-        def onerror(*args):
-            raise
+# version vulnerable to race conditions
+def _rmtree_unsafe(path, onerror):
     try:
         if os.path.islink(path):
             # symlinks to directories are forbidden, see bug #1669
@@ -268,7 +356,7 @@ def rmtree(path, ignore_errors=False, onerror=None):
     names = []
     try:
         names = os.listdir(path)
-    except os.error as err:
+    except os.error:
         onerror(os.listdir, path, sys.exc_info())
     for name in names:
         fullname = os.path.join(path, name)
@@ -277,17 +365,121 @@ def rmtree(path, ignore_errors=False, onerror=None):
         except os.error:
             mode = 0
         if stat.S_ISDIR(mode):
-            rmtree(fullname, ignore_errors, onerror)
+            _rmtree_unsafe(fullname, onerror)
         else:
             try:
-                os.remove(fullname)
-            except os.error as err:
-                onerror(os.remove, fullname, sys.exc_info())
+                os.unlink(fullname)
+            except os.error:
+                onerror(os.unlink, fullname, sys.exc_info())
     try:
         os.rmdir(path)
     except os.error:
         onerror(os.rmdir, path, sys.exc_info())
 
+# Version using fd-based APIs to protect against races
+def _rmtree_safe_fd(topfd, path, onerror):
+    names = []
+    try:
+        names = os.listdir(topfd)
+    except OSError as err:
+        err.filename = path
+        onerror(os.listdir, path, sys.exc_info())
+    for name in names:
+        fullname = os.path.join(path, name)
+        try:
+            orig_st = os.stat(name, dir_fd=topfd, follow_symlinks=False)
+            mode = orig_st.st_mode
+        except OSError:
+            mode = 0
+        if stat.S_ISDIR(mode):
+            try:
+                dirfd = os.open(name, os.O_RDONLY, dir_fd=topfd)
+            except OSError:
+                onerror(os.open, fullname, sys.exc_info())
+            else:
+                try:
+                    if os.path.samestat(orig_st, os.fstat(dirfd)):
+                        _rmtree_safe_fd(dirfd, fullname, onerror)
+                        try:
+                            os.rmdir(name, dir_fd=topfd)
+                        except OSError:
+                            onerror(os.rmdir, fullname, sys.exc_info())
+                    else:
+                        try:
+                            # This can only happen if someone replaces
+                            # a directory with a symlink after the call to
+                            # stat.S_ISDIR above.
+                            raise OSError("Cannot call rmtree on a symbolic "
+                                          "link")
+                        except OSError:
+                            onerror(os.path.islink, fullname, sys.exc_info())
+                finally:
+                    os.close(dirfd)
+        else:
+            try:
+                os.unlink(name, dir_fd=topfd)
+            except OSError:
+                onerror(os.unlink, fullname, sys.exc_info())
+
+_use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <=
+                     os.supports_dir_fd and
+                     os.listdir in os.supports_fd and
+                     os.stat in os.supports_follow_symlinks)
+
+def rmtree(path, ignore_errors=False, onerror=None):
+    """Recursively delete a directory tree.
+
+    If ignore_errors is set, errors are ignored; otherwise, if onerror
+    is set, it is called to handle the error with arguments (func,
+    path, exc_info) where func is platform and implementation dependent;
+    path is the argument to that function that caused it to fail; and
+    exc_info is a tuple returned by sys.exc_info().  If ignore_errors
+    is false and onerror is None, an exception is raised.
+
+    """
+    if ignore_errors:
+        def onerror(*args):
+            pass
+    elif onerror is None:
+        def onerror(*args):
+            raise
+    if _use_fd_functions:
+        # While the unsafe rmtree works fine on bytes, the fd based does not.
+        if isinstance(path, bytes):
+            path = os.fsdecode(path)
+        # Note: To guard against symlink races, we use the standard
+        # lstat()/open()/fstat() trick.
+        try:
+            orig_st = os.lstat(path)
+        except Exception:
+            onerror(os.lstat, path, sys.exc_info())
+            return
+        try:
+            fd = os.open(path, os.O_RDONLY)
+        except Exception:
+            onerror(os.lstat, path, sys.exc_info())
+            return
+        try:
+            if os.path.samestat(orig_st, os.fstat(fd)):
+                _rmtree_safe_fd(fd, path, onerror)
+                try:
+                    os.rmdir(path)
+                except os.error:
+                    onerror(os.rmdir, path, sys.exc_info())
+            else:
+                try:
+                    # symlinks to directories are forbidden, see bug #1669
+                    raise OSError("Cannot call rmtree on a symbolic link")
+                except OSError:
+                    onerror(os.path.islink, path, sys.exc_info())
+        finally:
+            os.close(fd)
+    else:
+        return _rmtree_unsafe(path, onerror)
+
+# Allow introspection of whether or not the hardening against symlink
+# attacks is supported on the current platform
+rmtree.avoids_symlink_attacks = _use_fd_functions
 
 def _basename(path):
     # A basename() variant which first strips the trailing slash, if present.
@@ -296,7 +488,8 @@ def _basename(path):
 
 def move(src, dst):
     """Recursively move a file or directory to another location. This is
-    similar to the Unix "mv" command.
+    similar to the Unix "mv" command. Return the file or directory's
+    destination.
 
     If the destination is a directory or a symlink to a directory, the source
     is moved inside the directory. The destination path must not already
@@ -306,7 +499,10 @@ def move(src, dst):
     overwritten depending on os.rename() semantics.
 
     If the destination is on our current filesystem, then rename() is used.
-    Otherwise, src is copied to the destination and then removed.
+    Otherwise, src is copied to the destination and then removed. Symlinks are
+    recreated under the new name if os.rename() fails because of cross
+    filesystem renames.
+
     A lot more could be done here...  A look at a mv.c shows a lot of
     the issues this implementation glosses over.
 
@@ -324,8 +520,12 @@ def move(src, dst):
             raise Error("Destination path '%s' already exists" % real_dst)
     try:
         os.rename(src, real_dst)
-    except OSError as exc:
-        if os.path.isdir(src):
+    except OSError:
+        if os.path.islink(src):
+            linkto = os.readlink(src)
+            os.symlink(linkto, real_dst)
+            os.unlink(src)
+        elif os.path.isdir(src):
             if _destinsrc(src, dst):
                 raise Error("Cannot move a directory '%s' into itself '%s'." % (src, dst))
             copytree(src, real_dst, symlinks=True)
@@ -333,6 +533,7 @@ def move(src, dst):
         else:
             copy2(src, real_dst)
             os.unlink(src)
+    return real_dst
 
 def _destinsrc(src, dst):
     src = abspath(src)
@@ -391,7 +592,7 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
         compress_ext['bzip2'] = '.bz2'
 
     # flags for compression program, each element of list will be an argument
-    if compress is not None and compress not in compress_ext.keys():
+    if compress is not None and compress not in compress_ext:
         raise ValueError("bad value for 'compress', or compression format not "
                          "supported : {0}".format(compress))
 
@@ -496,7 +697,7 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None):
 _ARCHIVE_FORMATS = {
     'gztar': (_make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"),
     'tar':   (_make_tarball, [('compress', None)], "uncompressed tar file"),
-    'zip':   (_make_zipfile, [],"ZIP file")
+    'zip':   (_make_zipfile, [], "ZIP file")
     }
 
 if _BZ2_SUPPORTED:
@@ -529,7 +730,7 @@ def register_archive_format(name, function, extra_args=None, description=''):
     if not isinstance(extra_args, (tuple, list)):
         raise TypeError('extra_args needs to be a sequence')
     for element in extra_args:
-        if not isinstance(element, (tuple, list)) or len(element) !=2 :
+        if not isinstance(element, (tuple, list)) or len(element) !=2:
             raise TypeError('extra_args elements are : (arg_name, value)')
 
     _ARCHIVE_FORMATS[name] = (function, extra_args, description)
@@ -681,7 +882,7 @@ def _unpack_zipfile(filename, extract_dir):
             if not name.endswith('/'):
                 # file
                 data = zip.read(info.filename)
-                f = open(target,'wb')
+                f = open(target, 'wb')
                 try:
                     f.write(data)
                 finally:
@@ -755,3 +956,170 @@ def unpack_archive(filename, extract_dir=None, format=None):
         func = _UNPACK_FORMATS[format][1]
         kwargs = dict(_UNPACK_FORMATS[format][2])
         func(filename, extract_dir, **kwargs)
+
+
+if hasattr(os, 'statvfs'):
+
+    __all__.append('disk_usage')
+    _ntuple_diskusage = collections.namedtuple('usage', 'total used free')
+
+    def disk_usage(path):
+        """Return disk usage statistics about the given path.
+
+        Returned value is a named tuple with attributes 'total', 'used' and
+        'free', which are the amount of total, used and free space, in bytes.
+        """
+        st = os.statvfs(path)
+        free = st.f_bavail * st.f_frsize
+        total = st.f_blocks * st.f_frsize
+        used = (st.f_blocks - st.f_bfree) * st.f_frsize
+        return _ntuple_diskusage(total, used, free)
+
+elif os.name == 'nt':
+
+    import nt
+    __all__.append('disk_usage')
+    _ntuple_diskusage = collections.namedtuple('usage', 'total used free')
+
+    def disk_usage(path):
+        """Return disk usage statistics about the given path.
+
+        Returned valus is a named tuple with attributes 'total', 'used' and
+        'free', which are the amount of total, used and free space, in bytes.
+        """
+        total, free = nt._getdiskusage(path)
+        used = total - free
+        return _ntuple_diskusage(total, used, free)
+
+
+def chown(path, user=None, group=None):
+    """Change owner user and group of the given path.
+
+    user and group can be the uid/gid or the user/group names, and in that case,
+    they are converted to their respective uid/gid.
+    """
+
+    if user is None and group is None:
+        raise ValueError("user and/or group must be set")
+
+    _user = user
+    _group = group
+
+    # -1 means don't change it
+    if user is None:
+        _user = -1
+    # user can either be an int (the uid) or a string (the system username)
+    elif isinstance(user, str):
+        _user = _get_uid(user)
+        if _user is None:
+            raise LookupError("no such user: {!r}".format(user))
+
+    if group is None:
+        _group = -1
+    elif not isinstance(group, int):
+        _group = _get_gid(group)
+        if _group is None:
+            raise LookupError("no such group: {!r}".format(group))
+
+    os.chown(path, _user, _group)
+
+def get_terminal_size(fallback=(80, 24)):
+    """Get the size of the terminal window.
+
+    For each of the two dimensions, the environment variable, COLUMNS
+    and LINES respectively, is checked. If the variable is defined and
+    the value is a positive integer, it is used.
+
+    When COLUMNS or LINES is not defined, which is the common case,
+    the terminal connected to sys.__stdout__ is queried
+    by invoking os.get_terminal_size.
+
+    If the terminal size cannot be successfully queried, either because
+    the system doesn't support querying, or because we are not
+    connected to a terminal, the value given in fallback parameter
+    is used. Fallback defaults to (80, 24) which is the default
+    size used by many terminal emulators.
+
+    The value returned is a named tuple of type os.terminal_size.
+    """
+    # columns, lines are the working values
+    try:
+        columns = int(os.environ['COLUMNS'])
+    except (KeyError, ValueError):
+        columns = 0
+
+    try:
+        lines = int(os.environ['LINES'])
+    except (KeyError, ValueError):
+        lines = 0
+
+    # only query if necessary
+    if columns <= 0 or lines <= 0:
+        try:
+            size = os.get_terminal_size(sys.__stdout__.fileno())
+        except (NameError, OSError):
+            size = os.terminal_size(fallback)
+        if columns <= 0:
+            columns = size.columns
+        if lines <= 0:
+            lines = size.lines
+
+    return os.terminal_size((columns, lines))
+
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+
+    """
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    path = (path or os.environ.get("PATH", os.defpath)).split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if not os.curdir in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if not normdir in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None
diff --git a/Lib/site.py b/Lib/site.py
index a2c0becbc1..b751006c88 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -13,6 +13,19 @@ prefixes directly, as well as with lib/site-packages appended.  The
 resulting directories, if they exist, are appended to sys.path, and
 also inspected for path configuration files.
 
+If a file named "pyvenv.cfg" exists one directory above sys.executable,
+sys.prefix and sys.exec_prefix are set to that directory and
+it is also checked for site-packages and site-python (sys.base_prefix and
+sys.base_exec_prefix will always be the "real" prefixes of the Python
+installation). If "pyvenv.cfg" (a bootstrap configuration file) contains
+the key "include-system-site-packages" set to anything other than "false"
+(case-insensitive), the system-level prefixes will still also be
+searched for site-packages; otherwise they won't.
+
+All of the resulting site-specific directories, if they exist, are
+appended to sys.path, and also inspected for path configuration
+files.
+
 A path configuration file is a file whose name has the form
 <package>.pth; its contents are additional directories (one per line)
 to be added to sys.path.  Non-existing directories (or
@@ -54,8 +67,8 @@ ImportError exception, it is silently ignored.
 
 import sys
 import os
+import re
 import builtins
-import traceback
 
 # Prefixes for site-packages; add additional prefixes like /usr/local here
 PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -82,7 +95,8 @@ def makepath(*paths):
 def abs_paths():
     """Set all module __file__ and __cached__ attributes to an absolute path"""
     for m in set(sys.modules.values()):
-        if hasattr(m, '__loader__'):
+        if (getattr(getattr(m, '__loader__', None), '__module__', None) !=
+                '_frozen_importlib'):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
             m.__file__ = os.path.abspath(m.__file__)
@@ -138,7 +152,7 @@ def addpackage(sitedir, name, known_paths):
         reset = 0
     fullname = os.path.join(sitedir, name)
     try:
-        f = open(fullname, "rU")
+        f = open(fullname, "r")
     except IOError:
         return
     with f:
@@ -154,9 +168,10 @@ def addpackage(sitedir, name, known_paths):
                 if not dircase in known_paths and os.path.exists(dir):
                     sys.path.append(dir)
                     known_paths.add(dircase)
-            except Exception as err:
+            except Exception:
                 print("Error processing line {:d} of {}:\n".format(n+1, fullname),
                       file=sys.stderr)
+                import traceback
                 for record in traceback.format_exception(*sys.exc_info()):
                     for line in record.splitlines():
                         print('  '+line, file=sys.stderr)
@@ -178,6 +193,7 @@ def addsitedir(sitedir, known_paths=None):
     sitedir, sitedircase = makepath(sitedir)
     if not sitedircase in known_paths:
         sys.path.append(sitedir)        # Add path component
+        known_paths.add(sitedircase)
     try:
         names = os.listdir(sitedir)
     except os.error:
@@ -241,7 +257,6 @@ def getusersitepackages():
         return USER_SITE
 
     from sysconfig import get_path
-    import os
 
     if sys.platform == 'darwin':
         from sysconfig import get_config_var
@@ -266,18 +281,21 @@ def addusersitepackages(known_paths):
         addsitedir(user_site, known_paths)
     return known_paths
 
-def getsitepackages():
+def getsitepackages(prefixes=None):
     """Returns a list containing all global site-packages directories
     (and possibly site-python).
 
-    For each directory present in the global ``PREFIXES``, this function
-    will find its `site-packages` subdirectory depending on the system
-    environment, and will return a list of full paths.
+    For each directory present in ``prefixes`` (or the global ``PREFIXES``),
+    this function will find its `site-packages` subdirectory depending on the
+    system environment, and will return a list of full paths.
     """
     sitepackages = []
     seen = set()
 
-    for prefix in PREFIXES:
+    if prefixes is None:
+        prefixes = PREFIXES
+
+    for prefix in prefixes:
         if not prefix or prefix in seen:
             continue
         seen.add(prefix)
@@ -303,9 +321,9 @@ def getsitepackages():
                             sys.version[:3], "site-packages"))
     return sitepackages
 
-def addsitepackages(known_paths):
+def addsitepackages(known_paths, prefixes=None):
     """Add site-packages (and possibly site-python) to sys.path"""
-    for sitedir in getsitepackages():
+    for sitedir in getsitepackages(prefixes):
         if os.path.isdir(sitedir):
             addsitedir(sitedir, known_paths)
 
@@ -385,7 +403,7 @@ class _Printer(object):
             for filename in self.__files:
                 filename = os.path.join(dir, filename)
                 try:
-                    fp = open(filename, "rU")
+                    fp = open(filename, "r")
                     data = fp.read()
                     fp.close()
                     break
@@ -475,6 +493,59 @@ def aliasmbcs():
                 encodings.aliases.aliases[enc] = 'mbcs'
 
 
+CONFIG_LINE = re.compile(r'^(?P<key>(\w|[-_])+)\s*=\s*(?P<value>.*)\s*$')
+
+def venv(known_paths):
+    global PREFIXES, ENABLE_USER_SITE
+
+    env = os.environ
+    if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
+        executable = os.environ['__PYVENV_LAUNCHER__']
+    else:
+        executable = sys.executable
+    exe_dir, _ = os.path.split(os.path.abspath(executable))
+    site_prefix = os.path.dirname(exe_dir)
+    sys._home = None
+    conf_basename = 'pyvenv.cfg'
+    candidate_confs = [
+        conffile for conffile in (
+            os.path.join(exe_dir, conf_basename),
+            os.path.join(site_prefix, conf_basename)
+            )
+        if os.path.isfile(conffile)
+        ]
+
+    if candidate_confs:
+        virtual_conf = candidate_confs[0]
+        system_site = "true"
+        with open(virtual_conf) as f:
+            for line in f:
+                line = line.strip()
+                m = CONFIG_LINE.match(line)
+                if m:
+                    d = m.groupdict()
+                    key, value = d['key'].lower(), d['value']
+                    if key == 'include-system-site-packages':
+                        system_site = value.lower()
+                    elif key == 'home':
+                        sys._home = value
+
+        sys.prefix = sys.exec_prefix = site_prefix
+
+        # Doing this here ensures venv takes precedence over user-site
+        addsitepackages(known_paths, [sys.prefix])
+
+        # addsitepackages will process site_prefix again if its in PREFIXES,
+        # but that's ok; known_paths will prevent anything being added twice
+        if system_site == "true":
+            PREFIXES.insert(0, sys.prefix)
+        else:
+            PREFIXES = [sys.prefix]
+            ENABLE_USER_SITE = False
+
+    return known_paths
+
+
 def execsitecustomize():
     """Run custom site specific code, if available."""
     try:
@@ -508,10 +579,16 @@ def execusercustomize():
 
 
 def main():
+    """Add standard site-specific directories to the module search path.
+
+    This function is called automatically when this module is imported,
+    unless the python interpreter was started with the -S flag.
+    """
     global ENABLE_USER_SITE
 
     abs_paths()
     known_paths = removeduppaths()
+    known_paths = venv(known_paths)
     if ENABLE_USER_SITE is None:
         ENABLE_USER_SITE = check_enableusersite()
     known_paths = addusersitepackages(known_paths)
@@ -526,7 +603,10 @@ def main():
     if ENABLE_USER_SITE:
         execusercustomize()
 
-main()
+# Prevent edition of sys.path when python was started with -S and
+# site is imported later.
+if not sys.flags.no_site:
+    main()
 
 def _script():
     help = """\
diff --git a/Lib/smtpd.py b/Lib/smtpd.py
index 8cd405c5f6..778d6d6280 100755
--- a/Lib/smtpd.py
+++ b/Lib/smtpd.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python3
-"""An RFC 2821 smtp proxy.
+"""An RFC 5321 smtp proxy.
 
 Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]]
 
@@ -20,6 +20,11 @@ Options:
         Use `classname' as the concrete SMTP proxy class.  Uses `PureProxy' by
         default.
 
+    --size limit
+    -s limit
+        Restrict the total size of the incoming message to "limit" number of
+        bytes via the RFC 1870 SIZE extension.  Defaults to 33554432 bytes.
+
     --debug
     -d
         Turn on debugging prints.
@@ -35,10 +40,9 @@ given then 8025 is used.  If remotehost is not given then `localhost' is used,
 and if remoteport is not given, then 25 is used.
 """
 
-
 # Overview:
 #
-# This file implements the minimal SMTP protocol as defined in RFC 821.  It
+# This file implements the minimal SMTP protocol as defined in RFC 5321.  It
 # has a hierarchy of classes which implement the backend functionality for the
 # smtpd.  A number of classes are provided:
 #
@@ -66,7 +70,7 @@ and if remoteport is not given, then 25 is used.
 #
 # - support mailbox delivery
 # - alias files
-# - ESMTP
+# - Handle more ESMTP extensions
 # - handle error codes from the backend smtpd
 
 import sys
@@ -77,12 +81,14 @@ import time
 import socket
 import asyncore
 import asynchat
+import collections
 from warnings import warn
+from email._header_value_parser import get_addr_spec, get_angle_addr
 
 __all__ = ["SMTPServer","DebuggingServer","PureProxy","MailmanProxy"]
 
 program = sys.argv[0]
-__version__ = 'Python SMTP proxy version 0.2'
+__version__ = 'Python SMTP proxy version 0.3'
 
 
 class Devnull:
@@ -94,9 +100,9 @@ DEBUGSTREAM = Devnull()
 NEWLINE = '\n'
 EMPTYSTRING = ''
 COMMASPACE = ', '
+DATA_SIZE_DEFAULT = 33554432
 
 
-
 def usage(code, msg=''):
     print(__doc__ % globals(), file=sys.stderr)
     if msg:
@@ -104,19 +110,23 @@ def usage(code, msg=''):
     sys.exit(code)
 
 
-
 class SMTPChannel(asynchat.async_chat):
     COMMAND = 0
     DATA = 1
 
-    data_size_limit = 33554432
     command_size_limit = 512
+    command_size_limits = collections.defaultdict(lambda x=command_size_limit: x)
+    command_size_limits.update({
+        'MAIL': command_size_limit + 26,
+        })
+    max_command_size_limit = max(command_size_limits.values())
 
-    def __init__(self, server, conn, addr):
+    def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT):
         asynchat.async_chat.__init__(self, conn)
         self.smtp_server = server
         self.conn = conn
         self.addr = addr
+        self.data_size_limit = data_size_limit
         self.received_lines = []
         self.smtp_state = self.COMMAND
         self.seen_greeting = ''
@@ -137,127 +147,128 @@ class SMTPChannel(asynchat.async_chat):
         print('Peer:', repr(self.peer), file=DEBUGSTREAM)
         self.push('220 %s %s' % (self.fqdn, __version__))
         self.set_terminator(b'\r\n')
+        self.extended_smtp = False
 
     # properties for backwards-compatibility
     @property
     def __server(self):
         warn("Access to __server attribute on SMTPChannel is deprecated, "
-            "use 'smtp_server' instead", PendingDeprecationWarning, 2)
+            "use 'smtp_server' instead", DeprecationWarning, 2)
         return self.smtp_server
     @__server.setter
     def __server(self, value):
         warn("Setting __server attribute on SMTPChannel is deprecated, "
-            "set 'smtp_server' instead", PendingDeprecationWarning, 2)
+            "set 'smtp_server' instead", DeprecationWarning, 2)
         self.smtp_server = value
 
     @property
     def __line(self):
         warn("Access to __line attribute on SMTPChannel is deprecated, "
-            "use 'received_lines' instead", PendingDeprecationWarning, 2)
+            "use 'received_lines' instead", DeprecationWarning, 2)
         return self.received_lines
     @__line.setter
     def __line(self, value):
         warn("Setting __line attribute on SMTPChannel is deprecated, "
-            "set 'received_lines' instead", PendingDeprecationWarning, 2)
+            "set 'received_lines' instead", DeprecationWarning, 2)
         self.received_lines = value
 
     @property
     def __state(self):
         warn("Access to __state attribute on SMTPChannel is deprecated, "
-            "use 'smtp_state' instead", PendingDeprecationWarning, 2)
+            "use 'smtp_state' instead", DeprecationWarning, 2)
         return self.smtp_state
     @__state.setter
     def __state(self, value):
         warn("Setting __state attribute on SMTPChannel is deprecated, "
-            "set 'smtp_state' instead", PendingDeprecationWarning, 2)
+            "set 'smtp_state' instead", DeprecationWarning, 2)
         self.smtp_state = value
 
     @property
     def __greeting(self):
         warn("Access to __greeting attribute on SMTPChannel is deprecated, "
-            "use 'seen_greeting' instead", PendingDeprecationWarning, 2)
+            "use 'seen_greeting' instead", DeprecationWarning, 2)
         return self.seen_greeting
     @__greeting.setter
     def __greeting(self, value):
         warn("Setting __greeting attribute on SMTPChannel is deprecated, "
-            "set 'seen_greeting' instead", PendingDeprecationWarning, 2)
+            "set 'seen_greeting' instead", DeprecationWarning, 2)
         self.seen_greeting = value
 
     @property
     def __mailfrom(self):
         warn("Access to __mailfrom attribute on SMTPChannel is deprecated, "
-            "use 'mailfrom' instead", PendingDeprecationWarning, 2)
+            "use 'mailfrom' instead", DeprecationWarning, 2)
         return self.mailfrom
     @__mailfrom.setter
     def __mailfrom(self, value):
         warn("Setting __mailfrom attribute on SMTPChannel is deprecated, "
-            "set 'mailfrom' instead", PendingDeprecationWarning, 2)
+            "set 'mailfrom' instead", DeprecationWarning, 2)
         self.mailfrom = value
 
     @property
     def __rcpttos(self):
         warn("Access to __rcpttos attribute on SMTPChannel is deprecated, "
-            "use 'rcpttos' instead", PendingDeprecationWarning, 2)
+            "use 'rcpttos' instead", DeprecationWarning, 2)
         return self.rcpttos
     @__rcpttos.setter
     def __rcpttos(self, value):
         warn("Setting __rcpttos attribute on SMTPChannel is deprecated, "
-            "set 'rcpttos' instead", PendingDeprecationWarning, 2)
+            "set 'rcpttos' instead", DeprecationWarning, 2)
         self.rcpttos = value
 
     @property
     def __data(self):
         warn("Access to __data attribute on SMTPChannel is deprecated, "
-            "use 'received_data' instead", PendingDeprecationWarning, 2)
+            "use 'received_data' instead", DeprecationWarning, 2)
         return self.received_data
     @__data.setter
     def __data(self, value):
         warn("Setting __data attribute on SMTPChannel is deprecated, "
-            "set 'received_data' instead", PendingDeprecationWarning, 2)
+            "set 'received_data' instead", DeprecationWarning, 2)
         self.received_data = value
 
     @property
     def __fqdn(self):
         warn("Access to __fqdn attribute on SMTPChannel is deprecated, "
-            "use 'fqdn' instead", PendingDeprecationWarning, 2)
+            "use 'fqdn' instead", DeprecationWarning, 2)
         return self.fqdn
     @__fqdn.setter
     def __fqdn(self, value):
         warn("Setting __fqdn attribute on SMTPChannel is deprecated, "
-            "set 'fqdn' instead", PendingDeprecationWarning, 2)
+            "set 'fqdn' instead", DeprecationWarning, 2)
         self.fqdn = value
 
     @property
     def __peer(self):
         warn("Access to __peer attribute on SMTPChannel is deprecated, "
-            "use 'peer' instead", PendingDeprecationWarning, 2)
+            "use 'peer' instead", DeprecationWarning, 2)
         return self.peer
     @__peer.setter
     def __peer(self, value):
         warn("Setting __peer attribute on SMTPChannel is deprecated, "
-            "set 'peer' instead", PendingDeprecationWarning, 2)
+            "set 'peer' instead", DeprecationWarning, 2)
         self.peer = value
 
     @property
     def __conn(self):
         warn("Access to __conn attribute on SMTPChannel is deprecated, "
-            "use 'conn' instead", PendingDeprecationWarning, 2)
+            "use 'conn' instead", DeprecationWarning, 2)
         return self.conn
     @__conn.setter
     def __conn(self, value):
         warn("Setting __conn attribute on SMTPChannel is deprecated, "
-            "set 'conn' instead", PendingDeprecationWarning, 2)
+            "set 'conn' instead", DeprecationWarning, 2)
         self.conn = value
 
     @property
     def __addr(self):
         warn("Access to __addr attribute on SMTPChannel is deprecated, "
-            "use 'addr' instead", PendingDeprecationWarning, 2)
+            "use 'addr' instead", DeprecationWarning, 2)
         return self.addr
     @__addr.setter
     def __addr(self, value):
         warn("Setting __addr attribute on SMTPChannel is deprecated, "
-            "set 'addr' instead", PendingDeprecationWarning, 2)
+            "set 'addr' instead", DeprecationWarning, 2)
         self.addr = value
 
     # Overrides base class for convenience
@@ -268,14 +279,14 @@ class SMTPChannel(asynchat.async_chat):
     def collect_incoming_data(self, data):
         limit = None
         if self.smtp_state == self.COMMAND:
-            limit = self.command_size_limit
+            limit = self.max_command_size_limit
         elif self.smtp_state == self.DATA:
             limit = self.data_size_limit
         if limit and self.num_bytes > limit:
             return
         elif limit:
             self.num_bytes += len(data)
-        self.received_lines.append(str(data, "utf8"))
+        self.received_lines.append(str(data, "utf-8"))
 
     # Implementation of base class abstract method
     def found_terminator(self):
@@ -283,11 +294,7 @@ class SMTPChannel(asynchat.async_chat):
         print('Data:', repr(line), file=DEBUGSTREAM)
         self.received_lines = []
         if self.smtp_state == self.COMMAND:
-            if self.num_bytes > self.command_size_limit:
-                self.push('500 Error: line too long')
-                self.num_bytes = 0
-                return
-            self.num_bytes = 0
+            sz, self.num_bytes = self.num_bytes, 0
             if not line:
                 self.push('500 Error: bad syntax')
                 return
@@ -299,9 +306,14 @@ class SMTPChannel(asynchat.async_chat):
             else:
                 command = line[:i].upper()
                 arg = line[i+1:].strip()
+            max_sz = (self.command_size_limits[command]
+                        if self.extended_smtp else self.command_size_limit)
+            if sz > max_sz:
+                self.push('500 Error: line too long')
+                return
             method = getattr(self, 'smtp_' + command, None)
             if not method:
-                self.push('502 Error: command "%s" not implemented' % command)
+                self.push('500 Error: command "%s" not recognized' % command)
                 return
             method(arg)
             return
@@ -310,12 +322,12 @@ class SMTPChannel(asynchat.async_chat):
                 self.push('451 Internal confusion')
                 self.num_bytes = 0
                 return
-            if self.num_bytes > self.data_size_limit:
+            if self.data_size_limit and self.num_bytes > self.data_size_limit:
                 self.push('552 Error: Too much mail data')
                 self.num_bytes = 0
                 return
             # Remove extraneous carriage returns and de-transparency according
-            # to RFC 821, Section 4.5.2.
+            # to RFC 5321, Section 4.5.2.
             data = []
             for text in line.split('\r\n'):
                 if text and text[0] == '.':
@@ -333,7 +345,7 @@ class SMTPChannel(asynchat.async_chat):
             self.num_bytes = 0
             self.set_terminator(b'\r\n')
             if not status:
-                self.push('250 Ok')
+                self.push('250 OK')
             else:
                 self.push(status)
 
@@ -346,58 +358,188 @@ class SMTPChannel(asynchat.async_chat):
             self.push('503 Duplicate HELO/EHLO')
         else:
             self.seen_greeting = arg
+            self.extended_smtp = False
             self.push('250 %s' % self.fqdn)
 
+    def smtp_EHLO(self, arg):
+        if not arg:
+            self.push('501 Syntax: EHLO hostname')
+            return
+        if self.seen_greeting:
+            self.push('503 Duplicate HELO/EHLO')
+        else:
+            self.seen_greeting = arg
+            self.extended_smtp = True
+            self.push('250-%s' % self.fqdn)
+            if self.data_size_limit:
+                self.push('250-SIZE %s' % self.data_size_limit)
+            self.push('250 HELP')
+
     def smtp_NOOP(self, arg):
         if arg:
             self.push('501 Syntax: NOOP')
         else:
-            self.push('250 Ok')
+            self.push('250 OK')
 
     def smtp_QUIT(self, arg):
         # args is ignored
         self.push('221 Bye')
         self.close_when_done()
 
-    # factored
-    def __getaddr(self, keyword, arg):
-        address = None
+    def _strip_command_keyword(self, keyword, arg):
         keylen = len(keyword)
         if arg[:keylen].upper() == keyword:
-            address = arg[keylen:].strip()
-            if not address:
-                pass
-            elif address[0] == '<' and address[-1] == '>' and address != '<>':
-                # Addresses can be in the form <person@dom.com> but watch out
-                # for null address, e.g. <>
-                address = address[1:-1]
-        return address
+            return arg[keylen:].strip()
+        return ''
+
+    def _getaddr(self, arg):
+        if not arg:
+            return '', ''
+        if arg.lstrip().startswith('<'):
+            address, rest = get_angle_addr(arg)
+        else:
+            address, rest = get_addr_spec(arg)
+        if not address:
+            return address, rest
+        return address.addr_spec, rest
+
+    def _getparams(self, params):
+        # Return any parameters that appear to be syntactically valid according
+        # to RFC 1869, ignore all others.  (Postel rule: accept what we can.)
+        params = [param.split('=', 1) for param in params.split()
+                                      if '=' in param]
+        return {k: v for k, v in params if k.isalnum()}
+
+    def smtp_HELP(self, arg):
+        if arg:
+            extended = ' [SP <mail parameters]'
+            lc_arg = arg.upper()
+            if lc_arg == 'EHLO':
+                self.push('250 Syntax: EHLO hostname')
+            elif lc_arg == 'HELO':
+                self.push('250 Syntax: HELO hostname')
+            elif lc_arg == 'MAIL':
+                msg = '250 Syntax: MAIL FROM: <address>'
+                if self.extended_smtp:
+                    msg += extended
+                self.push(msg)
+            elif lc_arg == 'RCPT':
+                msg = '250 Syntax: RCPT TO: <address>'
+                if self.extended_smtp:
+                    msg += extended
+                self.push(msg)
+            elif lc_arg == 'DATA':
+                self.push('250 Syntax: DATA')
+            elif lc_arg == 'RSET':
+                self.push('250 Syntax: RSET')
+            elif lc_arg == 'NOOP':
+                self.push('250 Syntax: NOOP')
+            elif lc_arg == 'QUIT':
+                self.push('250 Syntax: QUIT')
+            elif lc_arg == 'VRFY':
+                self.push('250 Syntax: VRFY <address>')
+            else:
+                self.push('501 Supported commands: EHLO HELO MAIL RCPT '
+                          'DATA RSET NOOP QUIT VRFY')
+        else:
+            self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA '
+                      'RSET NOOP QUIT VRFY')
+
+    def smtp_VRFY(self, arg):
+        if arg:
+            address, params = self._getaddr(arg)
+            if address:
+                self.push('252 Cannot VRFY user, but will accept message '
+                          'and attempt delivery')
+            else:
+                self.push('502 Could not VRFY %s' % arg)
+        else:
+            self.push('501 Syntax: VRFY <address>')
 
     def smtp_MAIL(self, arg):
+        if not self.seen_greeting:
+            self.push('503 Error: send HELO first');
+            return
         print('===> MAIL', arg, file=DEBUGSTREAM)
-        address = self.__getaddr('FROM:', arg) if arg else None
+        syntaxerr = '501 Syntax: MAIL FROM: <address>'
+        if self.extended_smtp:
+            syntaxerr += ' [SP <mail-parameters>]'
+        if arg is None:
+            self.push(syntaxerr)
+            return
+        arg = self._strip_command_keyword('FROM:', arg)
+        address, params = self._getaddr(arg)
+        if not address:
+            self.push(syntaxerr)
+            return
+        if not self.extended_smtp and params:
+            self.push(syntaxerr)
+            return
         if not address:
-            self.push('501 Syntax: MAIL FROM:<address>')
+            self.push(syntaxerr)
             return
         if self.mailfrom:
             self.push('503 Error: nested MAIL command')
             return
+        params = self._getparams(params.upper())
+        if params is None:
+            self.push(syntaxerr)
+            return
+        size = params.pop('SIZE', None)
+        if size:
+            if not size.isdigit():
+                self.push(syntaxerr)
+                return
+            elif self.data_size_limit and int(size) > self.data_size_limit:
+                self.push('552 Error: message size exceeds fixed maximum message size')
+                return
+        if len(params.keys()) > 0:
+            self.push('555 MAIL FROM parameters not recognized or not implemented')
+            return
         self.mailfrom = address
         print('sender:', self.mailfrom, file=DEBUGSTREAM)
-        self.push('250 Ok')
+        self.push('250 OK')
 
     def smtp_RCPT(self, arg):
+        if not self.seen_greeting:
+            self.push('503 Error: send HELO first');
+            return
         print('===> RCPT', arg, file=DEBUGSTREAM)
         if not self.mailfrom:
             self.push('503 Error: need MAIL command')
             return
-        address = self.__getaddr('TO:', arg) if arg else None
+        syntaxerr = '501 Syntax: RCPT TO: <address>'
+        if self.extended_smtp:
+            syntaxerr += ' [SP <mail-parameters>]'
+        if arg is None:
+            self.push(syntaxerr)
+            return
+        arg = self._strip_command_keyword('TO:', arg)
+        address, params = self._getaddr(arg)
+        if not address:
+            self.push(syntaxerr)
+            return
+        if params:
+            if self.extended_smtp:
+                params = self._getparams(params.upper())
+                if params is None:
+                    self.push(syntaxerr)
+                    return
+            else:
+                self.push(syntaxerr)
+                return
+        if not address:
+            self.push(syntaxerr)
+            return
+        if params and len(params.keys()) > 0:
+            self.push('555 RCPT TO parameters not recognized or not implemented')
+            return
         if not address:
             self.push('501 Syntax: RCPT TO: <address>')
             return
         self.rcpttos.append(address)
         print('recips:', self.rcpttos, file=DEBUGSTREAM)
-        self.push('250 Ok')
+        self.push('250 OK')
 
     def smtp_RSET(self, arg):
         if arg:
@@ -408,9 +550,12 @@ class SMTPChannel(asynchat.async_chat):
         self.rcpttos = []
         self.received_data = ''
         self.smtp_state = self.COMMAND
-        self.push('250 Ok')
+        self.push('250 OK')
 
     def smtp_DATA(self, arg):
+        if not self.seen_greeting:
+            self.push('503 Error: send HELO first');
+            return
         if not self.rcpttos:
             self.push('503 Error: need RCPT command')
             return
@@ -421,15 +566,20 @@ class SMTPChannel(asynchat.async_chat):
         self.set_terminator(b'\r\n.\r\n')
         self.push('354 End data with <CR><LF>.<CR><LF>')
 
+    # Commands that have not been implemented
+    def smtp_EXPN(self, arg):
+        self.push('502 EXPN not implemented')
+
 
-
 class SMTPServer(asyncore.dispatcher):
     # SMTPChannel class to use for managing client connections
     channel_class = SMTPChannel
 
-    def __init__(self, localaddr, remoteaddr):
+    def __init__(self, localaddr, remoteaddr,
+                 data_size_limit=DATA_SIZE_DEFAULT):
         self._localaddr = localaddr
         self._remoteaddr = remoteaddr
+        self.data_size_limit = data_size_limit
         asyncore.dispatcher.__init__(self)
         try:
             self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -447,7 +597,7 @@ class SMTPServer(asyncore.dispatcher):
 
     def handle_accepted(self, conn, addr):
         print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM)
-        channel = self.channel_class(self, conn, addr)
+        channel = self.channel_class(self, conn, addr, self.data_size_limit)
 
     # API for "doing something useful with the message"
     def process_message(self, peer, mailfrom, rcpttos, data):
@@ -475,7 +625,6 @@ class SMTPServer(asyncore.dispatcher):
         raise NotImplementedError
 
 
-
 class DebuggingServer(SMTPServer):
     # Do something with the gathered message
     def process_message(self, peer, mailfrom, rcpttos, data):
@@ -491,7 +640,6 @@ class DebuggingServer(SMTPServer):
         print('------------ END MESSAGE ------------')
 
 
-
 class PureProxy(SMTPServer):
     def process_message(self, peer, mailfrom, rcpttos, data):
         lines = data.split('\n')
@@ -532,7 +680,6 @@ class PureProxy(SMTPServer):
         return refused
 
 
-
 class MailmanProxy(PureProxy):
     def process_message(self, peer, mailfrom, rcpttos, data):
         from io import StringIO
@@ -611,19 +758,18 @@ class MailmanProxy(PureProxy):
                 msg.Enqueue(mlist, torequest=1)
 
 
-
 class Options:
     setuid = 1
     classname = 'PureProxy'
+    size_limit = None
 
 
-
 def parseargs():
     global DEBUGSTREAM
     try:
         opts, args = getopt.getopt(
-            sys.argv[1:], 'nVhc:d',
-            ['class=', 'nosetuid', 'version', 'help', 'debug'])
+            sys.argv[1:], 'nVhc:s:d',
+            ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug'])
     except getopt.error as e:
         usage(1, e)
 
@@ -640,6 +786,13 @@ def parseargs():
             options.classname = arg
         elif opt in ('-d', '--debug'):
             DEBUGSTREAM = sys.stderr
+        elif opt in ('-s', '--size'):
+            try:
+                int_size = int(arg)
+                options.size_limit = int_size
+            except:
+                print('Invalid size: ' + arg, file=sys.stderr)
+                sys.exit(1)
 
     # parse the rest of the arguments
     if len(args) < 1:
@@ -674,7 +827,6 @@ def parseargs():
     return options
 
 
-
 if __name__ == '__main__':
     options = parseargs()
     # Become nobody
@@ -687,7 +839,8 @@ if __name__ == '__main__':
         import __main__ as mod
     class_ = getattr(mod, classname)
     proxy = class_((options.localhost, options.localport),
-                   (options.remotehost, options.remoteport))
+                   (options.remotehost, options.remoteport),
+                   options.size_limit)
     if options.setuid:
         try:
             import pwd
diff --git a/Lib/smtplib.py b/Lib/smtplib.py
index e06a9be741..c949d77b51 100644
--- a/Lib/smtplib.py
+++ b/Lib/smtplib.py
@@ -133,24 +133,18 @@ class SMTPAuthenticationError(SMTPResponseException):
     combination provided.
     """
 
-def quoteaddr(addr):
+def quoteaddr(addrstring):
     """Quote a subset of the email addresses defined by RFC 821.
 
     Should be able to handle anything email.utils.parseaddr can handle.
     """
-    m = (None, None)
-    try:
-        m = email.utils.parseaddr(addr)[1]
-    except AttributeError:
-        pass
-    if m == (None, None):  # Indicates parse failure or AttributeError
-        # something weird here.. punt -ddm
-        return "<%s>" % addr
-    elif m is None:
-        # the sender wants an empty return address
-        return "<>"
-    else:
-        return "<%s>" % m
+    displayname, addr = email.utils.parseaddr(addrstring)
+    if (displayname, addr) == ('', ''):
+        # parseaddr couldn't parse it, use it as is and hope for the best.
+        if addrstring.strip().startswith('<'):
+            return addrstring
+        return "<%s>" % addrstring
+    return "<%s>" % addr
 
 def _addr_only(addrstring):
     displayname, addr = email.utils.parseaddr(addrstring)
@@ -180,27 +174,6 @@ try:
 except ImportError:
     _have_ssl = False
 else:
-    class SSLFakeFile:
-        """A fake file like object that really wraps a SSLObject.
-
-        It only supports what is needed in smtplib.
-        """
-        def __init__(self, sslobj):
-            self.sslobj = sslobj
-
-        def readline(self):
-            str = b""
-            chr = None
-            while chr != b"\n":
-                chr = self.sslobj.read(1)
-                if not chr:
-                    break
-                str += chr
-            return str
-
-        def close(self):
-            pass
-
     _have_ssl = True
 
 
@@ -242,7 +215,8 @@ class SMTP:
     default_port = SMTP_PORT
 
     def __init__(self, host='', port=0, local_hostname=None,
-                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                 source_address=None):
         """Initialize a new instance.
 
         If specified, `host' is the name of the remote host to which to
@@ -250,11 +224,16 @@ class SMTP:
         By default, smtplib.SMTP_PORT is used.  An SMTPConnectError is raised
         if the specified `host' doesn't respond correctly.  If specified,
         `local_hostname` is used as the FQDN of the local host.  By default,
-        the local hostname is found using socket.getfqdn().
+        the local hostname is found using socket.getfqdn(). The
+        `source_address` parameter takes a 2-tuple (host, port) for the socket
+        to bind to as its source address before connecting. If the host is ''
+        and port is 0, the OS default behavior will be used.
 
         """
         self.timeout = timeout
         self.esmtp_features = {}
+        self.source_address = source_address
+
         if host:
             (code, msg) = self.connect(host, port)
             if code != 220:
@@ -277,6 +256,19 @@ class SMTP:
                     pass
                 self.local_hostname = '[%s]' % addr
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        try:
+            code, message = self.docmd("QUIT")
+            if code != 221:
+                raise SMTPResponseException(code, message)
+        except SMTPServerDisconnected:
+            pass
+        finally:
+            self.close()
+
     def set_debuglevel(self, debuglevel):
         """Set the debug output level.
 
@@ -290,10 +282,12 @@ class SMTP:
         # This makes it simpler for SMTP_SSL to use the SMTP connect code
         # and just alter the socket connection bit.
         if self.debuglevel > 0:
-            print('connect:', (host, port), file=stderr)
-        return socket.create_connection((host, port), timeout)
+            print('connect: to', (host, port), self.source_address,
+                                 file=stderr)
+        return socket.create_connection((host, port), timeout,
+                                        self.source_address)
 
-    def connect(self, host='localhost', port=0):
+    def connect(self, host='localhost', port=0, source_address=None):
         """Connect to a host on a given port.
 
         If the hostname ends with a colon (`:') followed by a number, and
@@ -304,6 +298,10 @@ class SMTP:
         specified during instantiation.
 
         """
+
+        if source_address:
+            self.source_address = source_address
+
         if not port and (host.find(':') == host.rfind(':')):
             i = host.rfind(':')
             if i >= 0:
@@ -317,6 +315,7 @@ class SMTP:
         if self.debuglevel > 0:
             print('connect:', (host, port), file=stderr)
         self.sock = self._get_socket(host, port, self.timeout)
+        self.file = None
         (code, msg) = self.getreply()
         if self.debuglevel > 0:
             print("connect:", msg, file=stderr)
@@ -388,7 +387,8 @@ class SMTP:
 
         errmsg = b"\n".join(resp)
         if self.debuglevel > 0:
-            print('reply: retcode (%s); Msg: %s' % (errcode, errmsg), file=stderr)
+            print('reply: retcode (%s); Msg: %s' % (errcode, errmsg),
+                                                    file=stderr)
         return errcode, errmsg
 
     def docmd(self, cmd, args=""):
@@ -632,7 +632,7 @@ class SMTP:
         # We could not login sucessfully. Return result of last attempt.
         raise SMTPAuthenticationError(code, resp)
 
-    def starttls(self, keyfile=None, certfile=None):
+    def starttls(self, keyfile=None, certfile=None, context=None):
         """Puts the connection to the SMTP server into TLS mode.
 
         If there has been no previous EHLO or HELO command this session, this
@@ -656,8 +656,17 @@ class SMTP:
         if resp == 220:
             if not _have_ssl:
                 raise RuntimeError("No SSL support included in this Python")
-            self.sock = ssl.wrap_socket(self.sock, keyfile, certfile)
-            self.file = SSLFakeFile(self.sock)
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
+            if context is not None:
+                self.sock = context.wrap_socket(self.sock)
+            else:
+                self.sock = ssl.wrap_socket(self.sock, keyfile, certfile)
+            self.file = None
             # RFC 3207:
             # The client MUST discard any knowledge obtained from
             # the server, such as the list of SMTP service extensions,
@@ -786,7 +795,8 @@ class SMTP:
         # TODO implement heuristics to guess the correct Resent-* block with an
         # option allowing the user to enable the heuristics.  (It should be
         # possible to guess correctly almost all of the time.)
-        resent =msg.get_all('Resent-Date')
+
+        resent = msg.get_all('Resent-Date')
         if resent is None:
             header_prefix = ''
         elif len(resent) == 1:
@@ -795,13 +805,13 @@ class SMTP:
             raise ValueError("message has more than one 'Resent-' header block")
         if from_addr is None:
             # Prefer the sender field per RFC 2822:3.6.2.
-            from_addr = (msg[header_prefix+'Sender']
-                           if (header_prefix+'Sender') in msg
-                           else msg[header_prefix+'From'])
+            from_addr = (msg[header_prefix + 'Sender']
+                           if (header_prefix + 'Sender') in msg
+                           else msg[header_prefix + 'From'])
         if to_addrs is None:
-            addr_fields = [f for f in (msg[header_prefix+'To'],
-                                       msg[header_prefix+'Bcc'],
-                                       msg[header_prefix+'Cc']) if f is not None]
+            addr_fields = [f for f in (msg[header_prefix + 'To'],
+                                       msg[header_prefix + 'Bcc'],
+                                       msg[header_prefix + 'Cc']) if f is not None]
             to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)]
         # Make a local copy so we can delete the bcc headers.
         msg_copy = copy.copy(msg)
@@ -835,26 +845,41 @@ if _have_ssl:
         """ This is a subclass derived from SMTP that connects over an SSL encrypted
         socket (to use this class you need a socket module that was compiled with SSL
         support). If host is not specified, '' (the local host) is used. If port is
-        omitted, the standard SMTP-over-SSL port (465) is used. keyfile and certfile
+        omitted, the standard SMTP-over-SSL port (465) is used. The optional
+        source_address takes a two-tuple (host,port) for socket to bind to. keyfile and certfile
         are also optional - they can contain a PEM formatted private key and
-        certificate chain file for the SSL connection.
+        certificate chain file for the SSL connection. context also optional, can contain
+        a SSLContext, and is an alternative to keyfile and certfile; If it is specified both
+        keyfile and certfile must be None.
         """
 
         default_port = SMTP_SSL_PORT
 
         def __init__(self, host='', port=0, local_hostname=None,
                      keyfile=None, certfile=None,
-                     timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+                     timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None, context=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
-            SMTP.__init__(self, host, port, local_hostname, timeout)
+            self.context = context
+            SMTP.__init__(self, host, port, local_hostname, timeout,
+                    source_address)
 
         def _get_socket(self, host, port, timeout):
             if self.debuglevel > 0:
                 print('connect:', (host, port), file=stderr)
-            new_socket = socket.create_connection((host, port), timeout)
-            new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile)
-            self.file = SSLFakeFile(new_socket)
+            new_socket = socket.create_connection((host, port), timeout,
+                    self.source_address)
+            if self.context is not None:
+                new_socket = self.context.wrap_socket(new_socket)
+            else:
+                new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile)
             return new_socket
 
     __all__.append("SMTP_SSL")
@@ -879,18 +904,21 @@ class LMTP(SMTP):
 
     ehlo_msg = "lhlo"
 
-    def __init__(self, host='', port=LMTP_PORT, local_hostname=None):
+    def __init__(self, host='', port=LMTP_PORT, local_hostname=None,
+            source_address=None):
         """Initialize a new instance."""
-        SMTP.__init__(self, host, port, local_hostname)
+        SMTP.__init__(self, host, port, local_hostname=local_hostname,
+                      source_address=source_address)
 
-    def connect(self, host='localhost', port=0):
+    def connect(self, host='localhost', port=0, source_address=None):
         """Connect to the LMTP daemon, on either a Unix or a TCP socket."""
         if host[0] != '/':
-            return SMTP.connect(self, host, port)
+            return SMTP.connect(self, host, port, source_address=source_address)
 
         # Handle Unix-domain sockets.
         try:
             self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            self.file = None
             self.sock.connect(host)
         except socket.error:
             if self.debuglevel > 0:
diff --git a/Lib/socket.py b/Lib/socket.py
index ea56a67d55..d4f1b65e46 100644
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -12,6 +12,7 @@ Functions:
 socket() -- create a new socket object
 socketpair() -- create a pair of new socket objects [*]
 fromfd() -- create a socket object from an open file descriptor [*]
+fromshare() -- create a socket object from data received from socket.share() [*]
 gethostname() -- return the current hostname
 gethostbyname() -- map a hostname to its IP number
 gethostbyaddr() -- map an IP number or hostname to DNS info
@@ -53,7 +54,6 @@ try:
 except ImportError:
     errno = None
 EBADF = getattr(errno, 'EBADF', 9)
-EINTR = getattr(errno, 'EINTR', 4)
 EAGAIN = getattr(errno, 'EAGAIN', 11)
 EWOULDBLOCK = getattr(errno, 'EWOULDBLOCK', 11)
 
@@ -112,6 +112,9 @@ class socket(_socket.socket):
                                 s[7:])
         return s
 
+    def __getstate__(self):
+        raise TypeError("Cannot serialize socket object")
+
     def dup(self):
         """dup() -> socket object
 
@@ -207,7 +210,6 @@ class socket(_socket.socket):
         self._closed = True
         return super().detach()
 
-
 def fromfd(fd, family, type, proto=0):
     """ fromfd(fd, family, type[, proto]) -> socket object
 
@@ -217,6 +219,14 @@ def fromfd(fd, family, type, proto=0):
     nfd = dup(fd)
     return socket(family, type, proto, nfd)
 
+if hasattr(_socket.socket, "share"):
+    def fromshare(info):
+        """ fromshare(info) -> socket object
+
+        Create a socket object from a the bytes object returned by
+        socket.share(pid).
+        """
+        return socket(0, 0, 0, info)
 
 if hasattr(_socket, "socketpair"):
 
@@ -288,11 +298,10 @@ class SocketIO(io.RawIOBase):
             except timeout:
                 self._timeout_occurred = True
                 raise
+            except InterruptedError:
+                continue
             except error as e:
-                n = e.args[0]
-                if n == EINTR:
-                    continue
-                if n in _blocking_errnos:
+                if e.args[0] in _blocking_errnos:
                     return None
                 raise
 
diff --git a/Lib/socketserver.py b/Lib/socketserver.py
index 8f80a7dc31..8332fdf66d 100644
--- a/Lib/socketserver.py
+++ b/Lib/socketserver.py
@@ -153,8 +153,8 @@ def _eintr_retry(func, *args):
     while True:
         try:
             return func(*args)
-        except (OSError, select.error) as e:
-            if e.args[0] != errno.EINTR:
+        except OSError as e:
+            if e.errno != errno.EINTR:
                 raise
 
 class BaseServer:
@@ -180,6 +180,7 @@ class BaseServer:
     - process_request(request, client_address)
     - shutdown_request(request)
     - close_request(request)
+    - service_actions()
     - handle_error()
 
     Methods for derived classes:
@@ -236,6 +237,8 @@ class BaseServer:
                                        poll_interval)
                 if self in r:
                     self._handle_request_noblock()
+
+                self.service_actions()
         finally:
             self.__shutdown_request = False
             self.__is_shut_down.set()
@@ -250,6 +253,14 @@ class BaseServer:
         self.__shutdown_request = True
         self.__is_shut_down.wait()
 
+    def service_actions(self):
+        """Called by the serve_forever() loop.
+
+        May be overridden by a subclass / Mixin to implement any code that
+        needs to be run during the loop.
+        """
+        pass
+
     # The distinction between handling, getting, processing and
     # finishing a request is fairly arbitrary.  Remember:
     #
@@ -550,9 +561,15 @@ class ForkingMixIn:
         """
         self.collect_children()
 
+    def service_actions(self):
+        """Collect the zombie child processes regularly in the ForkingMixIn.
+
+        service_actions is called in the BaseServer's serve_forver loop.
+        """
+        self.collect_children()
+
     def process_request(self, request, client_address):
         """Fork a new subprocess to process the request."""
-        self.collect_children()
         pid = os.fork()
         if pid:
             # Parent process
@@ -560,6 +577,7 @@ class ForkingMixIn:
                 self.active_children = []
             self.active_children.append(pid)
             self.close_request(request)
+            return
         else:
             # Child process.
             # This must never return, hence os._exit()!
diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index 202bd38876..b7ec1ad0d9 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/dbapi.py: tests for DB-API compliance
 #
 # Copyright (C) 2004-2010 Gerhard Häring <gh@ghaering.de>
diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py
index 7f6f3473f3..9e833aedc5 100644
--- a/Lib/sqlite3/test/factory.py
+++ b/Lib/sqlite3/test/factory.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/factory.py: tests for the various factories in pysqlite
 #
 # Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
@@ -178,6 +178,8 @@ class TextFactoryTests(unittest.TestCase):
         self.assertTrue(row[0].endswith("reich"), "column must contain original data")
 
     def CheckOptimizedUnicode(self):
+        # In py3k, str objects are always returned when text_factory
+        # is OptimizedUnicode
         self.con.text_factory = sqlite.OptimizedUnicode
         austria = "Österreich"
         germany = "Deutchland"
diff --git a/Lib/sqlite3/test/hooks.py b/Lib/sqlite3/test/hooks.py
index 9544149ff6..087edb03df 100644
--- a/Lib/sqlite3/test/hooks.py
+++ b/Lib/sqlite3/test/hooks.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/hooks.py: tests for various SQLite-specific hooks
 #
 # Copyright (C) 2006-2007 Gerhard Häring <gh@ghaering.de>
@@ -195,10 +195,60 @@ class ProgressTests(unittest.TestCase):
         con.execute("select 1 union select 2 union select 3").fetchall()
         self.assertEqual(action, 0, "progress handler was not cleared")
 
+class TraceCallbackTests(unittest.TestCase):
+    def CheckTraceCallbackUsed(self):
+        """
+        Test that the trace callback is invoked once it is set.
+        """
+        con = sqlite.connect(":memory:")
+        traced_statements = []
+        def trace(statement):
+            traced_statements.append(statement)
+        con.set_trace_callback(trace)
+        con.execute("create table foo(a, b)")
+        self.assertTrue(traced_statements)
+        self.assertTrue(any("create table foo" in stmt for stmt in traced_statements))
+
+    def CheckClearTraceCallback(self):
+        """
+        Test that setting the trace callback to None clears the previously set callback.
+        """
+        con = sqlite.connect(":memory:")
+        traced_statements = []
+        def trace(statement):
+            traced_statements.append(statement)
+        con.set_trace_callback(trace)
+        con.set_trace_callback(None)
+        con.execute("create table foo(a, b)")
+        self.assertFalse(traced_statements, "trace callback was not cleared")
+
+    def CheckUnicodeContent(self):
+        """
+        Test that the statement can contain unicode literals.
+        """
+        unicode_value = '\xf6\xe4\xfc\xd6\xc4\xdc\xdf\u20ac'
+        con = sqlite.connect(":memory:")
+        traced_statements = []
+        def trace(statement):
+            traced_statements.append(statement)
+        con.set_trace_callback(trace)
+        con.execute("create table foo(x)")
+        # Can't execute bound parameters as their values don't appear
+        # in traced statements before SQLite 3.6.21
+        # (cf. http://www.sqlite.org/draft/releaselog/3_6_21.html)
+        con.execute('insert into foo(x) values ("%s")' % unicode_value)
+        con.commit()
+        self.assertTrue(any(unicode_value in stmt for stmt in traced_statements),
+                        "Unicode data %s garbled in trace callback: %s"
+                        % (ascii(unicode_value), ', '.join(map(ascii, traced_statements))))
+
+
+
 def suite():
     collation_suite = unittest.makeSuite(CollationTests, "Check")
     progress_suite = unittest.makeSuite(ProgressTests, "Check")
-    return unittest.TestSuite((collation_suite, progress_suite))
+    trace_suite = unittest.makeSuite(TraceCallbackTests, "Check")
+    return unittest.TestSuite((collation_suite, progress_suite, trace_suite))
 
 def test():
     runner = unittest.TextTestRunner()
diff --git a/Lib/sqlite3/test/transactions.py b/Lib/sqlite3/test/transactions.py
index 70e96a12ed..feb4fa1696 100644
--- a/Lib/sqlite3/test/transactions.py
+++ b/Lib/sqlite3/test/transactions.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/transactions.py: tests transactions
 #
 # Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
diff --git a/Lib/sqlite3/test/types.py b/Lib/sqlite3/test/types.py
index 29413e14ec..3b4cb6d1d1 100644
--- a/Lib/sqlite3/test/types.py
+++ b/Lib/sqlite3/test/types.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/types.py: tests for type conversion and detection
 #
 # Copyright (C) 2005 Gerhard Häring <gh@ghaering.de>
@@ -85,7 +85,7 @@ class DeclTypesTests(unittest.TestCase):
             if isinstance(_val, bytes):
                 # sqlite3 always calls __init__ with a bytes created from a
                 # UTF-8 string when __conform__ was used to store the object.
-                _val = _val.decode('utf8')
+                _val = _val.decode('utf-8')
             self.val = _val
 
         def __cmp__(self, other):
diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py
index 9a6a828d81..69e2ec2991 100644
--- a/Lib/sqlite3/test/userfunctions.py
+++ b/Lib/sqlite3/test/userfunctions.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/userfunctions.py: tests for user-defined functions and
 #                                  aggregates.
 #
diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
index 46eac9c070..90e3a25f1a 100644
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -319,11 +319,13 @@ def _optimize_unicode(charset, fixup):
                 # XXX: could expand category
                 return charset # cannot compress
     except IndexError:
-        # non-BMP characters
+        # non-BMP characters; XXX now they should work
         return charset
     if negate:
         if sys.maxunicode != 65535:
             # XXX: negation does not work with big charsets
+            # XXX2: now they should work, but removing this will make the
+            # charmap 17 times bigger
             return charset
         for i in range(65536):
             charmap[i] = not charmap[i]
diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py
index 045a5ebfef..b195fd01dc 100644
--- a/Lib/sre_parse.py
+++ b/Lib/sre_parse.py
@@ -178,6 +178,7 @@ class SubPattern:
 
 class Tokenizer:
     def __init__(self, string):
+        self.istext = isinstance(string, str)
         self.string = string
         self.index = 0
         self.__next()
@@ -188,14 +189,14 @@ class Tokenizer:
         char = self.string[self.index:self.index+1]
         # Special case for the str8, since indexing returns a integer
         # XXX This is only needed for test_bug_926075 in test_re.py
-        if char and isinstance(char, bytes):
+        if char and not self.istext:
             char = chr(char[0])
         if char == "\\":
             try:
                 c = self.string[self.index + 1]
             except IndexError:
                 raise error("bogus escape (end of line)")
-            if isinstance(self.string, bytes):
+            if not self.istext:
                 c = chr(c)
             char = char + c
         self.index = self.index + len(char)
@@ -210,6 +211,15 @@ class Tokenizer:
         this = self.next
         self.__next()
         return this
+    def getwhile(self, n, charset):
+        result = ''
+        for _ in range(n):
+            c = self.next
+            if c not in charset:
+                break
+            result += c
+            self.__next()
+        return result
     def tell(self):
         return self.index, self.next
     def seek(self, index):
@@ -242,20 +252,30 @@ def _class_escape(source, escape):
         c = escape[1:2]
         if c == "x":
             # hexadecimal escape (exactly two digits)
-            while source.next in HEXDIGITS and len(escape) < 4:
-                escape = escape + source.get()
-            escape = escape[2:]
-            if len(escape) != 2:
-                raise error("bogus escape: %s" % repr("\\" + escape))
-            return LITERAL, int(escape, 16) & 0xff
+            escape += source.getwhile(2, HEXDIGITS)
+            if len(escape) != 4:
+                raise ValueError
+            return LITERAL, int(escape[2:], 16) & 0xff
+        elif c == "u" and source.istext:
+            # unicode escape (exactly four digits)
+            escape += source.getwhile(4, HEXDIGITS)
+            if len(escape) != 6:
+                raise ValueError
+            return LITERAL, int(escape[2:], 16)
+        elif c == "U" and source.istext:
+            # unicode escape (exactly eight digits)
+            escape += source.getwhile(8, HEXDIGITS)
+            if len(escape) != 10:
+                raise ValueError
+            c = int(escape[2:], 16)
+            chr(c) # raise ValueError for invalid code
+            return LITERAL, c
         elif c in OCTDIGITS:
             # octal escape (up to three digits)
-            while source.next in OCTDIGITS and len(escape) < 4:
-                escape = escape + source.get()
-            escape = escape[1:]
-            return LITERAL, int(escape, 8) & 0xff
+            escape += source.getwhile(2, OCTDIGITS)
+            return LITERAL, int(escape[1:], 8) & 0xff
         elif c in DIGITS:
-            raise error("bogus escape: %s" % repr(escape))
+            raise ValueError
         if len(escape) == 2:
             return LITERAL, ord(escape[1])
     except ValueError:
@@ -274,15 +294,27 @@ def _escape(source, escape, state):
         c = escape[1:2]
         if c == "x":
             # hexadecimal escape
-            while source.next in HEXDIGITS and len(escape) < 4:
-                escape = escape + source.get()
+            escape += source.getwhile(2, HEXDIGITS)
             if len(escape) != 4:
                 raise ValueError
             return LITERAL, int(escape[2:], 16) & 0xff
+        elif c == "u" and source.istext:
+            # unicode escape (exactly four digits)
+            escape += source.getwhile(4, HEXDIGITS)
+            if len(escape) != 6:
+                raise ValueError
+            return LITERAL, int(escape[2:], 16)
+        elif c == "U" and source.istext:
+            # unicode escape (exactly eight digits)
+            escape += source.getwhile(8, HEXDIGITS)
+            if len(escape) != 10:
+                raise ValueError
+            c = int(escape[2:], 16)
+            chr(c) # raise ValueError for invalid code
+            return LITERAL, c
         elif c == "0":
             # octal escape
-            while source.next in OCTDIGITS and len(escape) < 4:
-                escape = escape + source.get()
+            escape += source.getwhile(2, OCTDIGITS)
             return LITERAL, int(escape[1:], 8) & 0xff
         elif c in DIGITS:
             # octal escape *or* decimal group reference (sigh)
@@ -802,7 +834,7 @@ def parse_template(source, pattern):
     else:
         # The tokenizer implicitly decodes bytes objects as latin-1, we must
         # therefore re-encode the final representation.
-        encode = lambda x: x.encode('latin1')
+        encode = lambda x: x.encode('latin-1')
     for c, s in p:
         if c is MARK:
             groupsappend((i, s))
diff --git a/Lib/ssl.py b/Lib/ssl.py
index e901b640a6..6ff5c53884 100644
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -60,10 +60,25 @@ import re
 import _ssl             # if we can't import it, let the error propagate
 
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
-from _ssl import _SSLContext, SSLError
+from _ssl import _SSLContext
+from _ssl import (
+    SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError,
+    SSLSyscallError, SSLEOFError,
+    )
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import OP_ALL, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_TLSv1
-from _ssl import RAND_status, RAND_egd, RAND_add
+from _ssl import (
+    OP_ALL, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_TLSv1,
+    OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE
+    )
+try:
+    from _ssl import OP_NO_COMPRESSION
+except ImportError:
+    pass
+try:
+    from _ssl import OP_SINGLE_ECDH_USE
+except ImportError:
+    pass
+from _ssl import RAND_status, RAND_egd, RAND_add, RAND_bytes, RAND_pseudo_bytes
 from _ssl import (
     SSL_ERROR_ZERO_RETURN,
     SSL_ERROR_WANT_READ,
@@ -75,8 +90,9 @@ from _ssl import (
     SSL_ERROR_EOF,
     SSL_ERROR_INVALID_ERROR_CODE,
     )
-from _ssl import HAS_SNI
-from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN
+from _ssl import (PROTOCOL_SSLv3, PROTOCOL_SSLv23,
+                  PROTOCOL_TLSv1)
 from _ssl import _OPENSSL_API_VERSION
 
 _PROTOCOL_NAMES = {
@@ -94,11 +110,16 @@ else:
 
 from socket import getnameinfo as _getnameinfo
 from socket import error as socket_error
-from socket import socket, AF_INET, SOCK_STREAM
+from socket import socket, AF_INET, SOCK_STREAM, create_connection
 import base64        # for DER-to-PEM translation
 import traceback
 import errno
 
+if _ssl.HAS_TLS_UNIQUE:
+    CHANNEL_BINDING_TYPES = ['tls-unique']
+else:
+    CHANNEL_BINDING_TYPES = []
+
 # Disable weak or insecure ciphers by default
 # (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL')
 _DEFAULT_CIPHERS = 'DEFAULT:!aNULL:!eNULL:!LOW:!EXPORT:!SSLv2'
@@ -188,6 +209,17 @@ class SSLContext(_SSLContext):
                          server_hostname=server_hostname,
                          _context=self)
 
+    def set_npn_protocols(self, npn_protocols):
+        protos = bytearray()
+        for protocol in npn_protocols:
+            b = bytes(protocol, 'ascii')
+            if len(b) == 0 or len(b) > 255:
+                raise SSLError('NPN protocols must be 1 to 255 in length')
+            protos.append(len(b))
+            protos.extend(b)
+
+        self._set_npn_protocols(protos)
+
 
 class SSLSocket(socket):
     """This class implements a subtype of socket.socket that wraps
@@ -199,7 +231,7 @@ class SSLSocket(socket):
                  ssl_version=PROTOCOL_SSLv23, ca_certs=None,
                  do_handshake_on_connect=True,
                  family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None,
-                 suppress_ragged_eofs=True, ciphers=None,
+                 suppress_ragged_eofs=True, npn_protocols=None, ciphers=None,
                  server_hostname=None,
                  _context=None):
 
@@ -219,6 +251,8 @@ class SSLSocket(socket):
                 self.context.load_verify_locations(ca_certs)
             if certfile:
                 self.context.load_cert_chain(certfile, keyfile)
+            if npn_protocols:
+                self.context.set_npn_protocols(npn_protocols)
             if ciphers:
                 self.context.set_ciphers(ciphers)
             self.keyfile = keyfile
@@ -319,6 +353,13 @@ class SSLSocket(socket):
         self._checkClosed()
         return self._sslobj.peer_certificate(binary_form)
 
+    def selected_npn_protocol(self):
+        self._checkClosed()
+        if not self._sslobj or not _ssl.HAS_NPN:
+            return None
+        else:
+            return self._sslobj.selected_npn_protocol()
+
     def cipher(self):
         self._checkClosed()
         if not self._sslobj:
@@ -326,6 +367,13 @@ class SSLSocket(socket):
         else:
             return self._sslobj.cipher()
 
+    def compression(self):
+        self._checkClosed()
+        if not self._sslobj:
+            return None
+        else:
+            return self._sslobj.compression()
+
     def send(self, data, flags=0):
         self._checkClosed()
         if self._sslobj:
@@ -358,6 +406,12 @@ class SSLSocket(socket):
         else:
             return socket.sendto(self, data, flags_or_addr, addr)
 
+    def sendmsg(self, *args, **kwargs):
+        # Ensure programs don't send data unencrypted if they try to
+        # use this method.
+        raise NotImplementedError("sendmsg not allowed on instances of %s" %
+                                  self.__class__)
+
     def sendall(self, data, flags=0):
         self._checkClosed()
         if self._sslobj:
@@ -416,6 +470,14 @@ class SSLSocket(socket):
         else:
             return socket.recvfrom_into(self, buffer, nbytes, flags)
 
+    def recvmsg(self, *args, **kwargs):
+        raise NotImplementedError("recvmsg not allowed on instances of %s" %
+                                  self.__class__)
+
+    def recvmsg_into(self, *args, **kwargs):
+        raise NotImplementedError("recvmsg_into not allowed on instances of "
+                                  "%s" % self.__class__)
+
     def pending(self):
         self._checkClosed()
         if self._sslobj:
@@ -497,16 +559,28 @@ class SSLSocket(socket):
                     server_side=True)
         return newsock, addr
 
-    def __del__(self):
-        # sys.stderr.write("__del__ on %s\n" % repr(self))
-        self._real_close()
+    def get_channel_binding(self, cb_type="tls-unique"):
+        """Get channel binding data for current connection.  Raise ValueError
+        if the requested `cb_type` is not supported.  Return bytes of the data
+        or None if the data is not available (e.g. before the handshake).
+        """
+        if cb_type not in CHANNEL_BINDING_TYPES:
+            raise ValueError("Unsupported channel binding type")
+        if cb_type != "tls-unique":
+            raise NotImplementedError(
+                            "{0} channel binding type not implemented"
+                            .format(cb_type))
+        if self._sslobj is None:
+            return None
+        return self._sslobj.tls_unique_cb()
 
 
 def wrap_socket(sock, keyfile=None, certfile=None,
                 server_side=False, cert_reqs=CERT_NONE,
                 ssl_version=PROTOCOL_SSLv23, ca_certs=None,
                 do_handshake_on_connect=True,
-                suppress_ragged_eofs=True, ciphers=None):
+                suppress_ragged_eofs=True,
+                ciphers=None):
 
     return SSLSocket(sock=sock, keyfile=keyfile, certfile=certfile,
                      server_side=server_side, cert_reqs=cert_reqs,
@@ -561,9 +635,9 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv3, ca_certs=None):
         cert_reqs = CERT_REQUIRED
     else:
         cert_reqs = CERT_NONE
-    s = wrap_socket(socket(), ssl_version=ssl_version,
+    s = create_connection(addr)
+    s = wrap_socket(s, ssl_version=ssl_version,
                     cert_reqs=cert_reqs, ca_certs=ca_certs)
-    s.connect(addr)
     dercert = s.getpeercert(True)
     s.close()
     return DER_cert_to_PEM_cert(dercert)
diff --git a/Lib/stat.py b/Lib/stat.py
index 78ccd5e8ff..704adfe2e1 100644
--- a/Lib/stat.py
+++ b/Lib/stat.py
@@ -19,78 +19,131 @@ ST_CTIME = 9
 # Extract bits from the mode
 
 def S_IMODE(mode):
+    """Return the portion of the file's mode that can be set by
+    os.chmod().
+    """
     return mode & 0o7777
 
 def S_IFMT(mode):
+    """Return the portion of the file's mode that describes the
+    file type.
+    """
     return mode & 0o170000
 
 # Constants used as S_IFMT() for various file types
 # (not all are implemented on all systems)
 
-S_IFDIR  = 0o040000
-S_IFCHR  = 0o020000
-S_IFBLK  = 0o060000
-S_IFREG  = 0o100000
-S_IFIFO  = 0o010000
-S_IFLNK  = 0o120000
-S_IFSOCK = 0o140000
+S_IFDIR  = 0o040000  # directory
+S_IFCHR  = 0o020000  # character device
+S_IFBLK  = 0o060000  # block device
+S_IFREG  = 0o100000  # regular file
+S_IFIFO  = 0o010000  # fifo (named pipe)
+S_IFLNK  = 0o120000  # symbolic link
+S_IFSOCK = 0o140000  # socket file
 
 # Functions to test for each file type
 
 def S_ISDIR(mode):
+    """Return True if mode is from a directory."""
     return S_IFMT(mode) == S_IFDIR
 
 def S_ISCHR(mode):
+    """Return True if mode is from a character special device file."""
     return S_IFMT(mode) == S_IFCHR
 
 def S_ISBLK(mode):
+    """Return True if mode is from a block special device file."""
     return S_IFMT(mode) == S_IFBLK
 
 def S_ISREG(mode):
+    """Return True if mode is from a regular file."""
     return S_IFMT(mode) == S_IFREG
 
 def S_ISFIFO(mode):
+    """Return True if mode is from a FIFO (named pipe)."""
     return S_IFMT(mode) == S_IFIFO
 
 def S_ISLNK(mode):
+    """Return True if mode is from a symbolic link."""
     return S_IFMT(mode) == S_IFLNK
 
 def S_ISSOCK(mode):
+    """Return True if mode is from a socket."""
     return S_IFMT(mode) == S_IFSOCK
 
 # Names for permission bits
 
-S_ISUID = 0o4000
-S_ISGID = 0o2000
-S_ENFMT = S_ISGID
-S_ISVTX = 0o1000
-S_IREAD = 0o0400
-S_IWRITE = 0o0200
-S_IEXEC = 0o0100
-S_IRWXU = 0o0700
-S_IRUSR = 0o0400
-S_IWUSR = 0o0200
-S_IXUSR = 0o0100
-S_IRWXG = 0o0070
-S_IRGRP = 0o0040
-S_IWGRP = 0o0020
-S_IXGRP = 0o0010
-S_IRWXO = 0o0007
-S_IROTH = 0o0004
-S_IWOTH = 0o0002
-S_IXOTH = 0o0001
+S_ISUID = 0o4000  # set UID bit
+S_ISGID = 0o2000  # set GID bit
+S_ENFMT = S_ISGID # file locking enforcement
+S_ISVTX = 0o1000  # sticky bit
+S_IREAD = 0o0400  # Unix V7 synonym for S_IRUSR
+S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR
+S_IEXEC = 0o0100  # Unix V7 synonym for S_IXUSR
+S_IRWXU = 0o0700  # mask for owner permissions
+S_IRUSR = 0o0400  # read by owner
+S_IWUSR = 0o0200  # write by owner
+S_IXUSR = 0o0100  # execute by owner
+S_IRWXG = 0o0070  # mask for group permissions
+S_IRGRP = 0o0040  # read by group
+S_IWGRP = 0o0020  # write by group
+S_IXGRP = 0o0010  # execute by group
+S_IRWXO = 0o0007  # mask for others (not in group) permissions
+S_IROTH = 0o0004  # read by others
+S_IWOTH = 0o0002  # write by others
+S_IXOTH = 0o0001  # execute by others
 
 # Names for file flags
 
-UF_NODUMP    = 0x00000001
-UF_IMMUTABLE = 0x00000002
-UF_APPEND    = 0x00000004
-UF_OPAQUE    = 0x00000008
-UF_NOUNLINK  = 0x00000010
-UF_COMPRESSED = 0x00000020  # OS X: file is hfs-compressed
-UF_HIDDEN    = 0x00008000   # OS X: file should not be displayed
-SF_ARCHIVED  = 0x00010000
-SF_IMMUTABLE = 0x00020000
-SF_APPEND    = 0x00040000
-SF_NOUNLINK  = 0x00100000
-SF_SNAPSHOT  = 0x00200000
+UF_NODUMP    = 0x00000001  # do not dump file
+UF_IMMUTABLE = 0x00000002  # file may not be changed
+UF_APPEND    = 0x00000004  # file may only be appended to
+UF_OPAQUE    = 0x00000008  # directory is opaque when viewed through a union stack
+UF_NOUNLINK  = 0x00000010  # file may not be renamed or deleted
+UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed
+UF_HIDDEN    = 0x00008000  # OS X: file should not be displayed
+SF_ARCHIVED  = 0x00010000  # file may be archived
+SF_IMMUTABLE = 0x00020000  # file may not be changed
+SF_APPEND    = 0x00040000  # file may only be appended to
+SF_NOUNLINK  = 0x00100000  # file may not be renamed or deleted
+SF_SNAPSHOT  = 0x00200000  # file is a snapshot file
+
+
+_filemode_table = (
+    ((S_IFLNK,         "l"),
+     (S_IFREG,         "-"),
+     (S_IFBLK,         "b"),
+     (S_IFDIR,         "d"),
+     (S_IFCHR,         "c"),
+     (S_IFIFO,         "p")),
+
+    ((S_IRUSR,         "r"),),
+    ((S_IWUSR,         "w"),),
+    ((S_IXUSR|S_ISUID, "s"),
+     (S_ISUID,         "S"),
+     (S_IXUSR,         "x")),
+
+    ((S_IRGRP,         "r"),),
+    ((S_IWGRP,         "w"),),
+    ((S_IXGRP|S_ISGID, "s"),
+     (S_ISGID,         "S"),
+     (S_IXGRP,         "x")),
+
+    ((S_IROTH,         "r"),),
+    ((S_IWOTH,         "w"),),
+    ((S_IXOTH|S_ISVTX, "t"),
+     (S_ISVTX,         "T"),
+     (S_IXOTH,         "x"))
+)
+
+def filemode(mode):
+    """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
+    perm = []
+    for table in _filemode_table:
+        for bit, char in table:
+            if mode & bit == bit:
+                perm.append(char)
+                break
+        else:
+            perm.append("-")
+    return "".join(perm)
diff --git a/Lib/string.py b/Lib/string.py
index 0f4ede23e1..b57c79b75e 100644
--- a/Lib/string.py
+++ b/Lib/string.py
@@ -46,23 +46,7 @@ def capwords(s, sep=None):
 
 ####################################################################
 import re as _re
-
-class _multimap:
-    """Helper class for combining multiple mappings.
-
-    Used by .{safe_,}substitute() to combine the mapping and keyword
-    arguments.
-    """
-    def __init__(self, primary, secondary):
-        self._primary = primary
-        self._secondary = secondary
-
-    def __getitem__(self, key):
-        try:
-            return self._primary[key]
-        except KeyError:
-            return self._secondary[key]
-
+from collections import ChainMap
 
 class _TemplateMetaclass(type):
     pattern = r"""
@@ -100,7 +84,7 @@ class Template(metaclass=_TemplateMetaclass):
 
     def _invalid(self, mo):
         i = mo.start('invalid')
-        lines = self.template[:i].splitlines(True)
+        lines = self.template[:i].splitlines(keepends=True)
         if not lines:
             colno = 1
             lineno = 1
@@ -116,7 +100,7 @@ class Template(metaclass=_TemplateMetaclass):
         if not args:
             mapping = kws
         elif kws:
-            mapping = _multimap(kws, args[0])
+            mapping = ChainMap(kws, args[0])
         else:
             mapping = args[0]
         # Helper function for .sub()
@@ -142,7 +126,7 @@ class Template(metaclass=_TemplateMetaclass):
         if not args:
             mapping = kws
         elif kws:
-            mapping = _multimap(kws, args[0])
+            mapping = ChainMap(kws, args[0])
         else:
             mapping = args[0]
         # Helper function for .sub()
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 1a21455a9b..aa3e21754e 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -191,8 +191,10 @@ should prepare for OSErrors.
 
 A ValueError will be raised if Popen is called with invalid arguments.
 
-check_call() and check_output() will raise CalledProcessError, if the
-called process returns a non-zero return code.
+Exceptions defined within this module inherit from SubprocessError.
+check_call() and check_output() will raise CalledProcessError if the
+called process returns a non-zero return code.  TimeoutExpired
+be raised if a timeout was specified and expired.
 
 
 Security
@@ -340,15 +342,23 @@ mswindows = (sys.platform == "win32")
 
 import io
 import os
+import time
 import traceback
 import gc
 import signal
 import builtins
 import warnings
 import errno
+try:
+    from time import monotonic as _time
+except ImportError:
+    from time import time as _time
 
 # Exception classes used by this module.
-class CalledProcessError(Exception):
+class SubprocessError(Exception): pass
+
+
+class CalledProcessError(SubprocessError):
     """This exception is raised when a process run by check_call() or
     check_output() returns a non-zero exit status.
     The exit status will be stored in the returncode attribute;
@@ -362,10 +372,24 @@ class CalledProcessError(Exception):
         return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
 
 
+class TimeoutExpired(SubprocessError):
+    """This exception is raised when the timeout expires while waiting for a
+    child process.
+    """
+    def __init__(self, cmd, timeout, output=None):
+        self.cmd = cmd
+        self.timeout = timeout
+        self.output = output
+
+    def __str__(self):
+        return ("Command '%s' timed out after %s seconds" %
+                (self.cmd, self.timeout))
+
+
 if mswindows:
     import threading
     import msvcrt
-    import _subprocess
+    import _winapi
     class STARTUPINFO:
         dwFlags = 0
         hStdInput = None
@@ -377,53 +401,49 @@ if mswindows:
 else:
     import select
     _has_poll = hasattr(select, 'poll')
-    import fcntl
-    import pickle
-
-    try:
-        import _posixsubprocess
-    except ImportError:
-        _posixsubprocess = None
-        warnings.warn("The _posixsubprocess module is not being used. "
-                      "Child process reliability may suffer if your "
-                      "program uses threads.", RuntimeWarning)
+    import _posixsubprocess
+    _create_pipe = _posixsubprocess.cloexec_pipe
 
     # When select or poll has indicated that the file is writable,
     # we can write up to _PIPE_BUF bytes without risk of blocking.
     # POSIX defines PIPE_BUF as >= 512.
     _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
 
-    _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1)
-
-    def _set_cloexec(fd, cloexec):
-        old = fcntl.fcntl(fd, fcntl.F_GETFD)
-        if cloexec:
-            fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC)
-        else:
-            fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC)
-
-    if _posixsubprocess:
-        _create_pipe = _posixsubprocess.cloexec_pipe
-    else:
-        def _create_pipe():
-            fds = os.pipe()
-            _set_cloexec(fds[0], True)
-            _set_cloexec(fds[1], True)
-            return fds
 
 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
-           "getoutput", "check_output", "CalledProcessError"]
+           "getoutput", "check_output", "CalledProcessError", "DEVNULL"]
 
 if mswindows:
-    from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
-                             STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
-                             STD_ERROR_HANDLE, SW_HIDE,
-                             STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
+    from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
+                         STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
+                         STD_ERROR_HANDLE, SW_HIDE,
+                         STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
 
     __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
                     "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
                     "STD_ERROR_HANDLE", "SW_HIDE",
                     "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"])
+
+    class Handle(int):
+        closed = False
+
+        def Close(self, CloseHandle=_winapi.CloseHandle):
+            if not self.closed:
+                self.closed = True
+                CloseHandle(self)
+
+        def Detach(self):
+            if not self.closed:
+                self.closed = True
+                return int(self)
+            raise ValueError("already closed")
+
+        def __repr__(self):
+            return "Handle(%d)" % int(self)
+
+        __del__ = Close
+        __str__ = __repr__
+
 try:
     MAXFD = os.sysconf("SC_OPEN_MAX")
 except:
@@ -448,27 +468,63 @@ def _cleanup():
 
 PIPE = -1
 STDOUT = -2
+DEVNULL = -3
 
 
 def _eintr_retry_call(func, *args):
     while True:
         try:
             return func(*args)
-        except (OSError, IOError) as e:
-            if e.errno == errno.EINTR:
-                continue
-            raise
-
-
-def call(*popenargs, **kwargs):
-    """Run command with arguments.  Wait for command to complete, then
-    return the returncode attribute.
+        except InterruptedError:
+            continue
+
+
+# XXX This function is only used by multiprocessing and the test suite,
+# but it's here so that it can be imported when Python is compiled without
+# threads.
+
+def _args_from_interpreter_flags():
+    """Return a list of command-line arguments reproducing the current
+    settings in sys.flags and sys.warnoptions."""
+    flag_opt_map = {
+        'debug': 'd',
+        # 'inspect': 'i',
+        # 'interactive': 'i',
+        'optimize': 'O',
+        'dont_write_bytecode': 'B',
+        'no_user_site': 's',
+        'no_site': 'S',
+        'ignore_environment': 'E',
+        'verbose': 'v',
+        'bytes_warning': 'b',
+        'quiet': 'q',
+        'hash_randomization': 'R',
+    }
+    args = []
+    for flag, opt in flag_opt_map.items():
+        v = getattr(sys.flags, flag)
+        if v > 0:
+            args.append('-' + opt * v)
+    for opt in sys.warnoptions:
+        args.append('-W' + opt)
+    return args
+
+
+def call(*popenargs, timeout=None, **kwargs):
+    """Run command with arguments.  Wait for command to complete or
+    timeout, then return the returncode attribute.
 
     The arguments are the same as for the Popen constructor.  Example:
 
     retcode = call(["ls", "-l"])
     """
-    return Popen(*popenargs, **kwargs).wait()
+    with Popen(*popenargs, **kwargs) as p:
+        try:
+            return p.wait(timeout=timeout)
+        except:
+            p.kill()
+            p.wait()
+            raise
 
 
 def check_call(*popenargs, **kwargs):
@@ -477,7 +533,7 @@ def check_call(*popenargs, **kwargs):
     CalledProcessError.  The CalledProcessError object will have the
     return code in the returncode attribute.
 
-    The arguments are the same as for the Popen constructor.  Example:
+    The arguments are the same as for the call function.  Example:
 
     check_call(["ls", "-l"])
     """
@@ -490,7 +546,7 @@ def check_call(*popenargs, **kwargs):
     return 0
 
 
-def check_output(*popenargs, **kwargs):
+def check_output(*popenargs, timeout=None, **kwargs):
     r"""Run command with arguments and return its output as a byte string.
 
     If the exit code was non-zero it raises a CalledProcessError.  The
@@ -512,14 +568,20 @@ def check_output(*popenargs, **kwargs):
     """
     if 'stdout' in kwargs:
         raise ValueError('stdout argument not allowed, it will be overridden.')
-    process = Popen(*popenargs, stdout=PIPE, **kwargs)
-    output, unused_err = process.communicate()
-    retcode = process.poll()
-    if retcode:
-        cmd = kwargs.get("args")
-        if cmd is None:
-            cmd = popenargs[0]
-        raise CalledProcessError(retcode, cmd, output=output)
+    with Popen(*popenargs, stdout=PIPE, **kwargs) as process:
+        try:
+            output, unused_err = process.communicate(timeout=timeout)
+        except TimeoutExpired:
+            process.kill()
+            output, unused_err = process.communicate()
+            raise TimeoutExpired(process.args, timeout, output=output)
+        except:
+            process.kill()
+            process.wait()
+            raise
+        retcode = process.poll()
+        if retcode:
+            raise CalledProcessError(retcode, process.args, output=output)
     return output
 
 
@@ -614,11 +676,19 @@ def getstatusoutput(cmd):
     >>> subprocess.getstatusoutput('/bin/junk')
     (256, 'sh: /bin/junk: not found')
     """
-    pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
-    text = pipe.read()
-    sts = pipe.close()
-    if sts is None: sts = 0
-    if text[-1:] == '\n': text = text[:-1]
+    with os.popen('{ ' + cmd + '; } 2>&1', 'r') as pipe:
+        try:
+            text = pipe.read()
+            sts = pipe.close()
+        except:
+            process = pipe._proc
+            process.kill()
+            process.wait()
+            raise
+    if sts is None:
+        sts = 0
+    if text[-1:] == '\n':
+        text = text[:-1]
     return sts, text
 
 
@@ -650,6 +720,8 @@ class Popen(object):
         _cleanup()
 
         self._child_created = False
+        self._input = None
+        self._communication_started = False
         if bufsize is None:
             bufsize = 0  # Restore default
         if not isinstance(bufsize, int):
@@ -684,6 +756,7 @@ class Popen(object):
                 raise ValueError("creationflags is only supported on Windows "
                                  "platforms")
 
+        self.args = args
         self.stdin = None
         self.stdout = None
         self.stderr = None
@@ -724,7 +797,7 @@ class Popen(object):
 
         if p2cwrite != -1:
             self.stdin = io.open(p2cwrite, 'wb', bufsize)
-            if self.universal_newlines:
+            if universal_newlines:
                 self.stdin = io.TextIOWrapper(self.stdin, write_through=True)
         if c2pread != -1:
             self.stdout = io.open(c2pread, 'rb', bufsize)
@@ -737,7 +810,7 @@ class Popen(object):
 
         try:
             self._execute_child(args, executable, preexec_fn, close_fds,
-                                pass_fds, cwd, env, universal_newlines,
+                                pass_fds, cwd, env,
                                 startupinfo, creationflags, shell,
                                 p2cread, p2cwrite,
                                 c2pread, c2pwrite,
@@ -798,19 +871,28 @@ class Popen(object):
             # Child is still running, keep us alive until we can wait on it.
             _active.append(self)
 
+    def _get_devnull(self):
+        if not hasattr(self, '_devnull'):
+            self._devnull = os.open(os.devnull, os.O_RDWR)
+        return self._devnull
 
-    def communicate(self, input=None):
+    def communicate(self, input=None, timeout=None):
         """Interact with process: Send data to stdin.  Read data from
         stdout and stderr, until end-of-file is reached.  Wait for
-        process to terminate.  The optional input argument should be a
-        string to be sent to the child process, or None, if no data
+        process to terminate.  The optional input argument should be
+        bytes to be sent to the child process, or None, if no data
         should be sent to the child.
 
         communicate() returns a tuple (stdout, stderr)."""
 
-        # Optimization: If we are only using one pipe, or no pipe at
-        # all, using select() or threads is unnecessary.
-        if [self.stdin, self.stdout, self.stderr].count(None) >= 2:
+        if self._communication_started and input:
+            raise ValueError("Cannot send input after starting communication")
+
+        # Optimization: If we are not worried about timeouts, we haven't
+        # started communicating, and we have one or zero pipes, using select()
+        # or threads is unnecessary.
+        if (timeout is None and not self._communication_started and
+            [self.stdin, self.stdout, self.stderr].count(None) >= 2):
             stdout = None
             stderr = None
             if self.stdin:
@@ -828,15 +910,42 @@ class Popen(object):
                 stderr = _eintr_retry_call(self.stderr.read)
                 self.stderr.close()
             self.wait()
-            return (stdout, stderr)
+        else:
+            if timeout is not None:
+                endtime = _time() + timeout
+            else:
+                endtime = None
+
+            try:
+                stdout, stderr = self._communicate(input, endtime, timeout)
+            finally:
+                self._communication_started = True
 
-        return self._communicate(input)
+            sts = self.wait(timeout=self._remaining_time(endtime))
+
+        return (stdout, stderr)
 
 
     def poll(self):
         return self._internal_poll()
 
 
+    def _remaining_time(self, endtime):
+        """Convenience for _communicate when computing timeouts."""
+        if endtime is None:
+            return None
+        else:
+            return endtime - _time()
+
+
+    def _check_timeout(self, endtime, orig_timeout):
+        """Convenience for checking if a timeout has expired."""
+        if endtime is None:
+            return
+        if _time() > endtime:
+            raise TimeoutExpired(self.args, orig_timeout)
+
+
     if mswindows:
         #
         # Windows methods
@@ -853,11 +962,16 @@ class Popen(object):
             errread, errwrite = -1, -1
 
             if stdin is None:
-                p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
+                p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
                 if p2cread is None:
-                    p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    p2cread, _ = _winapi.CreatePipe(None, 0)
+                    p2cread = Handle(p2cread)
+                    _winapi.CloseHandle(_)
             elif stdin == PIPE:
-                p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
+                p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
+            elif stdin == DEVNULL:
+                p2cread = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
@@ -866,11 +980,16 @@ class Popen(object):
             p2cread = self._make_inheritable(p2cread)
 
             if stdout is None:
-                c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
+                c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
-                    _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    _, c2pwrite = _winapi.CreatePipe(None, 0)
+                    c2pwrite = Handle(c2pwrite)
+                    _winapi.CloseHandle(_)
             elif stdout == PIPE:
-                c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
+                c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
+            elif stdout == DEVNULL:
+                c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
@@ -879,13 +998,18 @@ class Popen(object):
             c2pwrite = self._make_inheritable(c2pwrite)
 
             if stderr is None:
-                errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
+                errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
                 if errwrite is None:
-                    _, errwrite = _subprocess.CreatePipe(None, 0)
+                    _, errwrite = _winapi.CreatePipe(None, 0)
+                    errwrite = Handle(errwrite)
+                    _winapi.CloseHandle(_)
             elif stderr == PIPE:
-                errread, errwrite = _subprocess.CreatePipe(None, 0)
+                errread, errwrite = _winapi.CreatePipe(None, 0)
+                errread, errwrite = Handle(errread), Handle(errwrite)
             elif stderr == STDOUT:
                 errwrite = c2pwrite
+            elif stderr == DEVNULL:
+                errwrite = msvcrt.get_osfhandle(self._get_devnull())
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
@@ -900,20 +1024,22 @@ class Popen(object):
 
         def _make_inheritable(self, handle):
             """Return a duplicate of handle, which is inheritable"""
-            return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
-                                handle, _subprocess.GetCurrentProcess(), 0, 1,
-                                _subprocess.DUPLICATE_SAME_ACCESS)
+            h = _winapi.DuplicateHandle(
+                _winapi.GetCurrentProcess(), handle,
+                _winapi.GetCurrentProcess(), 0, 1,
+                _winapi.DUPLICATE_SAME_ACCESS)
+            return Handle(h)
 
 
         def _find_w9xpopen(self):
             """Find and return absolut path to w9xpopen.exe"""
             w9xpopen = os.path.join(
-                            os.path.dirname(_subprocess.GetModuleFileName(0)),
+                            os.path.dirname(_winapi.GetModuleFileName(0)),
                                     "w9xpopen.exe")
             if not os.path.exists(w9xpopen):
                 # Eeek - file-not-found - possibly an embedding
                 # situation - see if we can locate it in sys.exec_prefix
-                w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
+                w9xpopen = os.path.join(os.path.dirname(sys.base_exec_prefix),
                                         "w9xpopen.exe")
                 if not os.path.exists(w9xpopen):
                     raise RuntimeError("Cannot locate w9xpopen.exe, which is "
@@ -923,7 +1049,7 @@ class Popen(object):
 
 
         def _execute_child(self, args, executable, preexec_fn, close_fds,
-                           pass_fds, cwd, env, universal_newlines,
+                           pass_fds, cwd, env,
                            startupinfo, creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
@@ -940,17 +1066,17 @@ class Popen(object):
             if startupinfo is None:
                 startupinfo = STARTUPINFO()
             if -1 not in (p2cread, c2pwrite, errwrite):
-                startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES
+                startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
                 startupinfo.hStdInput = p2cread
                 startupinfo.hStdOutput = c2pwrite
                 startupinfo.hStdError = errwrite
 
             if shell:
-                startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
-                startupinfo.wShowWindow = _subprocess.SW_HIDE
+                startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
+                startupinfo.wShowWindow = _winapi.SW_HIDE
                 comspec = os.environ.get("COMSPEC", "cmd.exe")
                 args = '{} /c "{}"'.format (comspec, args)
-                if (_subprocess.GetVersion() >= 0x80000000 or
+                if (_winapi.GetVersion() >= 0x80000000 or
                         os.path.basename(comspec).lower() == "command.com"):
                     # Win9x, or using command.com on NT. We need to
                     # use the w9xpopen intermediate program. For more
@@ -964,11 +1090,11 @@ class Popen(object):
                     # use at xxx" and a hopeful warning about the
                     # stability of your system.  Cost is Ctrl+C won't
                     # kill children.
-                    creationflags |= _subprocess.CREATE_NEW_CONSOLE
+                    creationflags |= _winapi.CREATE_NEW_CONSOLE
 
             # Start the process
             try:
-                hp, ht, pid, tid = _subprocess.CreateProcess(executable, args,
+                hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
                                          # no special security
                                          None, None,
                                          int(not close_fds),
@@ -995,17 +1121,19 @@ class Popen(object):
                     c2pwrite.Close()
                 if errwrite != -1:
                     errwrite.Close()
+                if hasattr(self, '_devnull'):
+                    os.close(self._devnull)
 
             # Retain the process handle, but close the thread handle
             self._child_created = True
-            self._handle = hp
+            self._handle = Handle(hp)
             self.pid = pid
-            ht.Close()
+            _winapi.CloseHandle(ht)
 
         def _internal_poll(self, _deadstate=None,
-                _WaitForSingleObject=_subprocess.WaitForSingleObject,
-                _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0,
-                _GetExitCodeProcess=_subprocess.GetExitCodeProcess):
+                _WaitForSingleObject=_winapi.WaitForSingleObject,
+                _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
+                _GetExitCodeProcess=_winapi.GetExitCodeProcess):
             """Check if child process has terminated.  Returns returncode
             attribute.
 
@@ -1019,13 +1147,21 @@ class Popen(object):
             return self.returncode
 
 
-        def wait(self):
+        def wait(self, timeout=None, endtime=None):
             """Wait for child process to terminate.  Returns returncode
             attribute."""
+            if endtime is not None:
+                timeout = self._remaining_time(endtime)
+            if timeout is None:
+                timeout_millis = _winapi.INFINITE
+            else:
+                timeout_millis = int(timeout * 1000)
             if self.returncode is None:
-                _subprocess.WaitForSingleObject(self._handle,
-                                                _subprocess.INFINITE)
-                self.returncode = _subprocess.GetExitCodeProcess(self._handle)
+                result = _winapi.WaitForSingleObject(self._handle,
+                                                    timeout_millis)
+                if result == _winapi.WAIT_TIMEOUT:
+                    raise TimeoutExpired(self.args, timeout)
+                self.returncode = _winapi.GetExitCodeProcess(self._handle)
             return self.returncode
 
 
@@ -1034,22 +1170,23 @@ class Popen(object):
             fh.close()
 
 
-        def _communicate(self, input):
-            stdout = None # Return
-            stderr = None # Return
-
-            if self.stdout:
-                stdout = []
-                stdout_thread = threading.Thread(target=self._readerthread,
-                                                 args=(self.stdout, stdout))
-                stdout_thread.daemon = True
-                stdout_thread.start()
-            if self.stderr:
-                stderr = []
-                stderr_thread = threading.Thread(target=self._readerthread,
-                                                 args=(self.stderr, stderr))
-                stderr_thread.daemon = True
-                stderr_thread.start()
+        def _communicate(self, input, endtime, orig_timeout):
+            # Start reader threads feeding into a list hanging off of this
+            # object, unless they've already been started.
+            if self.stdout and not hasattr(self, "_stdout_buff"):
+                self._stdout_buff = []
+                self.stdout_thread = \
+                        threading.Thread(target=self._readerthread,
+                                         args=(self.stdout, self._stdout_buff))
+                self.stdout_thread.daemon = True
+                self.stdout_thread.start()
+            if self.stderr and not hasattr(self, "_stderr_buff"):
+                self._stderr_buff = []
+                self.stderr_thread = \
+                        threading.Thread(target=self._readerthread,
+                                         args=(self.stderr, self._stderr_buff))
+                self.stderr_thread.daemon = True
+                self.stderr_thread.start()
 
             if self.stdin:
                 if input is not None:
@@ -1060,10 +1197,28 @@ class Popen(object):
                             raise
                 self.stdin.close()
 
+            # Wait for the reader threads, or time out.  If we time out, the
+            # threads remain reading and the fds left open in case the user
+            # calls communicate again.
+            if self.stdout is not None:
+                self.stdout_thread.join(self._remaining_time(endtime))
+                if self.stdout_thread.is_alive():
+                    raise TimeoutExpired(self.args, orig_timeout)
+            if self.stderr is not None:
+                self.stderr_thread.join(self._remaining_time(endtime))
+                if self.stderr_thread.is_alive():
+                    raise TimeoutExpired(self.args, orig_timeout)
+
+            # Collect the output from and close both pipes, now that we know
+            # both have been read successfully.
+            stdout = None
+            stderr = None
             if self.stdout:
-                stdout_thread.join()
+                stdout = self._stdout_buff
+                self.stdout.close()
             if self.stderr:
-                stderr_thread.join()
+                stderr = self._stderr_buff
+                self.stderr.close()
 
             # All data exchanged.  Translate lists into strings.
             if stdout is not None:
@@ -1071,7 +1226,6 @@ class Popen(object):
             if stderr is not None:
                 stderr = stderr[0]
 
-            self.wait()
             return (stdout, stderr)
 
         def send_signal(self, sig):
@@ -1090,14 +1244,12 @@ class Popen(object):
             """Terminates the process
             """
             try:
-                _subprocess.TerminateProcess(self._handle, 1)
-            except OSError as e:
+                _winapi.TerminateProcess(self._handle, 1)
+            except PermissionError:
                 # ERROR_ACCESS_DENIED (winerror 5) is received when the
                 # process already died.
-                if e.winerror != 5:
-                    raise
-                rc = _subprocess.GetExitCodeProcess(self._handle)
-                if rc == _subprocess.STILL_ACTIVE:
+                rc = _winapi.GetExitCodeProcess(self._handle)
+                if rc == _winapi.STILL_ACTIVE:
                     raise
                 self.returncode = rc
 
@@ -1119,6 +1271,8 @@ class Popen(object):
                 pass
             elif stdin == PIPE:
                 p2cread, p2cwrite = _create_pipe()
+            elif stdin == DEVNULL:
+                p2cread = self._get_devnull()
             elif isinstance(stdin, int):
                 p2cread = stdin
             else:
@@ -1129,6 +1283,8 @@ class Popen(object):
                 pass
             elif stdout == PIPE:
                 c2pread, c2pwrite = _create_pipe()
+            elif stdout == DEVNULL:
+                c2pwrite = self._get_devnull()
             elif isinstance(stdout, int):
                 c2pwrite = stdout
             else:
@@ -1141,6 +1297,8 @@ class Popen(object):
                 errread, errwrite = _create_pipe()
             elif stderr == STDOUT:
                 errwrite = c2pwrite
+            elif stderr == DEVNULL:
+                errwrite = self._get_devnull()
             elif isinstance(stderr, int):
                 errwrite = stderr
             else:
@@ -1163,7 +1321,7 @@ class Popen(object):
 
 
         def _execute_child(self, args, executable, preexec_fn, close_fds,
-                           pass_fds, cwd, env, universal_newlines,
+                           pass_fds, cwd, env,
                            startupinfo, creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
@@ -1171,7 +1329,7 @@ class Popen(object):
                            restore_signals, start_new_session):
             """Execute program (POSIX version)"""
 
-            if isinstance(args, str):
+            if isinstance(args, (str, bytes)):
                 args = [args]
             else:
                 args = list(args)
@@ -1191,153 +1349,34 @@ class Popen(object):
             errpipe_read, errpipe_write = _create_pipe()
             try:
                 try:
-
-                    if _posixsubprocess:
-                        # We must avoid complex work that could involve
-                        # malloc or free in the child process to avoid
-                        # potential deadlocks, thus we do all this here.
-                        # and pass it to fork_exec()
-
-                        if env is not None:
-                            env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
-                                        for k, v in env.items()]
-                        else:
-                            env_list = None  # Use execv instead of execve.
-                        executable = os.fsencode(executable)
-                        if os.path.dirname(executable):
-                            executable_list = (executable,)
-                        else:
-                            # This matches the behavior of os._execvpe().
-                            executable_list = tuple(
-                                os.path.join(os.fsencode(dir), executable)
-                                for dir in os.get_exec_path(env))
-                        fds_to_keep = set(pass_fds)
-                        fds_to_keep.add(errpipe_write)
-                        self.pid = _posixsubprocess.fork_exec(
-                                args, executable_list,
-                                close_fds, sorted(fds_to_keep), cwd, env_list,
-                                p2cread, p2cwrite, c2pread, c2pwrite,
-                                errread, errwrite,
-                                errpipe_read, errpipe_write,
-                                restore_signals, start_new_session, preexec_fn)
-                        self._child_created = True
+                    # We must avoid complex work that could involve
+                    # malloc or free in the child process to avoid
+                    # potential deadlocks, thus we do all this here.
+                    # and pass it to fork_exec()
+
+                    if env is not None:
+                        env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
+                                    for k, v in env.items()]
                     else:
-                        # Pure Python implementation: It is not thread safe.
-                        # This implementation may deadlock in the child if your
-                        # parent process has any other threads running.
-
-                        gc_was_enabled = gc.isenabled()
-                        # Disable gc to avoid bug where gc -> file_dealloc ->
-                        # write to stderr -> hang.  See issue1336
-                        gc.disable()
-                        try:
-                            self.pid = os.fork()
-                        except:
-                            if gc_was_enabled:
-                                gc.enable()
-                            raise
-                        self._child_created = True
-                        if self.pid == 0:
-                            # Child
-                            reached_preexec = False
-                            try:
-                                # Close parent's pipe ends
-                                if p2cwrite != -1:
-                                    os.close(p2cwrite)
-                                if c2pread != -1:
-                                    os.close(c2pread)
-                                if errread != -1:
-                                    os.close(errread)
-                                os.close(errpipe_read)
-
-                                # When duping fds, if there arises a situation
-                                # where one of the fds is either 0, 1 or 2, it
-                                # is possible that it is overwritten (#12607).
-                                if c2pwrite == 0:
-                                    c2pwrite = os.dup(c2pwrite)
-                                if errwrite == 0 or errwrite == 1:
-                                    errwrite = os.dup(errwrite)
-
-                                # Dup fds for child
-                                def _dup2(a, b):
-                                    # dup2() removes the CLOEXEC flag but
-                                    # we must do it ourselves if dup2()
-                                    # would be a no-op (issue #10806).
-                                    if a == b:
-                                        _set_cloexec(a, False)
-                                    elif a != -1:
-                                        os.dup2(a, b)
-                                _dup2(p2cread, 0)
-                                _dup2(c2pwrite, 1)
-                                _dup2(errwrite, 2)
-
-                                # Close pipe fds.  Make sure we don't close the
-                                # same fd more than once, or standard fds.
-                                closed = set()
-                                for fd in [p2cread, c2pwrite, errwrite]:
-                                    if fd > 2 and fd not in closed:
-                                        os.close(fd)
-                                        closed.add(fd)
-
-                                # Close all other fds, if asked for
-                                if close_fds:
-                                    fds_to_keep = set(pass_fds)
-                                    fds_to_keep.add(errpipe_write)
-                                    self._close_fds(fds_to_keep)
-
-
-                                if cwd is not None:
-                                    os.chdir(cwd)
-
-                                # This is a copy of Python/pythonrun.c
-                                # _Py_RestoreSignals().  If that were exposed
-                                # as a sys._py_restoresignals func it would be
-                                # better.. but this pure python implementation
-                                # isn't likely to be used much anymore.
-                                if restore_signals:
-                                    signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ')
-                                    for sig in signals:
-                                        if hasattr(signal, sig):
-                                            signal.signal(getattr(signal, sig),
-                                                          signal.SIG_DFL)
-
-                                if start_new_session and hasattr(os, 'setsid'):
-                                    os.setsid()
-
-                                reached_preexec = True
-                                if preexec_fn:
-                                    preexec_fn()
-
-                                if env is None:
-                                    os.execvp(executable, args)
-                                else:
-                                    os.execvpe(executable, args, env)
-
-                            except:
-                                try:
-                                    exc_type, exc_value = sys.exc_info()[:2]
-                                    if isinstance(exc_value, OSError):
-                                        errno_num = exc_value.errno
-                                    else:
-                                        errno_num = 0
-                                    if not reached_preexec:
-                                        exc_value = "noexec"
-                                    message = '%s:%x:%s' % (exc_type.__name__,
-                                                            errno_num, exc_value)
-                                    message = message.encode(errors="surrogatepass")
-                                    os.write(errpipe_write, message)
-                                except Exception:
-                                    # We MUST not allow anything odd happening
-                                    # above to prevent us from exiting below.
-                                    pass
-
-                            # This exitcode won't be reported to applications
-                            # so it really doesn't matter what we return.
-                            os._exit(255)
-
-                        # Parent
-                        if gc_was_enabled:
-                            gc.enable()
+                        env_list = None  # Use execv instead of execve.
+                    executable = os.fsencode(executable)
+                    if os.path.dirname(executable):
+                        executable_list = (executable,)
+                    else:
+                        # This matches the behavior of os._execvpe().
+                        executable_list = tuple(
+                            os.path.join(os.fsencode(dir), executable)
+                            for dir in os.get_exec_path(env))
+                    fds_to_keep = set(pass_fds)
+                    fds_to_keep.add(errpipe_write)
+                    self.pid = _posixsubprocess.fork_exec(
+                            args, executable_list,
+                            close_fds, sorted(fds_to_keep), cwd, env_list,
+                            p2cread, p2cwrite, c2pread, c2pwrite,
+                            errread, errwrite,
+                            errpipe_read, errpipe_write,
+                            restore_signals, start_new_session, preexec_fn)
+                    self._child_created = True
                 finally:
                     # be sure the FD is closed no matter what
                     os.close(errpipe_write)
@@ -1348,6 +1387,8 @@ class Popen(object):
                     os.close(c2pwrite)
                 if errwrite != -1 and errread != -1:
                     os.close(errwrite)
+                if hasattr(self, '_devnull'):
+                    os.close(self._devnull)
 
                 # Wait for exec to fail or succeed; possibly raising an
                 # exception (limited in size)
@@ -1437,29 +1478,61 @@ class Popen(object):
             return self.returncode
 
 
-        def wait(self):
+        def _try_wait(self, wait_flags):
+            try:
+                (pid, sts) = _eintr_retry_call(os.waitpid, self.pid, wait_flags)
+            except OSError as e:
+                if e.errno != errno.ECHILD:
+                    raise
+                # This happens if SIGCLD is set to be ignored or waiting
+                # for child processes has otherwise been disabled for our
+                # process.  This child is dead, we can't get the status.
+                pid = self.pid
+                sts = 0
+            return (pid, sts)
+
+
+        def wait(self, timeout=None, endtime=None):
             """Wait for child process to terminate.  Returns returncode
             attribute."""
-            while self.returncode is None:
-                try:
-                    pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
-                except OSError as e:
-                    if e.errno != errno.ECHILD:
-                        raise
-                    # This happens if SIGCLD is set to be ignored or waiting
-                    # for child processes has otherwise been disabled for our
-                    # process.  This child is dead, we can't get the status.
-                    pid = self.pid
-                    sts = 0
-                # Check the pid and loop as waitpid has been known to return
-                # 0 even without WNOHANG in odd situations.  issue14396.
-                if pid == self.pid:
-                    self._handle_exitstatus(sts)
+            if self.returncode is not None:
+                return self.returncode
+
+            # endtime is preferred to timeout.  timeout is only used for
+            # printing.
+            if endtime is not None or timeout is not None:
+                if endtime is None:
+                    endtime = _time() + timeout
+                elif timeout is None:
+                    timeout = self._remaining_time(endtime)
+
+            if endtime is not None:
+                # Enter a busy loop if we have a timeout.  This busy loop was
+                # cribbed from Lib/threading.py in Thread.wait() at r71065.
+                delay = 0.0005 # 500 us -> initial delay of 1 ms
+                while True:
+                    (pid, sts) = self._try_wait(os.WNOHANG)
+                    assert pid == self.pid or pid == 0
+                    if pid == self.pid:
+                        self._handle_exitstatus(sts)
+                        break
+                    remaining = self._remaining_time(endtime)
+                    if remaining <= 0:
+                        raise TimeoutExpired(self.args, timeout)
+                    delay = min(delay * 2, remaining, .05)
+                    time.sleep(delay)
+            else:
+                while self.returncode is None:
+                    (pid, sts) = self._try_wait(0)
+                    # Check the pid and loop as waitpid has been known to return
+                    # 0 even without WNOHANG in odd situations.  issue14396.
+                    if pid == self.pid:
+                        self._handle_exitstatus(sts)
             return self.returncode
 
 
-        def _communicate(self, input):
-            if self.stdin:
+        def _communicate(self, input, endtime, orig_timeout):
+            if self.stdin and not self._communication_started:
                 # Flush stdio buffer.  This might block, if the user has
                 # been writing to .stdin in an uncontrolled fashion.
                 self.stdin.flush()
@@ -1467,9 +1540,13 @@ class Popen(object):
                     self.stdin.close()
 
             if _has_poll:
-                stdout, stderr = self._communicate_with_poll(input)
+                stdout, stderr = self._communicate_with_poll(input, endtime,
+                                                             orig_timeout)
             else:
-                stdout, stderr = self._communicate_with_select(input)
+                stdout, stderr = self._communicate_with_select(input, endtime,
+                                                               orig_timeout)
+
+            self.wait(timeout=self._remaining_time(endtime))
 
             # All data exchanged.  Translate lists into strings.
             if stdout is not None:
@@ -1487,69 +1564,92 @@ class Popen(object):
                     stderr = self._translate_newlines(stderr,
                                                       self.stderr.encoding)
 
-            self.wait()
             return (stdout, stderr)
 
 
-        def _communicate_with_poll(self, input):
+        def _save_input(self, input):
+            # This method is called from the _communicate_with_*() methods
+            # so that if we time out while communicating, we can continue
+            # sending input if we retry.
+            if self.stdin and self._input is None:
+                self._input_offset = 0
+                self._input = input
+                if self.universal_newlines and input is not None:
+                    self._input = self._input.encode(self.stdin.encoding)
+
+
+        def _communicate_with_poll(self, input, endtime, orig_timeout):
             stdout = None # Return
             stderr = None # Return
-            fd2file = {}
-            fd2output = {}
+
+            if not self._communication_started:
+                self._fd2file = {}
 
             poller = select.poll()
             def register_and_append(file_obj, eventmask):
                 poller.register(file_obj.fileno(), eventmask)
-                fd2file[file_obj.fileno()] = file_obj
+                self._fd2file[file_obj.fileno()] = file_obj
 
             def close_unregister_and_remove(fd):
                 poller.unregister(fd)
-                fd2file[fd].close()
-                fd2file.pop(fd)
+                self._fd2file[fd].close()
+                self._fd2file.pop(fd)
 
             if self.stdin and input:
                 register_and_append(self.stdin, select.POLLOUT)
 
+            # Only create this mapping if we haven't already.
+            if not self._communication_started:
+                self._fd2output = {}
+                if self.stdout:
+                    self._fd2output[self.stdout.fileno()] = []
+                if self.stderr:
+                    self._fd2output[self.stderr.fileno()] = []
+
             select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI
             if self.stdout:
                 register_and_append(self.stdout, select_POLLIN_POLLPRI)
-                fd2output[self.stdout.fileno()] = stdout = []
+                stdout = self._fd2output[self.stdout.fileno()]
             if self.stderr:
                 register_and_append(self.stderr, select_POLLIN_POLLPRI)
-                fd2output[self.stderr.fileno()] = stderr = []
+                stderr = self._fd2output[self.stderr.fileno()]
 
-            input_offset = 0
-            if self.universal_newlines and isinstance(input, str):
-                input = input.encode(self.stdin.encoding)
-            while fd2file:
+            self._save_input(input)
+
+            while self._fd2file:
+                timeout = self._remaining_time(endtime)
+                if timeout is not None and timeout < 0:
+                    raise TimeoutExpired(self.args, orig_timeout)
                 try:
-                    ready = poller.poll()
+                    ready = poller.poll(timeout)
                 except select.error as e:
                     if e.args[0] == errno.EINTR:
                         continue
                     raise
+                self._check_timeout(endtime, orig_timeout)
 
                 # XXX Rewrite these to use non-blocking I/O on the
                 # file objects; they are no longer using C stdio!
 
                 for fd, mode in ready:
                     if mode & select.POLLOUT:
-                        chunk = input[input_offset : input_offset + _PIPE_BUF]
+                        chunk = self._input[self._input_offset :
+                                            self._input_offset + _PIPE_BUF]
                         try:
-                            input_offset += os.write(fd, chunk)
+                            self._input_offset += os.write(fd, chunk)
                         except OSError as e:
                             if e.errno == errno.EPIPE:
                                 close_unregister_and_remove(fd)
                             else:
                                 raise
                         else:
-                            if input_offset >= len(input):
+                            if self._input_offset >= len(self._input):
                                 close_unregister_and_remove(fd)
                     elif mode & select_POLLIN_POLLPRI:
                         data = os.read(fd, 4096)
                         if not data:
                             close_unregister_and_remove(fd)
-                        fd2output[fd].append(data)
+                        self._fd2output[fd].append(data)
                     else:
                         # Ignore hang up or errors.
                         close_unregister_and_remove(fd)
@@ -1557,63 +1657,83 @@ class Popen(object):
             return (stdout, stderr)
 
 
-        def _communicate_with_select(self, input):
-            read_set = []
-            write_set = []
+        def _communicate_with_select(self, input, endtime, orig_timeout):
+            if not self._communication_started:
+                self._read_set = []
+                self._write_set = []
+                if self.stdin and input:
+                    self._write_set.append(self.stdin)
+                if self.stdout:
+                    self._read_set.append(self.stdout)
+                if self.stderr:
+                    self._read_set.append(self.stderr)
+
+            self._save_input(input)
+
             stdout = None # Return
             stderr = None # Return
 
-            if self.stdin and input:
-                write_set.append(self.stdin)
             if self.stdout:
-                read_set.append(self.stdout)
-                stdout = []
+                if not self._communication_started:
+                    self._stdout_buff = []
+                stdout = self._stdout_buff
             if self.stderr:
-                read_set.append(self.stderr)
-                stderr = []
-
-            input_offset = 0
-            if self.universal_newlines and isinstance(input, str):
-                input = input.encode(self.stdin.encoding)
-            while read_set or write_set:
+                if not self._communication_started:
+                    self._stderr_buff = []
+                stderr = self._stderr_buff
+
+            while self._read_set or self._write_set:
+                timeout = self._remaining_time(endtime)
+                if timeout is not None and timeout < 0:
+                    raise TimeoutExpired(self.args, orig_timeout)
                 try:
-                    rlist, wlist, xlist = select.select(read_set, write_set, [])
+                    (rlist, wlist, xlist) = \
+                        select.select(self._read_set, self._write_set, [],
+                                      timeout)
                 except select.error as e:
                     if e.args[0] == errno.EINTR:
                         continue
                     raise
 
+                # According to the docs, returning three empty lists indicates
+                # that the timeout expired.
+                if not (rlist or wlist or xlist):
+                    raise TimeoutExpired(self.args, orig_timeout)
+                # We also check what time it is ourselves for good measure.
+                self._check_timeout(endtime, orig_timeout)
+
                 # XXX Rewrite these to use non-blocking I/O on the
                 # file objects; they are no longer using C stdio!
 
                 if self.stdin in wlist:
-                    chunk = input[input_offset : input_offset + _PIPE_BUF]
+                    chunk = self._input[self._input_offset :
+                                        self._input_offset + _PIPE_BUF]
                     try:
                         bytes_written = os.write(self.stdin.fileno(), chunk)
                     except OSError as e:
                         if e.errno == errno.EPIPE:
                             self.stdin.close()
-                            write_set.remove(self.stdin)
+                            self._write_set.remove(self.stdin)
                         else:
                             raise
                     else:
-                        input_offset += bytes_written
-                        if input_offset >= len(input):
+                        self._input_offset += bytes_written
+                        if self._input_offset >= len(self._input):
                             self.stdin.close()
-                            write_set.remove(self.stdin)
+                            self._write_set.remove(self.stdin)
 
                 if self.stdout in rlist:
                     data = os.read(self.stdout.fileno(), 1024)
                     if not data:
                         self.stdout.close()
-                        read_set.remove(self.stdout)
+                        self._read_set.remove(self.stdout)
                     stdout.append(data)
 
                 if self.stderr in rlist:
                     data = os.read(self.stderr.fileno(), 1024)
                     if not data:
                         self.stderr.close()
-                        read_set.remove(self.stderr)
+                        self._read_set.remove(self.stderr)
                     stderr.append(data)
 
             return (stdout, stderr)
@@ -1633,68 +1753,3 @@ class Popen(object):
             """Kill the process with SIGKILL
             """
             self.send_signal(signal.SIGKILL)
-
-
-def _demo_posix():
-    #
-    # Example 1: Simple redirection: Get process list
-    #
-    plist = Popen(["ps"], stdout=PIPE).communicate()[0]
-    print("Process list:")
-    print(plist)
-
-    #
-    # Example 2: Change uid before executing child
-    #
-    if os.getuid() == 0:
-        p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
-        p.wait()
-
-    #
-    # Example 3: Connecting several subprocesses
-    #
-    print("Looking for 'hda'...")
-    p1 = Popen(["dmesg"], stdout=PIPE)
-    p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
-    print(repr(p2.communicate()[0]))
-
-    #
-    # Example 4: Catch execution error
-    #
-    print()
-    print("Trying a weird file...")
-    try:
-        print(Popen(["/this/path/does/not/exist"]).communicate())
-    except OSError as e:
-        if e.errno == errno.ENOENT:
-            print("The file didn't exist.  I thought so...")
-            print("Child traceback:")
-            print(e.child_traceback)
-        else:
-            print("Error", e.errno)
-    else:
-        print("Gosh.  No error.", file=sys.stderr)
-
-
-def _demo_windows():
-    #
-    # Example 1: Connecting several subprocesses
-    #
-    print("Looking for 'PROMPT' in set output...")
-    p1 = Popen("set", stdout=PIPE, shell=True)
-    p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
-    print(repr(p2.communicate()[0]))
-
-    #
-    # Example 2: Simple execution of program
-    #
-    print("Executing calc...")
-    p = Popen("calc")
-    p.wait()
-
-
-if __name__ == "__main__":
-    if mswindows:
-        _demo_windows()
-    else:
-        _demo_posix()
diff --git a/Lib/symbol.py b/Lib/symbol.py
index 679e5c8cc7..34143b5d8e 100755
--- a/Lib/symbol.py
+++ b/Lib/symbol.py
@@ -91,6 +91,7 @@ comp_for = 333
 comp_if = 334
 encoding_decl = 335
 yield_expr = 336
+yield_arg = 337
 #--end constants--
 
 sym_name = {}
@@ -104,7 +105,7 @@ def main():
     import token
     if len(sys.argv) == 1:
         sys.argv = sys.argv + ["Include/graminit.h", "Lib/symbol.py"]
-    token.main()
+    token._main()
 
 if __name__ == "__main__":
     main()
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
index 253e2efae6..71da1db183 100644
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -1,8 +1,8 @@
-"""Provide access to Python's configuration information.
+"""Access to Python's configuration information."""
 
-"""
-import sys
 import os
+import re
+import sys
 from os.path import pardir, realpath
 
 __all__ = [
@@ -17,50 +17,50 @@ __all__ = [
     'get_python_version',
     'get_scheme_names',
     'parse_config_h',
-    ]
+]
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{base}/lib/python{py_version_short}',
+        'stdlib': '{installed_base}/lib/python{py_version_short}',
         'platstdlib': '{platbase}/lib/python{py_version_short}',
         'purelib': '{base}/lib/python{py_version_short}/site-packages',
         'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
         'include':
-            '{base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/python{py_version_short}{abiflags}',
         'platinclude':
-            '{platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/python{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{base}/lib/python',
+        'stdlib': '{installed_base}/lib/python',
         'platstdlib': '{base}/lib/python',
         'purelib': '{base}/lib/python',
         'platlib': '{base}/lib/python',
-        'include': '{base}/include/python',
-        'platinclude': '{base}/include/python',
+        'include': '{installed_base}/include/python',
+        'platinclude': '{installed_base}/include/python',
         'scripts': '{base}/bin',
-        'data'   : '{base}',
+        'data': '{base}',
         },
     'nt': {
-        'stdlib': '{base}/Lib',
+        'stdlib': '{installed_base}/Lib',
         'platstdlib': '{base}/Lib',
         'purelib': '{base}/Lib/site-packages',
         'platlib': '{base}/Lib/site-packages',
-        'include': '{base}/Include',
-        'platinclude': '{base}/Include',
+        'include': '{installed_base}/Include',
+        'platinclude': '{installed_base}/Include',
         'scripts': '{base}/Scripts',
-        'data'   : '{base}',
+        'data': '{base}',
         },
     'os2': {
-        'stdlib': '{base}/Lib',
+        'stdlib': '{installed_base}/Lib',
         'platstdlib': '{base}/Lib',
         'purelib': '{base}/Lib/site-packages',
         'platlib': '{base}/Lib/site-packages',
-        'include': '{base}/Include',
-        'platinclude': '{base}/Include',
+        'include': '{installed_base}/Include',
+        'platinclude': '{installed_base}/Include',
         'scripts': '{base}/Scripts',
-        'data'   : '{base}',
+        'data': '{base}',
         },
     'os2_home': {
         'stdlib': '{userbase}/lib/python{py_version_short}',
@@ -69,7 +69,7 @@ _INSTALL_SCHEMES = {
         'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
         'include': '{userbase}/include/python{py_version_short}',
         'scripts': '{userbase}/bin',
-        'data'   : '{userbase}',
+        'data': '{userbase}',
         },
     'nt_user': {
         'stdlib': '{userbase}/Python{py_version_nodot}',
@@ -78,7 +78,7 @@ _INSTALL_SCHEMES = {
         'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
         'include': '{userbase}/Python{py_version_nodot}/Include',
         'scripts': '{userbase}/Scripts',
-        'data'   : '{userbase}',
+        'data': '{userbase}',
         },
     'posix_user': {
         'stdlib': '{userbase}/lib/python{py_version_short}',
@@ -87,7 +87,7 @@ _INSTALL_SCHEMES = {
         'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
         'include': '{userbase}/include/python{py_version_short}',
         'scripts': '{userbase}/bin',
-        'data'   : '{userbase}',
+        'data': '{userbase}',
         },
     'osx_framework_user': {
         'stdlib': '{userbase}/lib/python',
@@ -96,20 +96,26 @@ _INSTALL_SCHEMES = {
         'platlib': '{userbase}/lib/python/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
-        'data'   : '{userbase}',
+        'data': '{userbase}',
         },
     }
 
 _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include',
                 'scripts', 'data')
+
+ # FIXME don't rely on sys.version here, its format is an implementation detail
+ # of CPython, use sys.version_info or sys.hexversion
 _PY_VERSION = sys.version.split()[0]
 _PY_VERSION_SHORT = sys.version[:3]
 _PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2]
 _PREFIX = os.path.normpath(sys.prefix)
+_BASE_PREFIX = os.path.normpath(sys.base_prefix)
 _EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
 _CONFIG_VARS = None
 _USER_BASE = None
 
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -132,19 +138,35 @@ if os.name == "nt" and "\\pc\\v" in _PROJECT_BASE[-10:].lower():
 if os.name == "nt" and "\\pcbuild\\amd64" in _PROJECT_BASE[-14:].lower():
     _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
 
-def is_python_build():
+# set for cross builds
+if "_PYTHON_PROJECT_BASE" in os.environ:
+    _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"])
+
+def _is_python_source_dir(d):
     for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)):
+        if os.path.isfile(os.path.join(d, "Modules", fn)):
             return True
     return False
 
-_PYTHON_BUILD = is_python_build()
+_sys_home = getattr(sys, '_home', None)
+if _sys_home and os.name == 'nt' and \
+    _sys_home.lower().endswith(('pcbuild', 'pcbuild\\amd64')):
+    _sys_home = os.path.dirname(_sys_home)
+    if _sys_home.endswith('pcbuild'):   # must be amd64
+        _sys_home = os.path.dirname(_sys_home)
+def is_python_build(check_home=False):
+    if check_home and _sys_home:
+        return _is_python_source_dir(_sys_home)
+    return _is_python_source_dir(_PROJECT_BASE)
+
+_PYTHON_BUILD = is_python_build(True)
 
 if _PYTHON_BUILD:
     for scheme in ('posix_prefix', 'posix_home'):
         _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include'
         _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.'
 
+
 def _subst_vars(s, local_vars):
     try:
         return s.format(**local_vars)
@@ -161,6 +183,7 @@ def _extend_dict(target_dict, other_dict):
             continue
         target_dict[key] = value
 
+
 def _expand_vars(scheme, vars):
     res = {}
     if vars is None:
@@ -173,29 +196,41 @@ def _expand_vars(scheme, vars):
         res[key] = os.path.normpath(_subst_vars(value, vars))
     return res
 
+
 def _get_default_scheme():
     if os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
 
+
 def _getuserbase():
     env_base = os.environ.get("PYTHONUSERBASE", None)
+
     def joinuser(*args):
         return os.path.expanduser(os.path.join(*args))
 
     # what about 'os2emx', 'riscos' ?
     if os.name == "nt":
         base = os.environ.get("APPDATA") or "~"
-        return env_base if env_base else joinuser(base, "Python")
+        if env_base:
+            return env_base
+        else:
+            return joinuser(base, "Python")
 
     if sys.platform == "darwin":
         framework = get_config_var("PYTHONFRAMEWORK")
         if framework:
-            return env_base if env_base else joinuser("~", "Library", framework, "%d.%d"%(
-                sys.version_info[:2]))
+            if env_base:
+                return env_base
+            else:
+                return joinuser("~", "Library", framework, "%d.%d" %
+                                sys.version_info[:2])
 
-    return env_base if env_base else joinuser("~", ".local")
+    if env_base:
+        return env_base
+    else:
+        return joinuser("~", ".local")
 
 
 def _parse_makefile(filename, vars=None):
@@ -205,7 +240,6 @@ def _parse_makefile(filename, vars=None):
     optional dictionary is passed in as the second argument, it is
     used instead of a new dictionary.
     """
-    import re
     # Regexes needed for parsing Makefile (and similar syntaxes,
     # like old-style Setup files).
     _variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
@@ -267,7 +301,8 @@ def _parse_makefile(filename, vars=None):
                     item = os.environ[n]
 
                 elif n in renamed_variables:
-                    if name.startswith('PY_') and name[3:] in renamed_variables:
+                    if (name.startswith('PY_') and
+                        name[3:] in renamed_variables):
                         item = ""
 
                     elif 'PY_' + n in notdone:
@@ -300,7 +335,6 @@ def _parse_makefile(filename, vars=None):
                             if name not in done:
                                 done[name] = value
 
-
             else:
                 # bogus variable reference (e.g. "prefix=$/opt/python");
                 # just drop it since we can't deal
@@ -320,14 +354,17 @@ def _parse_makefile(filename, vars=None):
 def get_makefile_filename():
     """Return the path of the Makefile."""
     if _PYTHON_BUILD:
-        return os.path.join(_PROJECT_BASE, "Makefile")
-    return os.path.join(get_path('stdlib'),
-                        'config-{}{}'.format(_PY_VERSION_SHORT, sys.abiflags),
-                        'Makefile')
-
+        return os.path.join(_sys_home or _PROJECT_BASE, "Makefile")
+    if hasattr(sys, 'abiflags'):
+        config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags)
+    else:
+        config_dir_name = 'config'
+    return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile')
 
-def _init_posix(vars):
-    """Initialize the module as appropriate for POSIX systems."""
+def _generate_posix_vars():
+    """Generate the Python module containing build-time variables."""
+    import pprint
+    vars = {}
     # load the installed Makefile:
     makefile = get_makefile_filename()
     try:
@@ -353,6 +390,46 @@ def _init_posix(vars):
     if _PYTHON_BUILD:
         vars['LDSHARED'] = vars['BLDSHARED']
 
+    # There's a chicken-and-egg situation on OS X with regards to the
+    # _sysconfigdata module after the changes introduced by #15298:
+    # get_config_vars() is called by get_platform() as part of the
+    # `make pybuilddir.txt` target -- which is a precursor to the
+    # _sysconfigdata.py module being constructed.  Unfortunately,
+    # get_config_vars() eventually calls _init_posix(), which attempts
+    # to import _sysconfigdata, which we won't have built yet.  In order
+    # for _init_posix() to work, if we're on Darwin, just mock up the
+    # _sysconfigdata module manually and populate it with the build vars.
+    # This is more than sufficient for ensuring the subsequent call to
+    # get_platform() succeeds.
+    name = '_sysconfigdata'
+    if 'darwin' in sys.platform:
+        import imp
+        module = imp.new_module(name)
+        module.build_time_vars = vars
+        sys.modules[name] = module
+
+    pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3])
+    if hasattr(sys, "gettotalrefcount"):
+        pybuilddir += '-pydebug'
+    os.makedirs(pybuilddir, exist_ok=True)
+    destfile = os.path.join(pybuilddir, name + '.py')
+
+    with open(destfile, 'w', encoding='utf8') as f:
+        f.write('# system configuration generated and used by'
+                ' the sysconfig module\n')
+        f.write('build_time_vars = ')
+        pprint.pprint(vars, stream=f)
+
+    # Create file used for sys.path fixup -- see Modules/getpath.c
+    with open('pybuilddir.txt', 'w', encoding='ascii') as f:
+        f.write(pybuilddir)
+
+def _init_posix(vars):
+    """Initialize the module as appropriate for POSIX systems."""
+    # _sysconfigdata is generated at build time, see _generate_posix_vars()
+    from _sysconfigdata import build_time_vars
+    vars.update(build_time_vars)
+
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""
     # set basic install directories
@@ -376,7 +453,6 @@ def parse_config_h(fp, vars=None):
     optional dictionary is passed in as the second argument, it is
     used instead of a new dictionary.
     """
-    import re
     if vars is None:
         vars = {}
     define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
@@ -389,8 +465,10 @@ def parse_config_h(fp, vars=None):
         m = define_rx.match(line)
         if m:
             n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
+            try:
+                v = int(v)
+            except ValueError:
+                pass
             vars[n] = v
         else:
             m = undef_rx.match(line)
@@ -398,27 +476,29 @@ def parse_config_h(fp, vars=None):
                 vars[m.group(1)] = 0
     return vars
 
+
 def get_config_h_filename():
     """Return the path of pyconfig.h."""
     if _PYTHON_BUILD:
         if os.name == "nt":
-            inc_dir = os.path.join(_PROJECT_BASE, "PC")
+            inc_dir = os.path.join(_sys_home or _PROJECT_BASE, "PC")
         else:
-            inc_dir = _PROJECT_BASE
+            inc_dir = _sys_home or _PROJECT_BASE
     else:
         inc_dir = get_path('platinclude')
     return os.path.join(inc_dir, 'pyconfig.h')
 
+
 def get_scheme_names():
     """Return a tuple containing the schemes names."""
-    schemes = list(_INSTALL_SCHEMES.keys())
-    schemes.sort()
-    return tuple(schemes)
+    return tuple(sorted(_INSTALL_SCHEMES))
+
 
 def get_path_names():
     """Return a tuple containing the paths names."""
     return _SCHEME_KEYS
 
+
 def get_paths(scheme=_get_default_scheme(), vars=None, expand=True):
     """Return a mapping containing an install scheme.
 
@@ -430,6 +510,7 @@ def get_paths(scheme=_get_default_scheme(), vars=None, expand=True):
     else:
         return _INSTALL_SCHEMES[scheme]
 
+
 def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True):
     """Return a path corresponding to the scheme.
 
@@ -437,17 +518,17 @@ def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True):
     """
     return get_paths(scheme, vars, expand)[name]
 
+
 def get_config_vars(*args):
     """With no arguments, return a dictionary of all configuration
     variables relevant for the current platform.
 
     On Unix, this means every variable defined in Python's installed Makefile;
-    On Windows and Mac OS it's a much smaller set.
+    On Windows it's a much smaller set.
 
     With arguments, return a list of values that result from looking up
     each argument in the configuration variable dictionary.
     """
-    import re
     global _CONFIG_VARS
     if _CONFIG_VARS is None:
         _CONFIG_VARS = {}
@@ -459,7 +540,9 @@ def get_config_vars(*args):
         _CONFIG_VARS['py_version'] = _PY_VERSION
         _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT
         _CONFIG_VARS['py_version_nodot'] = _PY_VERSION[0] + _PY_VERSION[2]
+        _CONFIG_VARS['installed_base'] = _BASE_PREFIX
         _CONFIG_VARS['base'] = _PREFIX
+        _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX
         _CONFIG_VARS['platbase'] = _EXEC_PREFIX
         _CONFIG_VARS['projectbase'] = _PROJECT_BASE
         try:
@@ -477,29 +560,22 @@ def get_config_vars(*args):
         # the init-function.
         _CONFIG_VARS['userbase'] = _getuserbase()
 
-        if 'srcdir' not in _CONFIG_VARS:
-            _CONFIG_VARS['srcdir'] = _PROJECT_BASE
-        else:
-            _CONFIG_VARS['srcdir'] = _safe_realpath(_CONFIG_VARS['srcdir'])
-
-
-        # Convert srcdir into an absolute path if it appears necessary.
-        # Normally it is relative to the build directory.  However, during
-        # testing, for example, we might be running a non-installed python
-        # from a different directory.
-        if _PYTHON_BUILD and os.name == "posix":
-            base = _PROJECT_BASE
-            try:
-                cwd = os.getcwd()
-            except OSError:
-                cwd = None
-            if (not os.path.isabs(_CONFIG_VARS['srcdir']) and
-                base != cwd):
-                # srcdir is relative and we are not in the same directory
-                # as the executable. Assume executable is in the build
-                # directory and make srcdir absolute.
-                srcdir = os.path.join(base, _CONFIG_VARS['srcdir'])
-                _CONFIG_VARS['srcdir'] = os.path.normpath(srcdir)
+        # Always convert srcdir to an absolute path
+        srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE)
+        if os.name == 'posix':
+            if _PYTHON_BUILD:
+                # If srcdir is a relative path (typically '.' or '..')
+                # then it should be interpreted relative to the directory
+                # containing Makefile.
+                base = os.path.dirname(get_makefile_filename())
+                srcdir = os.path.join(base, srcdir)
+            else:
+                # srcdir is not meaningful since the installation is
+                # spread about the filesystem.  We choose the
+                # directory containing the Makefile since we know it
+                # exists.
+                srcdir = os.path.dirname(get_makefile_filename())
+        _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir)
 
         # OS X platforms require special customization to handle
         # multi-architecture, multi-os-version installers
@@ -515,6 +591,7 @@ def get_config_vars(*args):
     else:
         return _CONFIG_VARS
 
+
 def get_config_var(name):
     """Return the value of a single variable using the dictionary returned by
     'get_config_vars()'.
@@ -523,6 +600,7 @@ def get_config_var(name):
     """
     return get_config_vars().get(name)
 
+
 def get_platform():
     """Return a string that identifies the current platform.
 
@@ -548,7 +626,6 @@ def get_platform():
 
     For other non-POSIX platforms, currently just returns 'sys.platform'.
     """
-    import re
     if os.name == 'nt':
         # sniff sys.version for architecture.
         prefix = " bit ("
@@ -564,10 +641,13 @@ def get_platform():
         return sys.platform
 
     if os.name != "posix" or not hasattr(os, 'uname'):
-        # XXX what about the architecture? NT is Intel or Alpha,
-        # Mac OS is M68k or PPC, etc.
+        # XXX what about the architecture? NT is Intel or Alpha
         return sys.platform
 
+    # Set for cross builds explicitly
+    if "_PYTHON_HOST_PLATFORM" in os.environ:
+        return os.environ["_PYTHON_HOST_PLATFORM"]
+
     # Try to distinguish various flavours of Unix
     osname, host, release, version, machine = os.uname()
 
@@ -598,7 +678,7 @@ def get_platform():
         return "%s-%s.%s" % (osname, version, release)
     elif osname[:6] == "cygwin":
         osname = "cygwin"
-        rel_re = re.compile (r'[\d.]+')
+        rel_re = re.compile(r'[\d.]+')
         m = rel_re.match(release)
         if m:
             release = m.group()
@@ -614,21 +694,27 @@ def get_platform():
 def get_python_version():
     return _PY_VERSION_SHORT
 
+
 def _print_dict(title, data):
     for index, (key, value) in enumerate(sorted(data.items())):
         if index == 0:
-            print('{0}: '.format(title))
-        print('\t{0} = "{1}"'.format(key, value))
+            print('%s: ' % (title))
+        print('\t%s = "%s"' % (key, value))
+
 
 def _main():
     """Display all information sysconfig detains."""
-    print('Platform: "{0}"'.format(get_platform()))
-    print('Python version: "{0}"'.format(get_python_version()))
-    print('Current installation scheme: "{0}"'.format(_get_default_scheme()))
-    print('')
+    if '--generate-posix-vars' in sys.argv:
+        _generate_posix_vars()
+        return
+    print('Platform: "%s"' % get_platform())
+    print('Python version: "%s"' % get_python_version())
+    print('Current installation scheme: "%s"' % _get_default_scheme())
+    print()
     _print_dict('Paths', get_paths())
-    print('')
+    print()
     _print_dict('Variables', get_config_vars())
 
+
 if __name__ == '__main__':
     _main()
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
index 54d0e0e173..11b4b68146 100644
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -29,8 +29,6 @@
 """Read from and write to tar format archives.
 """
 
-__version__ = "$Revision$"
-
 version     = "0.9.0"
 __author__  = "Lars Gust\u00e4bel (lars@gustaebel.de)"
 __date__    = "$Date: 2011-02-25 17:42:01 +0200 (Fri, 25 Feb 2011) $"
@@ -42,9 +40,9 @@ __credits__ = "Gustavo Niemeyer, Niels Gust\u00e4bel, Richard Townsend."
 #---------
 import sys
 import os
+import io
 import shutil
 import stat
-import errno
 import time
 import struct
 import copy
@@ -247,8 +245,8 @@ def calc_chksums(buf):
        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
+    unsigned_chksum = 256 + sum(struct.unpack_from("148B8x356B", buf))
+    signed_chksum = 256 + sum(struct.unpack_from("148b8x356b", buf))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
@@ -276,47 +274,13 @@ def copyfileobj(src, dst, length=None):
         dst.write(buf)
     return
 
-filemode_table = (
-    ((S_IFLNK,      "l"),
-     (S_IFREG,      "-"),
-     (S_IFBLK,      "b"),
-     (S_IFDIR,      "d"),
-     (S_IFCHR,      "c"),
-     (S_IFIFO,      "p")),
-
-    ((TUREAD,       "r"),),
-    ((TUWRITE,      "w"),),
-    ((TUEXEC|TSUID, "s"),
-     (TSUID,        "S"),
-     (TUEXEC,       "x")),
-
-    ((TGREAD,       "r"),),
-    ((TGWRITE,      "w"),),
-    ((TGEXEC|TSGID, "s"),
-     (TSGID,        "S"),
-     (TGEXEC,       "x")),
-
-    ((TOREAD,       "r"),),
-    ((TOWRITE,      "w"),),
-    ((TOEXEC|TSVTX, "t"),
-     (TSVTX,        "T"),
-     (TOEXEC,       "x"))
-)
-
 def filemode(mode):
-    """Convert a file's mode to a string of the form
-       -rwxrwxrwx.
-       Used by TarFile.list()
-    """
-    perm = []
-    for table in filemode_table:
-        for bit, char in table:
-            if mode & bit == bit:
-                perm.append(char)
-                break
-        else:
-            perm.append("-")
-    return "".join(perm)
+    """Deprecated in this location; use stat.filemode."""
+    import warnings
+    warnings.warn("deprecated in favor of stat.filemode",
+                  DeprecationWarning, 2)
+    return stat.filemode(mode)
+
 
 class TarError(Exception):
     """Base exception."""
@@ -423,10 +387,11 @@ class _Stream:
                 self.crc = zlib.crc32(b"")
                 if mode == "r":
                     self._init_read_gz()
+                    self.exception = zlib.error
                 else:
                     self._init_write_gz()
 
-            if comptype == "bz2":
+            elif comptype == "bz2":
                 try:
                     import bz2
                 except ImportError:
@@ -434,8 +399,25 @@ class _Stream:
                 if mode == "r":
                     self.dbuf = b""
                     self.cmp = bz2.BZ2Decompressor()
+                    self.exception = IOError
                 else:
                     self.cmp = bz2.BZ2Compressor()
+
+            elif comptype == "xz":
+                try:
+                    import lzma
+                except ImportError:
+                    raise CompressionError("lzma module is not available")
+                if mode == "r":
+                    self.dbuf = b""
+                    self.cmp = lzma.LZMADecompressor()
+                    self.exception = lzma.LZMAError
+                else:
+                    self.cmp = lzma.LZMACompressor()
+
+            elif comptype != "tar":
+                raise CompressionError("unknown compression type %r" % comptype)
+
         except:
             if not self._extfileobj:
                 self.fileobj.close()
@@ -587,7 +569,7 @@ class _Stream:
                 break
             try:
                 buf = self.cmp.decompress(buf)
-            except IOError:
+            except self.exception:
                 raise ReadError("invalid compressed data")
             self.dbuf += buf
             c += len(buf)
@@ -625,76 +607,19 @@ class _StreamProxy(object):
         return self.buf
 
     def getcomptype(self):
-        if self.buf.startswith(b"\037\213\010"):
+        if self.buf.startswith(b"\x1f\x8b\x08"):
             return "gz"
-        if self.buf[0:3] == b"BZh" and self.buf[4:10] == b"1AY&SY":
+        elif self.buf[0:3] == b"BZh" and self.buf[4:10] == b"1AY&SY":
             return "bz2"
-        return "tar"
+        elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")):
+            return "xz"
+        else:
+            return "tar"
 
     def close(self):
         self.fileobj.close()
 # class StreamProxy
 
-class _BZ2Proxy(object):
-    """Small proxy class that enables external file object
-       support for "r:bz2" and "w:bz2" modes. This is actually
-       a workaround for a limitation in bz2 module's BZ2File
-       class which (unlike gzip.GzipFile) has no support for
-       a file object argument.
-    """
-
-    blocksize = 16 * 1024
-
-    def __init__(self, fileobj, mode):
-        self.fileobj = fileobj
-        self.mode = mode
-        self.name = getattr(self.fileobj, "name", None)
-        self.init()
-
-    def init(self):
-        import bz2
-        self.pos = 0
-        if self.mode == "r":
-            self.bz2obj = bz2.BZ2Decompressor()
-            self.fileobj.seek(0)
-            self.buf = b""
-        else:
-            self.bz2obj = bz2.BZ2Compressor()
-
-    def read(self, size):
-        x = len(self.buf)
-        while x < size:
-            raw = self.fileobj.read(self.blocksize)
-            if not raw:
-                break
-            data = self.bz2obj.decompress(raw)
-            self.buf += data
-            x += len(data)
-
-        buf = self.buf[:size]
-        self.buf = self.buf[size:]
-        self.pos += len(buf)
-        return buf
-
-    def seek(self, pos):
-        if pos < self.pos:
-            self.init()
-        self.read(pos - self.pos)
-
-    def tell(self):
-        return self.pos
-
-    def write(self, data):
-        self.pos += len(data)
-        raw = self.bz2obj.compress(data)
-        self.fileobj.write(raw)
-
-    def close(self):
-        if self.mode == "w":
-            raw = self.bz2obj.flush()
-            self.fileobj.write(raw)
-# class _BZ2Proxy
-
 #------------------------
 # Extraction file object
 #------------------------
@@ -709,6 +634,8 @@ class _FileInFile(object):
         self.offset = offset
         self.size = size
         self.position = 0
+        self.name = getattr(fileobj, "name", None)
+        self.closed = False
 
         if blockinfo is None:
             blockinfo = [(0, size)]
@@ -727,10 +654,16 @@ class _FileInFile(object):
         if lastpos < self.size:
             self.map.append((False, lastpos, self.size, None))
 
+    def flush(self):
+        pass
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return False
+
     def seekable(self):
-        if not hasattr(self.fileobj, "seekable"):
-            # XXX gzip.GzipFile and bz2.BZ2File
-            return True
         return self.fileobj.seekable()
 
     def tell(self):
@@ -738,10 +671,21 @@ class _FileInFile(object):
         """
         return self.position
 
-    def seek(self, position):
+    def seek(self, position, whence=io.SEEK_SET):
         """Seek to a position in the file.
         """
-        self.position = position
+        if whence == io.SEEK_SET:
+            self.position = min(max(position, 0), self.size)
+        elif whence == io.SEEK_CUR:
+            if position < 0:
+                self.position = max(self.position + position, 0)
+            else:
+                self.position = min(self.position + position, self.size)
+        elif whence == io.SEEK_END:
+            self.position = max(min(self.size + position, self.size), 0)
+        else:
+            raise ValueError("Invalid argument")
+        return self.position
 
     def read(self, size=None):
         """Read data from the file.
@@ -770,145 +714,22 @@ class _FileInFile(object):
             size -= length
             self.position += length
         return buf
-#class _FileInFile
-
-
-class ExFileObject(object):
-    """File-like object for reading an archive member.
-       Is returned by TarFile.extractfile().
-    """
-    blocksize = 1024
-
-    def __init__(self, tarfile, tarinfo):
-        self.fileobj = _FileInFile(tarfile.fileobj,
-                                   tarinfo.offset_data,
-                                   tarinfo.size,
-                                   tarinfo.sparse)
-        self.name = tarinfo.name
-        self.mode = "r"
-        self.closed = False
-        self.size = tarinfo.size
-
-        self.position = 0
-        self.buffer = b""
-
-    def readable(self):
-        return True
-
-    def writable(self):
-        return False
-
-    def seekable(self):
-        return self.fileobj.seekable()
-
-    def read(self, size=None):
-        """Read at most size bytes from the file. If size is not
-           present or None, read all data until EOF is reached.
-        """
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-
-        buf = b""
-        if self.buffer:
-            if size is None:
-                buf = self.buffer
-                self.buffer = b""
-            else:
-                buf = self.buffer[:size]
-                self.buffer = self.buffer[size:]
-
-        if size is None:
-            buf += self.fileobj.read()
-        else:
-            buf += self.fileobj.read(size - len(buf))
-
-        self.position += len(buf)
-        return buf
-
-    # XXX TextIOWrapper uses the read1() method.
-    read1 = read
-
-    def readline(self, size=-1):
-        """Read one entire line from the file. If size is present
-           and non-negative, return a string with at most that
-           size, which may be an incomplete line.
-        """
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-
-        pos = self.buffer.find(b"\n") + 1
-        if pos == 0:
-            # no newline found.
-            while True:
-                buf = self.fileobj.read(self.blocksize)
-                self.buffer += buf
-                if not buf or b"\n" in buf:
-                    pos = self.buffer.find(b"\n") + 1
-                    if pos == 0:
-                        # no newline found.
-                        pos = len(self.buffer)
-                    break
-
-        if size != -1:
-            pos = min(size, pos)
-
-        buf = self.buffer[:pos]
-        self.buffer = self.buffer[pos:]
-        self.position += len(buf)
-        return buf
-
-    def readlines(self):
-        """Return a list with all remaining lines.
-        """
-        result = []
-        while True:
-            line = self.readline()
-            if not line: break
-            result.append(line)
-        return result
-
-    def tell(self):
-        """Return the current file position.
-        """
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-
-        return self.position
-
-    def seek(self, pos, whence=os.SEEK_SET):
-        """Seek to a position in the file.
-        """
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
 
-        if whence == os.SEEK_SET:
-            self.position = min(max(pos, 0), self.size)
-        elif whence == os.SEEK_CUR:
-            if pos < 0:
-                self.position = max(self.position + pos, 0)
-            else:
-                self.position = min(self.position + pos, self.size)
-        elif whence == os.SEEK_END:
-            self.position = max(min(self.size + pos, self.size), 0)
-        else:
-            raise ValueError("Invalid argument")
-
-        self.buffer = b""
-        self.fileobj.seek(self.position)
+    def readinto(self, b):
+        buf = self.read(len(b))
+        b[:len(buf)] = buf
+        return len(buf)
 
     def close(self):
-        """Close the file object.
-        """
         self.closed = True
+#class _FileInFile
 
-    def __iter__(self):
-        """Get an iterator over the file's lines.
-        """
-        while True:
-            line = self.readline()
-            if not line:
-                break
-            yield line
+class ExFileObject(io.BufferedReader):
+
+    def __init__(self, tarfile, tarinfo):
+        fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data,
+                tarinfo.size, tarinfo.sparse)
+        super().__init__(fileobj)
 #class ExFileObject
 
 #------------------
@@ -1087,7 +908,7 @@ class TarInfo(object):
     def create_pax_global_header(cls, pax_headers):
         """Return the object as a pax global header block sequence.
         """
-        return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf8")
+        return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8")
 
     def _posix_split_name(self, name):
         """Split a name longer than 100 chars into a prefix
@@ -1170,7 +991,7 @@ class TarInfo(object):
         binary = False
         for keyword, value in pax_headers.items():
             try:
-                value.encode("utf8", "strict")
+                value.encode("utf-8", "strict")
             except UnicodeEncodeError:
                 binary = True
                 break
@@ -1181,13 +1002,13 @@ class TarInfo(object):
             records += b"21 hdrcharset=BINARY\n"
 
         for keyword, value in pax_headers.items():
-            keyword = keyword.encode("utf8")
+            keyword = keyword.encode("utf-8")
             if binary:
                 # Try to restore the original byte representation of `value'.
                 # Needless to say, that the encoding must match the string.
                 value = value.encode(encoding, "surrogateescape")
             else:
-                value = value.encode("utf8")
+                value = value.encode("utf-8")
 
             l = len(keyword) + len(value) + 3   # ' ' + '=' + '\n'
             n = p = 0
@@ -1396,7 +1217,7 @@ class TarInfo(object):
         # the translation to UTF-8 fails.
         match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf)
         if match is not None:
-            pax_headers["hdrcharset"] = match.group(1).decode("utf8")
+            pax_headers["hdrcharset"] = match.group(1).decode("utf-8")
 
         # For the time being, we don't care about anything other than "BINARY".
         # The only other value that is currently allowed by the standard is
@@ -1405,7 +1226,7 @@ class TarInfo(object):
         if hdrcharset == "BINARY":
             encoding = tarfile.encoding
         else:
-            encoding = "utf8"
+            encoding = "utf-8"
 
         # Parse pax header information. A record looks like that:
         # "%d %s=%s\n" % (length, keyword, value). length is the size
@@ -1422,20 +1243,20 @@ class TarInfo(object):
             length = int(length)
             value = buf[match.end(2) + 1:match.start(1) + length - 1]
 
-            # Normally, we could just use "utf8" as the encoding and "strict"
+            # Normally, we could just use "utf-8" as the encoding and "strict"
             # as the error handler, but we better not take the risk. For
             # example, GNU tar <= 1.23 is known to store filenames it cannot
             # translate to UTF-8 as raw strings (unfortunately without a
             # hdrcharset=BINARY header).
             # We first try the strict standard encoding, and if that fails we
             # fall back on the user's encoding and error handler.
-            keyword = self._decode_pax_field(keyword, "utf8", "utf8",
+            keyword = self._decode_pax_field(keyword, "utf-8", "utf-8",
                     tarfile.errors)
             if keyword in PAX_NAME_FIELDS:
                 value = self._decode_pax_field(value, encoding, tarfile.encoding,
                         tarfile.errors)
             else:
-                value = self._decode_pax_field(value, "utf8", "utf8",
+                value = self._decode_pax_field(value, "utf-8", "utf-8",
                         tarfile.errors)
 
             pax_headers[keyword] = value
@@ -1595,7 +1416,7 @@ class TarFile(object):
 
     tarinfo = TarInfo           # The default TarInfo class to use.
 
-    fileobject = ExFileObject   # The default ExFileObject class to use.
+    fileobject = ExFileObject   # The file-object for extractfile().
 
     def __init__(self, name=None, mode="r", fileobj=None, format=None,
             tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
@@ -1714,18 +1535,22 @@ class TarFile(object):
            'r:'         open for reading exclusively uncompressed
            'r:gz'       open for reading with gzip compression
            'r:bz2'      open for reading with bzip2 compression
+           'r:xz'       open for reading with lzma compression
            'a' or 'a:'  open for appending, creating the file if necessary
            'w' or 'w:'  open for writing without compression
            'w:gz'       open for writing with gzip compression
            'w:bz2'      open for writing with bzip2 compression
+           'w:xz'       open for writing with lzma compression
 
            'r|*'        open a stream of tar blocks with transparent compression
            'r|'         open an uncompressed stream of tar blocks for reading
            'r|gz'       open a gzip compressed stream of tar blocks
            'r|bz2'      open a bzip2 compressed stream of tar blocks
+           'r|xz'       open an lzma compressed stream of tar blocks
            'w|'         open an uncompressed stream for writing
            'w|gz'       open a gzip compressed stream for writing
            'w|bz2'      open a bzip2 compressed stream for writing
+           'w|xz'       open an lzma compressed stream for writing
         """
 
         if not name and not fileobj:
@@ -1832,10 +1657,8 @@ class TarFile(object):
         except ImportError:
             raise CompressionError("bz2 module is not available")
 
-        if fileobj is not None:
-            fileobj = _BZ2Proxy(fileobj, mode)
-        else:
-            fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+        fileobj = bz2.BZ2File(fileobj or name, mode,
+                              compresslevel=compresslevel)
 
         try:
             t = cls.taropen(name, mode, fileobj, **kwargs)
@@ -1845,11 +1668,35 @@ class TarFile(object):
         t._extfileobj = False
         return t
 
+    @classmethod
+    def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs):
+        """Open lzma compressed tar archive name for reading or writing.
+           Appending is not allowed.
+        """
+        if mode not in ("r", "w"):
+            raise ValueError("mode must be 'r' or 'w'")
+
+        try:
+            import lzma
+        except ImportError:
+            raise CompressionError("lzma module is not available")
+
+        fileobj = lzma.LZMAFile(fileobj or name, mode, preset=preset)
+
+        try:
+            t = cls.taropen(name, mode, fileobj, **kwargs)
+        except (lzma.LZMAError, EOFError):
+            fileobj.close()
+            raise ReadError("not an lzma file")
+        t._extfileobj = False
+        return t
+
     # All *open() methods are registered here.
     OPEN_METH = {
         "tar": "taropen",   # uncompressed tar
         "gz":  "gzopen",    # gzip compressed tar
-        "bz2": "bz2open"    # bzip2 compressed tar
+        "bz2": "bz2open",   # bzip2 compressed tar
+        "xz":  "xzopen"     # lzma compressed tar
     }
 
     #--------------------------------------------------------------------------
@@ -2009,7 +1856,7 @@ class TarFile(object):
 
         for tarinfo in self:
             if verbose:
-                print(filemode(tarinfo.mode), end=' ')
+                print(stat.filemode(tarinfo.mode), end=' ')
                 print("%s/%s" % (tarinfo.uname or tarinfo.uid,
                                  tarinfo.gname or tarinfo.gid), end=' ')
                 if tarinfo.ischr() or tarinfo.isblk():
@@ -2191,12 +2038,9 @@ class TarFile(object):
 
     def extractfile(self, member):
         """Extract a member from the archive as a file object. `member' may be
-           a filename or a TarInfo object. If `member' is a regular file, a
-           file-like object is returned. If `member' is a link, a file-like
-           object is constructed from the link's target. If `member' is none of
-           the above, None is returned.
-           The file-like object is read-only and provides the following
-           methods: read(), readline(), readlines(), seek() and tell()
+           a filename or a TarInfo object. If `member' is a regular file or a
+           link, an io.BufferedReader object is returned. Otherwise, None is
+           returned.
         """
         self._check("r")
 
@@ -2205,12 +2049,8 @@ class TarFile(object):
         else:
             tarinfo = member
 
-        if tarinfo.isreg():
-            return self.fileobject(self, tarinfo)
-
-        elif tarinfo.type not in SUPPORTED_TYPES:
-            # If a member's type is unknown, it is treated as a
-            # regular file.
+        if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES:
+            # Members with unknown types are treated as regular files.
             return self.fileobject(self, tarinfo)
 
         elif tarinfo.islnk() or tarinfo.issym():
@@ -2282,9 +2122,8 @@ class TarFile(object):
             # Use a safe mode for the directory, the real mode is set
             # later in _extract_member().
             os.mkdir(targetpath, 0o700)
-        except EnvironmentError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
 
     def makefile(self, tarinfo, targetpath):
         """Make a file called targetpath.
diff --git a/Lib/tempfile.py b/Lib/tempfile.py
index b90e826936..4aad7b5d42 100644
--- a/Lib/tempfile.py
+++ b/Lib/tempfile.py
@@ -10,8 +10,6 @@ This module also provides some data items to the user:
 
   TMP_MAX  - maximum number of names that will be tried before
              giving up.
-  template - the default prefix for all temporary names.
-             You may change this to control the default prefix.
   tempdir  - If this is set to a string before the first use of
              any routine from this module, it will be considered as
              another candidate location to store temporary files.
@@ -45,7 +43,7 @@ else:
     def _set_cloexec(fd):
         try:
             flags = _fcntl.fcntl(fd, _fcntl.F_GETFD, 0)
-        except IOError:
+        except OSError:
             pass
         else:
             # flags read successfully, modify
@@ -74,6 +72,8 @@ if hasattr(_os, 'TMP_MAX'):
 else:
     TMP_MAX = 10000
 
+# Although it does not have an underscore for historical reasons, this
+# variable is an internal implementation detail (see issue 10354).
 template = "tmp"
 
 # Internal routines.
@@ -85,19 +85,16 @@ if hasattr(_os, "lstat"):
 elif hasattr(_os, "stat"):
     _stat = _os.stat
 else:
-    # Fallback.  All we need is something that raises os.error if the
+    # Fallback.  All we need is something that raises OSError if the
     # file doesn't exist.
     def _stat(fn):
-        try:
-            f = open(fn)
-        except IOError:
-            raise _os.error
+        f = open(fn)
         f.close()
 
 def _exists(fn):
     try:
         _stat(fn)
-    except _os.error:
+    except OSError:
         return False
     else:
         return True
@@ -149,7 +146,7 @@ def _candidate_tempdir_list():
     # As a last resort, the current directory.
     try:
         dirlist.append(_os.getcwd())
-    except (AttributeError, _os.error):
+    except (AttributeError, OSError):
         dirlist.append(_os.curdir)
 
     return dirlist
@@ -184,12 +181,13 @@ def _get_default_tempdir():
                 finally:
                     _os.unlink(filename)
                 return dir
-            except (OSError, IOError) as e:
-                if e.args[0] != _errno.EEXIST:
-                    break # no point trying more names in this directory
+            except FileExistsError:
                 pass
-    raise IOError(_errno.ENOENT,
-                  "No usable temporary directory found in %s" % dirlist)
+            except OSError:
+                break   # no point trying more names in this directory
+    raise FileNotFoundError(_errno.ENOENT,
+                            "No usable temporary directory found in %s" %
+                            dirlist)
 
 _name_sequence = None
 
@@ -219,12 +217,11 @@ def _mkstemp_inner(dir, pre, suf, flags):
             fd = _os.open(file, flags, 0o600)
             _set_cloexec(fd)
             return (fd, _os.path.abspath(file))
-        except OSError as e:
-            if e.errno == _errno.EEXIST:
-                continue # try again
-            raise
+        except FileExistsError:
+            continue    # try again
 
-    raise IOError(_errno.EEXIST, "No usable temporary file name found")
+    raise FileExistsError(_errno.EEXIST,
+                          "No usable temporary file name found")
 
 
 # User visible interfaces.
@@ -308,12 +305,11 @@ def mkdtemp(suffix="", prefix=template, dir=None):
         try:
             _os.mkdir(file, 0o700)
             return file
-        except OSError as e:
-            if e.errno == _errno.EEXIST:
-                continue # try again
-            raise
+        except FileExistsError:
+            continue    # try again
 
-    raise IOError(_errno.EEXIST, "No usable temporary directory name found")
+    raise FileExistsError(_errno.EEXIST,
+                          "No usable temporary directory name found")
 
 def mktemp(suffix="", prefix=template, dir=None):
     """User-callable function to return a unique temporary file name.  The
@@ -342,7 +338,8 @@ def mktemp(suffix="", prefix=template, dir=None):
         if not _exists(file):
             return file
 
-    raise IOError(_errno.EEXIST, "No usable temporary filename found")
+    raise FileExistsError(_errno.EEXIST,
+                          "No usable temporary filename found")
 
 
 class _TemporaryFileWrapper:
@@ -608,8 +605,13 @@ class SpooledTemporaryFile:
     def tell(self):
         return self._file.tell()
 
-    def truncate(self):
-        self._file.truncate()
+    def truncate(self, size=None):
+        if size is None:
+            self._file.truncate()
+        else:
+            if size > self._max_size:
+                self.rollover()
+            self._file.truncate(size)
 
     def write(self, s):
         file = self._file
@@ -682,7 +684,7 @@ class TemporaryDirectory(object):
     _islink = staticmethod(_os.path.islink)
     _remove = staticmethod(_os.remove)
     _rmdir = staticmethod(_os.rmdir)
-    _os_error = _os.error
+    _os_error = OSError
     _warn = _warnings.warn
 
     def _rmtree(self, path):
diff --git a/Lib/test/buffer_tests.py b/Lib/test/buffer_tests.py
index 6d20f7d9e1..cf54c28759 100644
--- a/Lib/test/buffer_tests.py
+++ b/Lib/test/buffer_tests.py
@@ -200,7 +200,13 @@ class MixinBytesBufferCommonTests(object):
                          self.marshal(b'abc\ndef\r\nghi\n\r').splitlines())
         self.assertEqual([b'', b'abc', b'def', b'ghi', b''],
                          self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines())
+        self.assertEqual([b'', b'abc', b'def', b'ghi', b''],
+                         self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(False))
+        self.assertEqual([b'\n', b'abc\n', b'def\r\n', b'ghi\n', b'\r'],
+                         self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(True))
+        self.assertEqual([b'', b'abc', b'def', b'ghi', b''],
+                         self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(keepends=False))
         self.assertEqual([b'\n', b'abc\n', b'def\r\n', b'ghi\n', b'\r'],
-                         self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(1))
+                         self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(keepends=True))
 
         self.assertRaises(TypeError, self.marshal(b'abc').splitlines, 42, 42)
diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README
index 2a73e1bbd9..0259a0688c 100644
--- a/Lib/test/crashers/README
+++ b/Lib/test/crashers/README
@@ -14,3 +14,7 @@ note if the cause is system or environment dependent and what the variables are.
 Once the crash is fixed, the test case should be moved into an appropriate test
 (even if it was originally from the test suite).  This ensures the regression
 doesn't happen again.  And if it does, it should be easier to track down.
+
+Also see Lib/test_crashers.py which exercises the crashers in this directory.
+In particular, make sure to add any new infinite loop crashers to the black
+list so it doesn't try to run them.
diff --git a/Lib/test/crashers/borrowed_ref_1.py b/Lib/test/crashers/borrowed_ref_1.py
deleted file mode 100644
index b82f4644bf..0000000000
--- a/Lib/test/crashers/borrowed_ref_1.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""
-_PyType_Lookup() returns a borrowed reference.
-This attacks the call in dictobject.c.
-"""
-
-class A(object):
-    pass
-
-class B(object):
-    def __del__(self):
-        print('hi')
-        del D.__missing__
-
-class D(dict):
-    class __missing__:
-        def __init__(self, *args):
-            pass
-
-
-d = D()
-a = A()
-a.cycle = a
-a.other = B()
-del a
-
-prev = None
-while 1:
-    d[5]
-    prev = (prev,)
diff --git a/Lib/test/crashers/borrowed_ref_2.py b/Lib/test/crashers/borrowed_ref_2.py
deleted file mode 100644
index 6e403eb344..0000000000
--- a/Lib/test/crashers/borrowed_ref_2.py
+++ /dev/null
@@ -1,38 +0,0 @@
-"""
-_PyType_Lookup() returns a borrowed reference.
-This attacks PyObject_GenericSetAttr().
-
-NB. on my machine this crashes in 2.5 debug but not release.
-"""
-
-class A(object):
-    pass
-
-class B(object):
-    def __del__(self):
-        print("hi")
-        del C.d
-
-class D(object):
-    def __set__(self, obj, value):
-        self.hello = 42
-
-class C(object):
-    d = D()
-
-    def g():
-        pass
-
-
-c = C()
-a = A()
-a.cycle = a
-a.other = B()
-
-lst = [None] * 1000000
-i = 0
-del a
-while 1:
-    c.d = 42         # segfaults in PyMethod_New(__func__=D.__set__, __self__=d)
-    lst[i] = c.g     # consume the free list of instancemethod objects
-    i += 1
diff --git a/Lib/test/crashers/compiler_recursion.py b/Lib/test/crashers/compiler_recursion.py
deleted file mode 100644
index 4954bdd8f0..0000000000
--- a/Lib/test/crashers/compiler_recursion.py
+++ /dev/null
@@ -1,5 +0,0 @@
-"""
-The compiler (>= 2.5) recurses happily.
-"""
-
-compile('()'*9**5, '?', 'exec')
diff --git a/Lib/test/crashers/loosing_mro_ref.py b/Lib/test/crashers/loosing_mro_ref.py
deleted file mode 100644
index b3bcd32c58..0000000000
--- a/Lib/test/crashers/loosing_mro_ref.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""
-There is a way to put keys of any type in a type's dictionary.
-I think this allows various kinds of crashes, but so far I have only
-found a convoluted attack of _PyType_Lookup(), which uses the mro of the
-type without holding a strong reference to it.  Probably works with
-super.__getattribute__() too, which uses the same kind of code.
-"""
-
-class MyKey(object):
-    def __hash__(self):
-        return hash('mykey')
-
-    def __eq__(self, other):
-        # the following line decrefs the previous X.__mro__
-        X.__bases__ = (Base2,)
-        # trash all tuples of length 3, to make sure that the items of
-        # the previous X.__mro__ are really garbage
-        z = []
-        for i in range(1000):
-            z.append((i, None, None))
-        return 0
-
-
-class Base(object):
-    mykey = 'from Base'
-
-class Base2(object):
-    mykey = 'from Base2'
-
-# you can't add a non-string key to X.__dict__, but it can be
-# there from the beginning :-)
-X = type('X', (Base,), {MyKey(): 5})
-
-print(X.mykey)
-# I get a segfault, or a slightly wrong assertion error in a debug build.
diff --git a/Lib/test/crashers/nasty_eq_vs_dict.py b/Lib/test/crashers/nasty_eq_vs_dict.py
deleted file mode 100644
index 85f7cafa23..0000000000
--- a/Lib/test/crashers/nasty_eq_vs_dict.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# from http://mail.python.org/pipermail/python-dev/2001-June/015239.html
-
-# if you keep changing a dictionary while looking up a key, you can
-# provoke an infinite recursion in C
-
-# At the time neither Tim nor Michael could be bothered to think of a
-# way to fix it.
-
-class Yuck:
-    def __init__(self):
-        self.i = 0
-
-    def make_dangerous(self):
-        self.i = 1
-
-    def __hash__(self):
-        # direct to slot 4 in table of size 8; slot 12 when size 16
-        return 4 + 8
-
-    def __eq__(self, other):
-        if self.i == 0:
-            # leave dict alone
-            pass
-        elif self.i == 1:
-            # fiddle to 16 slots
-            self.__fill_dict(6)
-            self.i = 2
-        else:
-            # fiddle to 8 slots
-            self.__fill_dict(4)
-            self.i = 1
-
-        return 1
-
-    def __fill_dict(self, n):
-        self.i = 0
-        dict.clear()
-        for i in range(n):
-            dict[i] = i
-        dict[self] = "OK!"
-
-y = Yuck()
-dict = {y: "OK!"}
-
-z = Yuck()
-y.make_dangerous()
-print(dict[z])
diff --git a/Lib/test/crashers/recursion_limit_too_high.py b/Lib/test/crashers/recursion_limit_too_high.py
deleted file mode 100644
index ec64936a5d..0000000000
--- a/Lib/test/crashers/recursion_limit_too_high.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# The following example may crash or not depending on the platform.
-# E.g. on 32-bit Intel Linux in a "standard" configuration it seems to
-# crash on Python 2.5 (but not 2.4 nor 2.3).  On Windows the import
-# eventually fails to find the module, possibly because we run out of
-# file handles.
-
-# The point of this example is to show that sys.setrecursionlimit() is a
-# hack, and not a robust solution.  This example simply exercises a path
-# where it takes many C-level recursions, consuming a lot of stack
-# space, for each Python-level recursion.  So 1000 times this amount of
-# stack space may be too much for standard platforms already.
-
-import sys
-if 'recursion_limit_too_high' in sys.modules:
-    del sys.modules['recursion_limit_too_high']
-import recursion_limit_too_high
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index bb18630339..931ef6fc03 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -979,7 +979,7 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase):
         # exempt such platforms (provided they return reasonable
         # results!).
         for insane in -1e200, 1e200:
-            self.assertRaises(ValueError, self.theclass.fromtimestamp,
+            self.assertRaises(OverflowError, self.theclass.fromtimestamp,
                               insane)
 
     def test_today(self):
@@ -1291,12 +1291,18 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase):
         self.assertTrue(self.theclass.min)
         self.assertTrue(self.theclass.max)
 
-    def test_strftime_out_of_range(self):
-        # For nasty technical reasons, we can't handle years before 1000.
-        cls = self.theclass
-        self.assertEqual(cls(1000, 1, 1).strftime("%Y"), "1000")
-        for y in 1, 49, 51, 99, 100, 999:
-            self.assertRaises(ValueError, cls(y, 1, 1).strftime, "%Y")
+    def test_strftime_y2k(self):
+        for y in (1, 49, 70, 99, 100, 999, 1000, 1970):
+            d = self.theclass(y, 1, 1)
+            # Issue 13305:  For years < 1000, the value is not always
+            # padded to 4 digits across platforms.  The C standard
+            # assumes year >= 1900, so it does not specify the number
+            # of digits.
+            if d.strftime("%Y") != '%04d' % y:
+                # Year 42 returns '42', not padded
+                self.assertEqual(d.strftime("%Y"), '%d' % y)
+                # '0042' is obtained anyway
+                self.assertEqual(d.strftime("%4Y"), '%04d' % y)
 
     def test_replace(self):
         cls = self.theclass
@@ -1731,13 +1737,74 @@ class TestDateTime(TestDate):
         got = self.theclass.utcfromtimestamp(ts)
         self.verify_field_equality(expected, got)
 
+    # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in
+    # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0).
+    @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0')
+    def test_timestamp_naive(self):
+        t = self.theclass(1970, 1, 1)
+        self.assertEqual(t.timestamp(), 18000.0)
+        t = self.theclass(1970, 1, 1, 1, 2, 3, 4)
+        self.assertEqual(t.timestamp(),
+                         18000.0 + 3600 + 2*60 + 3 + 4*1e-6)
+        # Missing hour may produce platform-dependent result
+        t = self.theclass(2012, 3, 11, 2, 30)
+        self.assertIn(self.theclass.fromtimestamp(t.timestamp()),
+                      [t - timedelta(hours=1), t + timedelta(hours=1)])
+        # Ambiguous hour defaults to DST
+        t = self.theclass(2012, 11, 4, 1, 30)
+        self.assertEqual(self.theclass.fromtimestamp(t.timestamp()), t)
+
+        # Timestamp may raise an overflow error on some platforms
+        for t in [self.theclass(1,1,1), self.theclass(9999,12,12)]:
+            try:
+                s = t.timestamp()
+            except OverflowError:
+                pass
+            else:
+                self.assertEqual(self.theclass.fromtimestamp(s), t)
+
+    def test_timestamp_aware(self):
+        t = self.theclass(1970, 1, 1, tzinfo=timezone.utc)
+        self.assertEqual(t.timestamp(), 0.0)
+        t = self.theclass(1970, 1, 1, 1, 2, 3, 4, tzinfo=timezone.utc)
+        self.assertEqual(t.timestamp(),
+                         3600 + 2*60 + 3 + 4*1e-6)
+        t = self.theclass(1970, 1, 1, 1, 2, 3, 4,
+                          tzinfo=timezone(timedelta(hours=-5), 'EST'))
+        self.assertEqual(t.timestamp(),
+                         18000 + 3600 + 2*60 + 3 + 4*1e-6)
     def test_microsecond_rounding(self):
-        # Test whether fromtimestamp "rounds up" floats that are less
-        # than 1/2 microsecond smaller than an integer.
         for fts in [self.theclass.fromtimestamp,
                     self.theclass.utcfromtimestamp]:
-            self.assertEqual(fts(0.9999999), fts(1))
-            self.assertEqual(fts(0.99999949).microsecond, 999999)
+            zero = fts(0)
+            self.assertEqual(zero.second, 0)
+            self.assertEqual(zero.microsecond, 0)
+            try:
+                minus_one = fts(-1e-6)
+            except OSError:
+                # localtime(-1) and gmtime(-1) is not supported on Windows
+                pass
+            else:
+                self.assertEqual(minus_one.second, 59)
+                self.assertEqual(minus_one.microsecond, 999999)
+
+                t = fts(-1e-8)
+                self.assertEqual(t, minus_one)
+                t = fts(-9e-7)
+                self.assertEqual(t, minus_one)
+                t = fts(-1e-7)
+                self.assertEqual(t, minus_one)
+
+            t = fts(1e-7)
+            self.assertEqual(t, zero)
+            t = fts(9e-7)
+            self.assertEqual(t, zero)
+            t = fts(0.99999949)
+            self.assertEqual(t.second, 0)
+            self.assertEqual(t.microsecond, 999999)
+            t = fts(0.9999999)
+            self.assertEqual(t.second, 0)
+            self.assertEqual(t.microsecond, 999999)
 
     def test_insane_fromtimestamp(self):
         # It's possible that some platform maps time_t to double,
@@ -1745,7 +1812,7 @@ class TestDateTime(TestDate):
         # exempt such platforms (provided they return reasonable
         # results!).
         for insane in -1e200, 1e200:
-            self.assertRaises(ValueError, self.theclass.fromtimestamp,
+            self.assertRaises(OverflowError, self.theclass.fromtimestamp,
                               insane)
 
     def test_insane_utcfromtimestamp(self):
@@ -1754,7 +1821,7 @@ class TestDateTime(TestDate):
         # exempt such platforms (provided they return reasonable
         # results!).
         for insane in -1e200, 1e200:
-            self.assertRaises(ValueError, self.theclass.utcfromtimestamp,
+            self.assertRaises(OverflowError, self.theclass.utcfromtimestamp,
                               insane)
     @unittest.skipIf(sys.platform == "win32", "Windows doesn't accept negative timestamps")
     def test_negative_float_fromtimestamp(self):
@@ -1907,7 +1974,7 @@ class TestDateTime(TestDate):
         # simply can't be applied to a naive object.
         dt = self.theclass.now()
         f = FixedOffset(44, "")
-        self.assertRaises(TypeError, dt.astimezone) # not enough args
+        self.assertRaises(ValueError, dt.astimezone) # naive
         self.assertRaises(TypeError, dt.astimezone, f, f) # too many args
         self.assertRaises(TypeError, dt.astimezone, dt) # arg wrong type
         self.assertRaises(ValueError, dt.astimezone, f) # naive
@@ -2479,7 +2546,7 @@ class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase):
         self.assertEqual(t1, t2)
         self.assertEqual(t1, t3)
         self.assertEqual(t2, t3)
-        self.assertRaises(TypeError, lambda: t4 == t5) # mixed tz-aware & naive
+        self.assertNotEqual(t4, t5) # mixed tz-aware & naive
         self.assertRaises(TypeError, lambda: t4 < t5) # mixed tz-aware & naive
         self.assertRaises(TypeError, lambda: t5 < t4) # mixed tz-aware & naive
 
@@ -2631,7 +2698,7 @@ class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase):
         t2 = t2.replace(tzinfo=FixedOffset(None, ""))
         self.assertEqual(t1, t2)
         t2 = t2.replace(tzinfo=FixedOffset(0, ""))
-        self.assertRaises(TypeError, lambda: t1 == t2)
+        self.assertNotEqual(t1, t2)
 
         # In time w/ identical tzinfo objects, utcoffset is ignored.
         class Varies(tzinfo):
@@ -2736,16 +2803,16 @@ class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase):
                            microsecond=1)
         self.assertTrue(t1 > t2)
 
-        # Make t2 naive and it should fail.
+        # Make t2 naive and it should differ.
         t2 = self.theclass.min
-        self.assertRaises(TypeError, lambda: t1 == t2)
+        self.assertNotEqual(t1, t2)
         self.assertEqual(t2, t2)
 
         # It's also naive if it has tzinfo but tzinfo.utcoffset() is None.
         class Naive(tzinfo):
             def utcoffset(self, dt): return None
         t2 = self.theclass(5, 6, 7, tzinfo=Naive())
-        self.assertRaises(TypeError, lambda: t1 == t2)
+        self.assertNotEqual(t1, t2)
         self.assertEqual(t2, t2)
 
         # OTOH, it's OK to compare two of these mixing the two ways of being
@@ -3188,8 +3255,6 @@ class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase):
         self.assertTrue(dt.tzinfo is f44m)
         # Replacing with degenerate tzinfo raises an exception.
         self.assertRaises(ValueError, dt.astimezone, fnone)
-        # Ditto with None tz.
-        self.assertRaises(TypeError, dt.astimezone, None)
         # Replacing with same tzinfo makes no change.
         x = dt.astimezone(dt.tzinfo)
         self.assertTrue(x.tzinfo is f44m)
@@ -3209,6 +3274,25 @@ class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase):
         self.assertTrue(got.tzinfo is expected.tzinfo)
         self.assertEqual(got, expected)
 
+    @support.run_with_tz('UTC')
+    def test_astimezone_default_utc(self):
+        dt = self.theclass.now(timezone.utc)
+        self.assertEqual(dt.astimezone(None), dt)
+        self.assertEqual(dt.astimezone(), dt)
+
+    # Note that offset in TZ variable has the opposite sign to that
+    # produced by %z directive.
+    @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0')
+    def test_astimezone_default_eastern(self):
+        dt = self.theclass(2012, 11, 4, 6, 30, tzinfo=timezone.utc)
+        local = dt.astimezone()
+        self.assertEqual(dt, local)
+        self.assertEqual(local.strftime("%z %Z"), "-0500 EST")
+        dt = self.theclass(2012, 11, 4, 5, 30, tzinfo=timezone.utc)
+        local = dt.astimezone()
+        self.assertEqual(dt, local)
+        self.assertEqual(local.strftime("%z %Z"), "-0400 EDT")
+
     def test_aware_subtract(self):
         cls = self.theclass
 
@@ -3262,7 +3346,7 @@ class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase):
         t2 = t2.replace(tzinfo=FixedOffset(None, ""))
         self.assertEqual(t1, t2)
         t2 = t2.replace(tzinfo=FixedOffset(0, ""))
-        self.assertRaises(TypeError, lambda: t1 == t2)
+        self.assertNotEqual(t1, t2)
 
         # In datetime w/ identical tzinfo objects, utcoffset is ignored.
         class Varies(tzinfo):
diff --git a/Lib/test/decimaltestdata/extra.decTest b/Lib/test/decimaltestdata/extra.decTest
index fe8b77a6dd..b630d8e3f9 100644
--- a/Lib/test/decimaltestdata/extra.decTest
+++ b/Lib/test/decimaltestdata/extra.decTest
@@ -222,12 +222,25 @@ extr1700 power 10 1e-999999999 -> 1.000000000000000 Inexact Rounded
 extr1701 power 100.0 -557.71e-742888888 -> 1.000000000000000 Inexact Rounded
 extr1702 power 10 1e-100 -> 1.000000000000000 Inexact Rounded
 
+-- Another one (see issue #12080).  Thanks again to Stefan Krah.
+extr1703 power 4 -1.2e-999999999 -> 1.000000000000000 Inexact Rounded
+
 -- A couple of interesting exact cases for power.  Note that the specification
 -- requires these to be reported as Inexact.
 extr1710 power 1e375 56e-3 -> 1.000000000000000E+21 Inexact Rounded
 extr1711 power 10000 0.75 -> 1000.000000000000 Inexact Rounded
 extr1712 power 1e-24 0.875 -> 1.000000000000000E-21 Inexact Rounded
 
+-- Some more exact cases, exercising power with negative second argument.
+extr1720 power 400 -0.5 -> 0.05000000000000000 Inexact Rounded
+extr1721 power 4096 -0.75 -> 0.001953125000000000 Inexact Rounded
+extr1722 power 625e4 -0.25 -> 0.02000000000000000 Inexact Rounded
+
+-- Nonexact cases, to exercise some of the early exit conditions from
+-- _power_exact.
+extr1730 power 2048 -0.75 -> 0.003284751622084822 Inexact Rounded
+
+
 -- Tests for the is_* boolean operations
 precision: 9
 maxExponent: 999
diff --git a/Lib/test/dh512.pem b/Lib/test/dh512.pem
new file mode 100644
index 0000000000..200d16cd89
--- /dev/null
+++ b/Lib/test/dh512.pem
@@ -0,0 +1,9 @@
+-----BEGIN DH PARAMETERS-----
+MEYCQQD1Kv884bEpQBgRjXyEpwpy1obEAxnIByl6ypUM2Zafq9AKUJsCRtMIPWak
+XUGfnHy9iUsiGSa6q6Jew1XpKgVfAgEC
+-----END DH PARAMETERS-----
+
+These are the 512 bit DH parameters from "Assigned Number for SKIP Protocols"
+(http://www.skip-vpn.org/spec/numbers.html).
+See there for how they were generated.
+Note that g is not a generator, but this is not a problem since p is a safe prime.
diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt
index 5037b335d9..1c1f69f36b 100644
--- a/Lib/test/exception_hierarchy.txt
+++ b/Lib/test/exception_hierarchy.txt
@@ -11,11 +11,6 @@ BaseException
       +-- AssertionError
       +-- AttributeError
       +-- BufferError
-      +-- EnvironmentError
-      |    +-- IOError
-      |    +-- OSError
-      |         +-- WindowsError (Windows)
-      |         +-- VMSError (VMS)
       +-- EOFError
       +-- ImportError
       +-- LookupError
@@ -24,6 +19,22 @@ BaseException
       +-- MemoryError
       +-- NameError
       |    +-- UnboundLocalError
+      +-- OSError
+      |    +-- BlockingIOError
+      |    +-- ChildProcessError
+      |    +-- ConnectionError
+      |    |    +-- BrokenPipeError
+      |    |    +-- ConnectionAbortedError
+      |    |    +-- ConnectionRefusedError
+      |    |    +-- ConnectionResetError
+      |    +-- FileExistsError
+      |    +-- FileNotFoundError
+      |    +-- InterruptedError
+      |    +-- IsADirectoryError
+      |    +-- NotADirectoryError
+      |    +-- PermissionError
+      |    +-- ProcessLookupError
+      |    +-- TimeoutError
       +-- ReferenceError
       +-- RuntimeError
       |    +-- NotImplementedError
diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py
index 1caab1c491..88527df25e 100644
--- a/Lib/test/fork_wait.py
+++ b/Lib/test/fork_wait.py
@@ -43,6 +43,7 @@ class ForkWait(unittest.TestCase):
         self.assertEqual(spid, cpid)
         self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8))
 
+    @support.reap_threads
     def test_wait(self):
         for i in range(NUM_THREADS):
             _thread.start_new(self.f, (i,))
@@ -69,7 +70,8 @@ class ForkWait(unittest.TestCase):
             os._exit(n)
         else:
             # Parent
-            self.wait_impl(cpid)
-            # Tell threads to die
-            self.stop = 1
-            time.sleep(2*SHORTSLEEP) # Wait for threads to die
+            try:
+                self.wait_impl(cpid)
+            finally:
+                # Tell threads to die
+                self.stop = 1
diff --git a/Lib/test/test_future1.py b/Lib/test/future_test1.py
index 297c2e087c..297c2e087c 100644
--- a/Lib/test/test_future1.py
+++ b/Lib/test/future_test1.py
diff --git a/Lib/test/test_future2.py b/Lib/test/future_test2.py
index 3d7fc860a3..3d7fc860a3 100644
--- a/Lib/test/test_future2.py
+++ b/Lib/test/future_test2.py
diff --git a/Lib/test/json_tests/test_dump.py b/Lib/test/json_tests/test_dump.py
index fee972eac4..237ee35b26 100644
--- a/Lib/test/json_tests/test_dump.py
+++ b/Lib/test/json_tests/test_dump.py
@@ -1,6 +1,7 @@
 from io import StringIO
 from test.json_tests import PyTest, CTest
 
+from test.support import bigmemtest, _1G
 
 class TestDump:
     def test_dump(self):
@@ -29,4 +30,20 @@ class TestDump:
 
 
 class TestPyDump(TestDump, PyTest): pass
-class TestCDump(TestDump, CTest): pass
+
+class TestCDump(TestDump, CTest):
+
+    # The size requirement here is hopefully over-estimated (actual
+    # memory consumption depending on implementation details, and also
+    # system memory management, since this may allocate a lot of
+    # small objects).
+
+    @bigmemtest(size=_1G, memuse=1)
+    def test_large_list(self, size):
+        N = int(30 * 1024 * 1024 * (size / _1G))
+        l = [1] * N
+        encoded = self.dumps(l)
+        self.assertEqual(len(encoded), N * 3)
+        self.assertEqual(encoded[:1], "[")
+        self.assertEqual(encoded[-2:], "1]")
+        self.assertEqual(encoded[1:-2], "1, " * (N - 1))
diff --git a/Lib/test/json_tests/test_scanstring.py b/Lib/test/json_tests/test_scanstring.py
index f82cdeead3..426c8dd3bb 100644
--- a/Lib/test/json_tests/test_scanstring.py
+++ b/Lib/test/json_tests/test_scanstring.py
@@ -9,14 +9,9 @@ class TestScanstring:
             scanstring('"z\\ud834\\udd20x"', 1, True),
             ('z\U0001d120x', 16))
 
-        if sys.maxunicode == 65535:
-            self.assertEqual(
-                scanstring('"z\U0001d120x"', 1, True),
-                ('z\U0001d120x', 6))
-        else:
-            self.assertEqual(
-                scanstring('"z\U0001d120x"', 1, True),
-                ('z\U0001d120x', 5))
+        self.assertEqual(
+            scanstring('"z\U0001d120x"', 1, True),
+            ('z\U0001d120x', 5))
 
         self.assertEqual(
             scanstring('"\\u007b"', 1, True),
diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem
new file mode 100644
index 0000000000..e90574881d
--- /dev/null
+++ b/Lib/test/keycert.passwd.pem
@@ -0,0 +1,33 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A
+
+kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c
+u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA
+AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr
+Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+
+YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P
+6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+
+noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1
+94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l
+7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo
+cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO
+zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt
+L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo
+2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ==
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw
+MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH
+Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k
+YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
+gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7
+6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt
+pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw
+FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd
+BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G
+lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1
+CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX
+-----END CERTIFICATE-----
diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py
index be054eadf6..42e118ba8f 100644
--- a/Lib/test/list_tests.py
+++ b/Lib/test/list_tests.py
@@ -418,6 +418,47 @@ class CommonTest(seq_tests.CommonTest):
 
         self.assertRaises(TypeError, u.reverse, 42)
 
+    def test_clear(self):
+        u = self.type2test([2, 3, 4])
+        u.clear()
+        self.assertEqual(u, [])
+
+        u = self.type2test([])
+        u.clear()
+        self.assertEqual(u, [])
+
+        u = self.type2test([])
+        u.append(1)
+        u.clear()
+        u.append(2)
+        self.assertEqual(u, [2])
+
+        self.assertRaises(TypeError, u.clear, None)
+
+    def test_copy(self):
+        u = self.type2test([1, 2, 3])
+        v = u.copy()
+        self.assertEqual(v, [1, 2, 3])
+
+        u = self.type2test([])
+        v = u.copy()
+        self.assertEqual(v, [])
+
+        # test that it's indeed a copy and not a reference
+        u = self.type2test(['a', 'b'])
+        v = u.copy()
+        v.append('i')
+        self.assertEqual(u, ['a', 'b'])
+        self.assertEqual(v, u + ['i'])
+
+        # test that it's a shallow, not a deep copy
+        u = self.type2test([1, 2, [3, 4], 5])
+        v = u.copy()
+        self.assertEqual(u, v)
+        self.assertIs(v[3], u[3])
+
+        self.assertRaises(TypeError, u.copy, None)
+
     def test_sort(self):
         u = self.type2test([1, 0])
         u.sort()
diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
index 094cc7a459..bfbf44e0db 100644
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -4,7 +4,7 @@ Various tests for synchronization primitives.
 
 import sys
 import time
-from _thread import start_new_thread, get_ident, TIMEOUT_MAX
+from _thread import start_new_thread, TIMEOUT_MAX
 import threading
 import unittest
 
@@ -31,7 +31,7 @@ class Bunch(object):
         self.finished = []
         self._can_exit = not wait_before_exit
         def task():
-            tid = get_ident()
+            tid = threading.get_ident()
             self.started.append(tid)
             try:
                 f()
@@ -255,6 +255,18 @@ class RLockTests(BaseLockTests):
         lock.release()
         self.assertRaises(RuntimeError, lock.release)
 
+    def test_release_save_unacquired(self):
+        # Cannot _release_save an unacquired lock
+        lock = self.locktype()
+        self.assertRaises(RuntimeError, lock._release_save)
+        lock.acquire()
+        lock.acquire()
+        lock.release()
+        lock.acquire()
+        lock.release()
+        lock.release()
+        self.assertRaises(RuntimeError, lock._release_save)
+
     def test_different_thread(self):
         # Cannot release from a different thread
         lock = self.locktype()
diff --git a/Lib/test/mailcap.txt b/Lib/test/mailcap.txt
new file mode 100644
index 0000000000..f61135d8e5
--- /dev/null
+++ b/Lib/test/mailcap.txt
@@ -0,0 +1,39 @@
+# Mailcap file for test_mailcap; based on RFC 1524
+# Referred to by test_mailcap.py
+
+#
+# This is a comment.
+#
+
+application/frame; showframe %s; print="cat %s | lp"
+application/postscript; ps-to-terminal %s;\
+    needsterminal
+application/postscript; ps-to-terminal %s; \
+    compose=idraw %s
+application/x-dvi; xdvi %s
+application/x-movie; movieplayer %s; compose=moviemaker %s; \
+       description="Movie"; \
+       x11-bitmap="/usr/lib/Zmail/bitmaps/movie.xbm"
+application/*; echo "This is \"%t\" but \
+       is 50 \% Greek to me" \; cat %s; copiousoutput
+
+audio/basic; showaudio %s; compose=audiocompose %s; edit=audiocompose %s;\
+description="An audio fragment"
+audio/* ; /usr/local/bin/showaudio %t
+
+image/rgb; display %s
+#image/gif; display %s
+image/x-xwindowdump; display %s
+
+# The continuation char shouldn't \
+# make a difference in a comment.
+
+message/external-body; showexternal %s %{access-type} %{name} %{site} \
+    %{directory} %{mode} %{server}; needsterminal; composetyped = extcompose %s; \
+    description="A reference to data stored in an external location"
+
+text/richtext; shownonascii iso-8859-8 -e richtext -p %s; test=test "`echo \
+    %{charset} | tr '[A-Z]' '[a-z]'`"  = iso-8859-8; copiousoutput
+
+video/mpeg; mpeg_play %s
+video/*; animate %s
diff --git a/Lib/test/math_testcases.txt b/Lib/test/math_testcases.txt
index 5e24335a87..9585188243 100644
--- a/Lib/test/math_testcases.txt
+++ b/Lib/test/math_testcases.txt
@@ -517,3 +517,117 @@ expm10306 expm1 1.79e308 -> inf overflow
 
 -- weaker version of expm10302
 expm10307 expm1 709.5 -> 1.3549863193146328e+308
+
+-------------------------
+-- log2: log to base 2 --
+-------------------------
+
+-- special values
+log20000 log2 0.0 -> -inf               divide-by-zero
+log20001 log2 -0.0 -> -inf              divide-by-zero
+log20002 log2 inf -> inf
+log20003 log2 -inf -> nan               invalid
+log20004 log2 nan -> nan
+
+-- exact value at 1.0
+log20010 log2 1.0 -> 0.0
+
+-- negatives
+log20020 log2 -5e-324 -> nan            invalid
+log20021 log2 -1.0 -> nan               invalid
+log20022 log2 -1.7e-308 -> nan          invalid
+
+-- exact values at powers of 2
+log20100 log2 2.0 -> 1.0
+log20101 log2 4.0 -> 2.0
+log20102 log2 8.0 -> 3.0
+log20103 log2 16.0 -> 4.0
+log20104 log2 32.0 -> 5.0
+log20105 log2 64.0 -> 6.0
+log20106 log2 128.0 -> 7.0
+log20107 log2 256.0 -> 8.0
+log20108 log2 512.0 -> 9.0
+log20109 log2 1024.0 -> 10.0
+log20110 log2 2048.0 -> 11.0
+
+log20200 log2 0.5 -> -1.0
+log20201 log2 0.25 -> -2.0
+log20202 log2 0.125 -> -3.0
+log20203 log2 0.0625 -> -4.0
+
+-- values close to 1.0
+log20300 log2 1.0000000000000002 -> 3.2034265038149171e-16
+log20301 log2 1.0000000001 -> 1.4426951601859516e-10
+log20302 log2 1.00001 -> 1.4426878274712997e-5
+
+log20310 log2 0.9999999999999999 -> -1.6017132519074588e-16
+log20311 log2 0.9999999999 -> -1.4426951603302210e-10
+log20312 log2 0.99999 -> -1.4427022544056922e-5
+
+-- tiny values
+log20400 log2 5e-324 -> -1074.0
+log20401 log2 1e-323 -> -1073.0
+log20402 log2 1.5e-323 -> -1072.4150374992789
+log20403 log2 2e-323 -> -1072.0
+
+log20410 log2 1e-308 -> -1023.1538532253076
+log20411 log2 2.2250738585072014e-308 -> -1022.0
+log20412 log2 4.4501477170144028e-308 -> -1021.0
+log20413 log2 1e-307 -> -1019.8319251304202
+
+-- huge values
+log20500 log2 1.7976931348623157e+308 -> 1024.0
+log20501 log2 1.7e+308 -> 1023.9193879716706
+log20502 log2 8.9884656743115795e+307 -> 1023.0
+
+-- selection of random values
+log20600 log2 -7.2174324841039838e+289 -> nan   invalid
+log20601 log2 -2.861319734089617e+265 -> nan    invalid
+log20602 log2 -4.3507646894008962e+257 -> nan   invalid
+log20603 log2 -6.6717265307520224e+234 -> nan   invalid
+log20604 log2 -3.9118023786619294e+229 -> nan   invalid
+log20605 log2 -1.5478221302505161e+206 -> nan   invalid
+log20606 log2 -1.4380485131364602e+200 -> nan   invalid
+log20607 log2 -3.7235198730382645e+185 -> nan   invalid
+log20608 log2 -1.0472242235095724e+184 -> nan   invalid
+log20609 log2 -5.0141781956163884e+160 -> nan   invalid
+log20610 log2 -2.1157958031160324e+124 -> nan   invalid
+log20611 log2 -7.9677558612567718e+90 -> nan    invalid
+log20612 log2 -5.5553906194063732e+45 -> nan    invalid
+log20613 log2 -16573900952607.953 -> nan        invalid
+log20614 log2 -37198371019.888618 -> nan        invalid
+log20615 log2 -6.0727115121422674e-32 -> nan    invalid
+log20616 log2 -2.5406841656526057e-38 -> nan    invalid
+log20617 log2 -4.9056766703267657e-43 -> nan    invalid
+log20618 log2 -2.1646786075228305e-71 -> nan    invalid
+log20619 log2 -2.470826790488573e-78 -> nan     invalid
+log20620 log2 -3.8661709303489064e-165 -> nan   invalid
+log20621 log2 -1.0516496976649986e-182 -> nan   invalid
+log20622 log2 -1.5935458614317996e-255 -> nan   invalid
+log20623 log2 -2.8750977267336654e-293 -> nan   invalid
+log20624 log2 -7.6079466794732585e-296 -> nan   invalid
+log20625 log2 3.2073253539988545e-307 -> -1018.1505544209213
+log20626 log2 1.674937885472249e-244 -> -809.80634755783126
+log20627 log2 1.0911259044931283e-214 -> -710.76679472274213
+log20628 log2 2.0275372624809709e-154 -> -510.55719818383272
+log20629 log2 7.3926087369631841e-115 -> -379.13564735312292
+log20630 log2 1.3480198206342423e-86 -> -285.25497445094436
+log20631 log2 8.9927384655719947e-83 -> -272.55127136401637
+log20632 log2 3.1452398713597487e-60 -> -197.66251564496875
+log20633 log2 7.0706573215457351e-55 -> -179.88420087782217
+log20634 log2 3.1258285390731669e-49 -> -161.13023800505653
+log20635 log2 8.2253046627829942e-41 -> -133.15898277355879
+log20636 log2 7.8691367397519897e+49 -> 165.75068202732419
+log20637 log2 2.9920561983925013e+64 -> 214.18453534573757
+log20638 log2 4.7827254553946841e+77 -> 258.04629628445673
+log20639 log2 3.1903566496481868e+105 -> 350.47616767491166
+log20640 log2 5.6195082449502419e+113 -> 377.86831861008250
+log20641 log2 9.9625658250651047e+125 -> 418.55752921228753
+log20642 log2 2.7358945220961532e+145 -> 483.13158636923413
+log20643 log2 2.785842387926931e+174 -> 579.49360214860280
+log20644 log2 2.4169172507252751e+193 -> 642.40529039289652
+log20645 log2 3.1689091206395632e+205 -> 682.65924573798395
+log20646 log2 2.535995592365391e+208 -> 692.30359597460460
+log20647 log2 6.2011236566089916e+233 -> 776.64177576730913
+log20648 log2 2.1843274820677632e+253 -> 841.57499717289647
+log20649 log2 8.7493931063474791e+297 -> 989.74182713073981
diff --git a/Lib/test/memory_watchdog.py b/Lib/test/memory_watchdog.py
new file mode 100644
index 0000000000..88cca8d323
--- /dev/null
+++ b/Lib/test/memory_watchdog.py
@@ -0,0 +1,28 @@
+"""Memory watchdog: periodically read the memory usage of the main test process
+and print it out, until terminated."""
+# stdin should refer to the process' /proc/<PID>/statm: we don't pass the
+# process' PID to avoid a race condition in case of - unlikely - PID recycling.
+# If the process crashes, reading from the /proc entry will fail with ESRCH.
+
+
+import os
+import sys
+import time
+
+
+try:
+    page_size = os.sysconf('SC_PAGESIZE')
+except (ValueError, AttributeError):
+    try:
+        page_size = os.sysconf('SC_PAGE_SIZE')
+    except (ValueError, AttributeError):
+        page_size = 4096
+
+while True:
+    sys.stdin.seek(0)
+    statm = sys.stdin.read()
+    data = int(statm.split()[5])
+    sys.stdout.write(" ... process data size: {data:.1f}G\n"
+                     .format(data=data * page_size / (1024 ** 3)))
+    sys.stdout.flush()
+    time.sleep(1)
diff --git a/Lib/test/mock_socket.py b/Lib/test/mock_socket.py
index 803693283b..d09e78c1d5 100644
--- a/Lib/test/mock_socket.py
+++ b/Lib/test/mock_socket.py
@@ -106,7 +106,8 @@ def socket(family=None, type=None, proto=None):
     return MockSocket()
 
 
-def create_connection(address, timeout=socket_module._GLOBAL_DEFAULT_TIMEOUT):
+def create_connection(address, timeout=socket_module._GLOBAL_DEFAULT_TIMEOUT,
+                      source_address=None):
     try:
         int_port = int(address[1])
     except ValueError:
diff --git a/Lib/test/test_multibytecodec_support.py b/Lib/test/multibytecodec_support.py
index ef63b6934d..26bac7be10 100644
--- a/Lib/test/test_multibytecodec_support.py
+++ b/Lib/test/multibytecodec_support.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# test_multibytecodec_support.py
+# multibytecodec_support.py
 #   Common Unittest Routines for CJK codecs
 #
 
@@ -108,12 +108,19 @@ class TestBase:
         self.assertEqual(self.encode(sin,
                                     "test.xmlcharnamereplace")[0], sout)
 
+    def test_callback_returns_bytes(self):
+        def myreplace(exc):
+            return (b"1234", exc.end)
+        codecs.register_error("test.cjktest", myreplace)
+        enc = self.encode("abc" + self.unmappedunicode + "def", "test.cjktest")[0]
+        self.assertEqual(enc, b"abc1234def")
+
     def test_callback_wrong_objects(self):
         def myreplace(exc):
             return (ret, exc.end)
         codecs.register_error("test.cjktest", myreplace)
 
-        for ret in ([1, 2, 3], [], None, object(), b'string', b''):
+        for ret in ([1, 2, 3], [], None, object()):
             self.assertRaises(TypeError, self.encode, self.unmappedunicode,
                               'test.cjktest')
 
@@ -264,21 +271,6 @@ class TestBase:
 
                 self.assertEqual(ostream.getvalue(), self.tstring[0])
 
-if len('\U00012345') == 2: # ucs2 build
-    _unichr = chr
-    def chr(v):
-        if v >= 0x10000:
-            return _unichr(0xd800 + ((v - 0x10000) >> 10)) + \
-                   _unichr(0xdc00 + ((v - 0x10000) & 0x3ff))
-        else:
-            return _unichr(v)
-    _ord = ord
-    def ord(c):
-        if len(c) == 2:
-            return 0x10000 + ((_ord(c[0]) - 0xd800) << 10) + \
-                          (ord(c[1]) - 0xdc00)
-        else:
-            return _ord(c)
 
 class TestBase_Mapping(unittest.TestCase):
     pass_enctest = []
diff --git a/Lib/test/namespace_pkgs/both_portions/foo/one.py b/Lib/test/namespace_pkgs/both_portions/foo/one.py
new file mode 100644
index 0000000000..3080f6f8f1
--- /dev/null
+++ b/Lib/test/namespace_pkgs/both_portions/foo/one.py
@@ -0,0 +1 @@
+attr = 'both_portions foo one'
diff --git a/Lib/test/namespace_pkgs/both_portions/foo/two.py b/Lib/test/namespace_pkgs/both_portions/foo/two.py
new file mode 100644
index 0000000000..4131d3d4be
--- /dev/null
+++ b/Lib/test/namespace_pkgs/both_portions/foo/two.py
@@ -0,0 +1 @@
+attr = 'both_portions foo two'
diff --git a/Lib/test/namespace_pkgs/missing_directory.zip b/Lib/test/namespace_pkgs/missing_directory.zip
new file mode 100644
index 0000000000..836a9106bc
--- /dev/null
+++ b/Lib/test/namespace_pkgs/missing_directory.zip
diff --git a/Lib/test/namespace_pkgs/module_and_namespace_package/a_test.py b/Lib/test/namespace_pkgs/module_and_namespace_package/a_test.py
new file mode 100644
index 0000000000..43cbedbbdb
--- /dev/null
+++ b/Lib/test/namespace_pkgs/module_and_namespace_package/a_test.py
@@ -0,0 +1 @@
+attr = 'in module'
diff --git a/Lib/email/test/__init__.py b/Lib/test/namespace_pkgs/module_and_namespace_package/a_test/empty
index e69de29bb2..e69de29bb2 100644
--- a/Lib/email/test/__init__.py
+++ b/Lib/test/namespace_pkgs/module_and_namespace_package/a_test/empty
diff --git a/Lib/test/namespace_pkgs/nested_portion1.zip b/Lib/test/namespace_pkgs/nested_portion1.zip
new file mode 100644
index 0000000000..8d22406f23
--- /dev/null
+++ b/Lib/test/namespace_pkgs/nested_portion1.zip
diff --git a/Lib/importlib/test/__init__.py b/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py
index e69de29bb2..e69de29bb2 100644
--- a/Lib/importlib/test/__init__.py
+++ b/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py
diff --git a/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py b/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py
new file mode 100644
index 0000000000..d8f5c831f2
--- /dev/null
+++ b/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py
@@ -0,0 +1 @@
+attr = 'portion1 foo one'
diff --git a/Lib/test/namespace_pkgs/portion1/foo/one.py b/Lib/test/namespace_pkgs/portion1/foo/one.py
new file mode 100644
index 0000000000..d8f5c831f2
--- /dev/null
+++ b/Lib/test/namespace_pkgs/portion1/foo/one.py
@@ -0,0 +1 @@
+attr = 'portion1 foo one'
diff --git a/Lib/test/namespace_pkgs/portion2/foo/two.py b/Lib/test/namespace_pkgs/portion2/foo/two.py
new file mode 100644
index 0000000000..d092e1e993
--- /dev/null
+++ b/Lib/test/namespace_pkgs/portion2/foo/two.py
@@ -0,0 +1 @@
+attr = 'portion2 foo two'
diff --git a/Lib/test/namespace_pkgs/project1/parent/child/one.py b/Lib/test/namespace_pkgs/project1/parent/child/one.py
new file mode 100644
index 0000000000..2776fcdfde
--- /dev/null
+++ b/Lib/test/namespace_pkgs/project1/parent/child/one.py
@@ -0,0 +1 @@
+attr = 'parent child one'
diff --git a/Lib/test/namespace_pkgs/project2/parent/child/two.py b/Lib/test/namespace_pkgs/project2/parent/child/two.py
new file mode 100644
index 0000000000..8b037bcb0e
--- /dev/null
+++ b/Lib/test/namespace_pkgs/project2/parent/child/two.py
@@ -0,0 +1 @@
+attr = 'parent child two'
diff --git a/Lib/test/namespace_pkgs/project3/parent/child/three.py b/Lib/test/namespace_pkgs/project3/parent/child/three.py
new file mode 100644
index 0000000000..f8abfe1c17
--- /dev/null
+++ b/Lib/test/namespace_pkgs/project3/parent/child/three.py
@@ -0,0 +1 @@
+attr = 'parent child three'
diff --git a/Lib/test/namespace_pkgs/top_level_portion1.zip b/Lib/test/namespace_pkgs/top_level_portion1.zip
new file mode 100644
index 0000000000..3b866c914a
--- /dev/null
+++ b/Lib/test/namespace_pkgs/top_level_portion1.zip
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 1ce01c3d07..5d12375267 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -4,10 +4,11 @@ import pickle
 import pickletools
 import sys
 import copyreg
+import weakref
 from http.cookies import SimpleCookie
 
 from test.support import (
-    TestFailed, TESTFN, run_with_locale,
+    TestFailed, TESTFN, run_with_locale, no_tracing,
     _2G, _4G, bigmemtest,
     )
 
@@ -18,7 +19,7 @@ from pickle import bytes_types
 # kind of outer loop.
 protocols = range(pickle.HIGHEST_PROTOCOL + 1)
 
-character_size = 4 if sys.maxunicode > 0xFFFF else 2
+ascii_char_size = 1
 
 
 # Return True if opcode code appears in the pickle, else False.
@@ -747,6 +748,18 @@ class AbstractPickleTests(unittest.TestCase):
                 u = self.loads(s)
                 self.assertEqual(t, u)
 
+    def test_ellipsis(self):
+        for proto in protocols:
+            s = self.dumps(..., proto)
+            u = self.loads(s)
+            self.assertEqual(..., u)
+
+    def test_notimplemented(self):
+        for proto in protocols:
+            s = self.dumps(NotImplemented, proto)
+            u = self.loads(s)
+            self.assertEqual(NotImplemented, u)
+
     # Tests for protocol 2
 
     def test_proto(self):
@@ -880,6 +893,25 @@ class AbstractPickleTests(unittest.TestCase):
                 self.assertEqual(B(x), B(y), detail)
                 self.assertEqual(x.__dict__, y.__dict__, detail)
 
+    def test_newobj_proxies(self):
+        # NEWOBJ should use the __class__ rather than the raw type
+        classes = myclasses[:]
+        # Cannot create weakproxies to these classes
+        for c in (MyInt, MyTuple):
+            classes.remove(c)
+        for proto in protocols:
+            for C in classes:
+                B = C.__base__
+                x = C(C.sample)
+                x.foo = 42
+                p = weakref.proxy(x)
+                s = self.dumps(p, proto)
+                y = self.loads(s)
+                self.assertEqual(type(y), type(x))  # rather than type(p)
+                detail = (proto, C, B, x, y, type(y))
+                self.assertEqual(B(x), B(y), detail)
+                self.assertEqual(x.__dict__, y.__dict__, detail)
+
     # Register a type with copyreg, with extension code extcode.  Pickle
     # an object of that type.  Check that the resulting pickle uses opcode
     # (EXT[124]) under proto 2, and not in proto 1.
@@ -1040,13 +1072,13 @@ class AbstractPickleTests(unittest.TestCase):
             y = self.loads(s)
             self.assertEqual(y._reduce_called, 1)
 
+    @no_tracing
     def test_bad_getattr(self):
         x = BadGetattr()
         for proto in 0, 1:
             self.assertRaises(RuntimeError, self.dumps, x, proto)
         # protocol 2 don't raise a RuntimeError.
         d = self.dumps(x, 2)
-        self.assertRaises(RuntimeError, self.loads, d)
 
     def test_reduce_bad_iterator(self):
         # Issue4176: crash when 4th and 5th items of __reduce__()
@@ -1136,6 +1168,15 @@ class AbstractPickleTests(unittest.TestCase):
         empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
         self.assertEqual(empty, '')
 
+    def test_int_pickling_efficiency(self):
+        # Test compacity of int representation (see issue #12744)
+        for proto in protocols:
+            sizes = [len(self.dumps(2**n, proto)) for n in range(70)]
+            # the size function is monotonic
+            self.assertEqual(sorted(sizes), sizes)
+            if proto >= 2:
+                self.assertLessEqual(sizes[-1], 14)
+
     def check_negative_32b_binXXX(self, dumped):
         if sys.maxsize > 2**32:
             self.skipTest("test is only meaningful on 32-bit builds")
@@ -1217,7 +1258,7 @@ class BigmemPickleTests(unittest.TestCase):
     # All protocols use 1-byte per printable ASCII character; we add another
     # byte because the encoded form has to be copied into the internal buffer.
 
-    @bigmemtest(size=_2G, memuse=2 + character_size, dry_run=False)
+    @bigmemtest(size=_2G, memuse=2 + ascii_char_size, dry_run=False)
     def test_huge_str_32b(self, size):
         data = "abcd" * (size // 4)
         try:
@@ -1234,7 +1275,7 @@ class BigmemPickleTests(unittest.TestCase):
     # BINUNICODE (protocols 1, 2 and 3) cannot carry more than
     # 2**32 - 1 bytes of utf-8 encoded unicode.
 
-    @bigmemtest(size=_4G, memuse=1 + character_size, dry_run=False)
+    @bigmemtest(size=_4G, memuse=1 + ascii_char_size, dry_run=False)
     def test_huge_str_64b(self, size):
         data = "a" * size
         try:
@@ -1578,6 +1619,105 @@ class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
                 self.assertEqual(unpickler.load(), data)
 
 
+# Tests for dispatch_table attribute
+
+REDUCE_A = 'reduce_A'
+
+class AAA(object):
+    def __reduce__(self):
+        return str, (REDUCE_A,)
+
+class BBB(object):
+    pass
+
+class AbstractDispatchTableTests(unittest.TestCase):
+
+    def test_default_dispatch_table(self):
+        # No dispatch_table attribute by default
+        f = io.BytesIO()
+        p = self.pickler_class(f, 0)
+        with self.assertRaises(AttributeError):
+            p.dispatch_table
+        self.assertFalse(hasattr(p, 'dispatch_table'))
+
+    def test_class_dispatch_table(self):
+        # A dispatch_table attribute can be specified class-wide
+        dt = self.get_dispatch_table()
+
+        class MyPickler(self.pickler_class):
+            dispatch_table = dt
+
+        def dumps(obj, protocol=None):
+            f = io.BytesIO()
+            p = MyPickler(f, protocol)
+            self.assertEqual(p.dispatch_table, dt)
+            p.dump(obj)
+            return f.getvalue()
+
+        self._test_dispatch_table(dumps, dt)
+
+    def test_instance_dispatch_table(self):
+        # A dispatch_table attribute can also be specified instance-wide
+        dt = self.get_dispatch_table()
+
+        def dumps(obj, protocol=None):
+            f = io.BytesIO()
+            p = self.pickler_class(f, protocol)
+            p.dispatch_table = dt
+            self.assertEqual(p.dispatch_table, dt)
+            p.dump(obj)
+            return f.getvalue()
+
+        self._test_dispatch_table(dumps, dt)
+
+    def _test_dispatch_table(self, dumps, dispatch_table):
+        def custom_load_dump(obj):
+            return pickle.loads(dumps(obj, 0))
+
+        def default_load_dump(obj):
+            return pickle.loads(pickle.dumps(obj, 0))
+
+        # pickling complex numbers using protocol 0 relies on copyreg
+        # so check pickling a complex number still works
+        z = 1 + 2j
+        self.assertEqual(custom_load_dump(z), z)
+        self.assertEqual(default_load_dump(z), z)
+
+        # modify pickling of complex
+        REDUCE_1 = 'reduce_1'
+        def reduce_1(obj):
+            return str, (REDUCE_1,)
+        dispatch_table[complex] = reduce_1
+        self.assertEqual(custom_load_dump(z), REDUCE_1)
+        self.assertEqual(default_load_dump(z), z)
+
+        # check picklability of AAA and BBB
+        a = AAA()
+        b = BBB()
+        self.assertEqual(custom_load_dump(a), REDUCE_A)
+        self.assertIsInstance(custom_load_dump(b), BBB)
+        self.assertEqual(default_load_dump(a), REDUCE_A)
+        self.assertIsInstance(default_load_dump(b), BBB)
+
+        # modify pickling of BBB
+        dispatch_table[BBB] = reduce_1
+        self.assertEqual(custom_load_dump(a), REDUCE_A)
+        self.assertEqual(custom_load_dump(b), REDUCE_1)
+        self.assertEqual(default_load_dump(a), REDUCE_A)
+        self.assertIsInstance(default_load_dump(b), BBB)
+
+        # revert pickling of BBB and modify pickling of AAA
+        REDUCE_2 = 'reduce_2'
+        def reduce_2(obj):
+            return str, (REDUCE_2,)
+        dispatch_table[AAA] = reduce_2
+        del dispatch_table[BBB]
+        self.assertEqual(custom_load_dump(a), REDUCE_2)
+        self.assertIsInstance(custom_load_dump(b), BBB)
+        self.assertEqual(default_load_dump(a), REDUCE_A)
+        self.assertIsInstance(default_load_dump(b), BBB)
+
+
 if __name__ == "__main__":
     # Print some stuff that can be used to rewrite DATA{0,1,2}
     from pickletools import dis
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
index 84beb8dee5..636282ebed 100755
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -20,6 +20,11 @@ python -E -Wd -m test [options] [test_name1 ...]
 Options:
 
 -h/--help       -- print this text and exit
+--timeout TIMEOUT
+                -- dump the traceback and exit if a test takes more
+                   than TIMEOUT seconds; disabled if TIMEOUT is negative
+                   or equals to zero
+--wait          -- wait for user input, e.g., allow a debugger to be attached
 
 Verbosity
 
@@ -44,6 +49,9 @@ Selecting tests
                 -- specify which special resource intensive tests to run
 -M/--memlimit LIMIT
                 -- run very large memory-consuming tests
+   --testdir DIR
+                -- execute test files in the specified directory (instead
+                   of the Python stdlib test suite)
 
 Special runs
 
@@ -125,6 +133,8 @@ resources to test.  Currently only the following are defined:
 
     all -       Enable all special resources.
 
+    none -      Disable all special resources (this is the default).
+
     audio -     Tests that use the audio device.  (There are known
                 cases of broken audio drivers that can crash Python or
                 even the Linux kernel.)
@@ -155,8 +165,11 @@ example, to run all the tests except for the gui tests, give the
 option '-uall,-gui'.
 """
 
+# We import importlib *ASAP* in order to test #15386
+import importlib
+
 import builtins
-import errno
+import faulthandler
 import getopt
 import io
 import json
@@ -166,6 +179,7 @@ import platform
 import random
 import re
 import shutil
+import signal
 import sys
 import sysconfig
 import tempfile
@@ -225,6 +239,7 @@ ENV_CHANGED = -1
 SKIPPED = -2
 RESOURCE_DENIED = -3
 INTERRUPTED = -4
+CHILD_ERROR = -5   # error in a child process
 
 from test import support
 
@@ -268,6 +283,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
     on the command line.
     """
 
+    # Display the Python traceback on fatal errors (e.g. segfault)
+    faulthandler.enable(all_threads=True)
+
+    # Display the Python traceback on SIGALRM or SIGUSR1 signal
+    signals = []
+    if hasattr(signal, 'SIGALRM'):
+        signals.append(signal.SIGALRM)
+    if hasattr(signal, 'SIGUSR1'):
+        signals.append(signal.SIGUSR1)
+    for signum in signals:
+        faulthandler.register(signum, chain=True)
+
     replace_stdout()
 
     support.record_original_stdout(sys.stdout)
@@ -278,7 +305,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
              'use=', 'threshold=', 'coverdir=', 'nocoverdir',
              'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
              'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug',
-             'start=', 'nowindows', 'header', 'failfast', 'match='])
+             'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait',
+             'failfast', 'match='])
     except getopt.error as msg:
         usage(msg)
 
@@ -289,6 +317,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         use_resources = []
     debug = False
     start = None
+    timeout = None
     for o, a in opts:
         if o in ('-h', '--help'):
             print(__doc__)
@@ -332,7 +361,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         elif o in ('-T', '--coverage'):
             trace = True
         elif o in ('-D', '--coverdir'):
-            coverdir = os.path.join(os.getcwd(), a)
+            # CWD is replaced with a temporary dir before calling main(), so we
+            # need  join it with the saved CWD so it goes where the user expects.
+            coverdir = os.path.join(support.SAVEDCWD, a)
         elif o in ('-N', '--nocoverdir'):
             coverdir = None
         elif o in ('-R', '--huntrleaks'):
@@ -350,9 +381,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                 huntrleaks[1] = int(huntrleaks[1])
             if len(huntrleaks) == 2 or not huntrleaks[2]:
                 huntrleaks[2:] = ["reflog.txt"]
-            # Avoid false positives due to the character cache in
-            # stringobject.c filling slowly with random data
-            warm_char_cache()
+            # Avoid false positives due to various caches
+            # filling slowly with random data:
+            warm_caches()
         elif o in ('-M', '--memlimit'):
             support.set_memlimit(a)
         elif o in ('-u', '--use'):
@@ -361,6 +392,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                 if r == 'all':
                     use_resources[:] = RESOURCE_NAMES
                     continue
+                if r == 'none':
+                    del use_resources[:]
+                    continue
                 remove = False
                 if r[0] == '-':
                     remove = True
@@ -391,18 +425,45 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
             forever = True
         elif o in ('-j', '--multiprocess'):
             use_mp = int(a)
+            if use_mp <= 0:
+                try:
+                    import multiprocessing
+                    # Use all cores + extras for tests that like to sleep
+                    use_mp = 2 + multiprocessing.cpu_count()
+                except (ImportError, NotImplementedError):
+                    use_mp = 3
+            if use_mp == 1:
+                use_mp = None
         elif o == '--header':
             header = True
         elif o == '--slaveargs':
             args, kwargs = json.loads(a)
             try:
                 result = runtest(*args, **kwargs)
+            except KeyboardInterrupt:
+                result = INTERRUPTED, ''
             except BaseException as e:
-                result = INTERRUPTED, e.__class__.__name__
+                traceback.print_exc()
+                result = CHILD_ERROR, str(e)
             sys.stdout.flush()
             print()   # Force a newline (just in case)
             print(json.dumps(result))
             sys.exit(0)
+        elif o == '--testdir':
+            # CWD is replaced with a temporary dir before calling main(), so we
+            # join it with the saved CWD so it ends up where the user expects.
+            testdir = os.path.join(support.SAVEDCWD, a)
+        elif o == '--timeout':
+            if hasattr(faulthandler, 'dump_tracebacks_later'):
+                timeout = float(a)
+                if timeout <= 0:
+                    timeout = None
+            else:
+                print("Warning: The timeout option requires "
+                      "faulthandler.dump_tracebacks_later")
+                timeout = None
+        elif o == '--wait':
+            input("Press any key to continue...")
         else:
             print(("No handler for option {}.  Please report this as a bug "
                    "at http://bugs.python.org.").format(o), file=sys.stderr)
@@ -481,7 +542,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         print("==  ", os.getcwd())
         print("Testing with flags:", sys.flags)
 
-    alltests = findtests(testdir, stdtests, nottests)
+    # if testdir is set, then we are not running the python tests suite, so
+    # don't add default tests to be executed or skipped (pass empty values)
+    if testdir:
+        alltests = findtests(testdir, list(), set())
+    else:
+        alltests = findtests(testdir, stdtests, nottests)
+
     selected = tests or args or alltests
     if single:
         selected = selected[:1]
@@ -501,7 +568,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         random.shuffle(selected)
     if trace:
         import trace, tempfile
-        tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix,
+        tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,
                                          tempfile.gettempdir()],
                              trace=False, count=True)
 
@@ -566,7 +633,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                         (test, verbose, quiet),
                         dict(huntrleaks=huntrleaks, use_resources=use_resources,
                              debug=debug, output_on_failure=verbose3,
-                             failfast=failfast, match_tests=match_tests)
+                             timeout=timeout, failfast=failfast,
+                             match_tests=match_tests)
                     )
                     # -E is needed by some tests, e.g. test_import
                     # Running the child from the same working directory ensures
@@ -578,10 +646,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                                    close_fds=(os.name != 'nt'),
                                    cwd=support.SAVEDCWD)
                     stdout, stderr = popen.communicate()
+                    retcode = popen.wait()
                     # Strip last refcount output line if it exists, since it
                     # comes from the shutdown of the interpreter in the subcommand.
                     stderr = debug_output_pat.sub("", stderr)
                     stdout, _, result = stdout.strip().rpartition("\n")
+                    if retcode != 0:
+                        result = (CHILD_ERROR, "Exit code %s" % retcode)
+                        output.put((test, stdout.rstrip(), stderr.rstrip(), result))
+                        return
                     if not result:
                         output.put((None, None, None, None))
                         return
@@ -614,8 +687,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                 sys.stdout.flush()
                 sys.stderr.flush()
                 if result[0] == INTERRUPTED:
-                    assert result[1] == 'KeyboardInterrupt'
-                    raise KeyboardInterrupt   # What else?
+                    raise KeyboardInterrupt
+                if result[0] == CHILD_ERROR:
+                    raise Exception("Child error on {}: {}".format(test, result[1]))
                 test_index += 1
         except KeyboardInterrupt:
             interrupted = True
@@ -632,13 +706,14 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
             if trace:
                 # If we're tracing code coverage, then we don't exit with status
                 # if on a false return value from main.
-                tracer.runctx('runtest(test, verbose, quiet)',
+                tracer.runctx('runtest(test, verbose, quiet, timeout=timeout)',
                               globals=globals(), locals=vars())
             else:
                 try:
                     result = runtest(test, verbose, quiet, huntrleaks, debug,
                                      output_on_failure=verbose3,
-                                     failfast=failfast, match_tests=match_tests)
+                                     timeout=timeout, failfast=failfast,
+                                     match_tests=match_tests)
                     accumulate_result(test, result)
                 except KeyboardInterrupt:
                     interrupted = True
@@ -709,7 +784,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
             sys.stdout.flush()
             try:
                 verbose = True
-                ok = runtest(test, True, quiet, huntrleaks, debug)
+                ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout)
             except KeyboardInterrupt:
                 # print a newline separate from the ^C
                 print()
@@ -734,6 +809,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
     sys.exit(len(bad) > 0 or interrupted)
 
 
+# small set of tests to determine if we have a basically functioning interpreter
+# (i.e. if any of these fail, then anything else is likely to follow)
 STDTESTS = [
     'test_grammar',
     'test_opcodes',
@@ -744,12 +821,11 @@ STDTESTS = [
     'test_unittest',
     'test_doctest',
     'test_doctest2',
+    'test_support'
 ]
 
-NOTTESTS = {
-    'test_future1',
-    'test_future2',
-}
+# set of tests that we don't want to be executed when using regrtest
+NOTTESTS = set()
 
 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
     """Return a list of all applicable test modules."""
@@ -758,9 +834,9 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
     tests = []
     others = set(stdtests) | nottests
     for name in names:
-        modname, ext = os.path.splitext(name)
-        if modname[:5] == "test_" and ext == ".py" and modname not in others:
-            tests.append(modname)
+        mod, ext = os.path.splitext(name)
+        if mod[:5] == "test_" and ext in (".py", "") and mod not in others:
+            tests.append(mod)
     return stdtests + sorted(tests)
 
 # We do not use a generator so multiple threads can call next().
@@ -785,17 +861,14 @@ class MultiprocessTests(object):
 def replace_stdout():
     """Set stdout encoder error handler to backslashreplace (as stderr error
     handler) to avoid UnicodeEncodeError when printing a traceback"""
-    if os.name == "nt":
-        # Replace sys.stdout breaks the stdout newlines on Windows: issue #8533
-        return
-
     import atexit
 
     stdout = sys.stdout
     sys.stdout = open(stdout.fileno(), 'w',
         encoding=stdout.encoding,
         errors="backslashreplace",
-        closefd=False)
+        closefd=False,
+        newline='\n')
 
     def restore_stdout():
         sys.stdout.close()
@@ -804,7 +877,8 @@ def replace_stdout():
 
 def runtest(test, verbose, quiet,
             huntrleaks=False, debug=False, use_resources=None,
-            output_on_failure=False, failfast=False, match_tests=None):
+            output_on_failure=False, failfast=False, match_tests=None,
+            timeout=None):
     """Run a single test.
 
     test -- the name of the test
@@ -814,6 +888,8 @@ def runtest(test, verbose, quiet,
     huntrleaks -- run multiple times to test for leaks; requires a debug
                   build; a triple corresponding to -R's three arguments
     output_on_failure -- if true, display test output on failure
+    timeout -- dump the traceback and exit if a test takes more than
+               timeout seconds
 
     Returns one of the test result constants:
         INTERRUPTED      KeyboardInterrupt when run under -j
@@ -826,6 +902,9 @@ def runtest(test, verbose, quiet,
 
     if use_resources is not None:
         support.use_resources = use_resources
+    use_timeout = (timeout is not None)
+    if use_timeout:
+        faulthandler.dump_tracebacks_later(timeout, exit=True)
     try:
         support.match_tests = match_tests
         if failfast:
@@ -864,6 +943,8 @@ def runtest(test, verbose, quiet,
                                    display_failure=not verbose)
         return result
     finally:
+        if use_timeout:
+            faulthandler.cancel_dump_tracebacks_later()
         cleanup_test_droppings(test, verbose)
 runtest.stringio = None
 
@@ -909,10 +990,10 @@ class saved_test_environment:
     resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr',
                  'os.environ', 'sys.path', 'sys.path_hooks', '__import__',
                  'warnings.filters', 'asyncore.socket_map',
-                 'logging._handlers', 'logging._handlerList',
-                 'shutil.archive_formats', 'shutil.unpack_formats',
+                 'logging._handlers', 'logging._handlerList', 'sys.gettrace',
                  'sys.warnoptions', 'threading._dangling',
                  'multiprocessing.process._dangling',
+                 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES',
                  'support.TESTFN',
                 )
 
@@ -961,6 +1042,11 @@ class saved_test_environment:
         sys.path_hooks = saved_hooks[1]
         sys.path_hooks[:] = saved_hooks[2]
 
+    def get_sys_gettrace(self):
+        return sys.gettrace()
+    def restore_sys_gettrace(self, trace_fxn):
+        sys.settrace(trace_fxn)
+
     def get___import__(self):
         return builtins.__import__
     def restore___import__(self, import_):
@@ -1044,6 +1130,24 @@ class saved_test_environment:
         multiprocessing.process._dangling.clear()
         multiprocessing.process._dangling.update(saved)
 
+    def get_sysconfig__CONFIG_VARS(self):
+        # make sure the dict is initialized
+        sysconfig.get_config_var('prefix')
+        return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS,
+                dict(sysconfig._CONFIG_VARS))
+    def restore_sysconfig__CONFIG_VARS(self, saved):
+        sysconfig._CONFIG_VARS = saved[1]
+        sysconfig._CONFIG_VARS.clear()
+        sysconfig._CONFIG_VARS.update(saved[2])
+
+    def get_sysconfig__INSTALL_SCHEMES(self):
+        return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES,
+                sysconfig._INSTALL_SCHEMES.copy())
+    def restore_sysconfig__INSTALL_SCHEMES(self, saved):
+        sysconfig._INSTALL_SCHEMES = saved[1]
+        sysconfig._INSTALL_SCHEMES.clear()
+        sysconfig._INSTALL_SCHEMES.update(saved[2])
+
     def get_support_TESTFN(self):
         if os.path.isfile(support.TESTFN):
             result = 'f'
@@ -1108,14 +1212,15 @@ def runtest_inner(test, verbose, quiet,
             start_time = time.time()
             the_package = __import__(abstest, globals(), locals(), [])
             the_module = getattr(the_package, test)
-            # Old tests run to completion simply as a side-effect of
-            # being imported.  For tests based on unittest or doctest,
-            # explicitly invoke their test_main() function (if it exists).
-            indirect_test = getattr(the_module, "test_main", None)
-            if indirect_test is not None:
-                indirect_test()
+            # If the test has a test_main, that will run the appropriate
+            # tests.  If not, use normal unittest test loading.
+            test_runner = getattr(the_module, "test_main", None)
+            if test_runner is None:
+                tests = unittest.TestLoader().loadTestsFromModule(the_module)
+                test_runner = lambda: support.run_unittest(tests)
+            test_runner()
             if huntrleaks:
-                refleak = dash_R(the_module, test, indirect_test,
+                refleak = dash_R(the_module, test, test_runner,
                     huntrleaks)
             test_time = time.time() - start_time
     except support.ResourceDenied as msg:
@@ -1198,7 +1303,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
         False if the test didn't leak references; True if we detected refleaks.
     """
     # This code is hackish and inelegant, but it seems to do the job.
-    import copyreg, _abcoll
+    import copyreg
+    import collections.abc
 
     if not hasattr(sys, 'gettotalrefcount'):
         raise Exception("Tracking reference leaks requires a debug build "
@@ -1215,7 +1321,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
     else:
         zdc = zipimport._zip_directory_cache.copy()
     abcs = {}
-    for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:
+    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
         if not isabstract(abc):
             continue
         for obj in abc.__subclasses__() + [abc]:
@@ -1261,7 +1367,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
     import gc, copyreg
     import _strptime, linecache
     import urllib.parse, urllib.request, mimetypes, doctest
-    import struct, filecmp, _abcoll
+    import struct, filecmp, collections.abc
     from distutils.dir_util import _path_created
     from weakref import WeakSet
 
@@ -1288,7 +1394,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
     sys._clear_type_cache()
 
     # Clear ABC registries, restoring previously saved ABC registries.
-    for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:
+    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
         if not isabstract(abc):
             continue
         for obj in abc.__subclasses__() + [abc]:
@@ -1324,10 +1430,15 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
     # Collect cyclic trash.
     gc.collect()
 
-def warm_char_cache():
+def warm_caches():
+    # char cache
     s = bytes(range(256))
     for i in range(256):
         s[i:i+1]
+    # unicode cache
+    x = [chr(i) for i in range(256)]
+    # int cache
+    x = list(range(-5, 257))
 
 def findtestdir(path=None):
     return path or os.path.dirname(__file__) or os.curdir
@@ -1374,13 +1485,14 @@ def printlist(x, width=70, indent=4):
 # Tests that are expected to be skipped everywhere except on one platform
 # are also handled separately.
 
-_expectations = {
-    'win32':
+_expectations = (
+    ('win32',
         """
         test__locale
         test_crypt
         test_curses
         test_dbm
+        test_devpoll
         test_fcntl
         test_fork1
         test_epoll
@@ -1403,15 +1515,16 @@ _expectations = {
         test_threadsignals
         test_wait3
         test_wait4
-        """,
-    'linux2':
+        """),
+    ('linux',
         """
         test_curses
+        test_devpoll
         test_largefile
         test_kqueue
         test_ossaudiodev
-        """,
-    'unixware7':
+        """),
+    ('unixware',
         """
         test_epoll
         test_largefile
@@ -1421,8 +1534,8 @@ _expectations = {
         test_pyexpat
         test_sax
         test_sundry
-        """,
-    'openunix8':
+        """),
+    ('openunix',
         """
         test_epoll
         test_largefile
@@ -1432,8 +1545,8 @@ _expectations = {
         test_pyexpat
         test_sax
         test_sundry
-        """,
-    'sco_sv3':
+        """),
+    ('sco_sv',
         """
         test_asynchat
         test_fork1
@@ -1452,11 +1565,12 @@ _expectations = {
         test_threaded_import
         test_threadedtempfile
         test_threading
-        """,
-    'darwin':
+        """),
+    ('darwin',
         """
         test__locale
         test_curses
+        test_devpoll
         test_epoll
         test_dbm_gnu
         test_gdb
@@ -1465,8 +1579,8 @@ _expectations = {
         test_minidom
         test_ossaudiodev
         test_poll
-        """,
-    'sunos5':
+        """),
+    ('sunos',
         """
         test_curses
         test_dbm
@@ -1477,8 +1591,8 @@ _expectations = {
         test_openpty
         test_zipfile
         test_zlib
-        """,
-    'hp-ux11':
+        """),
+    ('hp-ux',
         """
         test_curses
         test_epoll
@@ -1493,11 +1607,12 @@ _expectations = {
         test_sax
         test_zipfile
         test_zlib
-        """,
-    'cygwin':
+        """),
+    ('cygwin',
         """
         test_curses
         test_dbm
+        test_devpoll
         test_epoll
         test_ioctl
         test_kqueue
@@ -1505,8 +1620,8 @@ _expectations = {
         test_locale
         test_ossaudiodev
         test_socketserver
-        """,
-    'os2emx':
+        """),
+    ('os2emx',
         """
         test_audioop
         test_curses
@@ -1519,9 +1634,10 @@ _expectations = {
         test_pty
         test_resource
         test_signal
-        """,
-    'freebsd4':
+        """),
+    ('freebsd',
         """
+        test_devpoll
         test_epoll
         test_dbm_gnu
         test_locale
@@ -1536,8 +1652,8 @@ _expectations = {
         test_timeout
         test_urllibnet
         test_multiprocessing
-        """,
-    'aix5':
+        """),
+    ('aix',
         """
         test_bz2
         test_epoll
@@ -1551,10 +1667,11 @@ _expectations = {
         test_ttk_textonly
         test_zipimport
         test_zlib
-        """,
-    'openbsd3':
+        """),
+    ('openbsd',
         """
         test_ctypes
+        test_devpoll
         test_epoll
         test_dbm_gnu
         test_locale
@@ -1566,11 +1683,12 @@ _expectations = {
         test_ttk_guionly
         test_ttk_textonly
         test_multiprocessing
-        """,
-    'netbsd3':
+        """),
+    ('netbsd',
         """
         test_ctypes
         test_curses
+        test_devpoll
         test_epoll
         test_dbm_gnu
         test_locale
@@ -1581,12 +1699,8 @@ _expectations = {
         test_ttk_guionly
         test_ttk_textonly
         test_multiprocessing
-        """,
-}
-_expectations['freebsd5'] = _expectations['freebsd4']
-_expectations['freebsd6'] = _expectations['freebsd4']
-_expectations['freebsd7'] = _expectations['freebsd4']
-_expectations['freebsd8'] = _expectations['freebsd4']
+        """),
+)
 
 class _ExpectedSkips:
     def __init__(self):
@@ -1594,9 +1708,13 @@ class _ExpectedSkips:
         from test import test_timeout
 
         self.valid = False
-        if sys.platform in _expectations:
-            s = _expectations[sys.platform]
-            self.expected = set(s.split())
+        expected = None
+        for item in _expectations:
+            if sys.platform.startswith(item[0]):
+                expected = item[1]
+                break
+        if expected is not None:
+            self.expected = set(expected.split())
 
             # These are broken tests, for now skipped on every platform.
             # XXX Fix these!
@@ -1656,9 +1774,8 @@ def _make_temp_dir_for_build(TEMPDIR):
         TEMPDIR = os.path.abspath(TEMPDIR)
         try:
             os.mkdir(TEMPDIR)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
 
     # Define a writable temp dir that will be used as cwd while running
     # the tests. The name of the dir includes the pid to allow parallel
diff --git a/Lib/test/reperf.py b/Lib/test/reperf.py
index 7c6823471e..e93bacdb58 100644
--- a/Lib/test/reperf.py
+++ b/Lib/test/reperf.py
@@ -9,13 +9,13 @@ def main():
     timefunc(10, p.findall, s)
 
 def timefunc(n, func, *args, **kw):
-    t0 = time.clock()
+    t0 = time.perf_counter()
     try:
         for i in range(n):
             result = func(*args, **kw)
         return result
     finally:
-        t1 = time.clock()
+        t1 = time.perf_counter()
         if n > 1:
             print(n, "times", end=' ')
         print(func.__name__, "%.3f" % (t1-t0), "CPU seconds")
diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py
index ba446cd69b..b09f4bf49e 100644
--- a/Lib/test/script_helper.py
+++ b/Lib/test/script_helper.py
@@ -1,6 +1,7 @@
 # Common utility functions used by various script execution tests
 #  e.g. test_cmd_line, test_cmd_line_script and test_runpy
 
+import importlib
 import sys
 import os
 import os.path
@@ -59,11 +60,12 @@ def assert_python_failure(*args, **env_vars):
     """
     return _assert_python(False, *args, **env_vars)
 
-def spawn_python(*args):
+def spawn_python(*args, **kw):
     cmd_line = [sys.executable, '-E']
     cmd_line.extend(args)
     return subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
-                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                            **kw)
 
 def kill_python(p):
     p.stdin.close()
@@ -92,6 +94,7 @@ def make_script(script_dir, script_basename, source):
     script_file = open(script_name, 'w', encoding='utf-8')
     script_file.write(source)
     script_file.close()
+    importlib.invalidate_caches()
     return script_name
 
 def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None):
diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py
index f655c29eee..f185a8251c 100644
--- a/Lib/test/seq_tests.py
+++ b/Lib/test/seq_tests.py
@@ -4,6 +4,7 @@ Tests common to tuple, list and UserList.UserList
 
 import unittest
 import sys
+import pickle
 
 # Various iterables
 # This is used for checking the constructor (here and in test_deque.py)
@@ -388,3 +389,9 @@ class CommonTest(unittest.TestCase):
         self.assertEqual(a.index(0, -4*sys.maxsize, 4*sys.maxsize), 2)
         self.assertRaises(ValueError, a.index, 0, 4*sys.maxsize,-4*sys.maxsize)
         self.assertRaises(ValueError, a.index, 2, 0, -10)
+
+    def test_pickle(self):
+        lst = self.type2test([4, 5, 6, 7])
+        lst2 = pickle.loads(pickle.dumps(lst))
+        self.assertEqual(lst2, lst)
+        self.assertNotEqual(id(lst2), id(lst))
diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py
index 0ce88de4c1..af7c0b4bd1 100644
--- a/Lib/test/sortperf.py
+++ b/Lib/test/sortperf.py
@@ -57,9 +57,9 @@ def flush():
     sys.stdout.flush()
 
 def doit(L):
-    t0 = time.clock()
+    t0 = time.perf_counter()
     L.sort()
-    t1 = time.clock()
+    t1 = time.perf_counter()
     print("%6.2f" % (t1-t0), end=' ')
     flush()
 
diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem
new file mode 100644
index 0000000000..2524672e70
--- /dev/null
+++ b/Lib/test/ssl_key.passwd.pem
@@ -0,0 +1,18 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A
+
+kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c
+u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA
+AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr
+Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+
+YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P
+6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+
+noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1
+94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l
+7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo
+cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO
+zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt
+L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo
+2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ==
+-----END RSA PRIVATE KEY-----
diff --git a/Lib/test/ssl_servers.py b/Lib/test/ssl_servers.py
index 77c0542610..8686153a17 100644
--- a/Lib/test/ssl_servers.py
+++ b/Lib/test/ssl_servers.py
@@ -94,7 +94,12 @@ class StatsRequestHandler(BaseHTTPRequestHandler):
         """Serve a GET request."""
         sock = self.rfile.raw._sock
         context = sock.context
-        body = pprint.pformat(context.session_stats())
+        stats = {
+            'session_cache': context.session_stats(),
+            'cipher': sock.cipher(),
+            'compression': sock.compression(),
+            }
+        body = pprint.pformat(stats)
         body = body.encode('utf-8')
         self.send_response(200)
         self.send_header("Content-type", "text/plain; charset=utf-8")
@@ -172,6 +177,11 @@ if __name__ == "__main__":
                         action='store_false', help='be less verbose')
     parser.add_argument('-s', '--stats', dest='use_stats_handler', default=False,
                         action='store_true', help='always return stats page')
+    parser.add_argument('--curve-name', dest='curve_name', type=str,
+                        action='store',
+                        help='curve name for EC-based Diffie-Hellman')
+    parser.add_argument('--dh', dest='dh_file', type=str, action='store',
+                        help='PEM file containing DH parameters')
     args = parser.parse_args()
 
     support.verbose = args.verbose
@@ -182,6 +192,10 @@ if __name__ == "__main__":
         handler_class.root = os.getcwd()
     context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
     context.load_cert_chain(CERTFILE)
+    if args.curve_name:
+        context.set_ecdh_curve(args.curve_name)
+    if args.dh_file:
+        context.load_dh_params(args.dh_file)
 
     server = HTTPSServer(("", args.port), handler_class, context)
     if args.verbose:
diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py
index 6792179489..385b03992d 100644
--- a/Lib/test/string_tests.py
+++ b/Lib/test/string_tests.py
@@ -20,7 +20,7 @@ class BadSeq2(Sequence):
     def __init__(self): self.seq = ['a', 'b', 'c']
     def __len__(self): return 8
 
-class BaseTest(unittest.TestCase):
+class BaseTest:
     # These tests are for buffers of values (bytes) and not
     # specific to character interpretation, used for bytes objects
     # and various string implementations
@@ -29,6 +29,11 @@ class BaseTest(unittest.TestCase):
     # Change in subclasses to change the behaviour of fixtesttype()
     type2test = None
 
+    # Whether the "contained items" of the container are integers in
+    # range(0, 256) (i.e. bytes, bytearray) or strings of length 1
+    # (str)
+    contains_bytes = False
+
     # All tests pass their arguments to the testing methods
     # as str objects. fixtesttype() can be used to propagate
     # these arguments to the appropriate type
@@ -48,11 +53,12 @@ class BaseTest(unittest.TestCase):
             return obj
 
     # check that obj.method(*args) returns result
-    def checkequal(self, result, obj, methodname, *args):
+    def checkequal(self, result, obj, methodname, *args, **kwargs):
         result = self.fixtype(result)
         obj = self.fixtype(obj)
         args = self.fixtype(args)
-        realresult = getattr(obj, methodname)(*args)
+        kwargs = {k: self.fixtype(v) for k,v in kwargs.items()}
+        realresult = getattr(obj, methodname)(*args, **kwargs)
         self.assertEqual(
             result,
             realresult
@@ -117,7 +123,11 @@ class BaseTest(unittest.TestCase):
         self.checkequal(0, '', 'count', 'xx', sys.maxsize, 0)
 
         self.checkraises(TypeError, 'hello', 'count')
-        self.checkraises(TypeError, 'hello', 'count', 42)
+
+        if self.contains_bytes:
+            self.checkequal(0, 'hello', 'count', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'count', 42)
 
         # For a variety of combinations,
         #    verify that str.count() matches an equivalent function
@@ -163,7 +173,11 @@ class BaseTest(unittest.TestCase):
         self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a', None, 6)
 
         self.checkraises(TypeError, 'hello', 'find')
-        self.checkraises(TypeError, 'hello', 'find', 42)
+
+        if self.contains_bytes:
+            self.checkequal(-1, 'hello', 'find', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'find', 42)
 
         self.checkequal(0, '', 'find', '')
         self.checkequal(-1, '', 'find', '', 1, 1)
@@ -217,7 +231,11 @@ class BaseTest(unittest.TestCase):
         self.checkequal( 2, 'rrarrrrrrrrra', 'rfind', 'a', None, 6)
 
         self.checkraises(TypeError, 'hello', 'rfind')
-        self.checkraises(TypeError, 'hello', 'rfind', 42)
+
+        if self.contains_bytes:
+            self.checkequal(-1, 'hello', 'rfind', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'rfind', 42)
 
         # For a variety of combinations,
         #    verify that str.rfind() matches __contains__
@@ -245,6 +263,9 @@ class BaseTest(unittest.TestCase):
         # issue 7458
         self.checkequal(-1, 'ab', 'rfind', 'xxx', sys.maxsize + 1, 0)
 
+        # issue #15534
+        self.checkequal(0, '<......\u043c...', "rfind", "<")
+
     def test_index(self):
         self.checkequal(0, 'abcdefghiabc', 'index', '')
         self.checkequal(3, 'abcdefghiabc', 'index', 'def')
@@ -264,7 +285,11 @@ class BaseTest(unittest.TestCase):
         self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a', None, 6)
 
         self.checkraises(TypeError, 'hello', 'index')
-        self.checkraises(TypeError, 'hello', 'index', 42)
+
+        if self.contains_bytes:
+            self.checkraises(ValueError, 'hello', 'index', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'index', 42)
 
     def test_rindex(self):
         self.checkequal(12, 'abcdefghiabc', 'rindex', '')
@@ -286,7 +311,11 @@ class BaseTest(unittest.TestCase):
         self.checkequal( 2, 'rrarrrrrrrrra', 'rindex', 'a', None, 6)
 
         self.checkraises(TypeError, 'hello', 'rindex')
-        self.checkraises(TypeError, 'hello', 'rindex', 42)
+
+        if self.contains_bytes:
+            self.checkraises(ValueError, 'hello', 'rindex', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'rindex', 42)
 
     def test_lower(self):
         self.checkequal('hello', 'HeLLo', 'lower')
@@ -364,6 +393,17 @@ class BaseTest(unittest.TestCase):
         self.checkequal(['a']*18 + ['aBLAHa'], ('aBLAH'*20)[:-4],
                         'split', 'BLAH', 18)
 
+        # with keyword args
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', sep='|')
+        self.checkequal(['a', 'b|c|d'],
+                        'a|b|c|d', 'split', '|', maxsplit=1)
+        self.checkequal(['a', 'b|c|d'],
+                        'a|b|c|d', 'split', sep='|', maxsplit=1)
+        self.checkequal(['a', 'b|c|d'],
+                        'a|b|c|d', 'split', maxsplit=1, sep='|')
+        self.checkequal(['a', 'b c d'],
+                        'a b c d', 'split', maxsplit=1)
+
         # argument type
         self.checkraises(TypeError, 'hello', 'split', 42, 42, 42)
 
@@ -421,6 +461,17 @@ class BaseTest(unittest.TestCase):
         self.checkequal(['aBLAHa'] + ['a']*18, ('aBLAH'*20)[:-4],
                         'rsplit', 'BLAH', 18)
 
+        # with keyword args
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', sep='|')
+        self.checkequal(['a|b|c', 'd'],
+                        'a|b|c|d', 'rsplit', '|', maxsplit=1)
+        self.checkequal(['a|b|c', 'd'],
+                        'a|b|c|d', 'rsplit', sep='|', maxsplit=1)
+        self.checkequal(['a|b|c', 'd'],
+                        'a|b|c|d', 'rsplit', maxsplit=1, sep='|')
+        self.checkequal(['a b c', 'd'],
+                        'a b c d', 'rsplit', maxsplit=1)
+
         # argument type
         self.checkraises(TypeError, 'hello', 'rsplit', 42, 42, 42)
 
@@ -550,6 +601,8 @@ class BaseTest(unittest.TestCase):
         EQ("ReyKKjavik", "Reykjavik", "replace", "k", "KK", 1)
         EQ("Reykjavik", "Reykjavik", "replace", "k", "KK", 0)
         EQ("A----B----C----", "A.B.C.", "replace", ".", "----")
+        # issue #15534
+        EQ('...\u043c......&lt;', '...\u043c......<', "replace", "<", "&lt;")
 
         EQ("Reykjavik", "Reykjavik", "replace", "q", "KK")
 
@@ -644,7 +697,7 @@ class CommonTest(BaseTest):
 
         # check that titlecased chars are lowered correctly
         # \u1ffc is the titlecased char
-        self.checkequal('\u1ffc\u1ff3\u1ff3\u1ff3',
+        self.checkequal('\u03a9\u0399\u1ff3\u1ff3\u1ff3',
                         '\u1ff3\u1ff3\u1ffc\u1ffc', 'capitalize')
         # check with cased non-letter chars
         self.checkequal('\u24c5\u24e8\u24e3\u24d7\u24de\u24dd',
@@ -909,7 +962,14 @@ class MixinStrUnicodeUserStringTest:
         self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi\n", 'splitlines')
         self.checkequal(['abc', 'def', 'ghi', ''], "abc\ndef\r\nghi\n\r", 'splitlines')
         self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r", 'splitlines')
-        self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'], "\nabc\ndef\r\nghi\n\r", 'splitlines', 1)
+        self.checkequal(['', 'abc', 'def', 'ghi', ''],
+                        "\nabc\ndef\r\nghi\n\r", 'splitlines', False)
+        self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'],
+                        "\nabc\ndef\r\nghi\n\r", 'splitlines', True)
+        self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r",
+                        'splitlines', keepends=False)
+        self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'],
+                        "\nabc\ndef\r\nghi\n\r", 'splitlines', keepends=True)
 
         self.checkraises(TypeError, 'abc', 'splitlines', 42, 42)
 
@@ -1143,6 +1203,10 @@ class MixinStrUnicodeUserStringTest:
         self.checkraises(TypeError, '%10.*f', '__mod__', ('foo', 42.))
         self.checkraises(ValueError, '%10', '__mod__', (42,))
 
+        # Outrageously large width or precision should raise ValueError.
+        self.checkraises(ValueError, '%%%df' % (2**64), '__mod__', (3.2))
+        self.checkraises(ValueError, '%%.%df' % (2**64), '__mod__', (3.2))
+
         self.checkraises(OverflowError, '%*s', '__mod__',
                          (_testcapi.PY_SSIZE_T_MAX + 1, ''))
         self.checkraises(OverflowError, '%.*f', '__mod__',
@@ -1271,6 +1335,9 @@ class MixinStrUnicodeUserStringTest:
         self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith,
                                 x, None, None, None)
 
+        # issue #15534
+        self.checkequal(10, "...\u043c......<", "find", "<")
+
 
 class MixinStrUnicodeTest:
     # Additional tests that only work with str and unicode.
diff --git a/Lib/test/support.py b/Lib/test/support.py
index ddda380f35..19989acf61 100644
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -15,7 +15,7 @@ import shutil
 import warnings
 import unittest
 import importlib
-import collections
+import collections.abc
 import re
 import subprocess
 import imp
@@ -37,26 +37,41 @@ try:
 except ImportError:
     multiprocessing = None
 
+try:
+    import zlib
+except ImportError:
+    zlib = None
+
+try:
+    import bz2
+except ImportError:
+    bz2 = None
+
+try:
+    import lzma
+except ImportError:
+    lzma = None
 
 __all__ = [
-    "Error", "TestFailed", "ResourceDenied", "import_module",
-    "verbose", "use_resources", "max_memuse", "record_original_stdout",
+    "Error", "TestFailed", "ResourceDenied", "import_module", "verbose",
+    "use_resources", "max_memuse", "record_original_stdout",
     "get_original_stdout", "unload", "unlink", "rmtree", "forget",
-    "is_resource_enabled", "requires", "requires_mac_ver",
-    "find_unused_port", "bind_port",
-    "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd",
-    "findfile", "sortdict", "check_syntax_error", "open_urlresource",
-    "check_warnings", "CleanImport", "EnvironmentVarGuard",
-    "TransientResource", "captured_output", "captured_stdout",
-    "captured_stdin", "captured_stderr",
-    "time_out", "socket_peer_reset", "ioerror_peer_reset",
-    "run_with_locale", 'temp_umask', "transient_internet",
-    "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner",
-    "run_unittest", "run_doctest", "threading_setup", "threading_cleanup",
-    "reap_children", "cpython_only", "check_impl_detail", "get_attribute",
-    "swap_item", "swap_attr", "requires_IEEE_754",
+    "is_resource_enabled", "requires", "requires_freebsd_version",
+    "requires_linux_version", "requires_mac_ver", "find_unused_port",
+    "bind_port", "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD",
+    "temp_cwd", "findfile", "create_empty_file", "sortdict",
+    "check_syntax_error", "open_urlresource", "check_warnings", "CleanImport",
+    "EnvironmentVarGuard", "TransientResource", "captured_stdout",
+    "captured_stdin", "captured_stderr", "time_out", "socket_peer_reset",
+    "ioerror_peer_reset", "run_with_locale", 'temp_umask',
+    "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest",
+    "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
+    "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail",
+    "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754",
     "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink",
-    "import_fresh_module", "failfast", "run_with_tz", "suppress_crash_popup",
+    "skip_unless_xattr", "import_fresh_module", "requires_zlib",
+    "PIPE_MAX_SIZE", "failfast", "anticipate_failure", "run_with_tz",
+    "requires_bz2", "requires_lzma", "suppress_crash_popup",
     ]
 
 class Error(Exception):
@@ -127,6 +142,17 @@ def _save_and_block_module(name, orig_modules):
     return saved
 
 
+def anticipate_failure(condition):
+    """Decorator to mark a test that is known to be broken in some cases
+
+       Any use of this decorator should have a comment identifying the
+       associated tracker issue.
+    """
+    if condition:
+        return unittest.expectedFailure
+    return lambda f: f
+
+
 def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
     """Imports and returns a module, deliberately bypassing the sys.modules cache
     and importing a fresh copy of the module. Once the import is complete,
@@ -170,8 +196,7 @@ def get_attribute(obj, name):
     try:
         attribute = getattr(obj, name)
     except AttributeError:
-        raise unittest.SkipTest("module %s has no attribute %s" % (
-            obj.__name__, name))
+        raise unittest.SkipTest("object %r has no attribute %r" % (obj, name))
     else:
         return attribute
 
@@ -276,8 +301,7 @@ def rmtree(path):
     try:
         _rmtree(path)
     except OSError as error:
-        # Unix returns ENOENT, Windows returns ESRCH.
-        if error.errno not in (errno.ENOENT, errno.ESRCH):
+        if error.errno != errno.ENOENT:
             raise
 
 def make_legacy_pyc(source):
@@ -362,9 +386,52 @@ def requires(resource, msg=None):
         return
     if not is_resource_enabled(resource):
         if msg is None:
-            msg = "Use of the `%s' resource not enabled" % resource
+            msg = "Use of the %r resource not enabled" % resource
         raise ResourceDenied(msg)
 
+def _requires_unix_version(sysname, min_version):
+    """Decorator raising SkipTest if the OS is `sysname` and the version is less
+    than `min_version`.
+
+    For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if
+    the FreeBSD version is less than 7.2.
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kw):
+            if platform.system() == sysname:
+                version_txt = platform.release().split('-', 1)[0]
+                try:
+                    version = tuple(map(int, version_txt.split('.')))
+                except ValueError:
+                    pass
+                else:
+                    if version < min_version:
+                        min_version_txt = '.'.join(map(str, min_version))
+                        raise unittest.SkipTest(
+                            "%s version %s or higher required, not %s"
+                            % (sysname, min_version_txt, version_txt))
+        return wrapper
+    return decorator
+
+def requires_freebsd_version(*min_version):
+    """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version is
+    less than `min_version`.
+
+    For example, @requires_freebsd_version(7, 2) raises SkipTest if the FreeBSD
+    version is less than 7.2.
+    """
+    return _requires_unix_version('FreeBSD', min_version)
+
+def requires_linux_version(*min_version):
+    """Decorator raising SkipTest if the OS is Linux and the Linux version is
+    less than `min_version`.
+
+    For example, @requires_linux_version(2, 6, 32) raises SkipTest if the Linux
+    version is less than 2.6.32.
+    """
+    return _requires_unix_version('Linux', min_version)
+
 def requires_mac_ver(*min_version):
     """Decorator raising SkipTest if the OS is Mac OS X and the OS X
     version if less than min_version.
@@ -392,6 +459,7 @@ def requires_mac_ver(*min_version):
         return wrapper
     return decorator
 
+
 HOST = 'localhost'
 
 def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
@@ -487,29 +555,41 @@ def bind_port(sock, host=HOST):
     port = sock.getsockname()[1]
     return port
 
-FUZZ = 1e-6
-
-def fcmp(x, y): # fuzzy comparison function
-    if isinstance(x, float) or isinstance(y, float):
+def _is_ipv6_enabled():
+    """Check whether IPv6 is enabled on this host."""
+    if socket.has_ipv6:
+        sock = None
         try:
-            fuzz = (abs(x) + abs(y)) * FUZZ
-            if abs(x-y) <= fuzz:
-                return 0
-        except:
+            sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            sock.bind(('::1', 0))
+            return True
+        except (socket.error, socket.gaierror):
             pass
-    elif type(x) == type(y) and isinstance(x, (tuple, list)):
-        for i in range(min(len(x), len(y))):
-            outcome = fcmp(x[i], y[i])
-            if outcome != 0:
-                return outcome
-        return (len(x) > len(y)) - (len(x) < len(y))
-    return (x > y) - (x < y)
+        finally:
+            if sock:
+                sock.close()
+    return False
+
+IPV6_ENABLED = _is_ipv6_enabled()
+
+
+# A constant likely larger than the underlying OS pipe buffer size.
+# Windows limit seems to be around 512B, and most Unix kernels have a 64K pipe
+# buffer size: take 1M to be sure.
+PIPE_MAX_SIZE = 1024 * 1024
+
 
 # decorator for skipping tests on non-IEEE 754 platforms
 requires_IEEE_754 = unittest.skipUnless(
     float.__getformat__("double").startswith("IEEE"),
     "test requires IEEE 754 doubles")
 
+requires_zlib = unittest.skipUnless(zlib, 'requires zlib')
+
+requires_bz2 = unittest.skipUnless(bz2, 'requires bz2')
+
+requires_lzma = unittest.skipUnless(lzma, 'requires lzma')
+
 is_jython = sys.platform.startswith('java')
 
 # Filename used for testing
@@ -688,14 +768,15 @@ def temp_cwd(name='tempcwd', quiet=False, path=None):
             rmtree(name)
 
 
-@contextlib.contextmanager
-def temp_umask(umask):
-    """Context manager that temporarily sets the process umask."""
-    oldmask = os.umask(umask)
-    try:
-        yield
-    finally:
-        os.umask(oldmask)
+if hasattr(os, "umask"):
+    @contextlib.contextmanager
+    def temp_umask(umask):
+        """Context manager that temporarily sets the process umask."""
+        oldmask = os.umask(umask)
+        try:
+            yield
+        finally:
+            os.umask(oldmask)
 
 
 def findfile(file, here=__file__, subdir=None):
@@ -713,6 +794,11 @@ def findfile(file, here=__file__, subdir=None):
         if os.path.exists(fn): return fn
     return file
 
+def create_empty_file(filename):
+    """Create an empty file. If the file already exists, truncate it."""
+    fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
+    os.close(fd)
+
 def sortdict(dict):
     "Like repr(dict), but in sorted order."
     items = sorted(dict.items())
@@ -777,7 +863,7 @@ def open_urlresource(url, *args, **kw):
     f = check_valid_file(fn)
     if f is not None:
         return f
-    raise TestFailed('invalid resource "%s"' % fn)
+    raise TestFailed('invalid resource %r' % fn)
 
 
 class WarningsRecorder(object):
@@ -898,7 +984,7 @@ class CleanImport(object):
         sys.modules.update(self.original_modules)
 
 
-class EnvironmentVarGuard(collections.MutableMapping):
+class EnvironmentVarGuard(collections.abc.MutableMapping):
 
     """Class to help protect the environment variable properly.  Can be used as
     a context manager."""
@@ -1029,7 +1115,7 @@ def transient_internet(resource_name, *, timeout=30.0, errnos=()):
         ('WSANO_DATA', 11004),
     ]
 
-    denied = ResourceDenied("Resource '%s' is not available" % resource_name)
+    denied = ResourceDenied("Resource %r is not available" % resource_name)
     captured_errnos = errnos
     gai_errnos = []
     if not captured_errnos:
@@ -1118,6 +1204,16 @@ def gc_collect():
     gc.collect()
     gc.collect()
 
+@contextlib.contextmanager
+def disable_gc():
+    have_gc = gc.isenabled()
+    gc.disable()
+    try:
+        yield
+    finally:
+        if have_gc:
+            gc.enable()
+
 
 def python_is_optimized():
     """Find if Python was built with optimizations."""
@@ -1126,19 +1222,21 @@ def python_is_optimized():
     for opt in cflags.split():
         if opt.startswith('-O'):
             final_opt = opt
-    return final_opt and final_opt != '-O0'
+    return final_opt != '' and final_opt != '-O0'
 
 
-_header = '2P'
+_header = 'nP'
+_align = '0n'
 if hasattr(sys, "gettotalrefcount"):
     _header = '2P' + _header
-_vheader = _header + 'P'
+    _align = '0P'
+_vheader = _header + 'n'
 
 def calcobjsize(fmt):
-    return struct.calcsize(_header + fmt + '0P')
+    return struct.calcsize(_header + fmt + _align)
 
 def calcvobjsize(fmt):
-    return struct.calcsize(_vheader + fmt + '0P')
+    return struct.calcsize(_vheader + fmt + _align)
 
 
 _TPFLAGS_HAVE_GC = 1<<14
@@ -1212,7 +1310,7 @@ def run_with_tz(tz):
             try:
                 return func(*args, **kwds)
             finally:
-                if orig_tz == None:
+                if orig_tz is None:
                     del os.environ['TZ']
                 else:
                     os.environ['TZ'] = orig_tz
@@ -1257,41 +1355,35 @@ def set_memlimit(limit):
         raise ValueError('Memory limit %r too low to be useful' % (limit,))
     max_memuse = memlimit
 
-def _memory_watchdog(start_evt, finish_evt, period=10.0):
-    """A function which periodically watches the process' memory consumption
+class _MemoryWatchdog:
+    """An object which periodically watches the process' memory consumption
     and prints it out.
     """
-    # XXX: because of the GIL, and because the very long operations tested
-    # in most bigmem tests are uninterruptible, the loop below gets woken up
-    # much less often than expected.
-    # The polling code should be rewritten in raw C, without holding the GIL,
-    # and push results onto an anonymous pipe.
-    try:
-        page_size = os.sysconf('SC_PAGESIZE')
-    except (ValueError, AttributeError):
+
+    def __init__(self):
+        self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid())
+        self.started = False
+
+    def start(self):
         try:
-            page_size = os.sysconf('SC_PAGE_SIZE')
-        except (ValueError, AttributeError):
-            page_size = 4096
-    procfile = '/proc/{pid}/statm'.format(pid=os.getpid())
-    try:
-        f = open(procfile, 'rb')
-    except IOError as e:
-        warnings.warn('/proc not available for stats: {}'.format(e),
-                      RuntimeWarning)
-        sys.stderr.flush()
-        return
-    with f:
-        start_evt.set()
-        old_data = -1
-        while not finish_evt.wait(period):
-            f.seek(0)
-            statm = f.read().decode('ascii')
-            data = int(statm.split()[5])
-            if data != old_data:
-                old_data = data
-                print(" ... process data size: {data:.1f}G"
-                       .format(data=data * page_size / (1024 ** 3)))
+            f = open(self.procfile, 'r')
+        except OSError as e:
+            warnings.warn('/proc not available for stats: {}'.format(e),
+                          RuntimeWarning)
+            sys.stderr.flush()
+            return
+
+        watchdog_script = findfile("memory_watchdog.py")
+        self.mem_watchdog = subprocess.Popen([sys.executable, watchdog_script],
+                                             stdin=f, stderr=subprocess.DEVNULL)
+        f.close()
+        self.started = True
+
+    def stop(self):
+        if self.started:
+            self.mem_watchdog.terminate()
+            self.mem_watchdog.wait()
+
 
 def bigmemtest(size, memuse, dry_run=True):
     """Decorator for bigmem tests.
@@ -1318,27 +1410,20 @@ def bigmemtest(size, memuse, dry_run=True):
                     "not enough memory: %.1fG minimum needed"
                     % (size * memuse / (1024 ** 3)))
 
-            if real_max_memuse and verbose and threading:
+            if real_max_memuse and verbose:
                 print()
                 print(" ... expected peak memory use: {peak:.1f}G"
                       .format(peak=size * memuse / (1024 ** 3)))
-                sys.stdout.flush()
-                start_evt = threading.Event()
-                finish_evt = threading.Event()
-                t = threading.Thread(target=_memory_watchdog,
-                                     args=(start_evt, finish_evt, 0.5))
-                t.daemon = True
-                t.start()
-                start_evt.set()
+                watchdog = _MemoryWatchdog()
+                watchdog.start()
             else:
-                t = None
+                watchdog = None
 
             try:
                 return f(self, maxsize)
             finally:
-                if t:
-                    finish_evt.set()
-                    t.join()
+                if watchdog:
+                    watchdog.stop()
 
         wrapper.size = size
         wrapper.memuse = memuse
@@ -1420,6 +1505,33 @@ def check_impl_detail(**guards):
     return guards.get(platform.python_implementation().lower(), default)
 
 
+def no_tracing(func):
+    """Decorator to temporarily turn off tracing for the duration of a test."""
+    if not hasattr(sys, 'gettrace'):
+        return func
+    else:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            original_trace = sys.gettrace()
+            try:
+                sys.settrace(None)
+                return func(*args, **kwargs)
+            finally:
+                sys.settrace(original_trace)
+        return wrapper
+
+
+def refcount_test(test):
+    """Decorator for tests which involve reference counting.
+
+    To start, the decorator does not run the test if is not run by CPython.
+    After that, any trace function is unset during the test to prevent
+    unexpected refcounts caused by the trace function.
+
+    """
+    return no_tracing(cpython_only(test))
+
+
 def _filter_suite(suite, pred):
     """Recursively filter test cases in a suite based on a predicate."""
     newtests = []
@@ -1432,7 +1544,6 @@ def _filter_suite(suite, pred):
                 newtests.append(test)
     suite._tests = newtests
 
-
 def _run_suite(suite):
     """Run tests from a unittest.TestSuite-derived class."""
     if verbose:
@@ -1491,7 +1602,7 @@ requires_docstrings = unittest.skipUnless(HAVE_DOCSTRINGS,
 #=======================================================================
 # doctest driver.
 
-def run_doctest(module, verbosity=None):
+def run_doctest(module, verbosity=None, optionflags=0):
     """Run doctest on the given module.  Return (#failures, #tests).
 
     If optional argument verbosity is not specified (or is None), pass
@@ -1506,7 +1617,7 @@ def run_doctest(module, verbosity=None):
     else:
         verbosity = None
 
-    f, t = doctest.testmod(module, verbose=verbosity)
+    f, t = doctest.testmod(module, verbose=verbosity, optionflags=optionflags)
     if f:
         raise TestFailed("%d of %d doctests failed" % (f, t))
     if verbose:
@@ -1664,28 +1775,13 @@ def strip_python_stderr(stderr):
     This will typically be run on the result of the communicate() method
     of a subprocess.Popen object.
     """
-    stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip()
+    stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
     return stderr
 
 def args_from_interpreter_flags():
     """Return a list of command-line arguments reproducing the current
-    settings in sys.flags."""
-    flag_opt_map = {
-        'bytes_warning': 'b',
-        'dont_write_bytecode': 'B',
-        'hash_randomization': 'R',
-        'ignore_environment': 'E',
-        'no_user_site': 's',
-        'no_site': 'S',
-        'optimize': 'O',
-        'verbose': 'v',
-    }
-    args = []
-    for flag, opt in flag_opt_map.items():
-        v = getattr(sys.flags, flag)
-        if v > 0:
-            args.append('-' + opt * v)
-    return args
+    settings in sys.flags and sys.warnoptions."""
+    return subprocess._args_from_interpreter_flags()
 
 #============================================================
 # Support for assertions about logging.
@@ -1775,6 +1871,40 @@ def skip_unless_symlink(test):
     msg = "Requires functional symlink implementation"
     return test if ok else unittest.skip(msg)(test)
 
+_can_xattr = None
+def can_xattr():
+    global _can_xattr
+    if _can_xattr is not None:
+        return _can_xattr
+    if not hasattr(os, "setxattr"):
+        can = False
+    else:
+        tmp_fp, tmp_name = tempfile.mkstemp()
+        try:
+            with open(TESTFN, "wb") as fp:
+                try:
+                    # TESTFN & tempfile may use different file systems with
+                    # different capabilities
+                    os.setxattr(tmp_fp, b"user.test", b"")
+                    os.setxattr(fp.fileno(), b"user.test", b"")
+                    # Kernels < 2.6.39 don't respect setxattr flags.
+                    kernel_version = platform.release()
+                    m = re.match("2.6.(\d{1,2})", kernel_version)
+                    can = m is None or int(m.group(1)) >= 39
+                except OSError:
+                    can = False
+        finally:
+            unlink(TESTFN)
+            unlink(tmp_name)
+    _can_xattr = can
+    return can
+
+def skip_unless_xattr(test):
+    """Skip decorator for tests that require functional extended attributes"""
+    ok = can_xattr()
+    msg = "no non-broken extended attribute support"
+    return test if ok else unittest.skip(msg)(test)
+
 
 if sys.platform.startswith('win'):
     @contextlib.contextmanager
diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py
index 3fadb575f2..4231f37bc9 100644
--- a/Lib/test/test__locale.py
+++ b/Lib/test/test__locale.py
@@ -1,23 +1,25 @@
-from test.support import run_unittest
 from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, localeconv, Error)
 try:
     from _locale import (RADIXCHAR, THOUSEP, nl_langinfo)
 except ImportError:
     nl_langinfo = None
 
-import unittest
+import codecs
+import locale
 import sys
+import unittest
 from platform import uname
+from test.support import run_unittest
 
-if uname()[0] == "Darwin":
-    maj, min, mic = [int(part) for part in uname()[2].split(".")]
+if uname().system == "Darwin":
+    maj, min, mic = [int(part) for part in uname().release.split(".")]
     if (maj, min, mic) < (8, 0, 0):
         raise unittest.SkipTest("locale support broken for OS X < 10.4")
 
 candidate_locales = ['es_UY', 'fr_FR', 'fi_FI', 'es_CO', 'pt_PT', 'it_IT',
     'et_EE', 'es_PY', 'no_NO', 'nl_NL', 'lv_LV', 'el_GR', 'be_BY', 'fr_BE',
     'ro_RO', 'ru_UA', 'ru_RU', 'es_VE', 'ca_ES', 'se_NO', 'es_EC', 'id_ID',
-    'ka_GE', 'es_CL', 'hu_HU', 'wa_BE', 'lt_LT', 'sl_SI', 'hr_HR', 'es_AR',
+    'ka_GE', 'es_CL', 'wa_BE', 'hu_HU', 'lt_LT', 'sl_SI', 'hr_HR', 'es_AR',
     'es_ES', 'oc_FR', 'gl_ES', 'bg_BG', 'is_IS', 'mk_MK', 'de_AT', 'pt_BR',
     'da_DK', 'nn_NO', 'cs_CZ', 'de_LU', 'es_BO', 'sq_AL', 'sk_SK', 'fr_CH',
     'de_DE', 'sr_YU', 'br_FR', 'nl_BE', 'sv_FI', 'pl_PL', 'fr_CA', 'fo_FO',
@@ -25,6 +27,31 @@ candidate_locales = ['es_UY', 'fr_FR', 'fi_FI', 'es_CO', 'pt_PT', 'it_IT',
     'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'en_US',
     'es_ES.ISO8859-1', 'fr_FR.ISO8859-15', 'ru_RU.KOI8-R', 'ko_KR.eucKR']
 
+# Issue #13441: Skip some locales (e.g. cs_CZ and hu_HU) on Solaris to
+# workaround a mbstowcs() bug. For example, on Solaris, the hu_HU locale uses
+# the locale encoding ISO-8859-2, the thousauds separator is b'\xA0' and it is
+# decoded as U+30000020 (an invalid character) by mbstowcs().
+if sys.platform == 'sunos5':
+    old_locale = locale.setlocale(locale.LC_ALL)
+    try:
+        locales = []
+        for loc in candidate_locales:
+            try:
+                locale.setlocale(locale.LC_ALL, loc)
+            except Error:
+                continue
+            encoding = locale.getpreferredencoding(False)
+            try:
+                localeconv()
+            except Exception as err:
+                print("WARNING: Skip locale %s (encoding %s): [%s] %s"
+                      % (loc, encoding, type(err), err))
+            else:
+                locales.append(loc)
+        candidate_locales = locales
+    finally:
+        locale.setlocale(locale.LC_ALL, old_locale)
+
 # Workaround for MSVC6(debug) crash bug
 if "MSC v.1200" in sys.version:
     def accept(loc):
@@ -86,9 +113,10 @@ class _LocaleTests(unittest.TestCase):
                 setlocale(LC_CTYPE, loc)
             except Error:
                 continue
+            formatting = localeconv()
             for lc in ("decimal_point",
                         "thousands_sep"):
-                self.numeric_tester('localeconv', localeconv()[lc], lc, loc)
+                self.numeric_tester('localeconv', formatting[lc], lc, loc)
 
     @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available")
     def test_lc_numeric_basic(self):
diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py
index d86f97c068..653c957b81 100644
--- a/Lib/test/test_abc.py
+++ b/Lib/test/test_abc.py
@@ -10,14 +10,7 @@ import abc
 from inspect import isabstract
 
 
-class TestABC(unittest.TestCase):
-
-    def test_abstractmethod_basics(self):
-        @abc.abstractmethod
-        def foo(self): pass
-        self.assertTrue(foo.__isabstractmethod__)
-        def bar(self): pass
-        self.assertFalse(hasattr(bar, "__isabstractmethod__"))
+class TestLegacyAPI(unittest.TestCase):
 
     def test_abstractproperty_basics(self):
         @abc.abstractproperty
@@ -29,10 +22,12 @@ class TestABC(unittest.TestCase):
         class C(metaclass=abc.ABCMeta):
             @abc.abstractproperty
             def foo(self): return 3
+        self.assertRaises(TypeError, C)
         class D(C):
             @property
             def foo(self): return super().foo
         self.assertEqual(D().foo, 3)
+        self.assertFalse(getattr(D.foo, "__isabstractmethod__", False))
 
     def test_abstractclassmethod_basics(self):
         @abc.abstractclassmethod
@@ -40,7 +35,7 @@ class TestABC(unittest.TestCase):
         self.assertTrue(foo.__isabstractmethod__)
         @classmethod
         def bar(cls): pass
-        self.assertFalse(hasattr(bar, "__isabstractmethod__"))
+        self.assertFalse(getattr(bar, "__isabstractmethod__", False))
 
         class C(metaclass=abc.ABCMeta):
             @abc.abstractclassmethod
@@ -58,7 +53,7 @@ class TestABC(unittest.TestCase):
         self.assertTrue(foo.__isabstractmethod__)
         @staticmethod
         def bar(): pass
-        self.assertFalse(hasattr(bar, "__isabstractmethod__"))
+        self.assertFalse(getattr(bar, "__isabstractmethod__", False))
 
         class C(metaclass=abc.ABCMeta):
             @abc.abstractstaticmethod
@@ -98,6 +93,163 @@ class TestABC(unittest.TestCase):
             self.assertRaises(TypeError, F)  # because bar is abstract now
             self.assertTrue(isabstract(F))
 
+
+class TestABC(unittest.TestCase):
+
+    def test_abstractmethod_basics(self):
+        @abc.abstractmethod
+        def foo(self): pass
+        self.assertTrue(foo.__isabstractmethod__)
+        def bar(self): pass
+        self.assertFalse(hasattr(bar, "__isabstractmethod__"))
+
+    def test_abstractproperty_basics(self):
+        @property
+        @abc.abstractmethod
+        def foo(self): pass
+        self.assertTrue(foo.__isabstractmethod__)
+        def bar(self): pass
+        self.assertFalse(getattr(bar, "__isabstractmethod__", False))
+
+        class C(metaclass=abc.ABCMeta):
+            @property
+            @abc.abstractmethod
+            def foo(self): return 3
+        self.assertRaises(TypeError, C)
+        class D(C):
+            @C.foo.getter
+            def foo(self): return super().foo
+        self.assertEqual(D().foo, 3)
+
+    def test_abstractclassmethod_basics(self):
+        @classmethod
+        @abc.abstractmethod
+        def foo(cls): pass
+        self.assertTrue(foo.__isabstractmethod__)
+        @classmethod
+        def bar(cls): pass
+        self.assertFalse(getattr(bar, "__isabstractmethod__", False))
+
+        class C(metaclass=abc.ABCMeta):
+            @classmethod
+            @abc.abstractmethod
+            def foo(cls): return cls.__name__
+        self.assertRaises(TypeError, C)
+        class D(C):
+            @classmethod
+            def foo(cls): return super().foo()
+        self.assertEqual(D.foo(), 'D')
+        self.assertEqual(D().foo(), 'D')
+
+    def test_abstractstaticmethod_basics(self):
+        @staticmethod
+        @abc.abstractmethod
+        def foo(): pass
+        self.assertTrue(foo.__isabstractmethod__)
+        @staticmethod
+        def bar(): pass
+        self.assertFalse(getattr(bar, "__isabstractmethod__", False))
+
+        class C(metaclass=abc.ABCMeta):
+            @staticmethod
+            @abc.abstractmethod
+            def foo(): return 3
+        self.assertRaises(TypeError, C)
+        class D(C):
+            @staticmethod
+            def foo(): return 4
+        self.assertEqual(D.foo(), 4)
+        self.assertEqual(D().foo(), 4)
+
+    def test_abstractmethod_integration(self):
+        for abstractthing in [abc.abstractmethod, abc.abstractproperty,
+                              abc.abstractclassmethod,
+                              abc.abstractstaticmethod]:
+            class C(metaclass=abc.ABCMeta):
+                @abstractthing
+                def foo(self): pass  # abstract
+                def bar(self): pass  # concrete
+            self.assertEqual(C.__abstractmethods__, {"foo"})
+            self.assertRaises(TypeError, C)  # because foo is abstract
+            self.assertTrue(isabstract(C))
+            class D(C):
+                def bar(self): pass  # concrete override of concrete
+            self.assertEqual(D.__abstractmethods__, {"foo"})
+            self.assertRaises(TypeError, D)  # because foo is still abstract
+            self.assertTrue(isabstract(D))
+            class E(D):
+                def foo(self): pass
+            self.assertEqual(E.__abstractmethods__, set())
+            E()  # now foo is concrete, too
+            self.assertFalse(isabstract(E))
+            class F(E):
+                @abstractthing
+                def bar(self): pass  # abstract override of concrete
+            self.assertEqual(F.__abstractmethods__, {"bar"})
+            self.assertRaises(TypeError, F)  # because bar is abstract now
+            self.assertTrue(isabstract(F))
+
+    def test_descriptors_with_abstractmethod(self):
+        class C(metaclass=abc.ABCMeta):
+            @property
+            @abc.abstractmethod
+            def foo(self): return 3
+            @foo.setter
+            @abc.abstractmethod
+            def foo(self, val): pass
+        self.assertRaises(TypeError, C)
+        class D(C):
+            @C.foo.getter
+            def foo(self): return super().foo
+        self.assertRaises(TypeError, D)
+        class E(D):
+            @D.foo.setter
+            def foo(self, val): pass
+        self.assertEqual(E().foo, 3)
+        # check that the property's __isabstractmethod__ descriptor does the
+        # right thing when presented with a value that fails truth testing:
+        class NotBool(object):
+            def __nonzero__(self):
+                raise ValueError()
+            __len__ = __nonzero__
+        with self.assertRaises(ValueError):
+            class F(C):
+                def bar(self):
+                    pass
+                bar.__isabstractmethod__ = NotBool()
+                foo = property(bar)
+
+
+    def test_customdescriptors_with_abstractmethod(self):
+        class Descriptor:
+            def __init__(self, fget, fset=None):
+                self._fget = fget
+                self._fset = fset
+            def getter(self, callable):
+                return Descriptor(callable, self._fget)
+            def setter(self, callable):
+                return Descriptor(self._fget, callable)
+            @property
+            def __isabstractmethod__(self):
+                return (getattr(self._fget, '__isabstractmethod__', False)
+                        or getattr(self._fset, '__isabstractmethod__', False))
+        class C(metaclass=abc.ABCMeta):
+            @Descriptor
+            @abc.abstractmethod
+            def foo(self): return 3
+            @foo.setter
+            @abc.abstractmethod
+            def foo(self, val): pass
+        self.assertRaises(TypeError, C)
+        class D(C):
+            @C.foo.getter
+            def foo(self): return super().foo
+        self.assertRaises(TypeError, D)
+        class E(D):
+            @D.foo.setter
+            def foo(self, val): pass
+        self.assertFalse(E.foo.__isabstractmethod__)
+
     def test_metaclass_abc(self):
         # Metaclasses can be ABCs, too.
         class A(metaclass=abc.ABCMeta):
@@ -121,11 +273,32 @@ class TestABC(unittest.TestCase):
         self.assertFalse(issubclass(B, (A,)))
         self.assertNotIsInstance(b, A)
         self.assertNotIsInstance(b, (A,))
-        A.register(B)
+        B1 = A.register(B)
+        self.assertTrue(issubclass(B, A))
+        self.assertTrue(issubclass(B, (A,)))
+        self.assertIsInstance(b, A)
+        self.assertIsInstance(b, (A,))
+        self.assertIs(B1, B)
+        class C(B):
+            pass
+        c = C()
+        self.assertTrue(issubclass(C, A))
+        self.assertTrue(issubclass(C, (A,)))
+        self.assertIsInstance(c, A)
+        self.assertIsInstance(c, (A,))
+
+    def test_register_as_class_deco(self):
+        class A(metaclass=abc.ABCMeta):
+            pass
+        @A.register
+        class B(object):
+            pass
+        b = B()
         self.assertTrue(issubclass(B, A))
         self.assertTrue(issubclass(B, (A,)))
         self.assertIsInstance(b, A)
         self.assertIsInstance(b, (A,))
+        @A.register
         class C(B):
             pass
         c = C()
@@ -133,6 +306,7 @@ class TestABC(unittest.TestCase):
         self.assertTrue(issubclass(C, (A,)))
         self.assertIsInstance(c, A)
         self.assertIsInstance(c, (A,))
+        self.assertIs(C, A.register(C))
 
     def test_isinstance_invalidation(self):
         class A(metaclass=abc.ABCMeta):
diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py
index 2a396cdad1..253e6f082c 100644
--- a/Lib/test/test_abstract_numbers.py
+++ b/Lib/test/test_abstract_numbers.py
@@ -14,6 +14,7 @@ class TestNumbers(unittest.TestCase):
         self.assertEqual(7, int(7).real)
         self.assertEqual(0, int(7).imag)
         self.assertEqual(7, int(7).conjugate())
+        self.assertEqual(-7, int(-7).conjugate())
         self.assertEqual(7, int(7).numerator)
         self.assertEqual(1, int(7).denominator)
 
@@ -24,6 +25,7 @@ class TestNumbers(unittest.TestCase):
         self.assertEqual(7.3, float(7.3).real)
         self.assertEqual(0, float(7.3).imag)
         self.assertEqual(7.3, float(7.3).conjugate())
+        self.assertEqual(-7.3, float(-7.3).conjugate())
 
     def test_complex(self):
         self.assertFalse(issubclass(complex, Real))
diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py
index 0b19af6a10..9c0e7b96ce 100644
--- a/Lib/test/test_aifc.py
+++ b/Lib/test/test_aifc.py
@@ -1,4 +1,4 @@
-from test.support import findfile, run_unittest, TESTFN, captured_stdout, unlink
+from test.support import findfile, run_unittest, TESTFN, unlink
 import unittest
 import os
 import io
@@ -214,11 +214,8 @@ class AIFCLowLevelTest(unittest.TestCase):
         b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0)
         b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8
         b += b'MARK' + struct.pack('>LhB', 3, 1, 1)
-        with captured_stdout() as s:
+        with self.assertWarns(UserWarning):
             f = aifc.open(io.BytesIO(b))
-        self.assertEqual(
-            s.getvalue(),
-            'Warning: MARK chunk contains only 0 markers instead of 1\n')
         self.assertEqual(f.getmarkers(), None)
 
     def test_read_comm_kludge_compname_even(self):
@@ -226,9 +223,8 @@ class AIFCLowLevelTest(unittest.TestCase):
         b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0)
         b += b'NONE' + struct.pack('B', 4) + b'even' + b'\x00'
         b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8
-        with captured_stdout() as s:
+        with self.assertWarns(UserWarning):
             f = aifc.open(io.BytesIO(b))
-        self.assertEqual(s.getvalue(), 'Warning: bad COMM chunk size\n')
         self.assertEqual(f.getcompname(), b'even')
 
     def test_read_comm_kludge_compname_odd(self):
@@ -236,9 +232,8 @@ class AIFCLowLevelTest(unittest.TestCase):
         b += b'COMM' + struct.pack('>LhlhhLL', 18, 0, 0, 0, 0, 0, 0)
         b += b'NONE' + struct.pack('B', 3) + b'odd'
         b += b'SSND' + struct.pack('>L', 8) + b'\x00' * 8
-        with captured_stdout() as s:
+        with self.assertWarns(UserWarning):
             f = aifc.open(io.BytesIO(b))
-        self.assertEqual(s.getvalue(), 'Warning: bad COMM chunk size\n')
         self.assertEqual(f.getcompname(), b'odd')
 
     def test_write_params_raises(self):
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 22c26cc103..c06c940bf2 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -1323,20 +1323,21 @@ class TestParserDefaultSuppress(ParserTestCase):
 class TestParserDefault42(ParserTestCase):
     """Test actions with a parser-level default of 42"""
 
-    parser_signature = Sig(argument_default=42, version='1.0')
+    parser_signature = Sig(argument_default=42)
     argument_signatures = [
+        Sig('--version', action='version', version='1.0'),
         Sig('foo', nargs='?'),
         Sig('bar', nargs='*'),
         Sig('--baz', action='store_true'),
     ]
     failures = ['-x']
     successes = [
-        ('', NS(foo=42, bar=42, baz=42)),
-        ('a', NS(foo='a', bar=42, baz=42)),
-        ('a b', NS(foo='a', bar=['b'], baz=42)),
-        ('--baz', NS(foo=42, bar=42, baz=True)),
-        ('a --baz', NS(foo='a', bar=42, baz=True)),
-        ('--baz a b', NS(foo='a', bar=['b'], baz=True)),
+        ('', NS(foo=42, bar=42, baz=42, version=42)),
+        ('a', NS(foo='a', bar=42, baz=42, version=42)),
+        ('a b', NS(foo='a', bar=['b'], baz=42, version=42)),
+        ('--baz', NS(foo=42, bar=42, baz=True, version=42)),
+        ('a --baz', NS(foo='a', bar=42, baz=True, version=42)),
+        ('--baz a b', NS(foo='a', bar=['b'], baz=True, version=42)),
     ]
 
 
@@ -2927,10 +2928,9 @@ class TestHelpFormattingMetaclass(type):
                 parser_text = sfile.getvalue()
                 self._test(tester, parser_text)
 
-        # add tests for {format,print}_{usage,help,version}
+        # add tests for {format,print}_{usage,help}
         for func_suffix, std_name in [('usage', 'stdout'),
-                                      ('help', 'stdout'),
-                                      ('version', 'stderr')]:
+                                      ('help', 'stdout')]:
             AddTests(cls, func_suffix, std_name)
 
 bases = TestCase,
@@ -2941,8 +2941,9 @@ class TestHelpBiggerOptionals(HelpTestCase):
     """Make sure that argument help aligns when options are longer"""
 
     parser_signature = Sig(prog='PROG', description='DESCRIPTION',
-                           epilog='EPILOG', version='0.1')
+                           epilog='EPILOG')
     argument_signatures = [
+        Sig('-v', '--version', action='version', version='0.1'),
         Sig('-x', action='store_true', help='X HELP'),
         Sig('--y', help='Y HELP'),
         Sig('foo', help='FOO HELP'),
@@ -2977,8 +2978,9 @@ class TestHelpBiggerOptionalGroups(HelpTestCase):
     """Make sure that argument help aligns when options are longer"""
 
     parser_signature = Sig(prog='PROG', description='DESCRIPTION',
-                           epilog='EPILOG', version='0.1')
+                           epilog='EPILOG')
     argument_signatures = [
+        Sig('-v', '--version', action='version', version='0.1'),
         Sig('-x', action='store_true', help='X HELP'),
         Sig('--y', help='Y HELP'),
         Sig('foo', help='FOO HELP'),
@@ -3145,9 +3147,9 @@ HHAAHHH
 class TestHelpWrappingLongNames(HelpTestCase):
     """Make sure that text after long names starts on the next line"""
 
-    parser_signature = Sig(usage='USAGE', description= 'D D' * 30,
-                           version='V V'*30)
+    parser_signature = Sig(usage='USAGE', description= 'D D' * 30)
     argument_signatures = [
+        Sig('-v', '--version', action='version', version='V V' * 30),
         Sig('-x', metavar='X' * 25, help='XH XH' * 20),
         Sig('y', metavar='y' * 25, help='YH YH' * 20),
     ]
@@ -3750,8 +3752,9 @@ class TestHelpNoHelpOptional(HelpTestCase):
 class TestHelpVersionOptional(HelpTestCase):
     """Test that the --version argument can be suppressed help messages"""
 
-    parser_signature = Sig(prog='PROG', version='1.0')
+    parser_signature = Sig(prog='PROG')
     argument_signatures = [
+        Sig('-v', '--version', action='version', version='1.0'),
         Sig('--foo', help='foo help'),
         Sig('spam', help='spam help'),
     ]
@@ -3984,8 +3987,8 @@ class TestHelpVersionAction(HelpTestCase):
 class TestHelpSubparsersOrdering(HelpTestCase):
     """Test ordering of subcommands in help matches the code"""
     parser_signature = Sig(prog='PROG',
-                           description='display some subcommands',
-                           version='0.1')
+                           description='display some subcommands')
+    argument_signatures = [Sig('-v', '--version', action='version', version='0.1')]
 
     subparsers_signatures = [Sig(name=name)
                              for name in ('a', 'b', 'c', 'd', 'e')]
@@ -4013,8 +4016,8 @@ class TestHelpSubparsersOrdering(HelpTestCase):
 class TestHelpSubparsersWithHelpOrdering(HelpTestCase):
     """Test ordering of subcommands in help matches the code"""
     parser_signature = Sig(prog='PROG',
-                           description='display some subcommands',
-                           version='0.1')
+                           description='display some subcommands')
+    argument_signatures = [Sig('-v', '--version', action='version', version='0.1')]
 
     subcommand_data = (('a', 'a subcommand help'),
                        ('b', 'b subcommand help'),
@@ -4052,6 +4055,37 @@ class TestHelpSubparsersWithHelpOrdering(HelpTestCase):
         '''
 
 
+
+class TestHelpMetavarTypeFormatter(HelpTestCase):
+    """"""
+
+    def custom_type(string):
+        return string
+
+    parser_signature = Sig(prog='PROG', description='description',
+                           formatter_class=argparse.MetavarTypeHelpFormatter)
+    argument_signatures = [Sig('a', type=int),
+                           Sig('-b', type=custom_type),
+                           Sig('-c', type=float, metavar='SOME FLOAT')]
+    argument_group_signatures = []
+    usage = '''\
+        usage: PROG [-h] [-b custom_type] [-c SOME FLOAT] int
+        '''
+    help = usage + '''\
+
+        description
+
+        positional arguments:
+          int
+
+        optional arguments:
+          -h, --help      show this help message and exit
+          -b custom_type
+          -c SOME FLOAT
+        '''
+    version = ''
+
+
 # =====================================
 # Optional/Positional constructor tests
 # =====================================
@@ -4280,32 +4314,28 @@ class TestOptionalsHelpVersionActions(TestCase):
             parser.format_help(),
             self._get_error(parser.parse_args, args_str.split()).stdout)
 
-    def assertPrintVersionExit(self, parser, args_str):
-        self.assertEqual(
-            parser.format_version(),
-            self._get_error(parser.parse_args, args_str.split()).stderr)
-
     def assertArgumentParserError(self, parser, *args):
         self.assertRaises(ArgumentParserError, parser.parse_args, args)
 
     def test_version(self):
-        parser = ErrorRaisingArgumentParser(version='1.0')
+        parser = ErrorRaisingArgumentParser()
+        parser.add_argument('-v', '--version', action='version', version='1.0')
         self.assertPrintHelpExit(parser, '-h')
         self.assertPrintHelpExit(parser, '--help')
-        self.assertPrintVersionExit(parser, '-v')
-        self.assertPrintVersionExit(parser, '--version')
+        self.assertRaises(AttributeError, getattr, parser, 'format_version')
 
     def test_version_format(self):
-        parser = ErrorRaisingArgumentParser(prog='PPP', version='%(prog)s 3.5')
+        parser = ErrorRaisingArgumentParser(prog='PPP')
+        parser.add_argument('-v', '--version', action='version', version='%(prog)s 3.5')
         msg = self._get_error(parser.parse_args, ['-v']).stderr
         self.assertEqual('PPP 3.5\n', msg)
 
     def test_version_no_help(self):
-        parser = ErrorRaisingArgumentParser(add_help=False, version='1.0')
+        parser = ErrorRaisingArgumentParser(add_help=False)
+        parser.add_argument('-v', '--version', action='version', version='1.0')
         self.assertArgumentParserError(parser, '-h')
         self.assertArgumentParserError(parser, '--help')
-        self.assertPrintVersionExit(parser, '-v')
-        self.assertPrintVersionExit(parser, '--version')
+        self.assertRaises(AttributeError, getattr, parser, 'format_version')
 
     def test_version_action(self):
         parser = ErrorRaisingArgumentParser(prog='XXX')
@@ -4325,12 +4355,13 @@ class TestOptionalsHelpVersionActions(TestCase):
         parser.add_argument('-x', action='help')
         parser.add_argument('-y', action='version')
         self.assertPrintHelpExit(parser, '-x')
-        self.assertPrintVersionExit(parser, '-y')
         self.assertArgumentParserError(parser, '-v')
         self.assertArgumentParserError(parser, '--version')
+        self.assertRaises(AttributeError, getattr, parser, 'format_version')
 
     def test_help_version_extra_arguments(self):
-        parser = ErrorRaisingArgumentParser(version='1.0')
+        parser = ErrorRaisingArgumentParser()
+        parser.add_argument('--version', action='version', version='1.0')
         parser.add_argument('-x', action='store_true')
         parser.add_argument('y')
 
@@ -4342,8 +4373,7 @@ class TestOptionalsHelpVersionActions(TestCase):
                 format = '%s %%s %s' % (prefix, suffix)
             self.assertPrintHelpExit(parser, format % '-h')
             self.assertPrintHelpExit(parser, format % '--help')
-            self.assertPrintVersionExit(parser, format % '-v')
-            self.assertPrintVersionExit(parser, format % '--version')
+            self.assertRaises(AttributeError, getattr, parser, 'format_version')
 
 
 # ======================
@@ -4398,7 +4428,7 @@ class TestStrings(TestCase):
         parser = argparse.ArgumentParser(prog='PROG')
         string = (
             "ArgumentParser(prog='PROG', usage=None, description=None, "
-            "version=None, formatter_class=%r, conflict_handler='error', "
+            "formatter_class=%r, conflict_handler='error', "
             "add_help=True)" % argparse.HelpFormatter)
         self.assertStringEqual(parser, string)
 
@@ -4442,7 +4472,7 @@ class TestEncoding(TestCase):
     def _test_module_encoding(self, path):
         path, _ = os.path.splitext(path)
         path += ".py"
-        with codecs.open(path, 'r', 'utf8') as f:
+        with codecs.open(path, 'r', 'utf-8') as f:
             f.read()
 
     def test_argparse_module_encoding(self):
@@ -4484,6 +4514,67 @@ class TestArgumentTypeError(TestCase):
         else:
             self.fail()
 
+# =========================
+# MessageContentError tests
+# =========================
+
+class TestMessageContentError(TestCase):
+
+    def test_missing_argument_name_in_message(self):
+        parser = ErrorRaisingArgumentParser(prog='PROG', usage='')
+        parser.add_argument('req_pos', type=str)
+        parser.add_argument('-req_opt', type=int, required=True)
+        parser.add_argument('need_one', type=str, nargs='+')
+
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_args([])
+        msg = str(cm.exception)
+        self.assertRegex(msg, 'req_pos')
+        self.assertRegex(msg, 'req_opt')
+        self.assertRegex(msg, 'need_one')
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_args(['myXargument'])
+        msg = str(cm.exception)
+        self.assertNotIn(msg, 'req_pos')
+        self.assertRegex(msg, 'req_opt')
+        self.assertRegex(msg, 'need_one')
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_args(['myXargument', '-req_opt=1'])
+        msg = str(cm.exception)
+        self.assertNotIn(msg, 'req_pos')
+        self.assertNotIn(msg, 'req_opt')
+        self.assertRegex(msg, 'need_one')
+
+    def test_optional_optional_not_in_message(self):
+        parser = ErrorRaisingArgumentParser(prog='PROG', usage='')
+        parser.add_argument('req_pos', type=str)
+        parser.add_argument('--req_opt', type=int, required=True)
+        parser.add_argument('--opt_opt', type=bool, nargs='?',
+                            default=True)
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_args([])
+        msg = str(cm.exception)
+        self.assertRegex(msg, 'req_pos')
+        self.assertRegex(msg, 'req_opt')
+        self.assertNotIn(msg, 'opt_opt')
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_args(['--req_opt=1'])
+        msg = str(cm.exception)
+        self.assertRegex(msg, 'req_pos')
+        self.assertNotIn(msg, 'req_opt')
+        self.assertNotIn(msg, 'opt_opt')
+
+    def test_optional_positional_not_in_message(self):
+        parser = ErrorRaisingArgumentParser(prog='PROG', usage='')
+        parser.add_argument('req_pos')
+        parser.add_argument('optional_positional', nargs='?', default='eggs')
+        with self.assertRaises(ArgumentParserError) as cm:
+            parser.parse_args([])
+        msg = str(cm.exception)
+        self.assertRegex(msg, 'req_pos')
+        self.assertNotIn(msg, 'optional_positional')
+
+
 # ================================================
 # Check that the type function is called only once
 # ================================================
@@ -4782,13 +4873,7 @@ class TestImportStar(TestCase):
         self.assertEqual(sorted(items), sorted(argparse.__all__))
 
 def test_main():
-    # silence warnings about version argument - these are expected
-    with support.check_warnings(
-            ('The "version" argument to ArgumentParser is deprecated.',
-             DeprecationWarning),
-            ('The (format|print)_version method is deprecated',
-             DeprecationWarning)):
-        support.run_unittest(__name__)
+    support.run_unittest(__name__)
     # Remove global references to avoid looking like we have refleaks.
     RFile.seen = {}
     WFile.seen = set()
diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
index e26e9add05..e004c0ee46 100755
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -16,6 +16,24 @@ import warnings
 import array
 from array import _array_reconstructor as array_reconstructor
 
+try:
+    # Try to determine availability of long long independently
+    # of the array module under test
+    struct.calcsize('@q')
+    have_long_long = True
+except struct.error:
+    have_long_long = False
+
+try:
+    import ctypes
+    sizeof_wchar = ctypes.sizeof(ctypes.c_wchar)
+except ImportError:
+    import sys
+    if sys.platform == 'win32':
+        sizeof_wchar = 2
+    else:
+        sizeof_wchar = 4
+
 
 class ArraySubclass(array.array):
     pass
@@ -24,8 +42,9 @@ class ArraySubclassWithKwargs(array.array):
     def __init__(self, typecode, newarg=None):
         array.array.__init__(self)
 
-tests = [] # list to accumulate all tests
 typecodes = "ubBhHiIlLfd"
+if have_long_long:
+    typecodes += 'qQ'
 
 class BadConstructorTest(unittest.TestCase):
 
@@ -35,7 +54,6 @@ class BadConstructorTest(unittest.TestCase):
         self.assertRaises(TypeError, array.array, 'xx')
         self.assertRaises(ValueError, array.array, 'x')
 
-tests.append(BadConstructorTest)
 
 # Machine format codes.
 #
@@ -165,10 +183,7 @@ class ArrayReconstructorTest(unittest.TestCase):
                 msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase))
 
 
-tests.append(ArrayReconstructorTest)
-
-
-class BaseTest(unittest.TestCase):
+class BaseTest:
     # Required class attributes (provided by subclasses
     # typecode: the typecode to test
     # example: an initializer usable in the constructor for this type
@@ -209,10 +224,14 @@ class BaseTest(unittest.TestCase):
         self.assertEqual(bi[1], len(a))
 
     def test_byteswap(self):
-        a = array.array(self.typecode, self.example)
+        if self.typecode == 'u':
+            example = '\U00100100'
+        else:
+            example = self.example
+        a = array.array(self.typecode, example)
         self.assertRaises(TypeError, a.byteswap, 42)
         if a.itemsize in (1, 2, 4, 8):
-            b = array.array(self.typecode, self.example)
+            b = array.array(self.typecode, example)
             b.byteswap()
             if a.itemsize==1:
                 self.assertEqual(a, b)
@@ -272,6 +291,20 @@ class BaseTest(unittest.TestCase):
             self.assertEqual(a.x, b.x)
             self.assertEqual(type(a), type(b))
 
+    def test_iterator_pickle(self):
+        data = array.array(self.typecode, self.example)
+        orgit = iter(data)
+        d = pickle.dumps(orgit)
+        it = pickle.loads(d)
+        self.assertEqual(type(orgit), type(it))
+        self.assertEqual(list(it), list(data))
+
+        if len(data):
+            it = pickle.loads(d)
+            next(it)
+            d = pickle.dumps(it)
+            self.assertEqual(list(it), list(data)[1:])
+
     def test_insert(self):
         a = array.array(self.typecode, self.example)
         a.insert(0, self.example[0])
@@ -991,14 +1024,14 @@ class BaseTest(unittest.TestCase):
     @support.cpython_only
     def test_sizeof_with_buffer(self):
         a = array.array(self.typecode, self.example)
-        basesize = support.calcvobjsize('4Pi')
+        basesize = support.calcvobjsize('Pn2Pi')
         buffer_size = a.buffer_info()[1] * a.itemsize
         support.check_sizeof(self, a, basesize + buffer_size)
 
     @support.cpython_only
     def test_sizeof_without_buffer(self):
         a = array.array(self.typecode)
-        basesize = support.calcvobjsize('4Pi')
+        basesize = support.calcvobjsize('Pn2Pi')
         support.check_sizeof(self, a, basesize)
 
 
@@ -1009,7 +1042,7 @@ class StringTest(BaseTest):
         a = array.array(self.typecode, self.example)
         self.assertRaises(TypeError, a.__setitem__, 0, self.example[:2])
 
-class UnicodeTest(StringTest):
+class UnicodeTest(StringTest, unittest.TestCase):
     typecode = 'u'
     example = '\x01\u263a\x00\ufeff'
     smallerexample = '\x01\u263a\x00\ufefe'
@@ -1027,6 +1060,7 @@ class UnicodeTest(StringTest):
         a.fromunicode('\x11abc\xff\u1234')
         s = a.tounicode()
         self.assertEqual(s, '\xa0\xc2\u1234 \x11abc\xff\u1234')
+        self.assertEqual(a.itemsize, sizeof_wchar)
 
         s = '\x00="\'a\\b\x80\xff\u0000\u0001\u1234'
         a = array.array('u', s)
@@ -1036,7 +1070,17 @@ class UnicodeTest(StringTest):
 
         self.assertRaises(TypeError, a.fromunicode)
 
-tests.append(UnicodeTest)
+    def test_issue17223(self):
+        # this used to crash
+        if sizeof_wchar == 4:
+            # U+FFFFFFFF is an invalid code point in Unicode 6.0
+            invalid_str = b'\xff\xff\xff\xff'
+        else:
+            # invalid UTF-16 surrogate pair
+            invalid_str = b'\xff\xdf\x61\x00'
+        a = array.array('u', invalid_str)
+        self.assertRaises(ValueError, a.tounicode)
+        self.assertRaises(ValueError, str, a)
 
 class NumberTest(BaseTest):
 
@@ -1178,45 +1222,47 @@ class UnsignedNumberTest(NumberTest):
         )
 
 
-class ByteTest(SignedNumberTest):
+class ByteTest(SignedNumberTest, unittest.TestCase):
     typecode = 'b'
     minitemsize = 1
-tests.append(ByteTest)
 
-class UnsignedByteTest(UnsignedNumberTest):
+class UnsignedByteTest(UnsignedNumberTest, unittest.TestCase):
     typecode = 'B'
     minitemsize = 1
-tests.append(UnsignedByteTest)
 
-class ShortTest(SignedNumberTest):
+class ShortTest(SignedNumberTest, unittest.TestCase):
     typecode = 'h'
     minitemsize = 2
-tests.append(ShortTest)
 
-class UnsignedShortTest(UnsignedNumberTest):
+class UnsignedShortTest(UnsignedNumberTest, unittest.TestCase):
     typecode = 'H'
     minitemsize = 2
-tests.append(UnsignedShortTest)
 
-class IntTest(SignedNumberTest):
+class IntTest(SignedNumberTest, unittest.TestCase):
     typecode = 'i'
     minitemsize = 2
-tests.append(IntTest)
 
-class UnsignedIntTest(UnsignedNumberTest):
+class UnsignedIntTest(UnsignedNumberTest, unittest.TestCase):
     typecode = 'I'
     minitemsize = 2
-tests.append(UnsignedIntTest)
 
-class LongTest(SignedNumberTest):
+class LongTest(SignedNumberTest, unittest.TestCase):
     typecode = 'l'
     minitemsize = 4
-tests.append(LongTest)
 
-class UnsignedLongTest(UnsignedNumberTest):
+class UnsignedLongTest(UnsignedNumberTest, unittest.TestCase):
     typecode = 'L'
     minitemsize = 4
-tests.append(UnsignedLongTest)
+
+@unittest.skipIf(not have_long_long, 'need long long support')
+class LongLongTest(SignedNumberTest, unittest.TestCase):
+    typecode = 'q'
+    minitemsize = 8
+
+@unittest.skipIf(not have_long_long, 'need long long support')
+class UnsignedLongLongTest(UnsignedNumberTest, unittest.TestCase):
+    typecode = 'Q'
+    minitemsize = 8
 
 class FPTest(NumberTest):
     example = [-42.0, 0, 42, 1e5, -1e10]
@@ -1243,12 +1289,11 @@ class FPTest(NumberTest):
             b.byteswap()
             self.assertEqual(a, b)
 
-class FloatTest(FPTest):
+class FloatTest(FPTest, unittest.TestCase):
     typecode = 'f'
     minitemsize = 4
-tests.append(FloatTest)
 
-class DoubleTest(FPTest):
+class DoubleTest(FPTest, unittest.TestCase):
     typecode = 'd'
     minitemsize = 8
 
@@ -1269,22 +1314,6 @@ class DoubleTest(FPTest):
         else:
             self.fail("Array of size > maxsize created - MemoryError expected")
 
-tests.append(DoubleTest)
-
-def test_main(verbose=None):
-    import sys
-
-    support.run_unittest(*tests)
-
-    # verify reference counting
-    if verbose and hasattr(sys, "gettotalrefcount"):
-        import gc
-        counts = [None] * 5
-        for i in range(len(counts)):
-            support.run_unittest(*tests)
-            gc.collect()
-            counts[i] = sys.gettotalrefcount()
-        print(counts)
 
 if __name__ == "__main__":
-    test_main(verbose=True)
+    unittest.main()
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 2887092a7d..dc24126b21 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -1,6 +1,10 @@
-import sys, unittest
-from test import support
+import os
+import sys
+import unittest
 import ast
+import weakref
+
+from test import support
 
 def to_tuple(t):
     if t is None or isinstance(t, (str, int, complex)):
@@ -52,6 +56,9 @@ exec_tests = [
     "while v:pass",
     # If
     "if v:pass",
+    # With
+    "with x as y: pass",
+    "with x as y, z as q: pass",
     # Raise
     "raise Exception('string')",
     # TryExcept
@@ -191,6 +198,9 @@ class AST_Tests(unittest.TestCase):
     def test_AST_objects(self):
         x = ast.AST()
         self.assertEqual(x._fields, ())
+        x.foobar = 42
+        self.assertEqual(x.foobar, 42)
+        self.assertEqual(x.__dict__["foobar"], 42)
 
         with self.assertRaises(AttributeError):
             x.vararg
@@ -199,6 +209,17 @@ class AST_Tests(unittest.TestCase):
             # "_ast.AST constructor takes 0 positional arguments"
             ast.AST(2)
 
+    def test_AST_garbage_collection(self):
+        class X:
+            pass
+        a = ast.AST()
+        a.x = X()
+        a.x.a = a
+        ref = weakref.ref(a.x)
+        del a
+        support.gc_collect()
+        self.assertIsNone(ref())
+
     def test_snippets(self):
         for input, output, kind in ((exec_tests, exec_results, "exec"),
                                     (single_tests, single_results, "single"),
@@ -378,6 +399,14 @@ class AST_Tests(unittest.TestCase):
             compile(m, "<test>", "exec")
         self.assertIn("string must be of type str", str(cm.exception))
 
+    def test_empty_yield_from(self):
+        # Issue 16546: yield from value is not optional.
+        empty_yield_from = ast.parse("def f():\n yield from g()")
+        empty_yield_from.body[0].body[0].value.value = None
+        with self.assertRaises(ValueError) as cm:
+            compile(empty_yield_from, "<test>", "exec")
+        self.assertIn("field value is required", str(cm.exception))
+
 
 class ASTHelpers_Test(unittest.TestCase):
 
@@ -390,7 +419,9 @@ class ASTHelpers_Test(unittest.TestCase):
         try:
             1/0
         except Exception:
-            self.assertRaises(SyntaxError, ast.parse, r"'\U'")
+            with self.assertRaises(SyntaxError) as e:
+                ast.literal_eval(r"'\U'")
+            self.assertIsNotNone(e.exception.__context__)
 
     def test_dump(self):
         node = ast.parse('spam(eggs, "and cheese")')
@@ -504,8 +535,413 @@ class ASTHelpers_Test(unittest.TestCase):
         self.assertIn("invalid integer value: None", str(cm.exception))
 
 
+class ASTValidatorTests(unittest.TestCase):
+
+    def mod(self, mod, msg=None, mode="exec", *, exc=ValueError):
+        mod.lineno = mod.col_offset = 0
+        ast.fix_missing_locations(mod)
+        with self.assertRaises(exc) as cm:
+            compile(mod, "<test>", mode)
+        if msg is not None:
+            self.assertIn(msg, str(cm.exception))
+
+    def expr(self, node, msg=None, *, exc=ValueError):
+        mod = ast.Module([ast.Expr(node)])
+        self.mod(mod, msg, exc=exc)
+
+    def stmt(self, stmt, msg=None):
+        mod = ast.Module([stmt])
+        self.mod(mod, msg)
+
+    def test_module(self):
+        m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))])
+        self.mod(m, "must have Load context", "single")
+        m = ast.Expression(ast.Name("x", ast.Store()))
+        self.mod(m, "must have Load context", "eval")
+
+    def _check_arguments(self, fac, check):
+        def arguments(args=None, vararg=None, varargannotation=None,
+                      kwonlyargs=None, kwarg=None, kwargannotation=None,
+                      defaults=None, kw_defaults=None):
+            if args is None:
+                args = []
+            if kwonlyargs is None:
+                kwonlyargs = []
+            if defaults is None:
+                defaults = []
+            if kw_defaults is None:
+                kw_defaults = []
+            args = ast.arguments(args, vararg, varargannotation, kwonlyargs,
+                                 kwarg, kwargannotation, defaults, kw_defaults)
+            return fac(args)
+        args = [ast.arg("x", ast.Name("x", ast.Store()))]
+        check(arguments(args=args), "must have Load context")
+        check(arguments(varargannotation=ast.Num(3)),
+              "varargannotation but no vararg")
+        check(arguments(varargannotation=ast.Name("x", ast.Store()), vararg="x"),
+                         "must have Load context")
+        check(arguments(kwonlyargs=args), "must have Load context")
+        check(arguments(kwargannotation=ast.Num(42)),
+                       "kwargannotation but no kwarg")
+        check(arguments(kwargannotation=ast.Name("x", ast.Store()),
+                          kwarg="x"), "must have Load context")
+        check(arguments(defaults=[ast.Num(3)]),
+                       "more positional defaults than args")
+        check(arguments(kw_defaults=[ast.Num(4)]),
+                       "length of kwonlyargs is not the same as kw_defaults")
+        args = [ast.arg("x", ast.Name("x", ast.Load()))]
+        check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]),
+                       "must have Load context")
+        args = [ast.arg("a", ast.Name("x", ast.Load())),
+                ast.arg("b", ast.Name("y", ast.Load()))]
+        check(arguments(kwonlyargs=args,
+                          kw_defaults=[None, ast.Name("x", ast.Store())]),
+                          "must have Load context")
+
+    def test_funcdef(self):
+        a = ast.arguments([], None, None, [], None, None, [], [])
+        f = ast.FunctionDef("x", a, [], [], None)
+        self.stmt(f, "empty body on FunctionDef")
+        f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())],
+                            None)
+        self.stmt(f, "must have Load context")
+        f = ast.FunctionDef("x", a, [ast.Pass()], [],
+                            ast.Name("x", ast.Store()))
+        self.stmt(f, "must have Load context")
+        def fac(args):
+            return ast.FunctionDef("x", args, [ast.Pass()], [], None)
+        self._check_arguments(fac, self.stmt)
+
+    def test_classdef(self):
+        def cls(bases=None, keywords=None, starargs=None, kwargs=None,
+                body=None, decorator_list=None):
+            if bases is None:
+                bases = []
+            if keywords is None:
+                keywords = []
+            if body is None:
+                body = [ast.Pass()]
+            if decorator_list is None:
+                decorator_list = []
+            return ast.ClassDef("myclass", bases, keywords, starargs,
+                                kwargs, body, decorator_list)
+        self.stmt(cls(bases=[ast.Name("x", ast.Store())]),
+                  "must have Load context")
+        self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]),
+                  "must have Load context")
+        self.stmt(cls(starargs=ast.Name("x", ast.Store())),
+                  "must have Load context")
+        self.stmt(cls(kwargs=ast.Name("x", ast.Store())),
+                  "must have Load context")
+        self.stmt(cls(body=[]), "empty body on ClassDef")
+        self.stmt(cls(body=[None]), "None disallowed")
+        self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]),
+                  "must have Load context")
+
+    def test_delete(self):
+        self.stmt(ast.Delete([]), "empty targets on Delete")
+        self.stmt(ast.Delete([None]), "None disallowed")
+        self.stmt(ast.Delete([ast.Name("x", ast.Load())]),
+                  "must have Del context")
+
+    def test_assign(self):
+        self.stmt(ast.Assign([], ast.Num(3)), "empty targets on Assign")
+        self.stmt(ast.Assign([None], ast.Num(3)), "None disallowed")
+        self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Num(3)),
+                  "must have Store context")
+        self.stmt(ast.Assign([ast.Name("x", ast.Store())],
+                                ast.Name("y", ast.Store())),
+                  "must have Load context")
+
+    def test_augassign(self):
+        aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(),
+                            ast.Name("y", ast.Load()))
+        self.stmt(aug, "must have Store context")
+        aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(),
+                            ast.Name("y", ast.Store()))
+        self.stmt(aug, "must have Load context")
+
+    def test_for(self):
+        x = ast.Name("x", ast.Store())
+        y = ast.Name("y", ast.Load())
+        p = ast.Pass()
+        self.stmt(ast.For(x, y, [], []), "empty body on For")
+        self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []),
+                  "must have Store context")
+        self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []),
+                  "must have Load context")
+        e = ast.Expr(ast.Name("x", ast.Store()))
+        self.stmt(ast.For(x, y, [e], []), "must have Load context")
+        self.stmt(ast.For(x, y, [p], [e]), "must have Load context")
+
+    def test_while(self):
+        self.stmt(ast.While(ast.Num(3), [], []), "empty body on While")
+        self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []),
+                  "must have Load context")
+        self.stmt(ast.While(ast.Num(3), [ast.Pass()],
+                             [ast.Expr(ast.Name("x", ast.Store()))]),
+                             "must have Load context")
+
+    def test_if(self):
+        self.stmt(ast.If(ast.Num(3), [], []), "empty body on If")
+        i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], [])
+        self.stmt(i, "must have Load context")
+        i = ast.If(ast.Num(3), [ast.Expr(ast.Name("x", ast.Store()))], [])
+        self.stmt(i, "must have Load context")
+        i = ast.If(ast.Num(3), [ast.Pass()],
+                   [ast.Expr(ast.Name("x", ast.Store()))])
+        self.stmt(i, "must have Load context")
+
+    def test_with(self):
+        p = ast.Pass()
+        self.stmt(ast.With([], [p]), "empty items on With")
+        i = ast.withitem(ast.Num(3), None)
+        self.stmt(ast.With([i], []), "empty body on With")
+        i = ast.withitem(ast.Name("x", ast.Store()), None)
+        self.stmt(ast.With([i], [p]), "must have Load context")
+        i = ast.withitem(ast.Num(3), ast.Name("x", ast.Load()))
+        self.stmt(ast.With([i], [p]), "must have Store context")
+
+    def test_raise(self):
+        r = ast.Raise(None, ast.Num(3))
+        self.stmt(r, "Raise with cause but no exception")
+        r = ast.Raise(ast.Name("x", ast.Store()), None)
+        self.stmt(r, "must have Load context")
+        r = ast.Raise(ast.Num(4), ast.Name("x", ast.Store()))
+        self.stmt(r, "must have Load context")
+
+    def test_try(self):
+        p = ast.Pass()
+        t = ast.Try([], [], [], [p])
+        self.stmt(t, "empty body on Try")
+        t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p])
+        self.stmt(t, "must have Load context")
+        t = ast.Try([p], [], [], [])
+        self.stmt(t, "Try has neither except handlers nor finalbody")
+        t = ast.Try([p], [], [p], [p])
+        self.stmt(t, "Try has orelse but no except handlers")
+        t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], [])
+        self.stmt(t, "empty body on ExceptHandler")
+        e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])]
+        self.stmt(ast.Try([p], e, [], []), "must have Load context")
+        e = [ast.ExceptHandler(None, "x", [p])]
+        t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p])
+        self.stmt(t, "must have Load context")
+        t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))])
+        self.stmt(t, "must have Load context")
+
+    def test_assert(self):
+        self.stmt(ast.Assert(ast.Name("x", ast.Store()), None),
+                  "must have Load context")
+        assrt = ast.Assert(ast.Name("x", ast.Load()),
+                           ast.Name("y", ast.Store()))
+        self.stmt(assrt, "must have Load context")
+
+    def test_import(self):
+        self.stmt(ast.Import([]), "empty names on Import")
+
+    def test_importfrom(self):
+        imp = ast.ImportFrom(None, [ast.alias("x", None)], -42)
+        self.stmt(imp, "level less than -1")
+        self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom")
+
+    def test_global(self):
+        self.stmt(ast.Global([]), "empty names on Global")
+
+    def test_nonlocal(self):
+        self.stmt(ast.Nonlocal([]), "empty names on Nonlocal")
+
+    def test_expr(self):
+        e = ast.Expr(ast.Name("x", ast.Store()))
+        self.stmt(e, "must have Load context")
+
+    def test_boolop(self):
+        b = ast.BoolOp(ast.And(), [])
+        self.expr(b, "less than 2 values")
+        b = ast.BoolOp(ast.And(), [ast.Num(3)])
+        self.expr(b, "less than 2 values")
+        b = ast.BoolOp(ast.And(), [ast.Num(4), None])
+        self.expr(b, "None disallowed")
+        b = ast.BoolOp(ast.And(), [ast.Num(4), ast.Name("x", ast.Store())])
+        self.expr(b, "must have Load context")
+
+    def test_unaryop(self):
+        u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store()))
+        self.expr(u, "must have Load context")
+
+    def test_lambda(self):
+        a = ast.arguments([], None, None, [], None, None, [], [])
+        self.expr(ast.Lambda(a, ast.Name("x", ast.Store())),
+                  "must have Load context")
+        def fac(args):
+            return ast.Lambda(args, ast.Name("x", ast.Load()))
+        self._check_arguments(fac, self.expr)
+
+    def test_ifexp(self):
+        l = ast.Name("x", ast.Load())
+        s = ast.Name("y", ast.Store())
+        for args in (s, l, l), (l, s, l), (l, l, s):
+            self.expr(ast.IfExp(*args), "must have Load context")
+
+    def test_dict(self):
+        d = ast.Dict([], [ast.Name("x", ast.Load())])
+        self.expr(d, "same number of keys as values")
+        d = ast.Dict([None], [ast.Name("x", ast.Load())])
+        self.expr(d, "None disallowed")
+        d = ast.Dict([ast.Name("x", ast.Load())], [None])
+        self.expr(d, "None disallowed")
+
+    def test_set(self):
+        self.expr(ast.Set([None]), "None disallowed")
+        s = ast.Set([ast.Name("x", ast.Store())])
+        self.expr(s, "must have Load context")
+
+    def _check_comprehension(self, fac):
+        self.expr(fac([]), "comprehension with no generators")
+        g = ast.comprehension(ast.Name("x", ast.Load()),
+                              ast.Name("x", ast.Load()), [])
+        self.expr(fac([g]), "must have Store context")
+        g = ast.comprehension(ast.Name("x", ast.Store()),
+                              ast.Name("x", ast.Store()), [])
+        self.expr(fac([g]), "must have Load context")
+        x = ast.Name("x", ast.Store())
+        y = ast.Name("y", ast.Load())
+        g = ast.comprehension(x, y, [None])
+        self.expr(fac([g]), "None disallowed")
+        g = ast.comprehension(x, y, [ast.Name("x", ast.Store())])
+        self.expr(fac([g]), "must have Load context")
+
+    def _simple_comp(self, fac):
+        g = ast.comprehension(ast.Name("x", ast.Store()),
+                              ast.Name("x", ast.Load()), [])
+        self.expr(fac(ast.Name("x", ast.Store()), [g]),
+                  "must have Load context")
+        def wrap(gens):
+            return fac(ast.Name("x", ast.Store()), gens)
+        self._check_comprehension(wrap)
+
+    def test_listcomp(self):
+        self._simple_comp(ast.ListComp)
+
+    def test_setcomp(self):
+        self._simple_comp(ast.SetComp)
+
+    def test_generatorexp(self):
+        self._simple_comp(ast.GeneratorExp)
+
+    def test_dictcomp(self):
+        g = ast.comprehension(ast.Name("y", ast.Store()),
+                              ast.Name("p", ast.Load()), [])
+        c = ast.DictComp(ast.Name("x", ast.Store()),
+                         ast.Name("y", ast.Load()), [g])
+        self.expr(c, "must have Load context")
+        c = ast.DictComp(ast.Name("x", ast.Load()),
+                         ast.Name("y", ast.Store()), [g])
+        self.expr(c, "must have Load context")
+        def factory(comps):
+            k = ast.Name("x", ast.Load())
+            v = ast.Name("y", ast.Load())
+            return ast.DictComp(k, v, comps)
+        self._check_comprehension(factory)
+
+    def test_yield(self):
+        self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load")
+        self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load")
+
+    def test_compare(self):
+        left = ast.Name("x", ast.Load())
+        comp = ast.Compare(left, [ast.In()], [])
+        self.expr(comp, "no comparators")
+        comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)])
+        self.expr(comp, "different number of comparators and operands")
+        comp = ast.Compare(ast.Num("blah"), [ast.In()], [left])
+        self.expr(comp, "non-numeric", exc=TypeError)
+        comp = ast.Compare(left, [ast.In()], [ast.Num("blah")])
+        self.expr(comp, "non-numeric", exc=TypeError)
+
+    def test_call(self):
+        func = ast.Name("x", ast.Load())
+        args = [ast.Name("y", ast.Load())]
+        keywords = [ast.keyword("w", ast.Name("z", ast.Load()))]
+        stararg = ast.Name("p", ast.Load())
+        kwarg = ast.Name("q", ast.Load())
+        call = ast.Call(ast.Name("x", ast.Store()), args, keywords, stararg,
+                        kwarg)
+        self.expr(call, "must have Load context")
+        call = ast.Call(func, [None], keywords, stararg, kwarg)
+        self.expr(call, "None disallowed")
+        bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))]
+        call = ast.Call(func, args, bad_keywords, stararg, kwarg)
+        self.expr(call, "must have Load context")
+        call = ast.Call(func, args, keywords, ast.Name("z", ast.Store()), kwarg)
+        self.expr(call, "must have Load context")
+        call = ast.Call(func, args, keywords, stararg,
+                        ast.Name("w", ast.Store()))
+        self.expr(call, "must have Load context")
+
+    def test_num(self):
+        class subint(int):
+            pass
+        class subfloat(float):
+            pass
+        class subcomplex(complex):
+            pass
+        for obj in "0", "hello", subint(), subfloat(), subcomplex():
+            self.expr(ast.Num(obj), "non-numeric", exc=TypeError)
+
+    def test_attribute(self):
+        attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load())
+        self.expr(attr, "must have Load context")
+
+    def test_subscript(self):
+        sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Index(ast.Num(3)),
+                            ast.Load())
+        self.expr(sub, "must have Load context")
+        x = ast.Name("x", ast.Load())
+        sub = ast.Subscript(x, ast.Index(ast.Name("y", ast.Store())),
+                            ast.Load())
+        self.expr(sub, "must have Load context")
+        s = ast.Name("x", ast.Store())
+        for args in (s, None, None), (None, s, None), (None, None, s):
+            sl = ast.Slice(*args)
+            self.expr(ast.Subscript(x, sl, ast.Load()),
+                      "must have Load context")
+        sl = ast.ExtSlice([])
+        self.expr(ast.Subscript(x, sl, ast.Load()), "empty dims on ExtSlice")
+        sl = ast.ExtSlice([ast.Index(s)])
+        self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context")
+
+    def test_starred(self):
+        left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())],
+                        ast.Store())
+        assign = ast.Assign([left], ast.Num(4))
+        self.stmt(assign, "must have Store context")
+
+    def _sequence(self, fac):
+        self.expr(fac([None], ast.Load()), "None disallowed")
+        self.expr(fac([ast.Name("x", ast.Store())], ast.Load()),
+                  "must have Load context")
+
+    def test_list(self):
+        self._sequence(ast.List)
+
+    def test_tuple(self):
+        self._sequence(ast.Tuple)
+
+    def test_stdlib_validates(self):
+        stdlib = os.path.dirname(ast.__file__)
+        tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")]
+        tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"])
+        for module in tests:
+            fn = os.path.join(stdlib, module)
+            with open(fn, "r", encoding="utf-8") as fp:
+                source = fp.read()
+            mod = ast.parse(source)
+            compile(mod, fn, "exec")
+
+
 def test_main():
-    support.run_unittest(AST_Tests, ASTHelpers_Test)
+    support.run_unittest(AST_Tests, ASTHelpers_Test, ASTValidatorTests)
 
 def main():
     if __name__ != '__main__':
@@ -539,9 +975,11 @@ exec_results = [
 ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
 ('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
 ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
 ('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]),
-('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
-('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
+('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]),
+('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]),
 ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
 ('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
 ('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py
index 8989a632e8..878b26cb71 100644
--- a/Lib/test/test_asyncore.py
+++ b/Lib/test/test_asyncore.py
@@ -21,6 +21,8 @@ except ImportError:
 
 HOST = support.HOST
 
+HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX')
+
 class dummysocket:
     def __init__(self):
         self.closed = False
@@ -72,15 +74,16 @@ def capture_server(evt, buf, serv):
         pass
     else:
         n = 200
-        while n > 0:
-            r, w, e = select.select([conn], [], [])
+        start = time.time()
+        while n > 0 and time.time() - start < 3.0:
+            r, w, e = select.select([conn], [], [], 0.1)
             if r:
+                n -= 1
                 data = conn.recv(10)
                 # keep everything except for the newline terminator
                 buf.write(data.replace(b'\n', b''))
                 if b'\n' in data:
                     break
-            n -= 1
             time.sleep(0.01)
 
         conn.close()
@@ -88,6 +91,13 @@ def capture_server(evt, buf, serv):
         serv.close()
         evt.set()
 
+def bind_af_aware(sock, addr):
+    """Helper function to bind a socket according to its family."""
+    if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX:
+        # Make sure the path doesn't exist.
+        unlink(addr)
+    sock.bind(addr)
+
 
 class HelperFunctionTests(unittest.TestCase):
     def test_readwriteexc(self):
@@ -353,7 +363,7 @@ class DispatcherWithSendTests(unittest.TestCase):
     @support.reap_threads
     def test_send(self):
         evt = threading.Event()
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock = socket.socket()
         sock.settimeout(3)
         port = support.bind_port(sock)
 
@@ -368,7 +378,7 @@ class DispatcherWithSendTests(unittest.TestCase):
 
             data = b"Suppose there isn't a 16-ton weight?"
             d = dispatcherwithsend_noread()
-            d.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+            d.create_socket()
             d.connect((HOST, port))
 
             # give time for socket to connect
@@ -468,22 +478,22 @@ class BaseTestHandler(asyncore.dispatcher):
         raise
 
 
-class TCPServer(asyncore.dispatcher):
+class BaseServer(asyncore.dispatcher):
     """A server which listens on an address and dispatches the
     connection to a handler.
     """
 
-    def __init__(self, handler=BaseTestHandler, host=HOST, port=0):
+    def __init__(self, family, addr, handler=BaseTestHandler):
         asyncore.dispatcher.__init__(self)
-        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.create_socket(family)
         self.set_reuse_addr()
-        self.bind((host, port))
+        bind_af_aware(self.socket, addr)
         self.listen(5)
         self.handler = handler
 
     @property
     def address(self):
-        return self.socket.getsockname()[:2]
+        return self.socket.getsockname()
 
     def handle_accepted(self, sock, addr):
         self.handler(sock)
@@ -494,16 +504,16 @@ class TCPServer(asyncore.dispatcher):
 
 class BaseClient(BaseTestHandler):
 
-    def __init__(self, address):
+    def __init__(self, family, address):
         BaseTestHandler.__init__(self)
-        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.create_socket(family)
         self.connect(address)
 
     def handle_connect(self):
         pass
 
 
-class BaseTestAPI(unittest.TestCase):
+class BaseTestAPI:
 
     def tearDown(self):
         asyncore.close_all()
@@ -526,8 +536,8 @@ class BaseTestAPI(unittest.TestCase):
             def handle_connect(self):
                 self.flag = True
 
-        server = TCPServer()
-        client = TestClient(server.address)
+        server = BaseServer(self.family, self.addr)
+        client = TestClient(self.family, server.address)
         self.loop_waiting_for_flag(client)
 
     def test_handle_accept(self):
@@ -535,18 +545,18 @@ class BaseTestAPI(unittest.TestCase):
 
         class TestListener(BaseTestHandler):
 
-            def __init__(self):
+            def __init__(self, family, addr):
                 BaseTestHandler.__init__(self)
-                self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
-                self.bind((HOST, 0))
+                self.create_socket(family)
+                bind_af_aware(self.socket, addr)
                 self.listen(5)
-                self.address = self.socket.getsockname()[:2]
+                self.address = self.socket.getsockname()
 
             def handle_accept(self):
                 self.flag = True
 
-        server = TestListener()
-        client = BaseClient(server.address)
+        server = TestListener(self.family, self.addr)
+        client = BaseClient(self.family, server.address)
         self.loop_waiting_for_flag(server)
 
     def test_handle_accepted(self):
@@ -554,12 +564,12 @@ class BaseTestAPI(unittest.TestCase):
 
         class TestListener(BaseTestHandler):
 
-            def __init__(self):
+            def __init__(self, family, addr):
                 BaseTestHandler.__init__(self)
-                self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
-                self.bind((HOST, 0))
+                self.create_socket(family)
+                bind_af_aware(self.socket, addr)
                 self.listen(5)
-                self.address = self.socket.getsockname()[:2]
+                self.address = self.socket.getsockname()
 
             def handle_accept(self):
                 asyncore.dispatcher.handle_accept(self)
@@ -568,8 +578,8 @@ class BaseTestAPI(unittest.TestCase):
                 sock.close()
                 self.flag = True
 
-        server = TestListener()
-        client = BaseClient(server.address)
+        server = TestListener(self.family, self.addr)
+        client = BaseClient(self.family, server.address)
         self.loop_waiting_for_flag(server)
 
 
@@ -585,8 +595,8 @@ class BaseTestAPI(unittest.TestCase):
                 BaseTestHandler.__init__(self, conn)
                 self.send(b'x' * 1024)
 
-        server = TCPServer(TestHandler)
-        client = TestClient(server.address)
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
         self.loop_waiting_for_flag(client)
 
     def test_handle_write(self):
@@ -596,8 +606,8 @@ class BaseTestAPI(unittest.TestCase):
             def handle_write(self):
                 self.flag = True
 
-        server = TCPServer()
-        client = TestClient(server.address)
+        server = BaseServer(self.family, self.addr)
+        client = TestClient(self.family, server.address)
         self.loop_waiting_for_flag(client)
 
     def test_handle_close(self):
@@ -620,8 +630,40 @@ class BaseTestAPI(unittest.TestCase):
                 BaseTestHandler.__init__(self, conn)
                 self.close()
 
-        server = TCPServer(TestHandler)
-        client = TestClient(server.address)
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    def test_handle_close_after_conn_broken(self):
+        # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and
+        # #11265).
+
+        data = b'\0' * 128
+
+        class TestClient(BaseClient):
+
+            def handle_write(self):
+                self.send(data)
+
+            def handle_close(self):
+                self.flag = True
+                self.close()
+
+            def handle_expt(self):
+                self.flag = True
+                self.close()
+
+        class TestHandler(BaseTestHandler):
+
+            def handle_read(self):
+                self.recv(len(data))
+                self.close()
+
+            def writable(self):
+                return False
+
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
         self.loop_waiting_for_flag(client)
 
     @unittest.skipIf(sys.platform.startswith("sunos"),
@@ -630,9 +672,12 @@ class BaseTestAPI(unittest.TestCase):
         # Make sure handle_expt is called on OOB data received.
         # Note: this might fail on some platforms as OOB data is
         # tenuously supported and rarely used.
+        if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
+            self.skipTest("Not applicable to AF_UNIX sockets.")
 
         class TestClient(BaseClient):
             def handle_expt(self):
+                self.socket.recv(1024, socket.MSG_OOB)
                 self.flag = True
 
         class TestHandler(BaseTestHandler):
@@ -640,8 +685,8 @@ class BaseTestAPI(unittest.TestCase):
                 BaseTestHandler.__init__(self, conn)
                 self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB)
 
-        server = TCPServer(TestHandler)
-        client = TestClient(server.address)
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
         self.loop_waiting_for_flag(client)
 
     def test_handle_error(self):
@@ -658,13 +703,13 @@ class BaseTestAPI(unittest.TestCase):
                 else:
                     raise Exception("exception not raised")
 
-        server = TCPServer()
-        client = TestClient(server.address)
+        server = BaseServer(self.family, self.addr)
+        client = TestClient(self.family, server.address)
         self.loop_waiting_for_flag(client)
 
     def test_connection_attributes(self):
-        server = TCPServer()
-        client = BaseClient(server.address)
+        server = BaseServer(self.family, self.addr)
+        client = BaseClient(self.family, server.address)
 
         # we start disconnected
         self.assertFalse(server.connected)
@@ -694,25 +739,29 @@ class BaseTestAPI(unittest.TestCase):
 
     def test_create_socket(self):
         s = asyncore.dispatcher()
-        s.create_socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.assertEqual(s.socket.family, socket.AF_INET)
+        s.create_socket(self.family)
+        self.assertEqual(s.socket.family, self.family)
         SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0)
         self.assertEqual(s.socket.type, socket.SOCK_STREAM | SOCK_NONBLOCK)
 
     def test_bind(self):
+        if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
+            self.skipTest("Not applicable to AF_UNIX sockets.")
         s1 = asyncore.dispatcher()
-        s1.create_socket(socket.AF_INET, socket.SOCK_STREAM)
-        s1.bind((HOST, 0))
+        s1.create_socket(self.family)
+        s1.bind(self.addr)
         s1.listen(5)
         port = s1.socket.getsockname()[1]
 
         s2 = asyncore.dispatcher()
-        s2.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        s2.create_socket(self.family)
         # EADDRINUSE indicates the socket was correctly bound
-        self.assertRaises(socket.error, s2.bind, (HOST, port))
+        self.assertRaises(socket.error, s2.bind, (self.addr[0], port))
 
     def test_set_reuse_addr(self):
-        sock = socket.socket()
+        if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
+            self.skipTest("Not applicable to AF_UNIX sockets.")
+        sock = socket.socket(self.family)
         try:
             sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         except socket.error:
@@ -720,11 +769,11 @@ class BaseTestAPI(unittest.TestCase):
         else:
             # if SO_REUSEADDR succeeded for sock we expect asyncore
             # to do the same
-            s = asyncore.dispatcher(socket.socket())
+            s = asyncore.dispatcher(socket.socket(self.family))
             self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET,
                                                  socket.SO_REUSEADDR))
             s.socket.close()
-            s.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.create_socket(self.family)
             s.set_reuse_addr()
             self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET,
                                                  socket.SO_REUSEADDR))
@@ -735,13 +784,14 @@ class BaseTestAPI(unittest.TestCase):
     @support.reap_threads
     def test_quick_connect(self):
         # see: http://bugs.python.org/issue10340
-        server = TCPServer()
-        t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, count=500))
-        t.start()
-        self.addCleanup(t.join)
-
-        for x in range(20):
-            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if self.family in (socket.AF_INET, getattr(socket, "AF_INET6", object())):
+            server = BaseServer(self.family, self.addr)
+            t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1,
+                                                              count=500))
+            t.start()
+            self.addCleanup(t.join)
+
+            s = socket.socket(self.family, socket.SOCK_STREAM)
             s.settimeout(.2)
             s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
                          struct.pack('ii', 1, 0))
@@ -752,19 +802,45 @@ class BaseTestAPI(unittest.TestCase):
             finally:
                 s.close()
 
-class TestAPI_UseSelect(BaseTestAPI):
+class TestAPI_UseIPv4Sockets(BaseTestAPI):
+    family = socket.AF_INET
+    addr = (HOST, 0)
+
+@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 support required')
+class TestAPI_UseIPv6Sockets(BaseTestAPI):
+    family = socket.AF_INET6
+    addr = ('::1', 0)
+
+@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required')
+class TestAPI_UseUnixSockets(BaseTestAPI):
+    if HAS_UNIX_SOCKETS:
+        family = socket.AF_UNIX
+    addr = support.TESTFN
+
+    def tearDown(self):
+        unlink(self.addr)
+        BaseTestAPI.tearDown(self)
+
+class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase):
     use_poll = False
 
 @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required')
-class TestAPI_UsePoll(BaseTestAPI):
+class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase):
     use_poll = True
 
+class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase):
+    use_poll = False
 
-def test_main():
-    tests = [HelperFunctionTests, DispatcherTests, DispatcherWithSendTests,
-             DispatcherWithSendTests_UsePoll, TestAPI_UseSelect,
-             TestAPI_UsePoll, FileWrapperTest]
-    run_unittest(*tests)
+@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required')
+class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase):
+    use_poll = True
+
+class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase):
+    use_poll = False
+
+@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required')
+class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase):
+    use_poll = True
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py
index ca94504b1c..25694762a9 100644
--- a/Lib/test/test_base64.py
+++ b/Lib/test/test_base64.py
@@ -103,44 +103,53 @@ class BaseXYTestCase(unittest.TestCase):
 
     def test_b64decode(self):
         eq = self.assertEqual
-        eq(base64.b64decode(b"d3d3LnB5dGhvbi5vcmc="), b"www.python.org")
-        eq(base64.b64decode(b'AA=='), b'\x00')
-        eq(base64.b64decode(b"YQ=="), b"a")
-        eq(base64.b64decode(b"YWI="), b"ab")
-        eq(base64.b64decode(b"YWJj"), b"abc")
-        eq(base64.b64decode(b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE"
-                            b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT"
-                            b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ=="),
-           b"abcdefghijklmnopqrstuvwxyz"
-           b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-           b"0123456789!@#0^&*();:<>,. []{}")
-        eq(base64.b64decode(b''), b'')
+
+        tests = {b"d3d3LnB5dGhvbi5vcmc=": b"www.python.org",
+                 b'AA==': b'\x00',
+                 b"YQ==": b"a",
+                 b"YWI=": b"ab",
+                 b"YWJj": b"abc",
+                 b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE"
+                 b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT"
+                 b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==":
+
+                 b"abcdefghijklmnopqrstuvwxyz"
+                 b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                 b"0123456789!@#0^&*();:<>,. []{}",
+                 b'': b'',
+                 }
+        for data, res in tests.items():
+            eq(base64.b64decode(data), res)
+            eq(base64.b64decode(data.decode('ascii')), res)
+
         # Test with arbitrary alternative characters
-        eq(base64.b64decode(b'01a*b$cd', altchars=b'*$'), b'\xd3V\xbeo\xf7\x1d')
-        # Check if passing a str object raises an error
-        self.assertRaises(TypeError, base64.b64decode, "")
-        self.assertRaises(TypeError, base64.b64decode, b"", altchars="")
+        tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d',
+                          }
+        for (data, altchars), res in tests_altchars.items():
+            data_str = data.decode('ascii')
+            altchars_str = altchars.decode('ascii')
+
+            eq(base64.b64decode(data, altchars=altchars), res)
+            eq(base64.b64decode(data_str, altchars=altchars), res)
+            eq(base64.b64decode(data, altchars=altchars_str), res)
+            eq(base64.b64decode(data_str, altchars=altchars_str), res)
+
         # Test standard alphabet
-        eq(base64.standard_b64decode(b"d3d3LnB5dGhvbi5vcmc="), b"www.python.org")
-        eq(base64.standard_b64decode(b"YQ=="), b"a")
-        eq(base64.standard_b64decode(b"YWI="), b"ab")
-        eq(base64.standard_b64decode(b"YWJj"), b"abc")
-        eq(base64.standard_b64decode(b""), b"")
-        eq(base64.standard_b64decode(b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE"
-                                     b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT"
-                                     b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ=="),
-           b"abcdefghijklmnopqrstuvwxyz"
-           b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-           b"0123456789!@#0^&*();:<>,. []{}")
-        # Check if passing a str object raises an error
-        self.assertRaises(TypeError, base64.standard_b64decode, "")
-        self.assertRaises(TypeError, base64.standard_b64decode, b"", altchars="")
+        for data, res in tests.items():
+            eq(base64.standard_b64decode(data), res)
+            eq(base64.standard_b64decode(data.decode('ascii')), res)
+
         # Test with 'URL safe' alternative characters
-        eq(base64.urlsafe_b64decode(b'01a-b_cd'), b'\xd3V\xbeo\xf7\x1d')
-        self.assertRaises(TypeError, base64.urlsafe_b64decode, "")
+        tests_urlsafe = {b'01a-b_cd': b'\xd3V\xbeo\xf7\x1d',
+                         b'': b'',
+                         }
+        for data, res in tests_urlsafe.items():
+            eq(base64.urlsafe_b64decode(data), res)
+            eq(base64.urlsafe_b64decode(data.decode('ascii')), res)
 
     def test_b64decode_padding_error(self):
         self.assertRaises(binascii.Error, base64.b64decode, b'abc')
+        self.assertRaises(binascii.Error, base64.b64decode, 'abc')
 
     def test_b64decode_invalid_chars(self):
         # issue 1466065: Test some invalid characters.
@@ -155,8 +164,11 @@ class BaseXYTestCase(unittest.TestCase):
                  (b'YWJj\nYWI=', b'abcab'))
         for bstr, res in tests:
             self.assertEqual(base64.b64decode(bstr), res)
+            self.assertEqual(base64.b64decode(bstr.decode('ascii')), res)
             with self.assertRaises(binascii.Error):
                 base64.b64decode(bstr, validate=True)
+            with self.assertRaises(binascii.Error):
+                base64.b64decode(bstr.decode('ascii'), validate=True)
 
     def test_b32encode(self):
         eq = self.assertEqual
@@ -171,40 +183,63 @@ class BaseXYTestCase(unittest.TestCase):
 
     def test_b32decode(self):
         eq = self.assertEqual
-        eq(base64.b32decode(b''), b'')
-        eq(base64.b32decode(b'AA======'), b'\x00')
-        eq(base64.b32decode(b'ME======'), b'a')
-        eq(base64.b32decode(b'MFRA===='), b'ab')
-        eq(base64.b32decode(b'MFRGG==='), b'abc')
-        eq(base64.b32decode(b'MFRGGZA='), b'abcd')
-        eq(base64.b32decode(b'MFRGGZDF'), b'abcde')
-        self.assertRaises(TypeError, base64.b32decode, "")
+        tests = {b'': b'',
+                 b'AA======': b'\x00',
+                 b'ME======': b'a',
+                 b'MFRA====': b'ab',
+                 b'MFRGG===': b'abc',
+                 b'MFRGGZA=': b'abcd',
+                 b'MFRGGZDF': b'abcde',
+                 }
+        for data, res in tests.items():
+            eq(base64.b32decode(data), res)
+            eq(base64.b32decode(data.decode('ascii')), res)
 
     def test_b32decode_casefold(self):
         eq = self.assertEqual
-        eq(base64.b32decode(b'', True), b'')
-        eq(base64.b32decode(b'ME======', True), b'a')
-        eq(base64.b32decode(b'MFRA====', True), b'ab')
-        eq(base64.b32decode(b'MFRGG===', True), b'abc')
-        eq(base64.b32decode(b'MFRGGZA=', True), b'abcd')
-        eq(base64.b32decode(b'MFRGGZDF', True), b'abcde')
-        # Lower cases
-        eq(base64.b32decode(b'me======', True), b'a')
-        eq(base64.b32decode(b'mfra====', True), b'ab')
-        eq(base64.b32decode(b'mfrgg===', True), b'abc')
-        eq(base64.b32decode(b'mfrggza=', True), b'abcd')
-        eq(base64.b32decode(b'mfrggzdf', True), b'abcde')
-        # Expected exceptions
+        tests = {b'': b'',
+                 b'ME======': b'a',
+                 b'MFRA====': b'ab',
+                 b'MFRGG===': b'abc',
+                 b'MFRGGZA=': b'abcd',
+                 b'MFRGGZDF': b'abcde',
+                 # Lower cases
+                 b'me======': b'a',
+                 b'mfra====': b'ab',
+                 b'mfrgg===': b'abc',
+                 b'mfrggza=': b'abcd',
+                 b'mfrggzdf': b'abcde',
+                 }
+
+        for data, res in tests.items():
+            eq(base64.b32decode(data, True), res)
+            eq(base64.b32decode(data.decode('ascii'), True), res)
+
         self.assertRaises(TypeError, base64.b32decode, b'me======')
+        self.assertRaises(TypeError, base64.b32decode, 'me======')
+
         # Mapping zero and one
         eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe')
-        eq(base64.b32decode(b'M1023456', map01=b'L'), b'b\xdd\xad\xf3\xbe')
-        eq(base64.b32decode(b'M1023456', map01=b'I'), b'b\x1d\xad\xf3\xbe')
-        self.assertRaises(TypeError, base64.b32decode, b"", map01="")
+        eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe')
+
+        map_tests = {(b'M1023456', b'L'): b'b\xdd\xad\xf3\xbe',
+                     (b'M1023456', b'I'): b'b\x1d\xad\xf3\xbe',
+                     }
+        for (data, map01), res in map_tests.items():
+            data_str = data.decode('ascii')
+            map01_str = map01.decode('ascii')
+
+            eq(base64.b32decode(data, map01=map01), res)
+            eq(base64.b32decode(data_str, map01=map01), res)
+            eq(base64.b32decode(data, map01=map01_str), res)
+            eq(base64.b32decode(data_str, map01=map01_str), res)
 
     def test_b32decode_error(self):
-        self.assertRaises(binascii.Error, base64.b32decode, b'abc')
-        self.assertRaises(binascii.Error, base64.b32decode, b'ABCDEF==')
+        for data in [b'abc', b'ABCDEF==']:
+            with self.assertRaises(binascii.Error):
+                base64.b32decode(data)
+            with self.assertRaises(binascii.Error):
+                base64.b32decode(data.decode('ascii'))
 
     def test_b16encode(self):
         eq = self.assertEqual
@@ -215,12 +250,24 @@ class BaseXYTestCase(unittest.TestCase):
     def test_b16decode(self):
         eq = self.assertEqual
         eq(base64.b16decode(b'0102ABCDEF'), b'\x01\x02\xab\xcd\xef')
+        eq(base64.b16decode('0102ABCDEF'), b'\x01\x02\xab\xcd\xef')
         eq(base64.b16decode(b'00'), b'\x00')
+        eq(base64.b16decode('00'), b'\x00')
         # Lower case is not allowed without a flag
         self.assertRaises(binascii.Error, base64.b16decode, b'0102abcdef')
+        self.assertRaises(binascii.Error, base64.b16decode, '0102abcdef')
         # Case fold
         eq(base64.b16decode(b'0102abcdef', True), b'\x01\x02\xab\xcd\xef')
-        self.assertRaises(TypeError, base64.b16decode, "")
+        eq(base64.b16decode('0102abcdef', True), b'\x01\x02\xab\xcd\xef')
+
+    def test_decode_nonascii_str(self):
+        decode_funcs = (base64.b64decode,
+                        base64.standard_b64decode,
+                        base64.urlsafe_b64decode,
+                        base64.b32decode,
+                        base64.b16decode)
+        for f in decode_funcs:
+            self.assertRaises(ValueError, f, 'with non-ascii \xcb')
 
     def test_ErrorHeritage(self):
         self.assertTrue(issubclass(binascii.Error, ValueError))
diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py
index f3c6ebbf05..0e545950ee 100644
--- a/Lib/test/test_bigmem.py
+++ b/Lib/test/test_bigmem.py
@@ -1,3 +1,13 @@
+"""Bigmem tests - tests for the 32-bit boundary in containers.
+
+These tests try to exercise the 32-bit boundary that is sometimes, if
+rarely, exceeded in practice, but almost never tested.  They are really only
+meaningful on 64-bit builds on machines with a *lot* of memory, but the
+tests are always run, usually with very low memory limits to make sure the
+tests themselves don't suffer from bitrot.  To run them for real, pass a
+high memory limit to regrtest, with the -M option.
+"""
+
 from test import support
 from test.support import bigmemtest, _1G, _2G, _4G
 
@@ -6,20 +16,35 @@ import operator
 import sys
 import functools
 
+# These tests all use one of the bigmemtest decorators to indicate how much
+# memory they use and how much memory they need to be even meaningful.  The
+# decorators take two arguments: a 'memuse' indicator declaring
+# (approximate) bytes per size-unit the test will use (at peak usage), and a
+# 'minsize' indicator declaring a minimum *useful* size.  A test that
+# allocates a bytestring to test various operations near the end will have a
+# minsize of at least 2Gb (or it wouldn't reach the 32-bit limit, so the
+# test wouldn't be very useful) and a memuse of 1 (one byte per size-unit,
+# if it allocates only one big string at a time.)
+#
+# When run with a memory limit set, both decorators skip tests that need
+# more memory than available to be meaningful.  The precisionbigmemtest will
+# always pass minsize as size, even if there is much more memory available.
+# The bigmemtest decorator will scale size upward to fill available memory.
+#
 # Bigmem testing houserules:
 #
 #  - Try not to allocate too many large objects. It's okay to rely on
-#    refcounting semantics, but don't forget that 's = create_largestring()'
+#    refcounting semantics, and don't forget that 's = create_largestring()'
 #    doesn't release the old 's' (if it exists) until well after its new
 #    value has been created. Use 'del s' before the create_largestring call.
 #
-#  - Do *not* compare large objects using assertEqual or similar. It's a
-#    lengthy operation and the errormessage will be utterly useless due to
-#    its size. To make sure whether a result has the right contents, better
-#    to use the strip or count methods, or compare meaningful slices.
+#  - Do *not* compare large objects using assertEqual, assertIn or similar.
+#    It's a lengthy operation and the errormessage will be utterly useless
+#    due to its size.  To make sure whether a result has the right contents,
+#    better to use the strip or count methods, or compare meaningful slices.
 #
 #  - Don't forget to test for large indices, offsets and results and such,
-#    in addition to large sizes.
+#    in addition to large sizes. Anything that probes the 32-bit boundary.
 #
 #  - When repeating an object (say, a substring, or a small list) to create
 #    a large object, make the subobject of a length that is not a power of
@@ -37,13 +62,14 @@ import functools
 # fail as well. I do not know whether it is due to memory fragmentation
 # issues, or other specifics of the platform malloc() routine.
 
-character_size = 4 if sys.maxunicode > 0xFFFF else 2
+ascii_char_size = 1
+ucs2_char_size = 2
+ucs4_char_size = 4
 
 
 class BaseStrTest:
 
-    @bigmemtest(size=_2G, memuse=2)
-    def test_capitalize(self, size):
+    def _test_capitalize(self, size):
         _ = self.from_latin1
         SUBSTR = self.from_latin1(' abc def ghi')
         s = _('-') * size + SUBSTR
@@ -92,7 +118,7 @@ class BaseStrTest:
         _ = self.from_latin1
         s = _('-') * size
         tabsize = 8
-        self.assertEqual(s.expandtabs(), s)
+        self.assertTrue(s.expandtabs() == s)
         del s
         slen, remainder = divmod(size, tabsize)
         s = _('       \t') * slen
@@ -347,7 +373,7 @@ class BaseStrTest:
     # suffer for the list size. (Otherwise, it'd cost another 48 times
     # size in bytes!) Nevertheless, a list of size takes
     # 8*size bytes.
-    @bigmemtest(size=_2G + 5, memuse=10)
+    @bigmemtest(size=_2G + 5, memuse=2 * ascii_char_size + 8)
     def test_split_large(self, size):
         _ = self.from_latin1
         s = _(' a') * size + _(' ')
@@ -366,9 +392,9 @@ class BaseStrTest:
         # take up an inordinate amount of memory
         chunksize = int(size ** 0.5 + 2) // 2
         SUBSTR = _(' ') * chunksize + _('\n') + _(' ') * chunksize + _('\r\n')
-        s = SUBSTR * chunksize
+        s = SUBSTR * (chunksize * 2)
         l = s.splitlines()
-        self.assertEqual(len(l), chunksize * 2)
+        self.assertEqual(len(l), chunksize * 4)
         expected = _(' ') * chunksize
         for item in l:
             self.assertEqual(item, expected)
@@ -394,8 +420,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size)
         self.assertEqual(s.strip(), SUBSTR.strip())
 
-    @bigmemtest(size=_2G, memuse=2)
-    def test_swapcase(self, size):
+    def _test_swapcase(self, size):
         _ = self.from_latin1
         SUBSTR = _("aBcDeFG12.'\xa9\x00")
         sublen = len(SUBSTR)
@@ -406,8 +431,7 @@ class BaseStrTest:
         self.assertEqual(s[:sublen * 3], SUBSTR.swapcase() * 3)
         self.assertEqual(s[-sublen * 3:], SUBSTR.swapcase() * 3)
 
-    @bigmemtest(size=_2G, memuse=2)
-    def test_title(self, size):
+    def _test_title(self, size):
         _ = self.from_latin1
         SUBSTR = _('SpaaHAaaAaham')
         s = SUBSTR * (size // len(SUBSTR) + 2)
@@ -419,14 +443,7 @@ class BaseStrTest:
     def test_translate(self, size):
         _ = self.from_latin1
         SUBSTR = _('aZz.z.Aaz.')
-        if isinstance(SUBSTR, str):
-            trans = {
-                ord(_('.')): _('-'),
-                ord(_('a')): _('!'),
-                ord(_('Z')): _('$'),
-            }
-        else:
-            trans = bytes.maketrans(b'.aZ', b'-!$')
+        trans = bytes.maketrans(b'.aZ', b'-!$')
         sublen = len(SUBSTR)
         repeats = size // sublen + 2
         s = SUBSTR * repeats
@@ -519,19 +536,19 @@ class BaseStrTest:
         edge = _('-') * (size // 2)
         s = _('').join([edge, SUBSTR, edge])
         del edge
-        self.assertIn(SUBSTR, s)
-        self.assertNotIn(SUBSTR * 2, s)
-        self.assertIn(_('-'), s)
-        self.assertNotIn(_('a'), s)
+        self.assertTrue(SUBSTR in s)
+        self.assertFalse(SUBSTR * 2 in s)
+        self.assertTrue(_('-') in s)
+        self.assertFalse(_('a') in s)
         s += _('a')
-        self.assertIn(_('a'), s)
+        self.assertTrue(_('a') in s)
 
     @bigmemtest(size=_2G + 10, memuse=2)
     def test_compare(self, size):
         _ = self.from_latin1
         s1 = _('-') * size
         s2 = _('-') * size
-        self.assertEqual(s1, s2)
+        self.assertTrue(s1 == s2)
         del s2
         s2 = s1 + _('a')
         self.assertFalse(s1 == s2)
@@ -552,7 +569,7 @@ class BaseStrTest:
         h1 = hash(s)
         del s
         s = _('\x00') * (size + 1)
-        self.assertFalse(h1 == hash(s))
+        self.assertNotEqual(h1, hash(s))
 
 
 class StrTest(unittest.TestCase, BaseStrTest):
@@ -563,7 +580,6 @@ class StrTest(unittest.TestCase, BaseStrTest):
     def basic_encode_test(self, size, enc, c='.', expectedsize=None):
         if expectedsize is None:
             expectedsize = size
-
         try:
             s = c * size
             self.assertEqual(len(s.encode(enc)), expectedsize)
@@ -582,48 +598,64 @@ class StrTest(unittest.TestCase, BaseStrTest):
                 memuse = meth.memuse
             except AttributeError:
                 continue
-            meth.memuse = character_size * memuse
+            meth.memuse = ascii_char_size * memuse
             self._adjusted[name] = memuse
 
     def tearDown(self):
         for name, memuse in self._adjusted.items():
             getattr(type(self), name).memuse = memuse
 
-    # the utf8 encoder preallocates big time (4x the number of characters)
-    @bigmemtest(size=_2G + 2, memuse=character_size + 4)
+    @bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
+    def test_capitalize(self, size):
+        self._test_capitalize(size)
+
+    @bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
+    def test_title(self, size):
+        self._test_title(size)
+
+    @bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
+    def test_swapcase(self, size):
+        self._test_swapcase(size)
+
+    # Many codecs convert to the legacy representation first, explaining
+    # why we add 'ucs4_char_size' to the 'memuse' below.
+
+    @bigmemtest(size=_2G + 2, memuse=ascii_char_size + 1)
     def test_encode(self, size):
         return self.basic_encode_test(size, 'utf-8')
 
-    @bigmemtest(size=_4G // 6 + 2, memuse=character_size + 1)
+    @bigmemtest(size=_4G // 6 + 2, memuse=ascii_char_size + ucs4_char_size + 1)
     def test_encode_raw_unicode_escape(self, size):
         try:
             return self.basic_encode_test(size, 'raw_unicode_escape')
         except MemoryError:
             pass # acceptable on 32-bit
 
-    @bigmemtest(size=_4G // 5 + 70, memuse=character_size + 1)
+    @bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + ucs4_char_size + 1)
     def test_encode_utf7(self, size):
         try:
             return self.basic_encode_test(size, 'utf7')
         except MemoryError:
             pass # acceptable on 32-bit
 
-    @bigmemtest(size=_4G // 4 + 5, memuse=character_size + 4)
+    @bigmemtest(size=_4G // 4 + 5, memuse=ascii_char_size + ucs4_char_size + 4)
     def test_encode_utf32(self, size):
         try:
-            return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4)
+            return self.basic_encode_test(size, 'utf32', expectedsize=4 * size + 4)
         except MemoryError:
             pass # acceptable on 32-bit
 
-    @bigmemtest(size=_2G - 1, memuse=character_size + 1)
+    @bigmemtest(size=_2G - 1, memuse=ascii_char_size + 1)
     def test_encode_ascii(self, size):
         return self.basic_encode_test(size, 'ascii', c='A')
 
-    @bigmemtest(size=_2G + 10, memuse=character_size * 2)
+    # str % (...) uses a Py_UCS4 intermediate representation
+
+    @bigmemtest(size=_2G + 10, memuse=ascii_char_size * 2 + ucs4_char_size)
     def test_format(self, size):
         s = '-' * size
         sf = '%s' % (s,)
-        self.assertEqual(s, sf)
+        self.assertTrue(s == sf)
         del sf
         sf = '..%s..' % (s,)
         self.assertEqual(len(sf), len(s) + 4)
@@ -640,7 +672,7 @@ class StrTest(unittest.TestCase, BaseStrTest):
         self.assertEqual(s.count('.'), 3)
         self.assertEqual(s.count('-'), size * 2)
 
-    @bigmemtest(size=_2G + 10, memuse=character_size * 2)
+    @bigmemtest(size=_2G + 10, memuse=ascii_char_size * 2)
     def test_repr_small(self, size):
         s = '-' * size
         s = repr(s)
@@ -661,7 +693,7 @@ class StrTest(unittest.TestCase, BaseStrTest):
         self.assertEqual(s.count('\\'), size)
         self.assertEqual(s.count('0'), size * 2)
 
-    @bigmemtest(size=_2G + 10, memuse=character_size * 5)
+    @bigmemtest(size=_2G + 10, memuse=ascii_char_size * 5)
     def test_repr_large(self, size):
         s = '\x00' * size
         s = repr(s)
@@ -671,7 +703,13 @@ class StrTest(unittest.TestCase, BaseStrTest):
         self.assertEqual(s.count('\\'), size)
         self.assertEqual(s.count('0'), size * 2)
 
-    @bigmemtest(size=_2G // 5 + 1, memuse=character_size * 7)
+    # ascii() calls encode('ascii', 'backslashreplace'), which itself
+    # creates a temporary Py_UNICODE representation in addition to the
+    # original (Py_UCS2) one
+    # There's also some overallocation when resizing the ascii() result
+    # that isn't taken into account here.
+    @bigmemtest(size=_2G // 5 + 1, memuse=ucs2_char_size +
+                                          ucs4_char_size + ascii_char_size * 6)
     def test_unicode_repr(self, size):
         # Use an assigned, but not printable code point.
         # It is in the range of the low surrogates \uDC00-\uDFFF.
@@ -686,9 +724,7 @@ class StrTest(unittest.TestCase, BaseStrTest):
         finally:
             r = s = None
 
-    # The character takes 4 bytes even in UCS-2 builds because it will
-    # be decomposed into surrogates.
-    @bigmemtest(size=_2G // 5 + 1, memuse=4 + character_size * 9)
+    @bigmemtest(size=_2G // 5 + 1, memuse=ucs4_char_size * 2 + ascii_char_size * 10)
     def test_unicode_repr_wide(self, size):
         char = "\U0001DCBA"
         s = char * size
@@ -701,39 +737,76 @@ class StrTest(unittest.TestCase, BaseStrTest):
         finally:
             r = s = None
 
-    @bigmemtest(size=_4G // 5, memuse=character_size * (6 + 1))
-    def _test_unicode_repr_overflow(self, size):
-        # XXX not sure what this test is about
-        char = "\uDCBA"
-        s = char * size
-        try:
-            r = repr(s)
-            self.assertTrue(s == eval(r))
-        finally:
-            r = s = None
+    # The original test_translate is overriden here, so as to get the
+    # correct size estimate: str.translate() uses an intermediate Py_UCS4
+    # representation.
+
+    @bigmemtest(size=_2G, memuse=ascii_char_size * 2 + ucs4_char_size)
+    def test_translate(self, size):
+        _ = self.from_latin1
+        SUBSTR = _('aZz.z.Aaz.')
+        trans = {
+            ord(_('.')): _('-'),
+            ord(_('a')): _('!'),
+            ord(_('Z')): _('$'),
+        }
+        sublen = len(SUBSTR)
+        repeats = size // sublen + 2
+        s = SUBSTR * repeats
+        s = s.translate(trans)
+        self.assertEqual(len(s), repeats * sublen)
+        self.assertEqual(s[:sublen], SUBSTR.translate(trans))
+        self.assertEqual(s[-sublen:], SUBSTR.translate(trans))
+        self.assertEqual(s.count(_('.')), 0)
+        self.assertEqual(s.count(_('!')), repeats * 2)
+        self.assertEqual(s.count(_('z')), repeats * 3)
 
 
 class BytesTest(unittest.TestCase, BaseStrTest):
 
     def from_latin1(self, s):
-        return s.encode("latin1")
+        return s.encode("latin-1")
 
-    @bigmemtest(size=_2G + 2, memuse=1 + character_size)
+    @bigmemtest(size=_2G + 2, memuse=1 + ascii_char_size)
     def test_decode(self, size):
         s = self.from_latin1('.') * size
         self.assertEqual(len(s.decode('utf-8')), size)
 
+    @bigmemtest(size=_2G, memuse=2)
+    def test_capitalize(self, size):
+        self._test_capitalize(size)
+
+    @bigmemtest(size=_2G, memuse=2)
+    def test_title(self, size):
+        self._test_title(size)
+
+    @bigmemtest(size=_2G, memuse=2)
+    def test_swapcase(self, size):
+        self._test_swapcase(size)
+
 
 class BytearrayTest(unittest.TestCase, BaseStrTest):
 
     def from_latin1(self, s):
-        return bytearray(s.encode("latin1"))
+        return bytearray(s.encode("latin-1"))
 
-    @bigmemtest(size=_2G + 2, memuse=1 + character_size)
+    @bigmemtest(size=_2G + 2, memuse=1 + ascii_char_size)
     def test_decode(self, size):
         s = self.from_latin1('.') * size
         self.assertEqual(len(s.decode('utf-8')), size)
 
+    @bigmemtest(size=_2G, memuse=2)
+    def test_capitalize(self, size):
+        self._test_capitalize(size)
+
+    @bigmemtest(size=_2G, memuse=2)
+    def test_title(self, size):
+        self._test_title(size)
+
+    @bigmemtest(size=_2G, memuse=2)
+    def test_swapcase(self, size):
+        self._test_swapcase(size)
+
     test_hash = None
     test_split_large = None
 
@@ -752,7 +825,7 @@ class TupleTest(unittest.TestCase):
     def test_compare(self, size):
         t1 = ('',) * size
         t2 = ('',) * size
-        self.assertEqual(t1, t2)
+        self.assertTrue(t1 == t2)
         del t2
         t2 = ('',) * (size + 1)
         self.assertFalse(t1 == t2)
@@ -783,9 +856,9 @@ class TupleTest(unittest.TestCase):
     def test_contains(self, size):
         t = (1, 2, 3, 4, 5) * size
         self.assertEqual(len(t), size * 5)
-        self.assertIn(5, t)
-        self.assertNotIn((1, 2, 3, 4, 5), t)
-        self.assertNotIn(0, t)
+        self.assertTrue(5 in t)
+        self.assertFalse((1, 2, 3, 4, 5) in t)
+        self.assertFalse(0 in t)
 
     @bigmemtest(size=_2G + 10, memuse=8)
     def test_hash(self, size):
@@ -869,11 +942,11 @@ class TupleTest(unittest.TestCase):
         self.assertEqual(s[-5:], '0, 0)')
         self.assertEqual(s.count('0'), size)
 
-    @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
     def test_repr_small(self, size):
         return self.basic_test_repr(size)
 
-    @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
     def test_repr_large(self, size):
         return self.basic_test_repr(size)
 
@@ -888,7 +961,7 @@ class ListTest(unittest.TestCase):
     def test_compare(self, size):
         l1 = [''] * size
         l2 = [''] * size
-        self.assertEqual(l1, l2)
+        self.assertTrue(l1 == l2)
         del l2
         l2 = [''] * (size + 1)
         self.assertFalse(l1 == l2)
@@ -934,9 +1007,9 @@ class ListTest(unittest.TestCase):
     def test_contains(self, size):
         l = [1, 2, 3, 4, 5] * size
         self.assertEqual(len(l), size * 5)
-        self.assertIn(5, l)
-        self.assertNotIn([1, 2, 3, 4, 5], l)
-        self.assertNotIn(0, l)
+        self.assertTrue(5 in l)
+        self.assertFalse([1, 2, 3, 4, 5] in l)
+        self.assertFalse(0 in l)
 
     @bigmemtest(size=_2G + 10, memuse=8)
     def test_hash(self, size):
@@ -1044,11 +1117,11 @@ class ListTest(unittest.TestCase):
         self.assertEqual(s[-5:], '0, 0]')
         self.assertEqual(s.count('0'), size)
 
-    @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
     def test_repr_small(self, size):
         return self.basic_test_repr(size)
 
-    @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
     def test_repr_large(self, size):
         return self.basic_test_repr(size)
 
diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py
index 1e9e888f28..04d8f9d680 100644
--- a/Lib/test/test_binascii.py
+++ b/Lib/test/test_binascii.py
@@ -208,9 +208,9 @@ class BinASCIITest(unittest.TestCase):
             except Exception as err:
                 self.fail("{}({!r}) raises {!r}".format(func, empty, err))
 
-    def test_unicode_strings(self):
-        # Unicode strings are not accepted.
-        for func in all_functions:
+    def test_unicode_b2a(self):
+        # Unicode strings are not accepted by b2a_* functions.
+        for func in set(all_functions) - set(a2b_functions) | {'rledecode_hqx'}:
             try:
                 self.assertRaises(TypeError, getattr(binascii, func), "test")
             except Exception as err:
@@ -218,6 +218,34 @@ class BinASCIITest(unittest.TestCase):
         # crc_hqx needs 2 arguments
         self.assertRaises(TypeError, binascii.crc_hqx, "test", 0)
 
+    def test_unicode_a2b(self):
+        # Unicode strings are accepted by a2b_* functions.
+        MAX_ALL = 45
+        raw = self.rawdata[:MAX_ALL]
+        for fa, fb in zip(a2b_functions, b2a_functions):
+            if fa == 'rledecode_hqx':
+                # Takes non-ASCII data
+                continue
+            a2b = getattr(binascii, fa)
+            b2a = getattr(binascii, fb)
+            try:
+                a = b2a(self.type2test(raw))
+                binary_res = a2b(a)
+                a = a.decode('ascii')
+                res = a2b(a)
+            except Exception as err:
+                self.fail("{}/{} conversion raises {!r}".format(fb, fa, err))
+            if fb == 'b2a_hqx':
+                # b2a_hqx returns a tuple
+                res, _ = res
+                binary_res, _ = binary_res
+            self.assertEqual(res, raw, "{}/{} conversion: "
+                             "{!r} != {!r}".format(fb, fa, res, raw))
+            self.assertEqual(res, binary_res)
+            self.assertIsInstance(res, bytes)
+            # non-ASCII string
+            self.assertRaises(ValueError, a2b, "\x80")
+
 
 class ArrayBinASCIITest(BinASCIITest):
     def type2test(self, s):
diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py
index f95ed635c3..7b9bd19fcb 100644
--- a/Lib/test/test_bisect.py
+++ b/Lib/test/test_bisect.py
@@ -3,25 +3,8 @@ import unittest
 from test import support
 from collections import UserList
 
-# We do a bit of trickery here to be able to test both the C implementation
-# and the Python implementation of the module.
-
-# Make it impossible to import the C implementation anymore.
-sys.modules['_bisect'] = 0
-# We must also handle the case that bisect was imported before.
-if 'bisect' in sys.modules:
-    del sys.modules['bisect']
-
-# Now we can import the module and get the pure Python implementation.
-import bisect as py_bisect
-
-# Restore everything to normal.
-del sys.modules['_bisect']
-del sys.modules['bisect']
-
-# This is now the module with the C implementation.
-import bisect as c_bisect
-
+py_bisect = support.import_fresh_module('bisect', blocked=['_bisect'])
+c_bisect = support.import_fresh_module('bisect', fresh=['_bisect'])
 
 class Range(object):
     """A trivial range()-like object without any integer width limitations."""
@@ -45,9 +28,7 @@ class Range(object):
         self.last_insert = idx, item
 
 
-class TestBisect(unittest.TestCase):
-    module = None
-
+class TestBisect:
     def setUp(self):
         self.precomputedCases = [
             (self.module.bisect_right, [], 1, 0),
@@ -218,17 +199,15 @@ class TestBisect(unittest.TestCase):
         self.module.insort(a=data, x=25, lo=1, hi=3)
         self.assertEqual(data, [10, 20, 25, 25, 25, 30, 40, 50])
 
-class TestBisectPython(TestBisect):
+class TestBisectPython(TestBisect, unittest.TestCase):
     module = py_bisect
 
-class TestBisectC(TestBisect):
+class TestBisectC(TestBisect, unittest.TestCase):
     module = c_bisect
 
 #==============================================================================
 
-class TestInsort(unittest.TestCase):
-    module = None
-
+class TestInsort:
     def test_vsBuiltinSort(self, n=500):
         from random import choice
         for insorted in (list(), UserList()):
@@ -255,15 +234,14 @@ class TestInsort(unittest.TestCase):
         self.module.insort_right(lst, 5)
         self.assertEqual([5, 10], lst.data)
 
-class TestInsortPython(TestInsort):
+class TestInsortPython(TestInsort, unittest.TestCase):
     module = py_bisect
 
-class TestInsortC(TestInsort):
+class TestInsortC(TestInsort, unittest.TestCase):
     module = c_bisect
 
 #==============================================================================
 
-
 class LenOnly:
     "Dummy sequence class defining __len__ but not __getitem__."
     def __len__(self):
@@ -284,9 +262,7 @@ class CmpErr:
     __eq__ = __lt__
     __ne__ = __lt__
 
-class TestErrorHandling(unittest.TestCase):
-    module = None
-
+class TestErrorHandling:
     def test_non_sequence(self):
         for f in (self.module.bisect_left, self.module.bisect_right,
                   self.module.insort_left, self.module.insort_right):
@@ -313,58 +289,40 @@ class TestErrorHandling(unittest.TestCase):
                   self.module.insort_left, self.module.insort_right):
             self.assertRaises(TypeError, f, 10)
 
-class TestErrorHandlingPython(TestErrorHandling):
+class TestErrorHandlingPython(TestErrorHandling, unittest.TestCase):
     module = py_bisect
 
-class TestErrorHandlingC(TestErrorHandling):
+class TestErrorHandlingC(TestErrorHandling, unittest.TestCase):
     module = c_bisect
 
 #==============================================================================
 
-libreftest = """
-Example from the Library Reference:  Doc/library/bisect.rst
-
-The bisect() function is generally useful for categorizing numeric data.
-This example uses bisect() to look up a letter grade for an exam total
-(say) based on a set of ordered numeric breakpoints: 85 and up is an `A',
-75..84 is a `B', etc.
-
-    >>> grades = "FEDCBA"
-    >>> breakpoints = [30, 44, 66, 75, 85]
-    >>> from bisect import bisect
-    >>> def grade(total):
-    ...           return grades[bisect(breakpoints, total)]
-    ...
-    >>> grade(66)
-    'C'
-    >>> list(map(grade, [33, 99, 77, 44, 12, 88]))
-    ['E', 'A', 'B', 'D', 'F', 'A']
+class TestDocExample:
+    def test_grades(self):
+        def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'):
+            i = self.module.bisect(breakpoints, score)
+            return grades[i]
+
+        result = [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]]
+        self.assertEqual(result, ['F', 'A', 'C', 'C', 'B', 'A', 'A'])
+
+    def test_colors(self):
+        data = [('red', 5), ('blue', 1), ('yellow', 8), ('black', 0)]
+        data.sort(key=lambda r: r[1])
+        keys = [r[1] for r in data]
+        bisect_left = self.module.bisect_left
+        self.assertEqual(data[bisect_left(keys, 0)], ('black', 0))
+        self.assertEqual(data[bisect_left(keys, 1)], ('blue', 1))
+        self.assertEqual(data[bisect_left(keys, 5)], ('red', 5))
+        self.assertEqual(data[bisect_left(keys, 8)], ('yellow', 8))
+
+class TestDocExamplePython(TestDocExample, unittest.TestCase):
+    module = py_bisect
 
-"""
+class TestDocExampleC(TestDocExample, unittest.TestCase):
+    module = c_bisect
 
 #------------------------------------------------------------------------------
 
-__test__ = {'libreftest' : libreftest}
-
-def test_main(verbose=None):
-    from test import test_bisect
-
-    test_classes = [TestBisectPython, TestBisectC,
-                    TestInsortPython, TestInsortC,
-                    TestErrorHandlingPython, TestErrorHandlingC]
-
-    support.run_unittest(*test_classes)
-    support.run_doctest(test_bisect, verbose)
-
-    # verify reference counting
-    if verbose and hasattr(sys, "gettotalrefcount"):
-        import gc
-        counts = [None] * 5
-        for i in range(len(counts)):
-            support.run_unittest(*test_classes)
-            gc.collect()
-            counts[i] = sys.gettotalrefcount()
-        print(counts)
-
 if __name__ == "__main__":
-    test_main(verbose=True)
+    unittest.main()
diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py
index b296870b09..4bab28be77 100644
--- a/Lib/test/test_bool.py
+++ b/Lib/test/test_bool.py
@@ -330,6 +330,16 @@ class BoolTest(unittest.TestCase):
                 except (Exception) as e_len:
                     self.assertEqual(str(e_bool), str(e_len))
 
+    def test_real_and_imag(self):
+        self.assertEqual(True.real, 1)
+        self.assertEqual(True.imag, 0)
+        self.assertIs(type(True.real), int)
+        self.assertIs(type(True.imag), int)
+        self.assertEqual(False.real, 0)
+        self.assertEqual(False.imag, 0)
+        self.assertIs(type(False.real), int)
+        self.assertIs(type(False.imag), int)
+
 def test_main():
     support.run_unittest(BoolTest)
 
diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py
new file mode 100644
index 0000000000..747e2a287e
--- /dev/null
+++ b/Lib/test/test_buffer.py
@@ -0,0 +1,4291 @@
+#
+# The ndarray object from _testbuffer.c is a complete implementation of
+# a PEP-3118 buffer provider. It is independent from NumPy's ndarray
+# and the tests don't require NumPy.
+#
+# If NumPy is present, some tests check both ndarray implementations
+# against each other.
+#
+# Most ndarray tests also check that memoryview(ndarray) behaves in
+# the same way as the original. Thus, a substantial part of the
+# memoryview tests is now in this module.
+#
+
+import unittest
+from test import support
+from itertools import permutations, product
+from random import randrange, sample, choice
+from sysconfig import get_config_var
+import warnings
+import sys, array, io
+from decimal import Decimal
+from fractions import Fraction
+
+try:
+    from _testbuffer import *
+except ImportError:
+    ndarray = None
+
+try:
+    import struct
+except ImportError:
+    struct = None
+
+try:
+    import ctypes
+except ImportError:
+    ctypes = None
+
+try:
+    with warnings.catch_warnings():
+        from numpy import ndarray as numpy_array
+except ImportError:
+    numpy_array = None
+
+
+SHORT_TEST = True
+
+
+# ======================================================================
+#                    Random lists by format specifier
+# ======================================================================
+
+# Native format chars and their ranges.
+NATIVE = {
+    '?':0, 'c':0, 'b':0, 'B':0,
+    'h':0, 'H':0, 'i':0, 'I':0,
+    'l':0, 'L':0, 'n':0, 'N':0,
+    'f':0, 'd':0, 'P':0
+}
+
+# NumPy does not have 'n' or 'N':
+if numpy_array:
+    del NATIVE['n']
+    del NATIVE['N']
+
+if struct:
+    try:
+        # Add "qQ" if present in native mode.
+        struct.pack('Q', 2**64-1)
+        NATIVE['q'] = 0
+        NATIVE['Q'] = 0
+    except struct.error:
+        pass
+
+# Standard format chars and their ranges.
+STANDARD = {
+    '?':(0, 2),            'c':(0, 1<<8),
+    'b':(-(1<<7), 1<<7),   'B':(0, 1<<8),
+    'h':(-(1<<15), 1<<15), 'H':(0, 1<<16),
+    'i':(-(1<<31), 1<<31), 'I':(0, 1<<32),
+    'l':(-(1<<31), 1<<31), 'L':(0, 1<<32),
+    'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64),
+    'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023)
+}
+
+def native_type_range(fmt):
+    """Return range of a native type."""
+    if fmt == 'c':
+        lh = (0, 256)
+    elif fmt == '?':
+        lh = (0, 2)
+    elif fmt == 'f':
+        lh = (-(1<<63), 1<<63)
+    elif fmt == 'd':
+        lh = (-(1<<1023), 1<<1023)
+    else:
+        for exp in (128, 127, 64, 63, 32, 31, 16, 15, 8, 7):
+            try:
+                struct.pack(fmt, (1<<exp)-1)
+                break
+            except struct.error:
+                pass
+        lh = (-(1<<exp), 1<<exp) if exp & 1 else (0, 1<<exp)
+    return lh
+
+fmtdict = {
+    '':NATIVE,
+    '@':NATIVE,
+    '<':STANDARD,
+    '>':STANDARD,
+    '=':STANDARD,
+    '!':STANDARD
+}
+
+if struct:
+    for fmt in fmtdict['@']:
+        fmtdict['@'][fmt] = native_type_range(fmt)
+
+MEMORYVIEW = NATIVE.copy()
+ARRAY = NATIVE.copy()
+for k in NATIVE:
+    if not k in "bBhHiIlLfd":
+        del ARRAY[k]
+
+BYTEFMT = NATIVE.copy()
+for k in NATIVE:
+    if not k in "Bbc":
+        del BYTEFMT[k]
+
+fmtdict['m']  = MEMORYVIEW
+fmtdict['@m'] = MEMORYVIEW
+fmtdict['a']  = ARRAY
+fmtdict['b']  = BYTEFMT
+fmtdict['@b']  = BYTEFMT
+
+# Capabilities of the test objects:
+MODE = 0
+MULT = 1
+cap = {         # format chars                  # multiplier
+  'ndarray':    (['', '@', '<', '>', '=', '!'], ['', '1', '2', '3']),
+  'array':      (['a'],                         ['']),
+  'numpy':      ([''],                          ['']),
+  'memoryview': (['@m', 'm'],                   ['']),
+  'bytefmt':    (['@b', 'b'],                   ['']),
+}
+
+def randrange_fmt(mode, char, obj):
+    """Return random item for a type specified by a mode and a single
+       format character."""
+    x = randrange(*fmtdict[mode][char])
+    if char == 'c':
+        x = bytes(chr(x), 'latin1')
+    if char == '?':
+        x = bool(x)
+    if char == 'f' or char == 'd':
+        x = struct.pack(char, x)
+        x = struct.unpack(char, x)[0]
+    if obj == 'numpy' and x == b'\x00':
+        # http://projects.scipy.org/numpy/ticket/1925
+        x = b'\x01'
+    return x
+
+def gen_item(fmt, obj):
+    """Return single random item."""
+    mode, chars = fmt.split('#')
+    x = []
+    for c in chars:
+        x.append(randrange_fmt(mode, c, obj))
+    return x[0] if len(x) == 1 else tuple(x)
+
+def gen_items(n, fmt, obj):
+    """Return a list of random items (or a scalar)."""
+    if n == 0:
+        return gen_item(fmt, obj)
+    lst = [0] * n
+    for i in range(n):
+        lst[i] = gen_item(fmt, obj)
+    return lst
+
+def struct_items(n, obj):
+    mode = choice(cap[obj][MODE])
+    xfmt = mode + '#'
+    fmt = mode.strip('amb')
+    nmemb = randrange(2, 10) # number of struct members
+    for _ in range(nmemb):
+        char = choice(tuple(fmtdict[mode]))
+        multiplier = choice(cap[obj][MULT])
+        xfmt += (char * int(multiplier if multiplier else 1))
+        fmt += (multiplier + char)
+    items = gen_items(n, xfmt, obj)
+    item = gen_item(xfmt, obj)
+    return fmt, items, item
+
+def randitems(n, obj='ndarray', mode=None, char=None):
+    """Return random format, items, item."""
+    if mode is None:
+        mode = choice(cap[obj][MODE])
+    if char is None:
+        char = choice(tuple(fmtdict[mode]))
+    multiplier = choice(cap[obj][MULT])
+    fmt = mode + '#' + char * int(multiplier if multiplier else 1)
+    items = gen_items(n, fmt, obj)
+    item = gen_item(fmt, obj)
+    fmt = mode.strip('amb') + multiplier + char
+    return fmt, items, item
+
+def iter_mode(n, obj='ndarray'):
+    """Iterate through supported mode/char combinations."""
+    for mode in cap[obj][MODE]:
+        for char in fmtdict[mode]:
+            yield randitems(n, obj, mode, char)
+
+def iter_format(nitems, testobj='ndarray'):
+    """Yield (format, items, item) for all possible modes and format
+       characters plus one random compound format string."""
+    for t in iter_mode(nitems, testobj):
+        yield t
+    if testobj != 'ndarray':
+        raise StopIteration
+    yield struct_items(nitems, testobj)
+
+
+def is_byte_format(fmt):
+    return 'c' in fmt or 'b' in fmt or 'B' in fmt
+
+def is_memoryview_format(fmt):
+    """format suitable for memoryview"""
+    x = len(fmt)
+    return ((x == 1 or (x == 2 and fmt[0] == '@')) and
+            fmt[x-1] in MEMORYVIEW)
+
+NON_BYTE_FORMAT = [c for c in fmtdict['@'] if not is_byte_format(c)]
+
+
+# ======================================================================
+#       Multi-dimensional tolist(), slicing and slice assignments
+# ======================================================================
+
+def atomp(lst):
+    """Tuple items (representing structs) are regarded as atoms."""
+    return not isinstance(lst, list)
+
+def listp(lst):
+    return isinstance(lst, list)
+
+def prod(lst):
+    """Product of list elements."""
+    if len(lst) == 0:
+        return 0
+    x = lst[0]
+    for v in lst[1:]:
+        x *= v
+    return x
+
+def strides_from_shape(ndim, shape, itemsize, layout):
+    """Calculate strides of a contiguous array. Layout is 'C' or
+       'F' (Fortran)."""
+    if ndim == 0:
+        return ()
+    if layout == 'C':
+        strides = list(shape[1:]) + [itemsize]
+        for i in range(ndim-2, -1, -1):
+            strides[i] *= strides[i+1]
+    else:
+        strides = [itemsize] + list(shape[:-1])
+        for i in range(1, ndim):
+            strides[i] *= strides[i-1]
+    return strides
+
+def _ca(items, s):
+    """Convert flat item list to the nested list representation of a
+       multidimensional C array with shape 's'."""
+    if atomp(items):
+        return items
+    if len(s) == 0:
+        return items[0]
+    lst = [0] * s[0]
+    stride = len(items) // s[0] if s[0] else 0
+    for i in range(s[0]):
+        start = i*stride
+        lst[i] = _ca(items[start:start+stride], s[1:])
+    return lst
+
+def _fa(items, s):
+    """Convert flat item list to the nested list representation of a
+       multidimensional Fortran array with shape 's'."""
+    if atomp(items):
+        return items
+    if len(s) == 0:
+        return items[0]
+    lst = [0] * s[0]
+    stride = s[0]
+    for i in range(s[0]):
+        lst[i] = _fa(items[i::stride], s[1:])
+    return lst
+
+def carray(items, shape):
+    if listp(items) and not 0 in shape and prod(shape) != len(items):
+        raise ValueError("prod(shape) != len(items)")
+    return _ca(items, shape)
+
+def farray(items, shape):
+    if listp(items) and not 0 in shape and prod(shape) != len(items):
+        raise ValueError("prod(shape) != len(items)")
+    return _fa(items, shape)
+
+def indices(shape):
+    """Generate all possible tuples of indices."""
+    iterables = [range(v) for v in shape]
+    return product(*iterables)
+
+def getindex(ndim, ind, strides):
+    """Convert multi-dimensional index to the position in the flat list."""
+    ret = 0
+    for i in range(ndim):
+        ret += strides[i] * ind[i]
+    return ret
+
+def transpose(src, shape):
+    """Transpose flat item list that is regarded as a multi-dimensional
+       matrix defined by shape: dest...[k][j][i] = src[i][j][k]...  """
+    if not shape:
+        return src
+    ndim = len(shape)
+    sstrides = strides_from_shape(ndim, shape, 1, 'C')
+    dstrides = strides_from_shape(ndim, shape[::-1], 1, 'C')
+    dest = [0] * len(src)
+    for ind in indices(shape):
+        fr = getindex(ndim, ind, sstrides)
+        to = getindex(ndim, ind[::-1], dstrides)
+        dest[to] = src[fr]
+    return dest
+
+def _flatten(lst):
+    """flatten list"""
+    if lst == []:
+        return lst
+    if atomp(lst):
+        return [lst]
+    return _flatten(lst[0]) + _flatten(lst[1:])
+
+def flatten(lst):
+    """flatten list or return scalar"""
+    if atomp(lst): # scalar
+        return lst
+    return _flatten(lst)
+
+def slice_shape(lst, slices):
+    """Get the shape of lst after slicing: slices is a list of slice
+       objects."""
+    if atomp(lst):
+        return []
+    return [len(lst[slices[0]])] + slice_shape(lst[0], slices[1:])
+
+def multislice(lst, slices):
+    """Multi-dimensional slicing: slices is a list of slice objects."""
+    if atomp(lst):
+        return lst
+    return [multislice(sublst, slices[1:]) for sublst in lst[slices[0]]]
+
+def m_assign(llst, rlst, lslices, rslices):
+    """Multi-dimensional slice assignment: llst and rlst are the operands,
+       lslices and rslices are lists of slice objects. llst and rlst must
+       have the same structure.
+
+       For a two-dimensional example, this is not implemented in Python:
+
+         llst[0:3:2, 0:3:2] = rlst[1:3:1, 1:3:1]
+
+       Instead we write:
+
+         lslices = [slice(0,3,2), slice(0,3,2)]
+         rslices = [slice(1,3,1), slice(1,3,1)]
+         multislice_assign(llst, rlst, lslices, rslices)
+    """
+    if atomp(rlst):
+        return rlst
+    rlst = [m_assign(l, r, lslices[1:], rslices[1:])
+            for l, r in zip(llst[lslices[0]], rlst[rslices[0]])]
+    llst[lslices[0]] = rlst
+    return llst
+
+def cmp_structure(llst, rlst, lslices, rslices):
+    """Compare the structure of llst[lslices] and rlst[rslices]."""
+    lshape = slice_shape(llst, lslices)
+    rshape = slice_shape(rlst, rslices)
+    if (len(lshape) != len(rshape)):
+        return -1
+    for i in range(len(lshape)):
+        if lshape[i] != rshape[i]:
+            return -1
+        if lshape[i] == 0:
+            return 0
+    return 0
+
+def multislice_assign(llst, rlst, lslices, rslices):
+    """Return llst after assigning: llst[lslices] = rlst[rslices]"""
+    if cmp_structure(llst, rlst, lslices, rslices) < 0:
+        raise ValueError("lvalue and rvalue have different structures")
+    return m_assign(llst, rlst, lslices, rslices)
+
+
+# ======================================================================
+#                          Random structures
+# ======================================================================
+
+#
+# PEP-3118 is very permissive with respect to the contents of a
+# Py_buffer. In particular:
+#
+#   - shape can be zero
+#   - strides can be any integer, including zero
+#   - offset can point to any location in the underlying
+#     memory block, provided that it is a multiple of
+#     itemsize.
+#
+# The functions in this section test and verify random structures
+# in full generality. A structure is valid iff it fits in the
+# underlying memory block.
+#
+# The structure 't' (short for 'tuple') is fully defined by:
+#
+#   t = (memlen, itemsize, ndim, shape, strides, offset)
+#
+
+def verify_structure(memlen, itemsize, ndim, shape, strides, offset):
+    """Verify that the parameters represent a valid array within
+       the bounds of the allocated memory:
+           char *mem: start of the physical memory block
+           memlen: length of the physical memory block
+           offset: (char *)buf - mem
+    """
+    if offset % itemsize:
+        return False
+    if offset < 0 or offset+itemsize > memlen:
+        return False
+    if any(v % itemsize for v in strides):
+        return False
+
+    if ndim <= 0:
+        return ndim == 0 and not shape and not strides
+    if 0 in shape:
+        return True
+
+    imin = sum(strides[j]*(shape[j]-1) for j in range(ndim)
+               if strides[j] <= 0)
+    imax = sum(strides[j]*(shape[j]-1) for j in range(ndim)
+               if strides[j] > 0)
+
+    return 0 <= offset+imin and offset+imax+itemsize <= memlen
+
+def get_item(lst, indices):
+    for i in indices:
+        lst = lst[i]
+    return lst
+
+def memory_index(indices, t):
+    """Location of an item in the underlying memory."""
+    memlen, itemsize, ndim, shape, strides, offset = t
+    p = offset
+    for i in range(ndim):
+        p += strides[i]*indices[i]
+    return p
+
+def is_overlapping(t):
+    """The structure 't' is overlapping if at least one memory location
+       is visited twice while iterating through all possible tuples of
+       indices."""
+    memlen, itemsize, ndim, shape, strides, offset = t
+    visited = 1<<memlen
+    for ind in indices(shape):
+        i = memory_index(ind, t)
+        bit = 1<<i
+        if visited & bit:
+            return True
+        visited |= bit
+    return False
+
+def rand_structure(itemsize, valid, maxdim=5, maxshape=16, shape=()):
+    """Return random structure:
+           (memlen, itemsize, ndim, shape, strides, offset)
+       If 'valid' is true, the returned structure is valid, otherwise invalid.
+       If 'shape' is given, use that instead of creating a random shape.
+    """
+    if not shape:
+        ndim = randrange(maxdim+1)
+        if (ndim == 0):
+            if valid:
+                return itemsize, itemsize, ndim, (), (), 0
+            else:
+                nitems = randrange(1, 16+1)
+                memlen = nitems * itemsize
+                offset = -itemsize if randrange(2) == 0 else memlen
+                return memlen, itemsize, ndim, (), (), offset
+
+        minshape = 2
+        n = randrange(100)
+        if n >= 95 and valid:
+            minshape = 0
+        elif n >= 90:
+            minshape = 1
+        shape = [0] * ndim
+
+        for i in range(ndim):
+            shape[i] = randrange(minshape, maxshape+1)
+    else:
+        ndim = len(shape)
+
+    maxstride = 5
+    n = randrange(100)
+    zero_stride = True if n >= 95 and n & 1 else False
+
+    strides = [0] * ndim
+    strides[ndim-1] = itemsize * randrange(-maxstride, maxstride+1)
+    if not zero_stride and strides[ndim-1] == 0:
+        strides[ndim-1] = itemsize
+
+    for i in range(ndim-2, -1, -1):
+        maxstride *= shape[i+1] if shape[i+1] else 1
+        if zero_stride:
+            strides[i] = itemsize * randrange(-maxstride, maxstride+1)
+        else:
+            strides[i] = ((1,-1)[randrange(2)] *
+                          itemsize * randrange(1, maxstride+1))
+
+    imin = imax = 0
+    if not 0 in shape:
+        imin = sum(strides[j]*(shape[j]-1) for j in range(ndim)
+                   if strides[j] <= 0)
+        imax = sum(strides[j]*(shape[j]-1) for j in range(ndim)
+                   if strides[j] > 0)
+
+    nitems = imax - imin
+    if valid:
+        offset = -imin * itemsize
+        memlen = offset + (imax+1) * itemsize
+    else:
+        memlen = (-imin + imax) * itemsize
+        offset = -imin-itemsize if randrange(2) == 0 else memlen
+    return memlen, itemsize, ndim, shape, strides, offset
+
+def randslice_from_slicelen(slicelen, listlen):
+    """Create a random slice of len slicelen that fits into listlen."""
+    maxstart = listlen - slicelen
+    start = randrange(maxstart+1)
+    maxstep = (listlen - start) // slicelen if slicelen else 1
+    step = randrange(1, maxstep+1)
+    stop = start + slicelen * step
+    s = slice(start, stop, step)
+    _, _, _, control = slice_indices(s, listlen)
+    if control != slicelen:
+        raise RuntimeError
+    return s
+
+def randslice_from_shape(ndim, shape):
+    """Create two sets of slices for an array x with shape 'shape'
+       such that shapeof(x[lslices]) == shapeof(x[rslices])."""
+    lslices = [0] * ndim
+    rslices = [0] * ndim
+    for n in range(ndim):
+        l = shape[n]
+        slicelen = randrange(1, l+1) if l > 0 else 0
+        lslices[n] = randslice_from_slicelen(slicelen, l)
+        rslices[n] = randslice_from_slicelen(slicelen, l)
+    return tuple(lslices), tuple(rslices)
+
+def rand_aligned_slices(maxdim=5, maxshape=16):
+    """Create (lshape, rshape, tuple(lslices), tuple(rslices)) such that
+       shapeof(x[lslices]) == shapeof(y[rslices]), where x is an array
+       with shape 'lshape' and y is an array with shape 'rshape'."""
+    ndim = randrange(1, maxdim+1)
+    minshape = 2
+    n = randrange(100)
+    if n >= 95:
+        minshape = 0
+    elif n >= 90:
+        minshape = 1
+    all_random = True if randrange(100) >= 80 else False
+    lshape = [0]*ndim; rshape = [0]*ndim
+    lslices = [0]*ndim; rslices = [0]*ndim
+
+    for n in range(ndim):
+        small = randrange(minshape, maxshape+1)
+        big = randrange(minshape, maxshape+1)
+        if big < small:
+            big, small = small, big
+
+        # Create a slice that fits the smaller value.
+        if all_random:
+            start = randrange(-small, small+1)
+            stop = randrange(-small, small+1)
+            step = (1,-1)[randrange(2)] * randrange(1, small+2)
+            s_small = slice(start, stop, step)
+            _, _, _, slicelen = slice_indices(s_small, small)
+        else:
+            slicelen = randrange(1, small+1) if small > 0 else 0
+            s_small = randslice_from_slicelen(slicelen, small)
+
+        # Create a slice of the same length for the bigger value.
+        s_big = randslice_from_slicelen(slicelen, big)
+        if randrange(2) == 0:
+            rshape[n], lshape[n] = big, small
+            rslices[n], lslices[n] = s_big, s_small
+        else:
+            rshape[n], lshape[n] = small, big
+            rslices[n], lslices[n] = s_small, s_big
+
+    return lshape, rshape, tuple(lslices), tuple(rslices)
+
+def randitems_from_structure(fmt, t):
+    """Return a list of random items for structure 't' with format
+       'fmtchar'."""
+    memlen, itemsize, _, _, _, _ = t
+    return gen_items(memlen//itemsize, '#'+fmt, 'numpy')
+
+def ndarray_from_structure(items, fmt, t, flags=0):
+    """Return ndarray from the tuple returned by rand_structure()"""
+    memlen, itemsize, ndim, shape, strides, offset = t
+    return ndarray(items, shape=shape, strides=strides, format=fmt,
+                   offset=offset, flags=ND_WRITABLE|flags)
+
+def numpy_array_from_structure(items, fmt, t):
+    """Return numpy_array from the tuple returned by rand_structure()"""
+    memlen, itemsize, ndim, shape, strides, offset = t
+    buf = bytearray(memlen)
+    for j, v in enumerate(items):
+        struct.pack_into(fmt, buf, j*itemsize, v)
+    return numpy_array(buffer=buf, shape=shape, strides=strides,
+                       dtype=fmt, offset=offset)
+
+
+# ======================================================================
+#                          memoryview casts
+# ======================================================================
+
+def cast_items(exporter, fmt, itemsize, shape=None):
+    """Interpret the raw memory of 'exporter' as a list of items with
+       size 'itemsize'. If shape=None, the new structure is assumed to
+       be 1-D with n * itemsize = bytelen. If shape is given, the usual
+       constraint for contiguous arrays prod(shape) * itemsize = bytelen
+       applies. On success, return (items, shape). If the constraints
+       cannot be met, return (None, None). If a chunk of bytes is interpreted
+       as NaN as a result of float conversion, return ('nan', None)."""
+    bytelen = exporter.nbytes
+    if shape:
+        if prod(shape) * itemsize != bytelen:
+            return None, shape
+    elif shape == []:
+        if exporter.ndim == 0 or itemsize != bytelen:
+            return None, shape
+    else:
+        n, r = divmod(bytelen, itemsize)
+        shape = [n]
+        if r != 0:
+            return None, shape
+
+    mem = exporter.tobytes()
+    byteitems = [mem[i:i+itemsize] for i in range(0, len(mem), itemsize)]
+
+    items = []
+    for v in byteitems:
+        item = struct.unpack(fmt, v)[0]
+        if item != item:
+            return 'nan', shape
+        items.append(item)
+
+    return (items, shape) if shape != [] else (items[0], shape)
+
+def gencastshapes():
+    """Generate shapes to test casting."""
+    for n in range(32):
+        yield [n]
+    ndim = randrange(4, 6)
+    minshape = 1 if randrange(100) > 80 else 2
+    yield [randrange(minshape, 5) for _ in range(ndim)]
+    ndim = randrange(2, 4)
+    minshape = 1 if randrange(100) > 80 else 2
+    yield [randrange(minshape, 5) for _ in range(ndim)]
+
+
+# ======================================================================
+#                              Actual tests
+# ======================================================================
+
+def genslices(n):
+    """Generate all possible slices for a single dimension."""
+    return product(range(-n, n+1), range(-n, n+1), range(-n, n+1))
+
+def genslices_ndim(ndim, shape):
+    """Generate all possible slice tuples for 'shape'."""
+    iterables = [genslices(shape[n]) for n in range(ndim)]
+    return product(*iterables)
+
+def rslice(n, allow_empty=False):
+    """Generate random slice for a single dimension of length n.
+       If zero=True, the slices may be empty, otherwise they will
+       be non-empty."""
+    minlen = 0 if allow_empty or n == 0 else 1
+    slicelen = randrange(minlen, n+1)
+    return randslice_from_slicelen(slicelen, n)
+
+def rslices(n, allow_empty=False):
+    """Generate random slices for a single dimension."""
+    for _ in range(5):
+        yield rslice(n, allow_empty)
+
+def rslices_ndim(ndim, shape, iterations=5):
+    """Generate random slice tuples for 'shape'."""
+    # non-empty slices
+    for _ in range(iterations):
+        yield tuple(rslice(shape[n]) for n in range(ndim))
+    # possibly empty slices
+    for _ in range(iterations):
+        yield tuple(rslice(shape[n], allow_empty=True) for n in range(ndim))
+    # invalid slices
+    yield tuple(slice(0,1,0) for _ in range(ndim))
+
+def rpermutation(iterable, r=None):
+    pool = tuple(iterable)
+    r = len(pool) if r is None else r
+    yield tuple(sample(pool, r))
+
+def ndarray_print(nd):
+    """Print ndarray for debugging."""
+    try:
+        x = nd.tolist()
+    except (TypeError, NotImplementedError):
+        x = nd.tobytes()
+    if isinstance(nd, ndarray):
+        offset = nd.offset
+        flags = nd.flags
+    else:
+        offset = 'unknown'
+        flags = 'unknown'
+    print("ndarray(%s, shape=%s, strides=%s, suboffsets=%s, offset=%s, "
+          "format='%s', itemsize=%s, flags=%s)" %
+          (x, nd.shape, nd.strides, nd.suboffsets, offset,
+           nd.format, nd.itemsize, flags))
+    sys.stdout.flush()
+
+
+ITERATIONS = 100
+MAXDIM = 5
+MAXSHAPE = 10
+
+if SHORT_TEST:
+    ITERATIONS = 10
+    MAXDIM = 3
+    MAXSHAPE = 4
+    genslices = rslices
+    genslices_ndim = rslices_ndim
+    permutations = rpermutation
+
+
+@unittest.skipUnless(struct, 'struct module required for this test.')
+@unittest.skipUnless(ndarray, 'ndarray object required for this test')
+class TestBufferProtocol(unittest.TestCase):
+
+    def setUp(self):
+        # The suboffsets tests need sizeof(void *).
+        self.sizeof_void_p = get_sizeof_void_p()
+
+    def verify(self, result, obj=-1,
+                     itemsize={1}, fmt=-1, readonly={1},
+                     ndim={1}, shape=-1, strides=-1,
+                     lst=-1, sliced=False, cast=False):
+        # Verify buffer contents against expected values. Default values
+        # are deliberately initialized to invalid types.
+        if shape:
+            expected_len = prod(shape)*itemsize
+        else:
+            if not fmt: # array has been implicitly cast to unsigned bytes
+                expected_len = len(lst)
+            else: # ndim = 0
+                expected_len = itemsize
+
+        # Reconstruct suboffsets from strides. Support for slicing
+        # could be added, but is currently only needed for test_getbuf().
+        suboffsets = ()
+        if result.suboffsets:
+            self.assertGreater(ndim, 0)
+
+            suboffset0 = 0
+            for n in range(1, ndim):
+                if shape[n] == 0:
+                    break
+                if strides[n] <= 0:
+                    suboffset0 += -strides[n] * (shape[n]-1)
+
+            suboffsets = [suboffset0] + [-1 for v in range(ndim-1)]
+
+            # Not correct if slicing has occurred in the first dimension.
+            stride0 = self.sizeof_void_p
+            if strides[0] < 0:
+                stride0 = -stride0
+            strides = [stride0] + list(strides[1:])
+
+        self.assertIs(result.obj, obj)
+        self.assertEqual(result.nbytes, expected_len)
+        self.assertEqual(result.itemsize, itemsize)
+        self.assertEqual(result.format, fmt)
+        self.assertEqual(result.readonly, readonly)
+        self.assertEqual(result.ndim, ndim)
+        self.assertEqual(result.shape, tuple(shape))
+        if not (sliced and suboffsets):
+            self.assertEqual(result.strides, tuple(strides))
+        self.assertEqual(result.suboffsets, tuple(suboffsets))
+
+        if isinstance(result, ndarray) or is_memoryview_format(fmt):
+            rep = result.tolist() if fmt else result.tobytes()
+            self.assertEqual(rep, lst)
+
+        if not fmt: # array has been cast to unsigned bytes,
+            return  # the remaining tests won't work.
+
+        # PyBuffer_GetPointer() is the definition how to access an item.
+        # If PyBuffer_GetPointer(indices) is correct for all possible
+        # combinations of indices, the buffer is correct.
+        #
+        # Also test tobytes() against the flattened 'lst', with all items
+        # packed to bytes.
+        if not cast: # casts chop up 'lst' in different ways
+            b = bytearray()
+            buf_err = None
+            for ind in indices(shape):
+                try:
+                    item1 = get_pointer(result, ind)
+                    item2 = get_item(lst, ind)
+                    if isinstance(item2, tuple):
+                        x = struct.pack(fmt, *item2)
+                    else:
+                        x = struct.pack(fmt, item2)
+                    b.extend(x)
+                except BufferError:
+                    buf_err = True # re-exporter does not provide full buffer
+                    break
+                self.assertEqual(item1, item2)
+
+            if not buf_err:
+                # test tobytes()
+                self.assertEqual(result.tobytes(), b)
+
+                # lst := expected multi-dimensional logical representation
+                # flatten(lst) := elements in C-order
+                ff = fmt if fmt else 'B'
+                flattened = flatten(lst)
+
+                # Rules for 'A': if the array is already contiguous, return
+                # the array unaltered. Otherwise, return a contiguous 'C'
+                # representation.
+                for order in ['C', 'F', 'A']:
+                    expected = result
+                    if order == 'F':
+                        if not is_contiguous(result, 'A') or \
+                           is_contiguous(result, 'C'):
+                            # For constructing the ndarray, convert the
+                            # flattened logical representation to Fortran order.
+                            trans = transpose(flattened, shape)
+                            expected = ndarray(trans, shape=shape, format=ff,
+                                               flags=ND_FORTRAN)
+                    else: # 'C', 'A'
+                        if not is_contiguous(result, 'A') or \
+                           is_contiguous(result, 'F') and order == 'C':
+                            # The flattened list is already in C-order.
+                            expected = ndarray(flattened, shape=shape, format=ff)
+
+                    contig = get_contiguous(result, PyBUF_READ, order)
+                    self.assertEqual(contig.tobytes(), b)
+                    self.assertTrue(cmp_contig(contig, expected))
+
+                    if ndim == 0:
+                        continue
+
+                    nmemb = len(flattened)
+                    ro = 0 if readonly else ND_WRITABLE
+
+                    ### See comment in test_py_buffer_to_contiguous for an
+                    ### explanation why these tests are valid.
+
+                    # To 'C'
+                    contig = py_buffer_to_contiguous(result, 'C', PyBUF_FULL_RO)
+                    self.assertEqual(len(contig), nmemb * itemsize)
+                    initlst = [struct.unpack_from(fmt, contig, n*itemsize)
+                               for n in range(nmemb)]
+                    if len(initlst[0]) == 1:
+                        initlst = [v[0] for v in initlst]
+
+                    y = ndarray(initlst, shape=shape, flags=ro, format=fmt)
+                    self.assertEqual(memoryview(y), memoryview(result))
+
+                    # To 'F'
+                    contig = py_buffer_to_contiguous(result, 'F', PyBUF_FULL_RO)
+                    self.assertEqual(len(contig), nmemb * itemsize)
+                    initlst = [struct.unpack_from(fmt, contig, n*itemsize)
+                               for n in range(nmemb)]
+                    if len(initlst[0]) == 1:
+                        initlst = [v[0] for v in initlst]
+
+                    y = ndarray(initlst, shape=shape, flags=ro|ND_FORTRAN,
+                                format=fmt)
+                    self.assertEqual(memoryview(y), memoryview(result))
+
+                    # To 'A'
+                    contig = py_buffer_to_contiguous(result, 'A', PyBUF_FULL_RO)
+                    self.assertEqual(len(contig), nmemb * itemsize)
+                    initlst = [struct.unpack_from(fmt, contig, n*itemsize)
+                               for n in range(nmemb)]
+                    if len(initlst[0]) == 1:
+                        initlst = [v[0] for v in initlst]
+
+                    f = ND_FORTRAN if is_contiguous(result, 'F') else 0
+                    y = ndarray(initlst, shape=shape, flags=f|ro, format=fmt)
+                    self.assertEqual(memoryview(y), memoryview(result))
+
+        if is_memoryview_format(fmt):
+            try:
+                m = memoryview(result)
+            except BufferError: # re-exporter does not provide full information
+                return
+            ex = result.obj if isinstance(result, memoryview) else result
+            self.assertIs(m.obj, ex)
+            self.assertEqual(m.nbytes, expected_len)
+            self.assertEqual(m.itemsize, itemsize)
+            self.assertEqual(m.format, fmt)
+            self.assertEqual(m.readonly, readonly)
+            self.assertEqual(m.ndim, ndim)
+            self.assertEqual(m.shape, tuple(shape))
+            if not (sliced and suboffsets):
+                self.assertEqual(m.strides, tuple(strides))
+            self.assertEqual(m.suboffsets, tuple(suboffsets))
+
+            n = 1 if ndim == 0 else len(lst)
+            self.assertEqual(len(m), n)
+
+            rep = result.tolist() if fmt else result.tobytes()
+            self.assertEqual(rep, lst)
+            self.assertEqual(m, result)
+
+    def verify_getbuf(self, orig_ex, ex, req, sliced=False):
+        def simple_fmt(ex):
+            return ex.format == '' or ex.format == 'B'
+        def match(req, flag):
+            return ((req&flag) == flag)
+
+        if (# writable request to read-only exporter
+            (ex.readonly and match(req, PyBUF_WRITABLE)) or
+            # cannot match explicit contiguity request
+            (match(req, PyBUF_C_CONTIGUOUS) and not ex.c_contiguous) or
+            (match(req, PyBUF_F_CONTIGUOUS) and not ex.f_contiguous) or
+            (match(req, PyBUF_ANY_CONTIGUOUS) and not ex.contiguous) or
+            # buffer needs suboffsets
+            (not match(req, PyBUF_INDIRECT) and ex.suboffsets) or
+            # buffer without strides must be C-contiguous
+            (not match(req, PyBUF_STRIDES) and not ex.c_contiguous) or
+            # PyBUF_SIMPLE|PyBUF_FORMAT and PyBUF_WRITABLE|PyBUF_FORMAT
+            (not match(req, PyBUF_ND) and match(req, PyBUF_FORMAT))):
+
+            self.assertRaises(BufferError, ndarray, ex, getbuf=req)
+            return
+
+        if isinstance(ex, ndarray) or is_memoryview_format(ex.format):
+            lst = ex.tolist()
+        else:
+            nd = ndarray(ex, getbuf=PyBUF_FULL_RO)
+            lst = nd.tolist()
+
+        # The consumer may have requested default values or a NULL format.
+        ro = 0 if match(req, PyBUF_WRITABLE) else ex.readonly
+        fmt = ex.format
+        itemsize = ex.itemsize
+        ndim = ex.ndim
+        if not match(req, PyBUF_FORMAT):
+            # itemsize refers to the original itemsize before the cast.
+            # The equality product(shape) * itemsize = len still holds.
+            # The equality calcsize(format) = itemsize does _not_ hold.
+            fmt = ''
+            lst = orig_ex.tobytes() # Issue 12834
+        if not match(req, PyBUF_ND):
+            ndim = 1
+        shape = orig_ex.shape if match(req, PyBUF_ND) else ()
+        strides = orig_ex.strides if match(req, PyBUF_STRIDES) else ()
+
+        nd = ndarray(ex, getbuf=req)
+        self.verify(nd, obj=ex,
+                    itemsize=itemsize, fmt=fmt, readonly=ro,
+                    ndim=ndim, shape=shape, strides=strides,
+                    lst=lst, sliced=sliced)
+
+    def test_ndarray_getbuf(self):
+        requests = (
+            # distinct flags
+            PyBUF_INDIRECT, PyBUF_STRIDES, PyBUF_ND, PyBUF_SIMPLE,
+            PyBUF_C_CONTIGUOUS, PyBUF_F_CONTIGUOUS, PyBUF_ANY_CONTIGUOUS,
+            # compound requests
+            PyBUF_FULL, PyBUF_FULL_RO,
+            PyBUF_RECORDS, PyBUF_RECORDS_RO,
+            PyBUF_STRIDED, PyBUF_STRIDED_RO,
+            PyBUF_CONTIG, PyBUF_CONTIG_RO,
+        )
+        # items and format
+        items_fmt = (
+            ([True if x % 2 else False for x in range(12)], '?'),
+            ([1,2,3,4,5,6,7,8,9,10,11,12], 'b'),
+            ([1,2,3,4,5,6,7,8,9,10,11,12], 'B'),
+            ([(2**31-x) if x % 2 else (-2**31+x) for x in range(12)], 'l')
+        )
+        # shape, strides, offset
+        structure = (
+            ([], [], 0),
+            ([12], [], 0),
+            ([12], [-1], 11),
+            ([6], [2], 0),
+            ([6], [-2], 11),
+            ([3, 4], [], 0),
+            ([3, 4], [-4, -1], 11),
+            ([2, 2], [4, 1], 4),
+            ([2, 2], [-4, -1], 8)
+        )
+        # ndarray creation flags
+        ndflags = (
+            0, ND_WRITABLE, ND_FORTRAN, ND_FORTRAN|ND_WRITABLE,
+            ND_PIL, ND_PIL|ND_WRITABLE
+        )
+        # flags that can actually be used as flags
+        real_flags = (0, PyBUF_WRITABLE, PyBUF_FORMAT,
+                      PyBUF_WRITABLE|PyBUF_FORMAT)
+
+        for items, fmt in items_fmt:
+            itemsize = struct.calcsize(fmt)
+            for shape, strides, offset in structure:
+                strides = [v * itemsize for v in strides]
+                offset *= itemsize
+                for flags in ndflags:
+
+                    if strides and (flags&ND_FORTRAN):
+                        continue
+                    if not shape and (flags&ND_PIL):
+                        continue
+
+                    _items = items if shape else items[0]
+                    ex1 = ndarray(_items, format=fmt, flags=flags,
+                                  shape=shape, strides=strides, offset=offset)
+                    ex2 = ex1[::-2] if shape else None
+
+                    m1 = memoryview(ex1)
+                    if ex2:
+                        m2 = memoryview(ex2)
+                    if ex1.ndim == 0 or (ex1.ndim == 1 and shape and strides):
+                        self.assertEqual(m1, ex1)
+                    if ex2 and ex2.ndim == 1 and shape and strides:
+                        self.assertEqual(m2, ex2)
+
+                    for req in requests:
+                        for bits in real_flags:
+                            self.verify_getbuf(ex1, ex1, req|bits)
+                            self.verify_getbuf(ex1, m1, req|bits)
+                            if ex2:
+                                self.verify_getbuf(ex2, ex2, req|bits,
+                                                   sliced=True)
+                                self.verify_getbuf(ex2, m2, req|bits,
+                                                   sliced=True)
+
+        items = [1,2,3,4,5,6,7,8,9,10,11,12]
+
+        # ND_GETBUF_FAIL
+        ex = ndarray(items, shape=[12], flags=ND_GETBUF_FAIL)
+        self.assertRaises(BufferError, ndarray, ex)
+
+        # Request complex structure from a simple exporter. In this
+        # particular case the test object is not PEP-3118 compliant.
+        base = ndarray([9], [1])
+        ex = ndarray(base, getbuf=PyBUF_SIMPLE)
+        self.assertRaises(BufferError, ndarray, ex, getbuf=PyBUF_WRITABLE)
+        self.assertRaises(BufferError, ndarray, ex, getbuf=PyBUF_ND)
+        self.assertRaises(BufferError, ndarray, ex, getbuf=PyBUF_STRIDES)
+        self.assertRaises(BufferError, ndarray, ex, getbuf=PyBUF_C_CONTIGUOUS)
+        self.assertRaises(BufferError, ndarray, ex, getbuf=PyBUF_F_CONTIGUOUS)
+        self.assertRaises(BufferError, ndarray, ex, getbuf=PyBUF_ANY_CONTIGUOUS)
+        nd = ndarray(ex, getbuf=PyBUF_SIMPLE)
+
+    def test_ndarray_exceptions(self):
+        nd = ndarray([9], [1])
+        ndm = ndarray([9], [1], flags=ND_VAREXPORT)
+
+        # Initialization of a new ndarray or mutation of an existing array.
+        for c in (ndarray, nd.push, ndm.push):
+            # Invalid types.
+            self.assertRaises(TypeError, c, {1,2,3})
+            self.assertRaises(TypeError, c, [1,2,'3'])
+            self.assertRaises(TypeError, c, [1,2,(3,4)])
+            self.assertRaises(TypeError, c, [1,2,3], shape={3})
+            self.assertRaises(TypeError, c, [1,2,3], shape=[3], strides={1})
+            self.assertRaises(TypeError, c, [1,2,3], shape=[3], offset=[])
+            self.assertRaises(TypeError, c, [1], shape=[1], format={})
+            self.assertRaises(TypeError, c, [1], shape=[1], flags={})
+            self.assertRaises(TypeError, c, [1], shape=[1], getbuf={})
+
+            # ND_FORTRAN flag is only valid without strides.
+            self.assertRaises(TypeError, c, [1], shape=[1], strides=[1],
+                              flags=ND_FORTRAN)
+
+            # ND_PIL flag is only valid with ndim > 0.
+            self.assertRaises(TypeError, c, [1], shape=[], flags=ND_PIL)
+
+            # Invalid items.
+            self.assertRaises(ValueError, c, [], shape=[1])
+            self.assertRaises(ValueError, c, ['XXX'], shape=[1], format="L")
+            # Invalid combination of items and format.
+            self.assertRaises(struct.error, c, [1000], shape=[1], format="B")
+            self.assertRaises(ValueError, c, [1,(2,3)], shape=[2], format="B")
+            self.assertRaises(ValueError, c, [1,2,3], shape=[3], format="QL")
+
+            # Invalid ndim.
+            n = ND_MAX_NDIM+1
+            self.assertRaises(ValueError, c, [1]*n, shape=[1]*n)
+
+            # Invalid shape.
+            self.assertRaises(ValueError, c, [1], shape=[-1])
+            self.assertRaises(ValueError, c, [1,2,3], shape=['3'])
+            self.assertRaises(OverflowError, c, [1], shape=[2**128])
+            # prod(shape) * itemsize != len(items)
+            self.assertRaises(ValueError, c, [1,2,3,4,5], shape=[2,2], offset=3)
+
+            # Invalid strides.
+            self.assertRaises(ValueError, c, [1,2,3], shape=[3], strides=['1'])
+            self.assertRaises(OverflowError, c, [1], shape=[1],
+                              strides=[2**128])
+
+            # Invalid combination of strides and shape.
+            self.assertRaises(ValueError, c, [1,2], shape=[2,1], strides=[1])
+            # Invalid combination of strides and format.
+            self.assertRaises(ValueError, c, [1,2,3,4], shape=[2], strides=[3],
+                              format="L")
+
+            # Invalid offset.
+            self.assertRaises(ValueError, c, [1,2,3], shape=[3], offset=4)
+            self.assertRaises(ValueError, c, [1,2,3], shape=[1], offset=3,
+                              format="L")
+
+            # Invalid format.
+            self.assertRaises(ValueError, c, [1,2,3], shape=[3], format="")
+            self.assertRaises(struct.error, c, [(1,2,3)], shape=[1],
+                              format="@#$")
+
+            # Striding out of the memory bounds.
+            items = [1,2,3,4,5,6,7,8,9,10]
+            self.assertRaises(ValueError, c, items, shape=[2,3],
+                              strides=[-3, -2], offset=5)
+
+            # Constructing consumer: format argument invalid.
+            self.assertRaises(TypeError, c, bytearray(), format="Q")
+
+            # Constructing original base object: getbuf argument invalid.
+            self.assertRaises(TypeError, c, [1], shape=[1], getbuf=PyBUF_FULL)
+
+            # Shape argument is mandatory for original base objects.
+            self.assertRaises(TypeError, c, [1])
+
+
+        # PyBUF_WRITABLE request to read-only provider.
+        self.assertRaises(BufferError, ndarray, b'123', getbuf=PyBUF_WRITABLE)
+
+        # ND_VAREXPORT can only be specified during construction.
+        nd = ndarray([9], [1], flags=ND_VAREXPORT)
+        self.assertRaises(ValueError, nd.push, [1], [1], flags=ND_VAREXPORT)
+
+        # Invalid operation for consumers: push/pop
+        nd = ndarray(b'123')
+        self.assertRaises(BufferError, nd.push, [1], [1])
+        self.assertRaises(BufferError, nd.pop)
+
+        # ND_VAREXPORT not set: push/pop fail with exported buffers
+        nd = ndarray([9], [1])
+        nd.push([1], [1])
+        m = memoryview(nd)
+        self.assertRaises(BufferError, nd.push, [1], [1])
+        self.assertRaises(BufferError, nd.pop)
+        m.release()
+        nd.pop()
+
+        # Single remaining buffer: pop fails
+        self.assertRaises(BufferError, nd.pop)
+        del nd
+
+        # get_pointer()
+        self.assertRaises(TypeError, get_pointer, {}, [1,2,3])
+        self.assertRaises(TypeError, get_pointer, b'123', {})
+
+        nd = ndarray(list(range(100)), shape=[1]*100)
+        self.assertRaises(ValueError, get_pointer, nd, [5])
+
+        nd = ndarray(list(range(12)), shape=[3,4])
+        self.assertRaises(ValueError, get_pointer, nd, [2,3,4])
+        self.assertRaises(ValueError, get_pointer, nd, [3,3])
+        self.assertRaises(ValueError, get_pointer, nd, [-3,3])
+        self.assertRaises(OverflowError, get_pointer, nd, [1<<64,3])
+
+        # tolist() needs format
+        ex = ndarray([1,2,3], shape=[3], format='L')
+        nd = ndarray(ex, getbuf=PyBUF_SIMPLE)
+        self.assertRaises(ValueError, nd.tolist)
+
+        # memoryview_from_buffer()
+        ex1 = ndarray([1,2,3], shape=[3], format='L')
+        ex2 = ndarray(ex1)
+        nd = ndarray(ex2)
+        self.assertRaises(TypeError, nd.memoryview_from_buffer)
+
+        nd = ndarray([(1,)*200], shape=[1], format='L'*200)
+        self.assertRaises(TypeError, nd.memoryview_from_buffer)
+
+        n = ND_MAX_NDIM
+        nd = ndarray(list(range(n)), shape=[1]*n)
+        self.assertRaises(ValueError, nd.memoryview_from_buffer)
+
+        # get_contiguous()
+        nd = ndarray([1], shape=[1])
+        self.assertRaises(TypeError, get_contiguous, 1, 2, 3, 4, 5)
+        self.assertRaises(TypeError, get_contiguous, nd, "xyz", 'C')
+        self.assertRaises(OverflowError, get_contiguous, nd, 2**64, 'C')
+        self.assertRaises(TypeError, get_contiguous, nd, PyBUF_READ, 961)
+        self.assertRaises(UnicodeEncodeError, get_contiguous, nd, PyBUF_READ,
+                          '\u2007')
+        self.assertRaises(ValueError, get_contiguous, nd, PyBUF_READ, 'Z')
+        self.assertRaises(ValueError, get_contiguous, nd, 255, 'A')
+
+        # cmp_contig()
+        nd = ndarray([1], shape=[1])
+        self.assertRaises(TypeError, cmp_contig, 1, 2, 3, 4, 5)
+        self.assertRaises(TypeError, cmp_contig, {}, nd)
+        self.assertRaises(TypeError, cmp_contig, nd, {})
+
+        # is_contiguous()
+        nd = ndarray([1], shape=[1])
+        self.assertRaises(TypeError, is_contiguous, 1, 2, 3, 4, 5)
+        self.assertRaises(TypeError, is_contiguous, {}, 'A')
+        self.assertRaises(TypeError, is_contiguous, nd, 201)
+
+    def test_ndarray_linked_list(self):
+        for perm in permutations(range(5)):
+            m = [0]*5
+            nd = ndarray([1,2,3], shape=[3], flags=ND_VAREXPORT)
+            m[0] = memoryview(nd)
+
+            for i in range(1, 5):
+                nd.push([1,2,3], shape=[3])
+                m[i] = memoryview(nd)
+
+            for i in range(5):
+                m[perm[i]].release()
+
+            self.assertRaises(BufferError, nd.pop)
+            del nd
+
+    def test_ndarray_format_scalar(self):
+        # ndim = 0: scalar
+        for fmt, scalar, _ in iter_format(0):
+            itemsize = struct.calcsize(fmt)
+            nd = ndarray(scalar, shape=(), format=fmt)
+            self.verify(nd, obj=None,
+                        itemsize=itemsize, fmt=fmt, readonly=1,
+                        ndim=0, shape=(), strides=(),
+                        lst=scalar)
+
+    def test_ndarray_format_shape(self):
+        # ndim = 1, shape = [n]
+        nitems =  randrange(1, 10)
+        for fmt, items, _ in iter_format(nitems):
+            itemsize = struct.calcsize(fmt)
+            for flags in (0, ND_PIL):
+                nd = ndarray(items, shape=[nitems], format=fmt, flags=flags)
+                self.verify(nd, obj=None,
+                            itemsize=itemsize, fmt=fmt, readonly=1,
+                            ndim=1, shape=(nitems,), strides=(itemsize,),
+                            lst=items)
+
+    def test_ndarray_format_strides(self):
+        # ndim = 1, strides
+        nitems = randrange(1, 30)
+        for fmt, items, _ in iter_format(nitems):
+            itemsize = struct.calcsize(fmt)
+            for step in range(-5, 5):
+                if step == 0:
+                    continue
+
+                shape = [len(items[::step])]
+                strides = [step*itemsize]
+                offset = itemsize*(nitems-1) if step < 0 else 0
+
+                for flags in (0, ND_PIL):
+                    nd = ndarray(items, shape=shape, strides=strides,
+                                 format=fmt, offset=offset, flags=flags)
+                    self.verify(nd, obj=None,
+                                itemsize=itemsize, fmt=fmt, readonly=1,
+                                ndim=1, shape=shape, strides=strides,
+                                lst=items[::step])
+
+    def test_ndarray_fortran(self):
+        items = [1,2,3,4,5,6,7,8,9,10,11,12]
+        ex = ndarray(items, shape=(3, 4), strides=(1, 3))
+        nd = ndarray(ex, getbuf=PyBUF_F_CONTIGUOUS|PyBUF_FORMAT)
+        self.assertEqual(nd.tolist(), farray(items, (3, 4)))
+
+    def test_ndarray_multidim(self):
+        for ndim in range(5):
+            shape_t = [randrange(2, 10) for _ in range(ndim)]
+            nitems = prod(shape_t)
+            for shape in permutations(shape_t):
+
+                fmt, items, _ = randitems(nitems)
+                itemsize = struct.calcsize(fmt)
+
+                for flags in (0, ND_PIL):
+                    if ndim == 0 and flags == ND_PIL:
+                        continue
+
+                    # C array
+                    nd = ndarray(items, shape=shape, format=fmt, flags=flags)
+
+                    strides = strides_from_shape(ndim, shape, itemsize, 'C')
+                    lst = carray(items, shape)
+                    self.verify(nd, obj=None,
+                                itemsize=itemsize, fmt=fmt, readonly=1,
+                                ndim=ndim, shape=shape, strides=strides,
+                                lst=lst)
+
+                    if is_memoryview_format(fmt):
+                        # memoryview: reconstruct strides
+                        ex = ndarray(items, shape=shape, format=fmt)
+                        nd = ndarray(ex, getbuf=PyBUF_CONTIG_RO|PyBUF_FORMAT)
+                        self.assertTrue(nd.strides == ())
+                        mv = nd.memoryview_from_buffer()
+                        self.verify(mv, obj=None,
+                                    itemsize=itemsize, fmt=fmt, readonly=1,
+                                    ndim=ndim, shape=shape, strides=strides,
+                                    lst=lst)
+
+                    # Fortran array
+                    nd = ndarray(items, shape=shape, format=fmt,
+                                 flags=flags|ND_FORTRAN)
+
+                    strides = strides_from_shape(ndim, shape, itemsize, 'F')
+                    lst = farray(items, shape)
+                    self.verify(nd, obj=None,
+                                itemsize=itemsize, fmt=fmt, readonly=1,
+                                ndim=ndim, shape=shape, strides=strides,
+                                lst=lst)
+
+    def test_ndarray_index_invalid(self):
+        # not writable
+        nd = ndarray([1], shape=[1])
+        self.assertRaises(TypeError, nd.__setitem__, 1, 8)
+        mv = memoryview(nd)
+        self.assertEqual(mv, nd)
+        self.assertRaises(TypeError, mv.__setitem__, 1, 8)
+
+        # cannot be deleted
+        nd = ndarray([1], shape=[1], flags=ND_WRITABLE)
+        self.assertRaises(TypeError, nd.__delitem__, 1)
+        mv = memoryview(nd)
+        self.assertEqual(mv, nd)
+        self.assertRaises(TypeError, mv.__delitem__, 1)
+
+        # overflow
+        nd = ndarray([1], shape=[1], flags=ND_WRITABLE)
+        self.assertRaises(OverflowError, nd.__getitem__, 1<<64)
+        self.assertRaises(OverflowError, nd.__setitem__, 1<<64, 8)
+        mv = memoryview(nd)
+        self.assertEqual(mv, nd)
+        self.assertRaises(IndexError, mv.__getitem__, 1<<64)
+        self.assertRaises(IndexError, mv.__setitem__, 1<<64, 8)
+
+        # format
+        items = [1,2,3,4,5,6,7,8]
+        nd = ndarray(items, shape=[len(items)], format="B", flags=ND_WRITABLE)
+        self.assertRaises(struct.error, nd.__setitem__, 2, 300)
+        self.assertRaises(ValueError, nd.__setitem__, 1, (100, 200))
+        mv = memoryview(nd)
+        self.assertEqual(mv, nd)
+        self.assertRaises(ValueError, mv.__setitem__, 2, 300)
+        self.assertRaises(TypeError, mv.__setitem__, 1, (100, 200))
+
+        items = [(1,2), (3,4), (5,6)]
+        nd = ndarray(items, shape=[len(items)], format="LQ", flags=ND_WRITABLE)
+        self.assertRaises(ValueError, nd.__setitem__, 2, 300)
+        self.assertRaises(struct.error, nd.__setitem__, 1, (b'\x001', 200))
+
+    def test_ndarray_index_scalar(self):
+        # scalar
+        nd = ndarray(1, shape=(), flags=ND_WRITABLE)
+        mv = memoryview(nd)
+        self.assertEqual(mv, nd)
+
+        x = nd[()];  self.assertEqual(x, 1)
+        x = nd[...]; self.assertEqual(x.tolist(), nd.tolist())
+
+        x = mv[()];  self.assertEqual(x, 1)
+        x = mv[...]; self.assertEqual(x.tolist(), nd.tolist())
+
+        self.assertRaises(TypeError, nd.__getitem__, 0)
+        self.assertRaises(TypeError, mv.__getitem__, 0)
+        self.assertRaises(TypeError, nd.__setitem__, 0, 8)
+        self.assertRaises(TypeError, mv.__setitem__, 0, 8)
+
+        self.assertEqual(nd.tolist(), 1)
+        self.assertEqual(mv.tolist(), 1)
+
+        nd[()] = 9; self.assertEqual(nd.tolist(), 9)
+        mv[()] = 9; self.assertEqual(mv.tolist(), 9)
+
+        nd[...] = 5; self.assertEqual(nd.tolist(), 5)
+        mv[...] = 5; self.assertEqual(mv.tolist(), 5)
+
+    def test_ndarray_index_null_strides(self):
+        ex = ndarray(list(range(2*4)), shape=[2, 4], flags=ND_WRITABLE)
+        nd = ndarray(ex, getbuf=PyBUF_CONTIG)
+
+        # Sub-views are only possible for full exporters.
+        self.assertRaises(BufferError, nd.__getitem__, 1)
+        # Same for slices.
+        self.assertRaises(BufferError, nd.__getitem__, slice(3,5,1))
+
+    def test_ndarray_index_getitem_single(self):
+        # getitem
+        for fmt, items, _ in iter_format(5):
+            nd = ndarray(items, shape=[5], format=fmt)
+            for i in range(-5, 5):
+                self.assertEqual(nd[i], items[i])
+
+            self.assertRaises(IndexError, nd.__getitem__, -6)
+            self.assertRaises(IndexError, nd.__getitem__, 5)
+
+            if is_memoryview_format(fmt):
+                mv = memoryview(nd)
+                self.assertEqual(mv, nd)
+                for i in range(-5, 5):
+                    self.assertEqual(mv[i], items[i])
+
+                self.assertRaises(IndexError, mv.__getitem__, -6)
+                self.assertRaises(IndexError, mv.__getitem__, 5)
+
+        # getitem with null strides
+        for fmt, items, _ in iter_format(5):
+            ex = ndarray(items, shape=[5], flags=ND_WRITABLE, format=fmt)
+            nd = ndarray(ex, getbuf=PyBUF_CONTIG|PyBUF_FORMAT)
+
+            for i in range(-5, 5):
+                self.assertEqual(nd[i], items[i])
+
+            if is_memoryview_format(fmt):
+                mv = nd.memoryview_from_buffer()
+                self.assertIs(mv.__eq__(nd), NotImplemented)
+                for i in range(-5, 5):
+                    self.assertEqual(mv[i], items[i])
+
+        # getitem with null format
+        items = [1,2,3,4,5]
+        ex = ndarray(items, shape=[5])
+        nd = ndarray(ex, getbuf=PyBUF_CONTIG_RO)
+        for i in range(-5, 5):
+            self.assertEqual(nd[i], items[i])
+
+        # getitem with null shape/strides/format
+        items = [1,2,3,4,5]
+        ex = ndarray(items, shape=[5])
+        nd = ndarray(ex, getbuf=PyBUF_SIMPLE)
+
+        for i in range(-5, 5):
+            self.assertEqual(nd[i], items[i])
+
+    def test_ndarray_index_setitem_single(self):
+        # assign single value
+        for fmt, items, single_item in iter_format(5):
+            nd = ndarray(items, shape=[5], format=fmt, flags=ND_WRITABLE)
+            for i in range(5):
+                items[i] = single_item
+                nd[i] = single_item
+            self.assertEqual(nd.tolist(), items)
+
+            self.assertRaises(IndexError, nd.__setitem__, -6, single_item)
+            self.assertRaises(IndexError, nd.__setitem__, 5, single_item)
+
+            if not is_memoryview_format(fmt):
+                continue
+
+            nd = ndarray(items, shape=[5], format=fmt, flags=ND_WRITABLE)
+            mv = memoryview(nd)
+            self.assertEqual(mv, nd)
+            for i in range(5):
+                items[i] = single_item
+                mv[i] = single_item
+            self.assertEqual(mv.tolist(), items)
+
+            self.assertRaises(IndexError, mv.__setitem__, -6, single_item)
+            self.assertRaises(IndexError, mv.__setitem__, 5, single_item)
+
+
+        # assign single value: lobject = robject
+        for fmt, items, single_item in iter_format(5):
+            nd = ndarray(items, shape=[5], format=fmt, flags=ND_WRITABLE)
+            for i in range(-5, 4):
+                items[i] = items[i+1]
+                nd[i] = nd[i+1]
+            self.assertEqual(nd.tolist(), items)
+
+            if not is_memoryview_format(fmt):
+                continue
+
+            nd = ndarray(items, shape=[5], format=fmt, flags=ND_WRITABLE)
+            mv = memoryview(nd)
+            self.assertEqual(mv, nd)
+            for i in range(-5, 4):
+                items[i] = items[i+1]
+                mv[i] = mv[i+1]
+            self.assertEqual(mv.tolist(), items)
+
+    def test_ndarray_index_getitem_multidim(self):
+        shape_t = (2, 3, 5)
+        nitems = prod(shape_t)
+        for shape in permutations(shape_t):
+
+            fmt, items, _ = randitems(nitems)
+
+            for flags in (0, ND_PIL):
+                # C array
+                nd = ndarray(items, shape=shape, format=fmt, flags=flags)
+                lst = carray(items, shape)
+
+                for i in range(-shape[0], shape[0]):
+                    self.assertEqual(lst[i], nd[i].tolist())
+                    for j in range(-shape[1], shape[1]):
+                        self.assertEqual(lst[i][j], nd[i][j].tolist())
+                        for k in range(-shape[2], shape[2]):
+                            self.assertEqual(lst[i][j][k], nd[i][j][k])
+
+                # Fortran array
+                nd = ndarray(items, shape=shape, format=fmt,
+                             flags=flags|ND_FORTRAN)
+                lst = farray(items, shape)
+
+                for i in range(-shape[0], shape[0]):
+                    self.assertEqual(lst[i], nd[i].tolist())
+                    for j in range(-shape[1], shape[1]):
+                        self.assertEqual(lst[i][j], nd[i][j].tolist())
+                        for k in range(shape[2], shape[2]):
+                            self.assertEqual(lst[i][j][k], nd[i][j][k])
+
+    def test_ndarray_sequence(self):
+        nd = ndarray(1, shape=())
+        self.assertRaises(TypeError, eval, "1 in nd", locals())
+        mv = memoryview(nd)
+        self.assertEqual(mv, nd)
+        self.assertRaises(TypeError, eval, "1 in mv", locals())
+
+        for fmt, items, _ in iter_format(5):
+            nd = ndarray(items, shape=[5], format=fmt)
+            for i, v in enumerate(nd):
+                self.assertEqual(v, items[i])
+                self.assertTrue(v in nd)
+
+            if is_memoryview_format(fmt):
+                mv = memoryview(nd)
+                for i, v in enumerate(mv):
+                    self.assertEqual(v, items[i])
+                    self.assertTrue(v in mv)
+
+    def test_ndarray_slice_invalid(self):
+        items = [1,2,3,4,5,6,7,8]
+
+        # rvalue is not an exporter
+        xl = ndarray(items, shape=[8], flags=ND_WRITABLE)
+        ml = memoryview(xl)
+        self.assertRaises(TypeError, xl.__setitem__, slice(0,8,1), items)
+        self.assertRaises(TypeError, ml.__setitem__, slice(0,8,1), items)
+
+        # rvalue is not a full exporter
+        xl = ndarray(items, shape=[8], flags=ND_WRITABLE)
+        ex = ndarray(items, shape=[8], flags=ND_WRITABLE)
+        xr = ndarray(ex, getbuf=PyBUF_ND)
+        self.assertRaises(BufferError, xl.__setitem__, slice(0,8,1), xr)
+
+        # zero step
+        nd = ndarray(items, shape=[8], format="L", flags=ND_WRITABLE)
+        mv = memoryview(nd)
+        self.assertRaises(ValueError, nd.__getitem__, slice(0,1,0))
+        self.assertRaises(ValueError, mv.__getitem__, slice(0,1,0))
+
+        nd = ndarray(items, shape=[2,4], format="L", flags=ND_WRITABLE)
+        mv = memoryview(nd)
+
+        self.assertRaises(ValueError, nd.__getitem__,
+                          (slice(0,1,1), slice(0,1,0)))
+        self.assertRaises(ValueError, nd.__getitem__,
+                          (slice(0,1,0), slice(0,1,1)))
+        self.assertRaises(TypeError, nd.__getitem__, "@%$")
+        self.assertRaises(TypeError, nd.__getitem__, ("@%$", slice(0,1,1)))
+        self.assertRaises(TypeError, nd.__getitem__, (slice(0,1,1), {}))
+
+        # memoryview: not implemented
+        self.assertRaises(NotImplementedError, mv.__getitem__,
+                          (slice(0,1,1), slice(0,1,0)))
+        self.assertRaises(TypeError, mv.__getitem__, "@%$")
+
+        # differing format
+        xl = ndarray(items, shape=[8], format="B", flags=ND_WRITABLE)
+        xr = ndarray(items, shape=[8], format="b")
+        ml = memoryview(xl)
+        mr = memoryview(xr)
+        self.assertRaises(ValueError, xl.__setitem__, slice(0,1,1), xr[7:8])
+        self.assertEqual(xl.tolist(), items)
+        self.assertRaises(ValueError, ml.__setitem__, slice(0,1,1), mr[7:8])
+        self.assertEqual(ml.tolist(), items)
+
+        # differing itemsize
+        xl = ndarray(items, shape=[8], format="B", flags=ND_WRITABLE)
+        yr = ndarray(items, shape=[8], format="L")
+        ml = memoryview(xl)
+        mr = memoryview(xr)
+        self.assertRaises(ValueError, xl.__setitem__, slice(0,1,1), xr[7:8])
+        self.assertEqual(xl.tolist(), items)
+        self.assertRaises(ValueError, ml.__setitem__, slice(0,1,1), mr[7:8])
+        self.assertEqual(ml.tolist(), items)
+
+        # differing ndim
+        xl = ndarray(items, shape=[2, 4], format="b", flags=ND_WRITABLE)
+        xr = ndarray(items, shape=[8], format="b")
+        ml = memoryview(xl)
+        mr = memoryview(xr)
+        self.assertRaises(ValueError, xl.__setitem__, slice(0,1,1), xr[7:8])
+        self.assertEqual(xl.tolist(), [[1,2,3,4], [5,6,7,8]])
+        self.assertRaises(NotImplementedError, ml.__setitem__, slice(0,1,1),
+                          mr[7:8])
+
+        # differing shape
+        xl = ndarray(items, shape=[8], format="b", flags=ND_WRITABLE)
+        xr = ndarray(items, shape=[8], format="b")
+        ml = memoryview(xl)
+        mr = memoryview(xr)
+        self.assertRaises(ValueError, xl.__setitem__, slice(0,2,1), xr[7:8])
+        self.assertEqual(xl.tolist(), items)
+        self.assertRaises(ValueError, ml.__setitem__, slice(0,2,1), mr[7:8])
+        self.assertEqual(ml.tolist(), items)
+
+        # _testbuffer.c module functions
+        self.assertRaises(TypeError, slice_indices, slice(0,1,2), {})
+        self.assertRaises(TypeError, slice_indices, "###########", 1)
+        self.assertRaises(ValueError, slice_indices, slice(0,1,0), 4)
+
+        x = ndarray(items, shape=[8], format="b", flags=ND_PIL)
+        self.assertRaises(TypeError, x.add_suboffsets)
+
+        ex = ndarray(items, shape=[8], format="B")
+        x = ndarray(ex, getbuf=PyBUF_SIMPLE)
+        self.assertRaises(TypeError, x.add_suboffsets)
+
+    def test_ndarray_slice_zero_shape(self):
+        items = [1,2,3,4,5,6,7,8,9,10,11,12]
+
+        x = ndarray(items, shape=[12], format="L", flags=ND_WRITABLE)
+        y = ndarray(items, shape=[12], format="L")
+        x[4:4] = y[9:9]
+        self.assertEqual(x.tolist(), items)
+
+        ml = memoryview(x)
+        mr = memoryview(y)
+        self.assertEqual(ml, x)
+        self.assertEqual(ml, y)
+        ml[4:4] = mr[9:9]
+        self.assertEqual(ml.tolist(), items)
+
+        x = ndarray(items, shape=[3, 4], format="L", flags=ND_WRITABLE)
+        y = ndarray(items, shape=[4, 3], format="L")
+        x[1:2, 2:2] = y[1:2, 3:3]
+        self.assertEqual(x.tolist(), carray(items, [3, 4]))
+
+    def test_ndarray_slice_multidim(self):
+        shape_t = (2, 3, 5)
+        ndim = len(shape_t)
+        nitems = prod(shape_t)
+        for shape in permutations(shape_t):
+
+            fmt, items, _ = randitems(nitems)
+            itemsize = struct.calcsize(fmt)
+
+            for flags in (0, ND_PIL):
+                nd = ndarray(items, shape=shape, format=fmt, flags=flags)
+                lst = carray(items, shape)
+
+                for slices in rslices_ndim(ndim, shape):
+
+                    listerr = None
+                    try:
+                        sliced = multislice(lst, slices)
+                    except Exception as e:
+                        listerr = e.__class__
+
+                    nderr = None
+                    try:
+                        ndsliced = nd[slices]
+                    except Exception as e:
+                        nderr = e.__class__
+
+                    if nderr or listerr:
+                        self.assertIs(nderr, listerr)
+                    else:
+                        self.assertEqual(ndsliced.tolist(), sliced)
+
+    def test_ndarray_slice_redundant_suboffsets(self):
+        shape_t = (2, 3, 5, 2)
+        ndim = len(shape_t)
+        nitems = prod(shape_t)
+        for shape in permutations(shape_t):
+
+            fmt, items, _ = randitems(nitems)
+            itemsize = struct.calcsize(fmt)
+
+            nd = ndarray(items, shape=shape, format=fmt)
+            nd.add_suboffsets()
+            ex = ndarray(items, shape=shape, format=fmt)
+            ex.add_suboffsets()
+            mv = memoryview(ex)
+            lst = carray(items, shape)
+
+            for slices in rslices_ndim(ndim, shape):
+
+                listerr = None
+                try:
+                    sliced = multislice(lst, slices)
+                except Exception as e:
+                    listerr = e.__class__
+
+                nderr = None
+                try:
+                    ndsliced = nd[slices]
+                except Exception as e:
+                    nderr = e.__class__
+
+                if nderr or listerr:
+                    self.assertIs(nderr, listerr)
+                else:
+                    self.assertEqual(ndsliced.tolist(), sliced)
+
+    def test_ndarray_slice_assign_single(self):
+        for fmt, items, _ in iter_format(5):
+            for lslice in genslices(5):
+                for rslice in genslices(5):
+                    for flags in (0, ND_PIL):
+
+                        f = flags|ND_WRITABLE
+                        nd = ndarray(items, shape=[5], format=fmt, flags=f)
+                        ex = ndarray(items, shape=[5], format=fmt, flags=f)
+                        mv = memoryview(ex)
+
+                        lsterr = None
+                        diff_structure = None
+                        lst = items[:]
+                        try:
+                            lval = lst[lslice]
+                            rval = lst[rslice]
+                            lst[lslice] = lst[rslice]
+                            diff_structure = len(lval) != len(rval)
+                        except Exception as e:
+                            lsterr = e.__class__
+
+                        nderr = None
+                        try:
+                            nd[lslice] = nd[rslice]
+                        except Exception as e:
+                            nderr = e.__class__
+
+                        if diff_structure: # ndarray cannot change shape
+                            self.assertIs(nderr, ValueError)
+                        else:
+                            self.assertEqual(nd.tolist(), lst)
+                            self.assertIs(nderr, lsterr)
+
+                        if not is_memoryview_format(fmt):
+                            continue
+
+                        mverr = None
+                        try:
+                            mv[lslice] = mv[rslice]
+                        except Exception as e:
+                            mverr = e.__class__
+
+                        if diff_structure: # memoryview cannot change shape
+                            self.assertIs(mverr, ValueError)
+                        else:
+                            self.assertEqual(mv.tolist(), lst)
+                            self.assertEqual(mv, nd)
+                            self.assertIs(mverr, lsterr)
+                            self.verify(mv, obj=ex,
+                              itemsize=nd.itemsize, fmt=fmt, readonly=0,
+                              ndim=nd.ndim, shape=nd.shape, strides=nd.strides,
+                              lst=nd.tolist())
+
+    def test_ndarray_slice_assign_multidim(self):
+        shape_t = (2, 3, 5)
+        ndim = len(shape_t)
+        nitems = prod(shape_t)
+        for shape in permutations(shape_t):
+
+            fmt, items, _ = randitems(nitems)
+
+            for flags in (0, ND_PIL):
+                for _ in range(ITERATIONS):
+                    lslices, rslices = randslice_from_shape(ndim, shape)
+
+                    nd = ndarray(items, shape=shape, format=fmt,
+                                 flags=flags|ND_WRITABLE)
+                    lst = carray(items, shape)
+
+                    listerr = None
+                    try:
+                        result = multislice_assign(lst, lst, lslices, rslices)
+                    except Exception as e:
+                        listerr = e.__class__
+
+                    nderr = None
+                    try:
+                        nd[lslices] = nd[rslices]
+                    except Exception as e:
+                        nderr = e.__class__
+
+                    if nderr or listerr:
+                        self.assertIs(nderr, listerr)
+                    else:
+                        self.assertEqual(nd.tolist(), result)
+
+    def test_ndarray_random(self):
+        # construction of valid arrays
+        for _ in range(ITERATIONS):
+            for fmt in fmtdict['@']:
+                itemsize = struct.calcsize(fmt)
+
+                t = rand_structure(itemsize, True, maxdim=MAXDIM,
+                                   maxshape=MAXSHAPE)
+                self.assertTrue(verify_structure(*t))
+                items = randitems_from_structure(fmt, t)
+
+                x = ndarray_from_structure(items, fmt, t)
+                xlist = x.tolist()
+
+                mv = memoryview(x)
+                if is_memoryview_format(fmt):
+                    mvlist = mv.tolist()
+                    self.assertEqual(mvlist, xlist)
+
+                if t[2] > 0:
+                    # ndim > 0: test against suboffsets representation.
+                    y = ndarray_from_structure(items, fmt, t, flags=ND_PIL)
+                    ylist = y.tolist()
+                    self.assertEqual(xlist, ylist)
+
+                    mv = memoryview(y)
+                    if is_memoryview_format(fmt):
+                        self.assertEqual(mv, y)
+                        mvlist = mv.tolist()
+                        self.assertEqual(mvlist, ylist)
+
+                if numpy_array:
+                    shape = t[3]
+                    if 0 in shape:
+                        continue # http://projects.scipy.org/numpy/ticket/1910
+                    z = numpy_array_from_structure(items, fmt, t)
+                    self.verify(x, obj=None,
+                                itemsize=z.itemsize, fmt=fmt, readonly=0,
+                                ndim=z.ndim, shape=z.shape, strides=z.strides,
+                                lst=z.tolist())
+
+    def test_ndarray_random_invalid(self):
+        # exceptions during construction of invalid arrays
+        for _ in range(ITERATIONS):
+            for fmt in fmtdict['@']:
+                itemsize = struct.calcsize(fmt)
+
+                t = rand_structure(itemsize, False, maxdim=MAXDIM,
+                                   maxshape=MAXSHAPE)
+                self.assertFalse(verify_structure(*t))
+                items = randitems_from_structure(fmt, t)
+
+                nderr = False
+                try:
+                    x = ndarray_from_structure(items, fmt, t)
+                except Exception as e:
+                    nderr = e.__class__
+                self.assertTrue(nderr)
+
+                if numpy_array:
+                    numpy_err = False
+                    try:
+                        y = numpy_array_from_structure(items, fmt, t)
+                    except Exception as e:
+                        numpy_err = e.__class__
+
+                    if 0: # http://projects.scipy.org/numpy/ticket/1910
+                        self.assertTrue(numpy_err)
+
+    def test_ndarray_random_slice_assign(self):
+        # valid slice assignments
+        for _ in range(ITERATIONS):
+            for fmt in fmtdict['@']:
+                itemsize = struct.calcsize(fmt)
+
+                lshape, rshape, lslices, rslices = \
+                    rand_aligned_slices(maxdim=MAXDIM, maxshape=MAXSHAPE)
+                tl = rand_structure(itemsize, True, shape=lshape)
+                tr = rand_structure(itemsize, True, shape=rshape)
+                self.assertTrue(verify_structure(*tl))
+                self.assertTrue(verify_structure(*tr))
+                litems = randitems_from_structure(fmt, tl)
+                ritems = randitems_from_structure(fmt, tr)
+
+                xl = ndarray_from_structure(litems, fmt, tl)
+                xr = ndarray_from_structure(ritems, fmt, tr)
+                xl[lslices] = xr[rslices]
+                xllist = xl.tolist()
+                xrlist = xr.tolist()
+
+                ml = memoryview(xl)
+                mr = memoryview(xr)
+                self.assertEqual(ml.tolist(), xllist)
+                self.assertEqual(mr.tolist(), xrlist)
+
+                if tl[2] > 0 and tr[2] > 0:
+                    # ndim > 0: test against suboffsets representation.
+                    yl = ndarray_from_structure(litems, fmt, tl, flags=ND_PIL)
+                    yr = ndarray_from_structure(ritems, fmt, tr, flags=ND_PIL)
+                    yl[lslices] = yr[rslices]
+                    yllist = yl.tolist()
+                    yrlist = yr.tolist()
+                    self.assertEqual(xllist, yllist)
+                    self.assertEqual(xrlist, yrlist)
+
+                    ml = memoryview(yl)
+                    mr = memoryview(yr)
+                    self.assertEqual(ml.tolist(), yllist)
+                    self.assertEqual(mr.tolist(), yrlist)
+
+                if numpy_array:
+                    if 0 in lshape or 0 in rshape:
+                        continue # http://projects.scipy.org/numpy/ticket/1910
+
+                    zl = numpy_array_from_structure(litems, fmt, tl)
+                    zr = numpy_array_from_structure(ritems, fmt, tr)
+                    zl[lslices] = zr[rslices]
+
+                    if not is_overlapping(tl) and not is_overlapping(tr):
+                        # Slice assignment of overlapping structures
+                        # is undefined in NumPy.
+                        self.verify(xl, obj=None,
+                                    itemsize=zl.itemsize, fmt=fmt, readonly=0,
+                                    ndim=zl.ndim, shape=zl.shape,
+                                    strides=zl.strides, lst=zl.tolist())
+
+                    self.verify(xr, obj=None,
+                                itemsize=zr.itemsize, fmt=fmt, readonly=0,
+                                ndim=zr.ndim, shape=zr.shape,
+                                strides=zr.strides, lst=zr.tolist())
+
+    def test_ndarray_re_export(self):
+        items = [1,2,3,4,5,6,7,8,9,10,11,12]
+
+        nd = ndarray(items, shape=[3,4], flags=ND_PIL)
+        ex = ndarray(nd)
+
+        self.assertTrue(ex.flags & ND_PIL)
+        self.assertIs(ex.obj, nd)
+        self.assertEqual(ex.suboffsets, (0, -1))
+        self.assertFalse(ex.c_contiguous)
+        self.assertFalse(ex.f_contiguous)
+        self.assertFalse(ex.contiguous)
+
+    def test_ndarray_zero_shape(self):
+        # zeros in shape
+        for flags in (0, ND_PIL):
+            nd = ndarray([1,2,3], shape=[0], flags=flags)
+            mv = memoryview(nd)
+            self.assertEqual(mv, nd)
+            self.assertEqual(nd.tolist(), [])
+            self.assertEqual(mv.tolist(), [])
+
+            nd = ndarray([1,2,3], shape=[0,3,3], flags=flags)
+            self.assertEqual(nd.tolist(), [])
+
+            nd = ndarray([1,2,3], shape=[3,0,3], flags=flags)
+            self.assertEqual(nd.tolist(), [[], [], []])
+
+            nd = ndarray([1,2,3], shape=[3,3,0], flags=flags)
+            self.assertEqual(nd.tolist(),
+                             [[[], [], []], [[], [], []], [[], [], []]])
+
+    def test_ndarray_zero_strides(self):
+        # zero strides
+        for flags in (0, ND_PIL):
+            nd = ndarray([1], shape=[5], strides=[0], flags=flags)
+            mv = memoryview(nd)
+            self.assertEqual(mv, nd)
+            self.assertEqual(nd.tolist(), [1, 1, 1, 1, 1])
+            self.assertEqual(mv.tolist(), [1, 1, 1, 1, 1])
+
+    def test_ndarray_offset(self):
+        nd = ndarray(list(range(20)), shape=[3], offset=7)
+        self.assertEqual(nd.offset, 7)
+        self.assertEqual(nd.tolist(), [7,8,9])
+
+    def test_ndarray_memoryview_from_buffer(self):
+        for flags in (0, ND_PIL):
+            nd = ndarray(list(range(3)), shape=[3], flags=flags)
+            m = nd.memoryview_from_buffer()
+            self.assertEqual(m, nd)
+
+    def test_ndarray_get_pointer(self):
+        for flags in (0, ND_PIL):
+            nd = ndarray(list(range(3)), shape=[3], flags=flags)
+            for i in range(3):
+                self.assertEqual(nd[i], get_pointer(nd, [i]))
+
+    def test_ndarray_tolist_null_strides(self):
+        ex = ndarray(list(range(20)), shape=[2,2,5])
+
+        nd = ndarray(ex, getbuf=PyBUF_ND|PyBUF_FORMAT)
+        self.assertEqual(nd.tolist(), ex.tolist())
+
+        m = memoryview(ex)
+        self.assertEqual(m.tolist(), ex.tolist())
+
+    def test_ndarray_cmp_contig(self):
+
+        self.assertFalse(cmp_contig(b"123", b"456"))
+
+        x = ndarray(list(range(12)), shape=[3,4])
+        y = ndarray(list(range(12)), shape=[4,3])
+        self.assertFalse(cmp_contig(x, y))
+
+        x = ndarray([1], shape=[1], format="B")
+        self.assertTrue(cmp_contig(x, b'\x01'))
+        self.assertTrue(cmp_contig(b'\x01', x))
+
+    def test_ndarray_hash(self):
+
+        a = array.array('L', [1,2,3])
+        nd = ndarray(a)
+        self.assertRaises(ValueError, hash, nd)
+
+        # one-dimensional
+        b = bytes(list(range(12)))
+
+        nd = ndarray(list(range(12)), shape=[12])
+        self.assertEqual(hash(nd), hash(b))
+
+        # C-contiguous
+        nd = ndarray(list(range(12)), shape=[3,4])
+        self.assertEqual(hash(nd), hash(b))
+
+        nd = ndarray(list(range(12)), shape=[3,2,2])
+        self.assertEqual(hash(nd), hash(b))
+
+        # Fortran contiguous
+        b = bytes(transpose(list(range(12)), shape=[4,3]))
+        nd = ndarray(list(range(12)), shape=[3,4], flags=ND_FORTRAN)
+        self.assertEqual(hash(nd), hash(b))
+
+        b = bytes(transpose(list(range(12)), shape=[2,3,2]))
+        nd = ndarray(list(range(12)), shape=[2,3,2], flags=ND_FORTRAN)
+        self.assertEqual(hash(nd), hash(b))
+
+        # suboffsets
+        b = bytes(list(range(12)))
+        nd = ndarray(list(range(12)), shape=[2,2,3], flags=ND_PIL)
+        self.assertEqual(hash(nd), hash(b))
+
+        # non-byte formats
+        nd = ndarray(list(range(12)), shape=[2,2,3], format='L')
+        self.assertEqual(hash(nd), hash(nd.tobytes()))
+
+    def test_py_buffer_to_contiguous(self):
+
+        # The requests are used in _testbuffer.c:py_buffer_to_contiguous
+        # to generate buffers without full information for testing.
+        requests = (
+            # distinct flags
+            PyBUF_INDIRECT, PyBUF_STRIDES, PyBUF_ND, PyBUF_SIMPLE,
+            # compound requests
+            PyBUF_FULL, PyBUF_FULL_RO,
+            PyBUF_RECORDS, PyBUF_RECORDS_RO,
+            PyBUF_STRIDED, PyBUF_STRIDED_RO,
+            PyBUF_CONTIG, PyBUF_CONTIG_RO,
+        )
+
+        # no buffer interface
+        self.assertRaises(TypeError, py_buffer_to_contiguous, {}, 'F',
+                          PyBUF_FULL_RO)
+
+        # scalar, read-only request
+        nd = ndarray(9, shape=(), format="L", flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            for request in requests:
+                b = py_buffer_to_contiguous(nd, order, request)
+                self.assertEqual(b, nd.tobytes())
+
+        # zeros in shape
+        nd = ndarray([1], shape=[0], format="L", flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            for request in requests:
+                b = py_buffer_to_contiguous(nd, order, request)
+                self.assertEqual(b, b'')
+
+        nd = ndarray(list(range(8)), shape=[2, 0, 7], format="L",
+                     flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            for request in requests:
+                b = py_buffer_to_contiguous(nd, order, request)
+                self.assertEqual(b, b'')
+
+        ### One-dimensional arrays are trivial, since Fortran and C order
+        ### are the same.
+
+        # one-dimensional
+        for f in [0, ND_FORTRAN]:
+            nd = ndarray([1], shape=[1], format="h", flags=f|ND_WRITABLE)
+            ndbytes = nd.tobytes()
+            for order in ['C', 'F', 'A']:
+                for request in requests:
+                    b = py_buffer_to_contiguous(nd, order, request)
+                    self.assertEqual(b, ndbytes)
+
+            nd = ndarray([1, 2, 3], shape=[3], format="b", flags=f|ND_WRITABLE)
+            ndbytes = nd.tobytes()
+            for order in ['C', 'F', 'A']:
+                for request in requests:
+                    b = py_buffer_to_contiguous(nd, order, request)
+                    self.assertEqual(b, ndbytes)
+
+        # one-dimensional, non-contiguous input
+        nd = ndarray([1, 2, 3], shape=[2], strides=[2], flags=ND_WRITABLE)
+        ndbytes = nd.tobytes()
+        for order in ['C', 'F', 'A']:
+            for request in [PyBUF_STRIDES, PyBUF_FULL]:
+                b = py_buffer_to_contiguous(nd, order, request)
+                self.assertEqual(b, ndbytes)
+
+        nd = nd[::-1]
+        ndbytes = nd.tobytes()
+        for order in ['C', 'F', 'A']:
+            for request in requests:
+                try:
+                    b = py_buffer_to_contiguous(nd, order, request)
+                except BufferError:
+                    continue
+                self.assertEqual(b, ndbytes)
+
+        ###
+        ### Multi-dimensional arrays:
+        ###
+        ### The goal here is to preserve the logical representation of the
+        ### input array but change the physical representation if necessary.
+        ###
+        ### _testbuffer example:
+        ### ====================
+        ###
+        ###    C input array:
+        ###    --------------
+        ###       >>> nd = ndarray(list(range(12)), shape=[3, 4])
+        ###       >>> nd.tolist()
+        ###       [[0, 1, 2, 3],
+        ###        [4, 5, 6, 7],
+        ###        [8, 9, 10, 11]]
+        ###
+        ###    Fortran output:
+        ###    ---------------
+        ###       >>> py_buffer_to_contiguous(nd, 'F', PyBUF_FULL_RO)
+        ###       >>> b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
+        ###
+        ###    The return value corresponds to this input list for
+        ###    _testbuffer's ndarray:
+        ###       >>> nd = ndarray([0,4,8,1,5,9,2,6,10,3,7,11], shape=[3,4],
+        ###                        flags=ND_FORTRAN)
+        ###       >>> nd.tolist()
+        ###       [[0, 1, 2, 3],
+        ###        [4, 5, 6, 7],
+        ###        [8, 9, 10, 11]]
+        ###
+        ###    The logical array is the same, but the values in memory are now
+        ###    in Fortran order.
+        ###
+        ### NumPy example:
+        ### ==============
+        ###    _testbuffer's ndarray takes lists to initialize the memory.
+        ###    Here's the same sequence in NumPy:
+        ###
+        ###    C input:
+        ###    --------
+        ###       >>> nd = ndarray(buffer=bytearray(list(range(12))),
+        ###                        shape=[3, 4], dtype='B')
+        ###       >>> nd
+        ###       array([[ 0,  1,  2,  3],
+        ###              [ 4,  5,  6,  7],
+        ###              [ 8,  9, 10, 11]], dtype=uint8)
+        ###
+        ###    Fortran output:
+        ###    ---------------
+        ###       >>> fortran_buf = nd.tostring(order='F')
+        ###       >>> fortran_buf
+        ###       b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
+        ###
+        ###       >>> nd = ndarray(buffer=fortran_buf, shape=[3, 4],
+        ###                        dtype='B', order='F')
+        ###
+        ###       >>> nd
+        ###       array([[ 0,  1,  2,  3],
+        ###              [ 4,  5,  6,  7],
+        ###              [ 8,  9, 10, 11]], dtype=uint8)
+        ###
+
+        # multi-dimensional, contiguous input
+        lst = list(range(12))
+        for f in [0, ND_FORTRAN]:
+            nd = ndarray(lst, shape=[3, 4], flags=f|ND_WRITABLE)
+            if numpy_array:
+                na = numpy_array(buffer=bytearray(lst),
+                                 shape=[3, 4], dtype='B',
+                                 order='C' if f == 0 else 'F')
+
+            # 'C' request
+            if f == ND_FORTRAN: # 'F' to 'C'
+                x = ndarray(transpose(lst, [4, 3]), shape=[3, 4],
+                            flags=ND_WRITABLE)
+                expected = x.tobytes()
+            else:
+                expected = nd.tobytes()
+            for request in requests:
+                try:
+                    b = py_buffer_to_contiguous(nd, 'C', request)
+                except BufferError:
+                    continue
+
+                self.assertEqual(b, expected)
+
+                # Check that output can be used as the basis for constructing
+                # a C array that is logically identical to the input array.
+                y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
+                self.assertEqual(memoryview(y), memoryview(nd))
+
+                if numpy_array:
+                    self.assertEqual(b, na.tostring(order='C'))
+
+            # 'F' request
+            if f == 0: # 'C' to 'F'
+                x = ndarray(transpose(lst, [3, 4]), shape=[4, 3],
+                            flags=ND_WRITABLE)
+            else:
+                x = ndarray(lst, shape=[3, 4], flags=ND_WRITABLE)
+            expected = x.tobytes()
+            for request in [PyBUF_FULL, PyBUF_FULL_RO, PyBUF_INDIRECT,
+                            PyBUF_STRIDES, PyBUF_ND]:
+                try:
+                    b = py_buffer_to_contiguous(nd, 'F', request)
+                except BufferError:
+                    continue
+                self.assertEqual(b, expected)
+
+                # Check that output can be used as the basis for constructing
+                # a Fortran array that is logically identical to the input array.
+                y = ndarray([v for v in b], shape=[3, 4], flags=ND_FORTRAN|ND_WRITABLE)
+                self.assertEqual(memoryview(y), memoryview(nd))
+
+                if numpy_array:
+                    self.assertEqual(b, na.tostring(order='F'))
+
+            # 'A' request
+            if f == ND_FORTRAN:
+                x = ndarray(lst, shape=[3, 4], flags=ND_WRITABLE)
+                expected = x.tobytes()
+            else:
+                expected = nd.tobytes()
+            for request in [PyBUF_FULL, PyBUF_FULL_RO, PyBUF_INDIRECT,
+                            PyBUF_STRIDES, PyBUF_ND]:
+                try:
+                    b = py_buffer_to_contiguous(nd, 'A', request)
+                except BufferError:
+                    continue
+
+                self.assertEqual(b, expected)
+
+                # Check that output can be used as the basis for constructing
+                # an array with order=f that is logically identical to the input
+                # array.
+                y = ndarray([v for v in b], shape=[3, 4], flags=f|ND_WRITABLE)
+                self.assertEqual(memoryview(y), memoryview(nd))
+
+                if numpy_array:
+                    self.assertEqual(b, na.tostring(order='A'))
+
+        # multi-dimensional, non-contiguous input
+        nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL)
+
+        # 'C'
+        b = py_buffer_to_contiguous(nd, 'C', PyBUF_FULL_RO)
+        self.assertEqual(b, nd.tobytes())
+        y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
+        self.assertEqual(memoryview(y), memoryview(nd))
+
+        # 'F'
+        b = py_buffer_to_contiguous(nd, 'F', PyBUF_FULL_RO)
+        x = ndarray(transpose(lst, [3, 4]), shape=[4, 3], flags=ND_WRITABLE)
+        self.assertEqual(b, x.tobytes())
+        y = ndarray([v for v in b], shape=[3, 4], flags=ND_FORTRAN|ND_WRITABLE)
+        self.assertEqual(memoryview(y), memoryview(nd))
+
+        # 'A'
+        b = py_buffer_to_contiguous(nd, 'A', PyBUF_FULL_RO)
+        self.assertEqual(b, nd.tobytes())
+        y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
+        self.assertEqual(memoryview(y), memoryview(nd))
+
+    def test_memoryview_construction(self):
+
+        items_shape = [(9, []), ([1,2,3], [3]), (list(range(2*3*5)), [2,3,5])]
+
+        # NumPy style, C-contiguous:
+        for items, shape in items_shape:
+
+            # From PEP-3118 compliant exporter:
+            ex = ndarray(items, shape=shape)
+            m = memoryview(ex)
+            self.assertTrue(m.c_contiguous)
+            self.assertTrue(m.contiguous)
+
+            ndim = len(shape)
+            strides = strides_from_shape(ndim, shape, 1, 'C')
+            lst = carray(items, shape)
+
+            self.verify(m, obj=ex,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=ndim, shape=shape, strides=strides,
+                        lst=lst)
+
+            # From memoryview:
+            m2 = memoryview(m)
+            self.verify(m2, obj=ex,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=ndim, shape=shape, strides=strides,
+                        lst=lst)
+
+            # PyMemoryView_FromBuffer(): no strides
+            nd = ndarray(ex, getbuf=PyBUF_CONTIG_RO|PyBUF_FORMAT)
+            self.assertEqual(nd.strides, ())
+            m = nd.memoryview_from_buffer()
+            self.verify(m, obj=None,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=ndim, shape=shape, strides=strides,
+                        lst=lst)
+
+            # PyMemoryView_FromBuffer(): no format, shape, strides
+            nd = ndarray(ex, getbuf=PyBUF_SIMPLE)
+            self.assertEqual(nd.format, '')
+            self.assertEqual(nd.shape, ())
+            self.assertEqual(nd.strides, ())
+            m = nd.memoryview_from_buffer()
+
+            lst = [items] if ndim == 0 else items
+            self.verify(m, obj=None,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=1, shape=[ex.nbytes], strides=(1,),
+                        lst=lst)
+
+        # NumPy style, Fortran contiguous:
+        for items, shape in items_shape:
+
+            # From PEP-3118 compliant exporter:
+            ex = ndarray(items, shape=shape, flags=ND_FORTRAN)
+            m = memoryview(ex)
+            self.assertTrue(m.f_contiguous)
+            self.assertTrue(m.contiguous)
+
+            ndim = len(shape)
+            strides = strides_from_shape(ndim, shape, 1, 'F')
+            lst = farray(items, shape)
+
+            self.verify(m, obj=ex,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=ndim, shape=shape, strides=strides,
+                        lst=lst)
+
+            # From memoryview:
+            m2 = memoryview(m)
+            self.verify(m2, obj=ex,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=ndim, shape=shape, strides=strides,
+                        lst=lst)
+
+        # PIL style:
+        for items, shape in items_shape[1:]:
+
+            # From PEP-3118 compliant exporter:
+            ex = ndarray(items, shape=shape, flags=ND_PIL)
+            m = memoryview(ex)
+
+            ndim = len(shape)
+            lst = carray(items, shape)
+
+            self.verify(m, obj=ex,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=ndim, shape=shape, strides=ex.strides,
+                        lst=lst)
+
+            # From memoryview:
+            m2 = memoryview(m)
+            self.verify(m2, obj=ex,
+                        itemsize=1, fmt='B', readonly=1,
+                        ndim=ndim, shape=shape, strides=ex.strides,
+                        lst=lst)
+
+        # Invalid number of arguments:
+        self.assertRaises(TypeError, memoryview, b'9', 'x')
+        # Not a buffer provider:
+        self.assertRaises(TypeError, memoryview, {})
+        # Non-compliant buffer provider:
+        ex = ndarray([1,2,3], shape=[3])
+        nd = ndarray(ex, getbuf=PyBUF_SIMPLE)
+        self.assertRaises(BufferError, memoryview, nd)
+        nd = ndarray(ex, getbuf=PyBUF_CONTIG_RO|PyBUF_FORMAT)
+        self.assertRaises(BufferError, memoryview, nd)
+
+        # ndim > 64
+        nd = ndarray([1]*128, shape=[1]*128, format='L')
+        self.assertRaises(ValueError, memoryview, nd)
+        self.assertRaises(ValueError, nd.memoryview_from_buffer)
+        self.assertRaises(ValueError, get_contiguous, nd, PyBUF_READ, 'C')
+        self.assertRaises(ValueError, get_contiguous, nd, PyBUF_READ, 'F')
+        self.assertRaises(ValueError, get_contiguous, nd[::-1], PyBUF_READ, 'C')
+
+    def test_memoryview_cast_zero_shape(self):
+        # Casts are undefined if shape contains zeros. These arrays are
+        # regarded as C-contiguous by Numpy and PyBuffer_GetContiguous(),
+        # so they are not caught by the test for C-contiguity in memory_cast().
+        items = [1,2,3]
+        for shape in ([0,3,3], [3,0,3], [0,3,3]):
+            ex = ndarray(items, shape=shape)
+            self.assertTrue(ex.c_contiguous)
+            msrc = memoryview(ex)
+            self.assertRaises(TypeError, msrc.cast, 'c')
+
+    def test_memoryview_struct_module(self):
+
+        class INT(object):
+            def __init__(self, val):
+                self.val = val
+            def __int__(self):
+                return self.val
+
+        class IDX(object):
+            def __init__(self, val):
+                self.val = val
+            def __index__(self):
+                return self.val
+
+        def f(): return 7
+
+        values = [INT(9), IDX(9),
+                  2.2+3j, Decimal("-21.1"), 12.2, Fraction(5, 2),
+                  [1,2,3], {4,5,6}, {7:8}, (), (9,),
+                  True, False, None, NotImplemented,
+                  b'a', b'abc', bytearray(b'a'), bytearray(b'abc'),
+                  'a', 'abc', r'a', r'abc',
+                  f, lambda x: x]
+
+        for fmt, items, item in iter_format(10, 'memoryview'):
+            ex = ndarray(items, shape=[10], format=fmt, flags=ND_WRITABLE)
+            nd = ndarray(items, shape=[10], format=fmt, flags=ND_WRITABLE)
+            m = memoryview(ex)
+
+            struct.pack_into(fmt, nd, 0, item)
+            m[0] = item
+            self.assertEqual(m[0], nd[0])
+
+            itemsize = struct.calcsize(fmt)
+            if 'P' in fmt:
+                continue
+
+            for v in values:
+                struct_err = None
+                try:
+                    struct.pack_into(fmt, nd, itemsize, v)
+                except struct.error:
+                    struct_err = struct.error
+
+                mv_err = None
+                try:
+                    m[1] = v
+                except (TypeError, ValueError) as e:
+                    mv_err = e.__class__
+
+                if struct_err or mv_err:
+                    self.assertIsNot(struct_err, None)
+                    self.assertIsNot(mv_err, None)
+                else:
+                    self.assertEqual(m[1], nd[1])
+
+    def test_memoryview_cast_zero_strides(self):
+        # Casts are undefined if strides contains zeros. These arrays are
+        # (sometimes!) regarded as C-contiguous by Numpy, but not by
+        # PyBuffer_GetContiguous().
+        ex = ndarray([1,2,3], shape=[3], strides=[0])
+        self.assertFalse(ex.c_contiguous)
+        msrc = memoryview(ex)
+        self.assertRaises(TypeError, msrc.cast, 'c')
+
+    def test_memoryview_cast_invalid(self):
+        # invalid format
+        for sfmt in NON_BYTE_FORMAT:
+            sformat = '@' + sfmt if randrange(2) else sfmt
+            ssize = struct.calcsize(sformat)
+            for dfmt in NON_BYTE_FORMAT:
+                dformat = '@' + dfmt if randrange(2) else dfmt
+                dsize = struct.calcsize(dformat)
+                ex = ndarray(list(range(32)), shape=[32//ssize], format=sformat)
+                msrc = memoryview(ex)
+                self.assertRaises(TypeError, msrc.cast, dfmt, [32//dsize])
+
+        for sfmt, sitems, _ in iter_format(1):
+            ex = ndarray(sitems, shape=[1], format=sfmt)
+            msrc = memoryview(ex)
+            for dfmt, _, _ in iter_format(1):
+                if (not is_memoryview_format(sfmt) or
+                    not is_memoryview_format(dfmt)):
+                    self.assertRaises(ValueError, msrc.cast, dfmt,
+                                      [32//dsize])
+                else:
+                    if not is_byte_format(sfmt) and not is_byte_format(dfmt):
+                        self.assertRaises(TypeError, msrc.cast, dfmt,
+                                          [32//dsize])
+
+        # invalid shape
+        size_h = struct.calcsize('h')
+        size_d = struct.calcsize('d')
+        ex = ndarray(list(range(2*2*size_d)), shape=[2,2,size_d], format='h')
+        msrc = memoryview(ex)
+        self.assertRaises(TypeError, msrc.cast, shape=[2,2,size_h], format='d')
+
+        ex = ndarray(list(range(120)), shape=[1,2,3,4,5])
+        m = memoryview(ex)
+
+        # incorrect number of args
+        self.assertRaises(TypeError, m.cast)
+        self.assertRaises(TypeError, m.cast, 1, 2, 3)
+
+        # incorrect dest format type
+        self.assertRaises(TypeError, m.cast, {})
+
+        # incorrect dest format
+        self.assertRaises(ValueError, m.cast, "X")
+        self.assertRaises(ValueError, m.cast, "@X")
+        self.assertRaises(ValueError, m.cast, "@XY")
+
+        # dest format not implemented
+        self.assertRaises(ValueError, m.cast, "=B")
+        self.assertRaises(ValueError, m.cast, "!L")
+        self.assertRaises(ValueError, m.cast, "<P")
+        self.assertRaises(ValueError, m.cast, ">l")
+        self.assertRaises(ValueError, m.cast, "BI")
+        self.assertRaises(ValueError, m.cast, "xBI")
+
+        # src format not implemented
+        ex = ndarray([(1,2), (3,4)], shape=[2], format="II")
+        m = memoryview(ex)
+        self.assertRaises(NotImplementedError, m.__getitem__, 0)
+        self.assertRaises(NotImplementedError, m.__setitem__, 0, 8)
+        self.assertRaises(NotImplementedError, m.tolist)
+
+        # incorrect shape type
+        ex = ndarray(list(range(120)), shape=[1,2,3,4,5])
+        m = memoryview(ex)
+        self.assertRaises(TypeError, m.cast, "B", shape={})
+
+        # incorrect shape elements
+        ex = ndarray(list(range(120)), shape=[2*3*4*5])
+        m = memoryview(ex)
+        self.assertRaises(OverflowError, m.cast, "B", shape=[2**64])
+        self.assertRaises(ValueError, m.cast, "B", shape=[-1])
+        self.assertRaises(ValueError, m.cast, "B", shape=[2,3,4,5,6,7,-1])
+        self.assertRaises(ValueError, m.cast, "B", shape=[2,3,4,5,6,7,0])
+        self.assertRaises(TypeError, m.cast, "B", shape=[2,3,4,5,6,7,'x'])
+
+        # N-D -> N-D cast
+        ex = ndarray(list([9 for _ in range(3*5*7*11)]), shape=[3,5,7,11])
+        m = memoryview(ex)
+        self.assertRaises(TypeError, m.cast, "I", shape=[2,3,4,5])
+
+        # cast with ndim > 64
+        nd = ndarray(list(range(128)), shape=[128], format='I')
+        m = memoryview(nd)
+        self.assertRaises(ValueError, m.cast, 'I', [1]*128)
+
+        # view->len not a multiple of itemsize
+        ex = ndarray(list([9 for _ in range(3*5*7*11)]), shape=[3*5*7*11])
+        m = memoryview(ex)
+        self.assertRaises(TypeError, m.cast, "I", shape=[2,3,4,5])
+
+        # product(shape) * itemsize != buffer size
+        ex = ndarray(list([9 for _ in range(3*5*7*11)]), shape=[3*5*7*11])
+        m = memoryview(ex)
+        self.assertRaises(TypeError, m.cast, "B", shape=[2,3,4,5])
+
+        # product(shape) * itemsize overflow
+        nd = ndarray(list(range(128)), shape=[128], format='I')
+        m1 = memoryview(nd)
+        nd = ndarray(list(range(128)), shape=[128], format='B')
+        m2 = memoryview(nd)
+        if sys.maxsize == 2**63-1:
+            self.assertRaises(TypeError, m1.cast, 'B',
+                              [7, 7, 73, 127, 337, 92737, 649657])
+            self.assertRaises(ValueError, m1.cast, 'B',
+                              [2**20, 2**20, 2**10, 2**10, 2**3])
+            self.assertRaises(ValueError, m2.cast, 'I',
+                              [2**20, 2**20, 2**10, 2**10, 2**1])
+        else:
+            self.assertRaises(TypeError, m1.cast, 'B',
+                              [1, 2147483647])
+            self.assertRaises(ValueError, m1.cast, 'B',
+                              [2**10, 2**10, 2**5, 2**5, 2**1])
+            self.assertRaises(ValueError, m2.cast, 'I',
+                              [2**10, 2**10, 2**5, 2**3, 2**1])
+
+    def test_memoryview_cast(self):
+        bytespec = (
+          ('B', lambda ex: list(ex.tobytes())),
+          ('b', lambda ex: [x-256 if x > 127 else x for x in list(ex.tobytes())]),
+          ('c', lambda ex: [bytes(chr(x), 'latin-1') for x in list(ex.tobytes())]),
+        )
+
+        def iter_roundtrip(ex, m, items, fmt):
+            srcsize = struct.calcsize(fmt)
+            for bytefmt, to_bytelist in bytespec:
+
+                m2 = m.cast(bytefmt)
+                lst = to_bytelist(ex)
+                self.verify(m2, obj=ex,
+                            itemsize=1, fmt=bytefmt, readonly=0,
+                            ndim=1, shape=[31*srcsize], strides=(1,),
+                            lst=lst, cast=True)
+
+                m3 = m2.cast(fmt)
+                self.assertEqual(m3, ex)
+                lst = ex.tolist()
+                self.verify(m3, obj=ex,
+                            itemsize=srcsize, fmt=fmt, readonly=0,
+                            ndim=1, shape=[31], strides=(srcsize,),
+                            lst=lst, cast=True)
+
+        # cast from ndim = 0 to ndim = 1
+        srcsize = struct.calcsize('I')
+        ex = ndarray(9, shape=[], format='I')
+        destitems, destshape = cast_items(ex, 'B', 1)
+        m = memoryview(ex)
+        m2 = m.cast('B')
+        self.verify(m2, obj=ex,
+                    itemsize=1, fmt='B', readonly=1,
+                    ndim=1, shape=destshape, strides=(1,),
+                    lst=destitems, cast=True)
+
+        # cast from ndim = 1 to ndim = 0
+        destsize = struct.calcsize('I')
+        ex = ndarray([9]*destsize, shape=[destsize], format='B')
+        destitems, destshape = cast_items(ex, 'I', destsize, shape=[])
+        m = memoryview(ex)
+        m2 = m.cast('I', shape=[])
+        self.verify(m2, obj=ex,
+                    itemsize=destsize, fmt='I', readonly=1,
+                    ndim=0, shape=(), strides=(),
+                    lst=destitems, cast=True)
+
+        # array.array: roundtrip to/from bytes
+        for fmt, items, _ in iter_format(31, 'array'):
+            ex = array.array(fmt, items)
+            m = memoryview(ex)
+            iter_roundtrip(ex, m, items, fmt)
+
+        # ndarray: roundtrip to/from bytes
+        for fmt, items, _ in iter_format(31, 'memoryview'):
+            ex = ndarray(items, shape=[31], format=fmt, flags=ND_WRITABLE)
+            m = memoryview(ex)
+            iter_roundtrip(ex, m, items, fmt)
+
+    def test_memoryview_cast_1D_ND(self):
+        # Cast between C-contiguous buffers. At least one buffer must
+        # be 1D, at least one format must be 'c', 'b' or 'B'.
+        for _tshape in gencastshapes():
+            for char in fmtdict['@']:
+                tfmt = ('', '@')[randrange(2)] + char
+                tsize = struct.calcsize(tfmt)
+                n = prod(_tshape) * tsize
+                obj = 'memoryview' if is_byte_format(tfmt) else 'bytefmt'
+                for fmt, items, _ in iter_format(n, obj):
+                    size = struct.calcsize(fmt)
+                    shape = [n] if n > 0 else []
+                    tshape = _tshape + [size]
+
+                    ex = ndarray(items, shape=shape, format=fmt)
+                    m = memoryview(ex)
+
+                    titems, tshape = cast_items(ex, tfmt, tsize, shape=tshape)
+
+                    if titems is None:
+                        self.assertRaises(TypeError, m.cast, tfmt, tshape)
+                        continue
+                    if titems == 'nan':
+                        continue # NaNs in lists are a recipe for trouble.
+
+                    # 1D -> ND
+                    nd = ndarray(titems, shape=tshape, format=tfmt)
+
+                    m2 = m.cast(tfmt, shape=tshape)
+                    ndim = len(tshape)
+                    strides = nd.strides
+                    lst = nd.tolist()
+                    self.verify(m2, obj=ex,
+                                itemsize=tsize, fmt=tfmt, readonly=1,
+                                ndim=ndim, shape=tshape, strides=strides,
+                                lst=lst, cast=True)
+
+                    # ND -> 1D
+                    m3 = m2.cast(fmt)
+                    m4 = m2.cast(fmt, shape=shape)
+                    ndim = len(shape)
+                    strides = ex.strides
+                    lst = ex.tolist()
+
+                    self.verify(m3, obj=ex,
+                                itemsize=size, fmt=fmt, readonly=1,
+                                ndim=ndim, shape=shape, strides=strides,
+                                lst=lst, cast=True)
+
+                    self.verify(m4, obj=ex,
+                                itemsize=size, fmt=fmt, readonly=1,
+                                ndim=ndim, shape=shape, strides=strides,
+                                lst=lst, cast=True)
+
+    def test_memoryview_tolist(self):
+
+        # Most tolist() tests are in self.verify() etc.
+
+        a = array.array('h', list(range(-6, 6)))
+        m = memoryview(a)
+        self.assertEqual(m, a)
+        self.assertEqual(m.tolist(), a.tolist())
+
+        a = a[2::3]
+        m = m[2::3]
+        self.assertEqual(m, a)
+        self.assertEqual(m.tolist(), a.tolist())
+
+        ex = ndarray(list(range(2*3*5*7*11)), shape=[11,2,7,3,5], format='L')
+        m = memoryview(ex)
+        self.assertEqual(m.tolist(), ex.tolist())
+
+        ex = ndarray([(2, 5), (7, 11)], shape=[2], format='lh')
+        m = memoryview(ex)
+        self.assertRaises(NotImplementedError, m.tolist)
+
+        ex = ndarray([b'12345'], shape=[1], format="s")
+        m = memoryview(ex)
+        self.assertRaises(NotImplementedError, m.tolist)
+
+        ex = ndarray([b"a",b"b",b"c",b"d",b"e",b"f"], shape=[2,3], format='s')
+        m = memoryview(ex)
+        self.assertRaises(NotImplementedError, m.tolist)
+
+    def test_memoryview_repr(self):
+        m = memoryview(bytearray(9))
+        r = m.__repr__()
+        self.assertTrue(r.startswith("<memory"))
+
+        m.release()
+        r = m.__repr__()
+        self.assertTrue(r.startswith("<released"))
+
+    def test_memoryview_sequence(self):
+
+        for fmt in ('d', 'f'):
+            inf = float(3e400)
+            ex = array.array(fmt, [1.0, inf, 3.0])
+            m = memoryview(ex)
+            self.assertIn(1.0, m)
+            self.assertIn(5e700, m)
+            self.assertIn(3.0, m)
+
+        ex = ndarray(9.0, [], format='f')
+        m = memoryview(ex)
+        self.assertRaises(TypeError, eval, "9.0 in m", locals())
+
+    def test_memoryview_index(self):
+
+        # ndim = 0
+        ex = ndarray(12.5, shape=[], format='d')
+        m = memoryview(ex)
+        self.assertEqual(m[()], 12.5)
+        self.assertEqual(m[...], m)
+        self.assertEqual(m[...], ex)
+        self.assertRaises(TypeError, m.__getitem__, 0)
+
+        ex = ndarray((1,2,3), shape=[], format='iii')
+        m = memoryview(ex)
+        self.assertRaises(NotImplementedError, m.__getitem__, ())
+
+        # range
+        ex = ndarray(list(range(7)), shape=[7], flags=ND_WRITABLE)
+        m = memoryview(ex)
+
+        self.assertRaises(IndexError, m.__getitem__, 2**64)
+        self.assertRaises(TypeError, m.__getitem__, 2.0)
+        self.assertRaises(TypeError, m.__getitem__, 0.0)
+
+        # out of bounds
+        self.assertRaises(IndexError, m.__getitem__, -8)
+        self.assertRaises(IndexError, m.__getitem__, 8)
+
+        # Not implemented: multidimensional sub-views
+        ex = ndarray(list(range(12)), shape=[3,4], flags=ND_WRITABLE)
+        m = memoryview(ex)
+
+        self.assertRaises(NotImplementedError, m.__getitem__, 0)
+        self.assertRaises(NotImplementedError, m.__setitem__, 0, 9)
+        self.assertRaises(NotImplementedError, m.__getitem__, 0)
+
+    def test_memoryview_assign(self):
+
+        # ndim = 0
+        ex = ndarray(12.5, shape=[], format='f', flags=ND_WRITABLE)
+        m = memoryview(ex)
+        m[()] = 22.5
+        self.assertEqual(m[()], 22.5)
+        m[...] = 23.5
+        self.assertEqual(m[()], 23.5)
+        self.assertRaises(TypeError, m.__setitem__, 0, 24.7)
+
+        # read-only
+        ex = ndarray(list(range(7)), shape=[7])
+        m = memoryview(ex)
+        self.assertRaises(TypeError, m.__setitem__, 2, 10)
+
+        # range
+        ex = ndarray(list(range(7)), shape=[7], flags=ND_WRITABLE)
+        m = memoryview(ex)
+
+        self.assertRaises(IndexError, m.__setitem__, 2**64, 9)
+        self.assertRaises(TypeError, m.__setitem__, 2.0, 10)
+        self.assertRaises(TypeError, m.__setitem__, 0.0, 11)
+
+        # out of bounds
+        self.assertRaises(IndexError, m.__setitem__, -8, 20)
+        self.assertRaises(IndexError, m.__setitem__, 8, 25)
+
+        # pack_single() success:
+        for fmt in fmtdict['@']:
+            if fmt == 'c' or fmt == '?':
+                continue
+            ex = ndarray([1,2,3], shape=[3], format=fmt, flags=ND_WRITABLE)
+            m = memoryview(ex)
+            i = randrange(-3, 3)
+            m[i] = 8
+            self.assertEqual(m[i], 8)
+            self.assertEqual(m[i], ex[i])
+
+        ex = ndarray([b'1', b'2', b'3'], shape=[3], format='c',
+                     flags=ND_WRITABLE)
+        m = memoryview(ex)
+        m[2] = b'9'
+        self.assertEqual(m[2], b'9')
+
+        ex = ndarray([True, False, True], shape=[3], format='?',
+                     flags=ND_WRITABLE)
+        m = memoryview(ex)
+        m[1] = True
+        self.assertEqual(m[1], True)
+
+        # pack_single() exceptions:
+        nd = ndarray([b'x'], shape=[1], format='c', flags=ND_WRITABLE)
+        m = memoryview(nd)
+        self.assertRaises(TypeError, m.__setitem__, 0, 100)
+
+        ex = ndarray(list(range(120)), shape=[1,2,3,4,5], flags=ND_WRITABLE)
+        m1 = memoryview(ex)
+
+        for fmt, _range in fmtdict['@'].items():
+            if (fmt == '?'): # PyObject_IsTrue() accepts anything
+                continue
+            if fmt == 'c': # special case tested above
+                continue
+            m2 = m1.cast(fmt)
+            lo, hi = _range
+            if fmt == 'd' or fmt == 'f':
+                lo, hi = -2**1024, 2**1024
+            if fmt != 'P': # PyLong_AsVoidPtr() accepts negative numbers
+                self.assertRaises(ValueError, m2.__setitem__, 0, lo-1)
+                self.assertRaises(TypeError, m2.__setitem__, 0, "xyz")
+            self.assertRaises(ValueError, m2.__setitem__, 0, hi)
+
+        # invalid item
+        m2 = m1.cast('c')
+        self.assertRaises(ValueError, m2.__setitem__, 0, b'\xff\xff')
+
+        # format not implemented
+        ex = ndarray(list(range(1)), shape=[1], format="xL", flags=ND_WRITABLE)
+        m = memoryview(ex)
+        self.assertRaises(NotImplementedError, m.__setitem__, 0, 1)
+
+        ex = ndarray([b'12345'], shape=[1], format="s", flags=ND_WRITABLE)
+        m = memoryview(ex)
+        self.assertRaises(NotImplementedError, m.__setitem__, 0, 1)
+
+        # Not implemented: multidimensional sub-views
+        ex = ndarray(list(range(12)), shape=[3,4], flags=ND_WRITABLE)
+        m = memoryview(ex)
+
+        self.assertRaises(NotImplementedError, m.__setitem__, 0, [2, 3])
+
+    def test_memoryview_slice(self):
+
+        ex = ndarray(list(range(12)), shape=[12], flags=ND_WRITABLE)
+        m = memoryview(ex)
+
+        # zero step
+        self.assertRaises(ValueError, m.__getitem__, slice(0,2,0))
+        self.assertRaises(ValueError, m.__setitem__, slice(0,2,0),
+                          bytearray([1,2]))
+
+        # invalid slice key
+        self.assertRaises(TypeError, m.__getitem__, ())
+
+        # multidimensional slices
+        ex = ndarray(list(range(12)), shape=[12], flags=ND_WRITABLE)
+        m = memoryview(ex)
+
+        self.assertRaises(NotImplementedError, m.__getitem__,
+                          (slice(0,2,1), slice(0,2,1)))
+        self.assertRaises(NotImplementedError, m.__setitem__,
+                          (slice(0,2,1), slice(0,2,1)), bytearray([1,2]))
+
+        # invalid slice tuple
+        self.assertRaises(TypeError, m.__getitem__, (slice(0,2,1), {}))
+        self.assertRaises(TypeError, m.__setitem__, (slice(0,2,1), {}),
+                          bytearray([1,2]))
+
+        # rvalue is not an exporter
+        self.assertRaises(TypeError, m.__setitem__, slice(0,1,1), [1])
+
+        # non-contiguous slice assignment
+        for flags in (0, ND_PIL):
+            ex1 = ndarray(list(range(12)), shape=[12], strides=[-1], offset=11,
+                          flags=ND_WRITABLE|flags)
+            ex2 = ndarray(list(range(24)), shape=[12], strides=[2], flags=flags)
+            m1 = memoryview(ex1)
+            m2 = memoryview(ex2)
+
+            ex1[2:5] = ex1[2:5]
+            m1[2:5] = m2[2:5]
+
+            self.assertEqual(m1, ex1)
+            self.assertEqual(m2, ex2)
+
+            ex1[1:3][::-1] = ex2[0:2][::1]
+            m1[1:3][::-1] = m2[0:2][::1]
+
+            self.assertEqual(m1, ex1)
+            self.assertEqual(m2, ex2)
+
+            ex1[4:1:-2][::-1] = ex1[1:4:2][::1]
+            m1[4:1:-2][::-1] = m1[1:4:2][::1]
+
+            self.assertEqual(m1, ex1)
+            self.assertEqual(m2, ex2)
+
+    def test_memoryview_array(self):
+
+        def cmptest(testcase, a, b, m, singleitem):
+            for i, _ in enumerate(a):
+                ai = a[i]
+                mi = m[i]
+                testcase.assertEqual(ai, mi)
+                a[i] = singleitem
+                if singleitem != ai:
+                    testcase.assertNotEqual(a, m)
+                    testcase.assertNotEqual(a, b)
+                else:
+                    testcase.assertEqual(a, m)
+                    testcase.assertEqual(a, b)
+                m[i] = singleitem
+                testcase.assertEqual(a, m)
+                testcase.assertEqual(b, m)
+                a[i] = ai
+                m[i] = mi
+
+        for n in range(1, 5):
+            for fmt, items, singleitem in iter_format(n, 'array'):
+                for lslice in genslices(n):
+                    for rslice in genslices(n):
+
+                        a = array.array(fmt, items)
+                        b = array.array(fmt, items)
+                        m = memoryview(b)
+
+                        self.assertEqual(m, a)
+                        self.assertEqual(m.tolist(), a.tolist())
+                        self.assertEqual(m.tobytes(), a.tobytes())
+                        self.assertEqual(len(m), len(a))
+
+                        cmptest(self, a, b, m, singleitem)
+
+                        array_err = None
+                        have_resize = None
+                        try:
+                            al = a[lslice]
+                            ar = a[rslice]
+                            a[lslice] = a[rslice]
+                            have_resize = len(al) != len(ar)
+                        except Exception as e:
+                            array_err = e.__class__
+
+                        m_err = None
+                        try:
+                            m[lslice] = m[rslice]
+                        except Exception as e:
+                            m_err = e.__class__
+
+                        if have_resize: # memoryview cannot change shape
+                            self.assertIs(m_err, ValueError)
+                        elif m_err or array_err:
+                            self.assertIs(m_err, array_err)
+                        else:
+                            self.assertEqual(m, a)
+                            self.assertEqual(m.tolist(), a.tolist())
+                            self.assertEqual(m.tobytes(), a.tobytes())
+                            cmptest(self, a, b, m, singleitem)
+
+    def test_memoryview_compare_special_cases(self):
+
+        a = array.array('L', [1, 2, 3])
+        b = array.array('L', [1, 2, 7])
+
+        # Ordering comparisons raise:
+        v = memoryview(a)
+        w = memoryview(b)
+        for attr in ('__lt__', '__le__', '__gt__', '__ge__'):
+            self.assertIs(getattr(v, attr)(w), NotImplemented)
+            self.assertIs(getattr(a, attr)(v), NotImplemented)
+
+        # Released views compare equal to themselves:
+        v = memoryview(a)
+        v.release()
+        self.assertEqual(v, v)
+        self.assertNotEqual(v, a)
+        self.assertNotEqual(a, v)
+
+        v = memoryview(a)
+        w = memoryview(a)
+        w.release()
+        self.assertNotEqual(v, w)
+        self.assertNotEqual(w, v)
+
+        # Operand does not implement the buffer protocol:
+        v = memoryview(a)
+        self.assertNotEqual(v, [1, 2, 3])
+
+        # NaNs
+        nd = ndarray([(0, 0)], shape=[1], format='l x d x', flags=ND_WRITABLE)
+        nd[0] = (-1, float('nan'))
+        self.assertNotEqual(memoryview(nd), nd)
+
+        # Depends on issue #15625: the struct module does not understand 'u'.
+        a = array.array('u', 'xyz')
+        v = memoryview(a)
+        self.assertNotEqual(a, v)
+        self.assertNotEqual(v, a)
+
+        # Some ctypes format strings are unknown to the struct module.
+        if ctypes:
+            # format: "T{>l:x:>l:y:}"
+            class BEPoint(ctypes.BigEndianStructure):
+                _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)]
+            point = BEPoint(100, 200)
+            a = memoryview(point)
+            b = memoryview(point)
+            self.assertNotEqual(a, b)
+            self.assertNotEqual(a, point)
+            self.assertNotEqual(point, a)
+            self.assertRaises(NotImplementedError, a.tolist)
+
+    def test_memoryview_compare_ndim_zero(self):
+
+        nd1 = ndarray(1729, shape=[], format='@L')
+        nd2 = ndarray(1729, shape=[], format='L', flags=ND_WRITABLE)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+        self.assertEqual(v, w)
+        self.assertEqual(w, v)
+        self.assertEqual(v, nd2)
+        self.assertEqual(nd2, v)
+        self.assertEqual(w, nd1)
+        self.assertEqual(nd1, w)
+
+        self.assertFalse(v.__ne__(w))
+        self.assertFalse(w.__ne__(v))
+
+        w[()] = 1728
+        self.assertNotEqual(v, w)
+        self.assertNotEqual(w, v)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(nd2, v)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(nd1, w)
+
+        self.assertFalse(v.__eq__(w))
+        self.assertFalse(w.__eq__(v))
+
+        nd = ndarray(list(range(12)), shape=[12], flags=ND_WRITABLE|ND_PIL)
+        ex = ndarray(list(range(12)), shape=[12], flags=ND_WRITABLE|ND_PIL)
+        m = memoryview(ex)
+
+        self.assertEqual(m, nd)
+        m[9] = 100
+        self.assertNotEqual(m, nd)
+
+        # struct module: equal
+        nd1 = ndarray((1729, 1.2, b'12345'), shape=[], format='Lf5s')
+        nd2 = ndarray((1729, 1.2, b'12345'), shape=[], format='hf5s',
+                      flags=ND_WRITABLE)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+        self.assertEqual(v, w)
+        self.assertEqual(w, v)
+        self.assertEqual(v, nd2)
+        self.assertEqual(nd2, v)
+        self.assertEqual(w, nd1)
+        self.assertEqual(nd1, w)
+
+        # struct module: not equal
+        nd1 = ndarray((1729, 1.2, b'12345'), shape=[], format='Lf5s')
+        nd2 = ndarray((-1729, 1.2, b'12345'), shape=[], format='hf5s',
+                      flags=ND_WRITABLE)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+        self.assertNotEqual(v, w)
+        self.assertNotEqual(w, v)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(nd2, v)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(nd1, w)
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+
+    def test_memoryview_compare_ndim_one(self):
+
+        # contiguous
+        nd1 = ndarray([-529, 576, -625, 676, -729], shape=[5], format='@h')
+        nd2 = ndarray([-529, 576, -625, 676, 729], shape=[5], format='@h')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # contiguous, struct module
+        nd1 = ndarray([-529, 576, -625, 676, -729], shape=[5], format='<i')
+        nd2 = ndarray([-529, 576, -625, 676, 729], shape=[5], format='>h')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # non-contiguous
+        nd1 = ndarray([-529, -625, -729], shape=[3], format='@h')
+        nd2 = ndarray([-529, 576, -625, 676, -729], shape=[5], format='@h')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd2[::2])
+        self.assertEqual(w[::2], nd1)
+        self.assertEqual(v, w[::2])
+        self.assertEqual(v[::-1], w[::-2])
+
+        # non-contiguous, struct module
+        nd1 = ndarray([-529, -625, -729], shape=[3], format='!h')
+        nd2 = ndarray([-529, 576, -625, 676, -729], shape=[5], format='<l')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd2[::2])
+        self.assertEqual(w[::2], nd1)
+        self.assertEqual(v, w[::2])
+        self.assertEqual(v[::-1], w[::-2])
+
+        # non-contiguous, suboffsets
+        nd1 = ndarray([-529, -625, -729], shape=[3], format='@h')
+        nd2 = ndarray([-529, 576, -625, 676, -729], shape=[5], format='@h',
+                      flags=ND_PIL)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd2[::2])
+        self.assertEqual(w[::2], nd1)
+        self.assertEqual(v, w[::2])
+        self.assertEqual(v[::-1], w[::-2])
+
+        # non-contiguous, suboffsets, struct module
+        nd1 = ndarray([-529, -625, -729], shape=[3], format='h  0c')
+        nd2 = ndarray([-529, 576, -625, 676, -729], shape=[5], format='>  h',
+                      flags=ND_PIL)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd2[::2])
+        self.assertEqual(w[::2], nd1)
+        self.assertEqual(v, w[::2])
+        self.assertEqual(v[::-1], w[::-2])
+
+    def test_memoryview_compare_zero_shape(self):
+
+        # zeros in shape
+        nd1 = ndarray([900, 961], shape=[0], format='@h')
+        nd2 = ndarray([-900, -961], shape=[0], format='@h')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+        # zeros in shape, struct module
+        nd1 = ndarray([900, 961], shape=[0], format='= h0c')
+        nd2 = ndarray([-900, -961], shape=[0], format='@   i')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+    def test_memoryview_compare_zero_strides(self):
+
+        # zero strides
+        nd1 = ndarray([900, 900, 900, 900], shape=[4], format='@L')
+        nd2 = ndarray([900], shape=[4], strides=[0], format='L')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+        # zero strides, struct module
+        nd1 = ndarray([(900, 900)]*4, shape=[4], format='@ Li')
+        nd2 = ndarray([(900, 900)], shape=[4], strides=[0], format='!L  h')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+    def test_memoryview_compare_random_formats(self):
+
+        # random single character native formats
+        n = 10
+        for char in fmtdict['@m']:
+            fmt, items, singleitem = randitems(n, 'memoryview', '@', char)
+            for flags in (0, ND_PIL):
+                nd = ndarray(items, shape=[n], format=fmt, flags=flags)
+                m = memoryview(nd)
+                self.assertEqual(m, nd)
+
+                nd = nd[::-3]
+                m = memoryview(nd)
+                self.assertEqual(m, nd)
+
+        # random formats
+        n = 10
+        for _ in range(100):
+            fmt, items, singleitem = randitems(n)
+            for flags in (0, ND_PIL):
+                nd = ndarray(items, shape=[n], format=fmt, flags=flags)
+                m = memoryview(nd)
+                self.assertEqual(m, nd)
+
+                nd = nd[::-3]
+                m = memoryview(nd)
+                self.assertEqual(m, nd)
+
+    def test_memoryview_compare_multidim_c(self):
+
+        # C-contiguous, different values
+        nd1 = ndarray(list(range(-15, 15)), shape=[3, 2, 5], format='@h')
+        nd2 = ndarray(list(range(0, 30)), shape=[3, 2, 5], format='@h')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # C-contiguous, different values, struct module
+        nd1 = ndarray([(0, 1, 2)]*30, shape=[3, 2, 5], format='=f q xxL')
+        nd2 = ndarray([(-1.2, 1, 2)]*30, shape=[3, 2, 5], format='< f 2Q')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # C-contiguous, different shape
+        nd1 = ndarray(list(range(30)), shape=[2, 3, 5], format='L')
+        nd2 = ndarray(list(range(30)), shape=[3, 2, 5], format='L')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # C-contiguous, different shape, struct module
+        nd1 = ndarray([(0, 1, 2)]*21, shape=[3, 7], format='! b B xL')
+        nd2 = ndarray([(0, 1, 2)]*21, shape=[7, 3], format='= Qx l xxL')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # C-contiguous, different format, struct module
+        nd1 = ndarray(list(range(30)), shape=[2, 3, 5], format='L')
+        nd2 = ndarray(list(range(30)), shape=[2, 3, 5], format='l')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+    def test_memoryview_compare_multidim_fortran(self):
+
+        # Fortran-contiguous, different values
+        nd1 = ndarray(list(range(-15, 15)), shape=[5, 2, 3], format='@h',
+                      flags=ND_FORTRAN)
+        nd2 = ndarray(list(range(0, 30)), shape=[5, 2, 3], format='@h',
+                      flags=ND_FORTRAN)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # Fortran-contiguous, different values, struct module
+        nd1 = ndarray([(2**64-1, -1)]*6, shape=[2, 3], format='=Qq',
+                      flags=ND_FORTRAN)
+        nd2 = ndarray([(-1, 2**64-1)]*6, shape=[2, 3], format='=qQ',
+                      flags=ND_FORTRAN)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # Fortran-contiguous, different shape
+        nd1 = ndarray(list(range(-15, 15)), shape=[2, 3, 5], format='l',
+                      flags=ND_FORTRAN)
+        nd2 = ndarray(list(range(-15, 15)), shape=[3, 2, 5], format='l',
+                      flags=ND_FORTRAN)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # Fortran-contiguous, different shape, struct module
+        nd1 = ndarray(list(range(-15, 15)), shape=[2, 3, 5], format='0ll',
+                      flags=ND_FORTRAN)
+        nd2 = ndarray(list(range(-15, 15)), shape=[3, 2, 5], format='l',
+                      flags=ND_FORTRAN)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # Fortran-contiguous, different format, struct module
+        nd1 = ndarray(list(range(30)), shape=[5, 2, 3], format='@h',
+                      flags=ND_FORTRAN)
+        nd2 = ndarray(list(range(30)), shape=[5, 2, 3], format='@b',
+                      flags=ND_FORTRAN)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+    def test_memoryview_compare_multidim_mixed(self):
+
+        # mixed C/Fortran contiguous
+        lst1 = list(range(-15, 15))
+        lst2 = transpose(lst1, [3, 2, 5])
+        nd1 = ndarray(lst1, shape=[3, 2, 5], format='@l')
+        nd2 = ndarray(lst2, shape=[3, 2, 5], format='l', flags=ND_FORTRAN)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, w)
+
+        # mixed C/Fortran contiguous, struct module
+        lst1 = [(-3.3, -22, b'x')]*30
+        lst1[5] = (-2.2, -22, b'x')
+        lst2 = transpose(lst1, [3, 2, 5])
+        nd1 = ndarray(lst1, shape=[3, 2, 5], format='d b c')
+        nd2 = ndarray(lst2, shape=[3, 2, 5], format='d h c', flags=ND_FORTRAN)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, w)
+
+        # different values, non-contiguous
+        ex1 = ndarray(list(range(40)), shape=[5, 8], format='@I')
+        nd1 = ex1[3:1:-1, ::-2]
+        ex2 = ndarray(list(range(40)), shape=[5, 8], format='I')
+        nd2 = ex2[1:3:1, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # same values, non-contiguous, struct module
+        ex1 = ndarray([(2**31-1, -2**31)]*22, shape=[11, 2], format='=ii')
+        nd1 = ex1[3:1:-1, ::-2]
+        ex2 = ndarray([(2**31-1, -2**31)]*22, shape=[11, 2], format='>ii')
+        nd2 = ex2[1:3:1, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+        # different shape
+        ex1 = ndarray(list(range(30)), shape=[2, 3, 5], format='b')
+        nd1 = ex1[1:3:, ::-2]
+        nd2 = ndarray(list(range(30)), shape=[3, 2, 5], format='b')
+        nd2 = ex2[1:3:, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # different shape, struct module
+        ex1 = ndarray(list(range(30)), shape=[2, 3, 5], format='B')
+        nd1 = ex1[1:3:, ::-2]
+        nd2 = ndarray(list(range(30)), shape=[3, 2, 5], format='b')
+        nd2 = ex2[1:3:, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # different format, struct module
+        ex1 = ndarray([(2, b'123')]*30, shape=[5, 3, 2], format='b3s')
+        nd1 = ex1[1:3:, ::-2]
+        nd2 = ndarray([(2, b'123')]*30, shape=[5, 3, 2], format='i3s')
+        nd2 = ex2[1:3:, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+    def test_memoryview_compare_multidim_zero_shape(self):
+
+        # zeros in shape
+        nd1 = ndarray(list(range(30)), shape=[0, 3, 2], format='i')
+        nd2 = ndarray(list(range(30)), shape=[5, 0, 2], format='@i')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # zeros in shape, struct module
+        nd1 = ndarray(list(range(30)), shape=[0, 3, 2], format='i')
+        nd2 = ndarray(list(range(30)), shape=[5, 0, 2], format='@i')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+    def test_memoryview_compare_multidim_zero_strides(self):
+
+        # zero strides
+        nd1 = ndarray([900]*80, shape=[4, 5, 4], format='@L')
+        nd2 = ndarray([900], shape=[4, 5, 4], strides=[0, 0, 0], format='L')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+        self.assertEqual(v.tolist(), w.tolist())
+
+        # zero strides, struct module
+        nd1 = ndarray([(1, 2)]*10, shape=[2, 5], format='=lQ')
+        nd2 = ndarray([(1, 2)], shape=[2, 5], strides=[0, 0], format='<lQ')
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+    def test_memoryview_compare_multidim_suboffsets(self):
+
+        # suboffsets
+        ex1 = ndarray(list(range(40)), shape=[5, 8], format='@I')
+        nd1 = ex1[3:1:-1, ::-2]
+        ex2 = ndarray(list(range(40)), shape=[5, 8], format='I', flags=ND_PIL)
+        nd2 = ex2[1:3:1, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # suboffsets, struct module
+        ex1 = ndarray([(2**64-1, -1)]*40, shape=[5, 8], format='=Qq',
+                      flags=ND_WRITABLE)
+        ex1[2][7] = (1, -2)
+        nd1 = ex1[3:1:-1, ::-2]
+
+        ex2 = ndarray([(2**64-1, -1)]*40, shape=[5, 8], format='>Qq',
+                      flags=ND_PIL|ND_WRITABLE)
+        ex2[2][7] = (1, -2)
+        nd2 = ex2[1:3:1, ::-2]
+
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+        # suboffsets, different shape
+        ex1 = ndarray(list(range(30)), shape=[2, 3, 5], format='b',
+                      flags=ND_PIL)
+        nd1 = ex1[1:3:, ::-2]
+        nd2 = ndarray(list(range(30)), shape=[3, 2, 5], format='b')
+        nd2 = ex2[1:3:, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # suboffsets, different shape, struct module
+        ex1 = ndarray([(2**8-1, -1)]*40, shape=[2, 3, 5], format='Bb',
+                      flags=ND_PIL|ND_WRITABLE)
+        nd1 = ex1[1:2:, ::-2]
+
+        ex2 = ndarray([(2**8-1, -1)]*40, shape=[3, 2, 5], format='Bb')
+        nd2 = ex2[1:2:, ::-2]
+
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # suboffsets, different format
+        ex1 = ndarray(list(range(30)), shape=[5, 3, 2], format='i', flags=ND_PIL)
+        nd1 = ex1[1:3:, ::-2]
+        ex2 = ndarray(list(range(30)), shape=[5, 3, 2], format='@I', flags=ND_PIL)
+        nd2 = ex2[1:3:, ::-2]
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, nd2)
+        self.assertEqual(w, nd1)
+        self.assertEqual(v, w)
+
+        # suboffsets, different format, struct module
+        ex1 = ndarray([(b'hello', b'', 1)]*27, shape=[3, 3, 3], format='5s0sP',
+                      flags=ND_PIL|ND_WRITABLE)
+        ex1[1][2][2] = (b'sushi', b'', 1)
+        nd1 = ex1[1:3:, ::-2]
+
+        ex2 = ndarray([(b'hello', b'', 1)]*27, shape=[3, 3, 3], format='5s0sP',
+                      flags=ND_PIL|ND_WRITABLE)
+        ex1[1][2][2] = (b'sushi', b'', 1)
+        nd2 = ex2[1:3:, ::-2]
+
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertNotEqual(v, nd2)
+        self.assertNotEqual(w, nd1)
+        self.assertNotEqual(v, w)
+
+        # initialize mixed C/Fortran + suboffsets
+        lst1 = list(range(-15, 15))
+        lst2 = transpose(lst1, [3, 2, 5])
+        nd1 = ndarray(lst1, shape=[3, 2, 5], format='@l', flags=ND_PIL)
+        nd2 = ndarray(lst2, shape=[3, 2, 5], format='l', flags=ND_FORTRAN|ND_PIL)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, w)
+
+        # initialize mixed C/Fortran + suboffsets, struct module
+        lst1 = [(b'sashimi', b'sliced', 20.05)]*30
+        lst1[11] = (b'ramen', b'spicy', 9.45)
+        lst2 = transpose(lst1, [3, 2, 5])
+
+        nd1 = ndarray(lst1, shape=[3, 2, 5], format='< 10p 9p d', flags=ND_PIL)
+        nd2 = ndarray(lst2, shape=[3, 2, 5], format='> 10p 9p d',
+                      flags=ND_FORTRAN|ND_PIL)
+        v = memoryview(nd1)
+        w = memoryview(nd2)
+
+        self.assertEqual(v, nd1)
+        self.assertEqual(w, nd2)
+        self.assertEqual(v, w)
+
+    def test_memoryview_compare_not_equal(self):
+
+        # items not equal
+        for byteorder in ['=', '<', '>', '!']:
+            x = ndarray([2**63]*120, shape=[3,5,2,2,2], format=byteorder+'Q')
+            y = ndarray([2**63]*120, shape=[3,5,2,2,2], format=byteorder+'Q',
+                        flags=ND_WRITABLE|ND_FORTRAN)
+            y[2][3][1][1][1] = 1
+            a = memoryview(x)
+            b = memoryview(y)
+            self.assertEqual(a, x)
+            self.assertEqual(b, y)
+            self.assertNotEqual(a, b)
+            self.assertNotEqual(a, y)
+            self.assertNotEqual(b, x)
+
+            x = ndarray([(2**63, 2**31, 2**15)]*120, shape=[3,5,2,2,2],
+                        format=byteorder+'QLH')
+            y = ndarray([(2**63, 2**31, 2**15)]*120, shape=[3,5,2,2,2],
+                        format=byteorder+'QLH', flags=ND_WRITABLE|ND_FORTRAN)
+            y[2][3][1][1][1] = (1, 1, 1)
+            a = memoryview(x)
+            b = memoryview(y)
+            self.assertEqual(a, x)
+            self.assertEqual(b, y)
+            self.assertNotEqual(a, b)
+            self.assertNotEqual(a, y)
+            self.assertNotEqual(b, x)
+
+    def test_memoryview_check_released(self):
+
+        a = array.array('d', [1.1, 2.2, 3.3])
+
+        m = memoryview(a)
+        m.release()
+
+        # PyMemoryView_FromObject()
+        self.assertRaises(ValueError, memoryview, m)
+        # memoryview.cast()
+        self.assertRaises(ValueError, m.cast, 'c')
+        # getbuffer()
+        self.assertRaises(ValueError, ndarray, m)
+        # memoryview.tolist()
+        self.assertRaises(ValueError, m.tolist)
+        # memoryview.tobytes()
+        self.assertRaises(ValueError, m.tobytes)
+        # sequence
+        self.assertRaises(ValueError, eval, "1.0 in m", locals())
+        # subscript
+        self.assertRaises(ValueError, m.__getitem__, 0)
+        # assignment
+        self.assertRaises(ValueError, m.__setitem__, 0, 1)
+
+        for attr in ('obj', 'nbytes', 'readonly', 'itemsize', 'format', 'ndim',
+                     'shape', 'strides', 'suboffsets', 'c_contiguous',
+                     'f_contiguous', 'contiguous'):
+            self.assertRaises(ValueError, m.__getattribute__, attr)
+
+        # richcompare
+        b = array.array('d', [1.1, 2.2, 3.3])
+        m1 = memoryview(a)
+        m2 = memoryview(b)
+
+        self.assertEqual(m1, m2)
+        m1.release()
+        self.assertNotEqual(m1, m2)
+        self.assertNotEqual(m1, a)
+        self.assertEqual(m1, m1)
+
+    def test_memoryview_tobytes(self):
+        # Many implicit tests are already in self.verify().
+
+        t = (-529, 576, -625, 676, -729)
+
+        nd = ndarray(t, shape=[5], format='@h')
+        m = memoryview(nd)
+        self.assertEqual(m, nd)
+        self.assertEqual(m.tobytes(), nd.tobytes())
+
+        nd = ndarray([t], shape=[1], format='>hQiLl')
+        m = memoryview(nd)
+        self.assertEqual(m, nd)
+        self.assertEqual(m.tobytes(), nd.tobytes())
+
+        nd = ndarray([t for _ in range(12)], shape=[2,2,3], format='=hQiLl')
+        m = memoryview(nd)
+        self.assertEqual(m, nd)
+        self.assertEqual(m.tobytes(), nd.tobytes())
+
+        nd = ndarray([t for _ in range(120)], shape=[5,2,2,3,2],
+                     format='<hQiLl')
+        m = memoryview(nd)
+        self.assertEqual(m, nd)
+        self.assertEqual(m.tobytes(), nd.tobytes())
+
+        # Unknown formats are handled: tobytes() purely depends on itemsize.
+        if ctypes:
+            # format: "T{>l:x:>l:y:}"
+            class BEPoint(ctypes.BigEndianStructure):
+                _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)]
+            point = BEPoint(100, 200)
+            a = memoryview(point)
+            self.assertEqual(a.tobytes(), bytes(point))
+
+    def test_memoryview_get_contiguous(self):
+        # Many implicit tests are already in self.verify().
+
+        # no buffer interface
+        self.assertRaises(TypeError, get_contiguous, {}, PyBUF_READ, 'F')
+
+        # writable request to read-only object
+        self.assertRaises(BufferError, get_contiguous, b'x', PyBUF_WRITE, 'C')
+
+        # writable request to non-contiguous object
+        nd = ndarray([1, 2, 3], shape=[2], strides=[2])
+        self.assertRaises(BufferError, get_contiguous, nd, PyBUF_WRITE, 'A')
+
+        # scalar, read-only request from read-only exporter
+        nd = ndarray(9, shape=(), format="L")
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_READ, order)
+            self.assertEqual(m, nd)
+            self.assertEqual(m[()], 9)
+
+        # scalar, read-only request from writable exporter
+        nd = ndarray(9, shape=(), format="L", flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_READ, order)
+            self.assertEqual(m, nd)
+            self.assertEqual(m[()], 9)
+
+        # scalar, writable request
+        for order in ['C', 'F', 'A']:
+            nd[()] = 9
+            m = get_contiguous(nd, PyBUF_WRITE, order)
+            self.assertEqual(m, nd)
+            self.assertEqual(m[()], 9)
+
+            m[()] = 10
+            self.assertEqual(m[()], 10)
+            self.assertEqual(nd[()], 10)
+
+        # zeros in shape
+        nd = ndarray([1], shape=[0], format="L", flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_READ, order)
+            self.assertRaises(IndexError, m.__getitem__, 0)
+            self.assertEqual(m, nd)
+            self.assertEqual(m.tolist(), [])
+
+        nd = ndarray(list(range(8)), shape=[2, 0, 7], format="L",
+                     flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_READ, order)
+            self.assertEqual(ndarray(m).tolist(), [[], []])
+
+        # one-dimensional
+        nd = ndarray([1], shape=[1], format="h", flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_WRITE, order)
+            self.assertEqual(m, nd)
+            self.assertEqual(m.tolist(), nd.tolist())
+
+        nd = ndarray([1, 2, 3], shape=[3], format="b", flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_WRITE, order)
+            self.assertEqual(m, nd)
+            self.assertEqual(m.tolist(), nd.tolist())
+
+        # one-dimensional, non-contiguous
+        nd = ndarray([1, 2, 3], shape=[2], strides=[2], flags=ND_WRITABLE)
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_READ, order)
+            self.assertEqual(m, nd)
+            self.assertEqual(m.tolist(), nd.tolist())
+            self.assertRaises(TypeError, m.__setitem__, 1, 20)
+            self.assertEqual(m[1], 3)
+            self.assertEqual(nd[1], 3)
+
+        nd = nd[::-1]
+        for order in ['C', 'F', 'A']:
+            m = get_contiguous(nd, PyBUF_READ, order)
+            self.assertEqual(m, nd)
+            self.assertEqual(m.tolist(), nd.tolist())
+            self.assertRaises(TypeError, m.__setitem__, 1, 20)
+            self.assertEqual(m[1], 1)
+            self.assertEqual(nd[1], 1)
+
+        # multi-dimensional, contiguous input
+        nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE)
+        for order in ['C', 'A']:
+            m = get_contiguous(nd, PyBUF_WRITE, order)
+            self.assertEqual(ndarray(m).tolist(), nd.tolist())
+
+        self.assertRaises(BufferError, get_contiguous, nd, PyBUF_WRITE, 'F')
+        m = get_contiguous(nd, PyBUF_READ, order)
+        self.assertEqual(ndarray(m).tolist(), nd.tolist())
+
+        nd = ndarray(list(range(12)), shape=[3, 4],
+                     flags=ND_WRITABLE|ND_FORTRAN)
+        for order in ['F', 'A']:
+            m = get_contiguous(nd, PyBUF_WRITE, order)
+            self.assertEqual(ndarray(m).tolist(), nd.tolist())
+
+        self.assertRaises(BufferError, get_contiguous, nd, PyBUF_WRITE, 'C')
+        m = get_contiguous(nd, PyBUF_READ, order)
+        self.assertEqual(ndarray(m).tolist(), nd.tolist())
+
+        # multi-dimensional, non-contiguous input
+        nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL)
+        for order in ['C', 'F', 'A']:
+            self.assertRaises(BufferError, get_contiguous, nd, PyBUF_WRITE,
+                              order)
+            m = get_contiguous(nd, PyBUF_READ, order)
+            self.assertEqual(ndarray(m).tolist(), nd.tolist())
+
+        # flags
+        nd = ndarray([1,2,3,4,5], shape=[3], strides=[2])
+        m = get_contiguous(nd, PyBUF_READ, 'C')
+        self.assertTrue(m.c_contiguous)
+
+    def test_memoryview_serializing(self):
+
+        # C-contiguous
+        size = struct.calcsize('i')
+        a = array.array('i', [1,2,3,4,5])
+        m = memoryview(a)
+        buf = io.BytesIO(m)
+        b = bytearray(5*size)
+        buf.readinto(b)
+        self.assertEqual(m.tobytes(), b)
+
+        # C-contiguous, multi-dimensional
+        size = struct.calcsize('L')
+        nd = ndarray(list(range(12)), shape=[2,3,2], format="L")
+        m = memoryview(nd)
+        buf = io.BytesIO(m)
+        b = bytearray(2*3*2*size)
+        buf.readinto(b)
+        self.assertEqual(m.tobytes(), b)
+
+        # Fortran contiguous, multi-dimensional
+        #size = struct.calcsize('L')
+        #nd = ndarray(list(range(12)), shape=[2,3,2], format="L",
+        #             flags=ND_FORTRAN)
+        #m = memoryview(nd)
+        #buf = io.BytesIO(m)
+        #b = bytearray(2*3*2*size)
+        #buf.readinto(b)
+        #self.assertEqual(m.tobytes(), b)
+
+    def test_memoryview_hash(self):
+
+        # bytes exporter
+        b = bytes(list(range(12)))
+        m = memoryview(b)
+        self.assertEqual(hash(b), hash(m))
+
+        # C-contiguous
+        mc = m.cast('c', shape=[3,4])
+        self.assertEqual(hash(mc), hash(b))
+
+        # non-contiguous
+        mx = m[::-2]
+        b = bytes(list(range(12))[::-2])
+        self.assertEqual(hash(mx), hash(b))
+
+        # Fortran contiguous
+        nd = ndarray(list(range(30)), shape=[3,2,5], flags=ND_FORTRAN)
+        m = memoryview(nd)
+        self.assertEqual(hash(m), hash(nd))
+
+        # multi-dimensional slice
+        nd = ndarray(list(range(30)), shape=[3,2,5])
+        x = nd[::2, ::, ::-1]
+        m = memoryview(x)
+        self.assertEqual(hash(m), hash(x))
+
+        # multi-dimensional slice with suboffsets
+        nd = ndarray(list(range(30)), shape=[2,5,3], flags=ND_PIL)
+        x = nd[::2, ::, ::-1]
+        m = memoryview(x)
+        self.assertEqual(hash(m), hash(x))
+
+        # equality-hash invariant
+        x = ndarray(list(range(12)), shape=[12], format='B')
+        a = memoryview(x)
+
+        y = ndarray(list(range(12)), shape=[12], format='b')
+        b = memoryview(y)
+
+        self.assertEqual(a, b)
+        self.assertEqual(hash(a), hash(b))
+
+        # non-byte formats
+        nd = ndarray(list(range(12)), shape=[2,2,3], format='L')
+        m = memoryview(nd)
+        self.assertRaises(ValueError, m.__hash__)
+
+        nd = ndarray(list(range(-6, 6)), shape=[2,2,3], format='h')
+        m = memoryview(nd)
+        self.assertRaises(ValueError, m.__hash__)
+
+        nd = ndarray(list(range(12)), shape=[2,2,3], format='= L')
+        m = memoryview(nd)
+        self.assertRaises(ValueError, m.__hash__)
+
+        nd = ndarray(list(range(-6, 6)), shape=[2,2,3], format='< h')
+        m = memoryview(nd)
+        self.assertRaises(ValueError, m.__hash__)
+
+    def test_memoryview_release(self):
+
+        # Create re-exporter from getbuffer(memoryview), then release the view.
+        a = bytearray([1,2,3])
+        m = memoryview(a)
+        nd = ndarray(m) # re-exporter
+        self.assertRaises(BufferError, m.release)
+        del nd
+        m.release()
+
+        a = bytearray([1,2,3])
+        m = memoryview(a)
+        nd1 = ndarray(m, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        nd2 = ndarray(nd1, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        self.assertIs(nd2.obj, m)
+        self.assertRaises(BufferError, m.release)
+        del nd1, nd2
+        m.release()
+
+        # chained views
+        a = bytearray([1,2,3])
+        m1 = memoryview(a)
+        m2 = memoryview(m1)
+        nd = ndarray(m2) # re-exporter
+        m1.release()
+        self.assertRaises(BufferError, m2.release)
+        del nd
+        m2.release()
+
+        a = bytearray([1,2,3])
+        m1 = memoryview(a)
+        m2 = memoryview(m1)
+        nd1 = ndarray(m2, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        nd2 = ndarray(nd1, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        self.assertIs(nd2.obj, m2)
+        m1.release()
+        self.assertRaises(BufferError, m2.release)
+        del nd1, nd2
+        m2.release()
+
+        # Allow changing layout while buffers are exported.
+        nd = ndarray([1,2,3], shape=[3], flags=ND_VAREXPORT)
+        m1 = memoryview(nd)
+
+        nd.push([4,5,6,7,8], shape=[5]) # mutate nd
+        m2 = memoryview(nd)
+
+        x = memoryview(m1)
+        self.assertEqual(x.tolist(), m1.tolist())
+
+        y = memoryview(m2)
+        self.assertEqual(y.tolist(), m2.tolist())
+        self.assertEqual(y.tolist(), nd.tolist())
+        m2.release()
+        y.release()
+
+        nd.pop() # pop the current view
+        self.assertEqual(x.tolist(), nd.tolist())
+
+        del nd
+        m1.release()
+        x.release()
+
+        # If multiple memoryviews share the same managed buffer, implicit
+        # release() in the context manager's __exit__() method should still
+        # work.
+        def catch22(b):
+            with memoryview(b) as m2:
+                pass
+
+        x = bytearray(b'123')
+        with memoryview(x) as m1:
+            catch22(m1)
+            self.assertEqual(m1[0], ord(b'1'))
+
+        x = ndarray(list(range(12)), shape=[2,2,3], format='l')
+        y = ndarray(x, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        z = ndarray(y, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        self.assertIs(z.obj, x)
+        with memoryview(z) as m:
+            catch22(m)
+            self.assertEqual(m[0:1].tolist(), [[[0, 1, 2], [3, 4, 5]]])
+
+        # Test garbage collection.
+        for flags in (0, ND_REDIRECT):
+            x = bytearray(b'123')
+            with memoryview(x) as m1:
+                del x
+                y = ndarray(m1, getbuf=PyBUF_FULL_RO, flags=flags)
+                with memoryview(y) as m2:
+                    del y
+                    z = ndarray(m2, getbuf=PyBUF_FULL_RO, flags=flags)
+                    with memoryview(z) as m3:
+                        del z
+                        catch22(m3)
+                        catch22(m2)
+                        catch22(m1)
+                        self.assertEqual(m1[0], ord(b'1'))
+                        self.assertEqual(m2[1], ord(b'2'))
+                        self.assertEqual(m3[2], ord(b'3'))
+                        del m3
+                    del m2
+                del m1
+
+            x = bytearray(b'123')
+            with memoryview(x) as m1:
+                del x
+                y = ndarray(m1, getbuf=PyBUF_FULL_RO, flags=flags)
+                with memoryview(y) as m2:
+                    del y
+                    z = ndarray(m2, getbuf=PyBUF_FULL_RO, flags=flags)
+                    with memoryview(z) as m3:
+                        del z
+                        catch22(m1)
+                        catch22(m2)
+                        catch22(m3)
+                        self.assertEqual(m1[0], ord(b'1'))
+                        self.assertEqual(m2[1], ord(b'2'))
+                        self.assertEqual(m3[2], ord(b'3'))
+                        del m1, m2, m3
+
+        # memoryview.release() fails if the view has exported buffers.
+        x = bytearray(b'123')
+        with self.assertRaises(BufferError):
+            with memoryview(x) as m:
+                ex = ndarray(m)
+                m[0] == ord(b'1')
+
+    def test_memoryview_redirect(self):
+
+        nd = ndarray([1.0 * x for x in range(12)], shape=[12], format='d')
+        a = array.array('d', [1.0 * x for x in range(12)])
+
+        for x in (nd, a):
+            y = ndarray(x, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+            z = ndarray(y, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+            m = memoryview(z)
+
+            self.assertIs(y.obj, x)
+            self.assertIs(z.obj, x)
+            self.assertIs(m.obj, x)
+
+            self.assertEqual(m, x)
+            self.assertEqual(m, y)
+            self.assertEqual(m, z)
+
+            self.assertEqual(m[1:3], x[1:3])
+            self.assertEqual(m[1:3], y[1:3])
+            self.assertEqual(m[1:3], z[1:3])
+            del y, z
+            self.assertEqual(m[1:3], x[1:3])
+
+    def test_memoryview_from_static_exporter(self):
+
+        fmt = 'B'
+        lst = [0,1,2,3,4,5,6,7,8,9,10,11]
+
+        # exceptions
+        self.assertRaises(TypeError, staticarray, 1, 2, 3)
+
+        # view.obj==x
+        x = staticarray()
+        y = memoryview(x)
+        self.verify(y, obj=x,
+                    itemsize=1, fmt=fmt, readonly=1,
+                    ndim=1, shape=[12], strides=[1],
+                    lst=lst)
+        for i in range(12):
+            self.assertEqual(y[i], i)
+        del x
+        del y
+
+        x = staticarray()
+        y = memoryview(x)
+        del y
+        del x
+
+        x = staticarray()
+        y = ndarray(x, getbuf=PyBUF_FULL_RO)
+        z = ndarray(y, getbuf=PyBUF_FULL_RO)
+        m = memoryview(z)
+        self.assertIs(y.obj, x)
+        self.assertIs(m.obj, z)
+        self.verify(m, obj=z,
+                    itemsize=1, fmt=fmt, readonly=1,
+                    ndim=1, shape=[12], strides=[1],
+                    lst=lst)
+        del x, y, z, m
+
+        x = staticarray()
+        y = ndarray(x, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        z = ndarray(y, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        m = memoryview(z)
+        self.assertIs(y.obj, x)
+        self.assertIs(z.obj, x)
+        self.assertIs(m.obj, x)
+        self.verify(m, obj=x,
+                    itemsize=1, fmt=fmt, readonly=1,
+                    ndim=1, shape=[12], strides=[1],
+                    lst=lst)
+        del x, y, z, m
+
+        # view.obj==NULL
+        x = staticarray(legacy_mode=True)
+        y = memoryview(x)
+        self.verify(y, obj=None,
+                    itemsize=1, fmt=fmt, readonly=1,
+                    ndim=1, shape=[12], strides=[1],
+                    lst=lst)
+        for i in range(12):
+            self.assertEqual(y[i], i)
+        del x
+        del y
+
+        x = staticarray(legacy_mode=True)
+        y = memoryview(x)
+        del y
+        del x
+
+        x = staticarray(legacy_mode=True)
+        y = ndarray(x, getbuf=PyBUF_FULL_RO)
+        z = ndarray(y, getbuf=PyBUF_FULL_RO)
+        m = memoryview(z)
+        self.assertIs(y.obj, None)
+        self.assertIs(m.obj, z)
+        self.verify(m, obj=z,
+                    itemsize=1, fmt=fmt, readonly=1,
+                    ndim=1, shape=[12], strides=[1],
+                    lst=lst)
+        del x, y, z, m
+
+        x = staticarray(legacy_mode=True)
+        y = ndarray(x, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        z = ndarray(y, getbuf=PyBUF_FULL_RO, flags=ND_REDIRECT)
+        m = memoryview(z)
+        # Clearly setting view.obj==NULL is inferior, since it
+        # messes up the redirection chain:
+        self.assertIs(y.obj, None)
+        self.assertIs(z.obj, y)
+        self.assertIs(m.obj, y)
+        self.verify(m, obj=y,
+                    itemsize=1, fmt=fmt, readonly=1,
+                    ndim=1, shape=[12], strides=[1],
+                    lst=lst)
+        del x, y, z, m
+
+    def test_memoryview_getbuffer_undefined(self):
+
+        # getbufferproc does not adhere to the new documentation
+        nd = ndarray([1,2,3], [3], flags=ND_GETBUF_FAIL|ND_GETBUF_UNDEFINED)
+        self.assertRaises(BufferError, memoryview, nd)
+
+    def test_issue_7385(self):
+        x = ndarray([1,2,3], shape=[3], flags=ND_GETBUF_FAIL)
+        self.assertRaises(BufferError, memoryview, x)
+
+
+def test_main():
+    support.run_unittest(TestBufferProtocol)
+
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_bufio.py b/Lib/test/test_bufio.py
index 5ab6f5a0da..6338ad8e3c 100644
--- a/Lib/test/test_bufio.py
+++ b/Lib/test/test_bufio.py
@@ -11,7 +11,7 @@ import _pyio as pyio # Python implementation.
 lengths = list(range(1, 257)) + [512, 1000, 1024, 2048, 4096, 8192, 10000,
                                  16384, 32768, 65536, 1000000]
 
-class BufferSizeTest(unittest.TestCase):
+class BufferSizeTest:
     def try_one(self, s):
         # Write s + "\n" + s to file, then open it and ensure that successive
         # .readline()s deliver what we wrote.
@@ -62,15 +62,12 @@ class BufferSizeTest(unittest.TestCase):
         self.drive_one(bytes(1000))
 
 
-class CBufferSizeTest(BufferSizeTest):
+class CBufferSizeTest(BufferSizeTest, unittest.TestCase):
     open = io.open
 
-class PyBufferSizeTest(BufferSizeTest):
+class PyBufferSizeTest(BufferSizeTest, unittest.TestCase):
     open = staticmethod(pyio.open)
 
 
-def test_main():
-    support.run_unittest(CBufferSizeTest, PyBufferSizeTest)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index f4d3400b6e..c342a4329f 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -1,19 +1,21 @@
 # Python test set -- built-in functions
 
-import platform
-import unittest
-import sys
-import warnings
+import ast
+import builtins
 import collections
 import io
+import locale
 import os
-import ast
-import types
-import builtins
+import pickle
+import platform
 import random
+import sys
 import traceback
-from test.support import fcmp, TESTFN, unlink,  run_unittest, check_warnings
+import types
+import unittest
+import warnings
 from operator import neg
+from test.support import TESTFN, unlink,  run_unittest, check_warnings
 try:
     import pty, signal
 except ImportError:
@@ -110,7 +112,30 @@ class TestFailingIter:
     def __iter__(self):
         raise RuntimeError
 
+def filter_char(arg):
+    return ord(arg) > ord("d")
+
+def map_char(arg):
+    return chr(ord(arg)+1)
+
 class BuiltinTest(unittest.TestCase):
+    # Helper to check picklability
+    def check_iter_pickle(self, it, seq):
+        itorg = it
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(type(itorg), type(it))
+        self.assertEqual(list(it), seq)
+
+        #test the iterator after dropping one from it
+        it = pickle.loads(d)
+        try:
+            next(it)
+        except StopIteration:
+            return
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(list(it), seq[1:])
 
     def test_import(self):
         __import__('sys')
@@ -257,8 +282,7 @@ class BuiltinTest(unittest.TestCase):
         self.assertEqual(chr(0xff), '\xff')
         self.assertRaises(ValueError, chr, 1<<24)
         self.assertEqual(chr(sys.maxunicode),
-                         str(('\\U%08x' % (sys.maxunicode)).encode("ascii"),
-                             'unicode-escape'))
+                         str('\\U0010ffff'.encode("ascii"), 'unicode-escape'))
         self.assertRaises(TypeError, chr)
         self.assertEqual(chr(0x0000FFFF), "\U0000FFFF")
         self.assertEqual(chr(0x00010000), "\U00010000")
@@ -380,7 +404,15 @@ class BuiltinTest(unittest.TestCase):
         f = Foo()
         self.assertTrue(dir(f) == ["ga", "kan", "roo"])
 
-        # dir(obj__dir__not_list)
+        # dir(obj__dir__tuple)
+        class Foo(object):
+            def __dir__(self):
+                return ("b", "c", "a")
+        res = dir(Foo())
+        self.assertIsInstance(res, list)
+        self.assertTrue(res == ["a", "b", "c"])
+
+        # dir(obj__dir__not_sequence)
         class Foo(object):
             def __dir__(self):
                 return 7
@@ -393,6 +425,8 @@ class BuiltinTest(unittest.TestCase):
         except:
             self.assertEqual(len(dir(sys.exc_info()[2])), 4)
 
+        # test that object has a __dir__()
+        self.assertEqual(sorted([].__dir__()), dir([]))
 
     def test_divmod(self):
         self.assertEqual(divmod(12, 7), (1, 5))
@@ -402,10 +436,13 @@ class BuiltinTest(unittest.TestCase):
 
         self.assertEqual(divmod(-sys.maxsize-1, -1), (sys.maxsize+1, 0))
 
-        self.assertTrue(not fcmp(divmod(3.25, 1.0), (3.0, 0.25)))
-        self.assertTrue(not fcmp(divmod(-3.25, 1.0), (-4.0, 0.75)))
-        self.assertTrue(not fcmp(divmod(3.25, -1.0), (-4.0, -0.75)))
-        self.assertTrue(not fcmp(divmod(-3.25, -1.0), (3.0, -0.25)))
+        for num, denom, exp_result in [ (3.25, 1.0, (3.0, 0.25)),
+                                        (-3.25, 1.0, (-4.0, 0.75)),
+                                        (3.25, -1.0, (-4.0, -0.75)),
+                                        (-3.25, -1.0, (3.0, -0.25))]:
+            result = divmod(num, denom)
+            self.assertAlmostEqual(result[0], exp_result[0])
+            self.assertAlmostEqual(result[1], exp_result[1])
 
         self.assertRaises(TypeError, divmod)
 
@@ -520,6 +557,39 @@ class BuiltinTest(unittest.TestCase):
             del l['__builtins__']
         self.assertEqual((g, l), ({'a': 1}, {'b': 2}))
 
+    def test_exec_globals(self):
+        code = compile("print('Hello World!')", "", "exec")
+        # no builtin function
+        self.assertRaisesRegex(NameError, "name 'print' is not defined",
+                               exec, code, {'__builtins__': {}})
+        # __builtins__ must be a mapping type
+        self.assertRaises(TypeError,
+                          exec, code, {'__builtins__': 123})
+
+        # no __build_class__ function
+        code = compile("class A: pass", "", "exec")
+        self.assertRaisesRegex(NameError, "__build_class__ not found",
+                               exec, code, {'__builtins__': {}})
+
+        class frozendict_error(Exception):
+            pass
+
+        class frozendict(dict):
+            def __setitem__(self, key, value):
+                raise frozendict_error("frozendict is readonly")
+
+        # read-only builtins
+        frozen_builtins = frozendict(__builtins__)
+        code = compile("__builtins__['superglobal']=2; print(superglobal)", "test", "exec")
+        self.assertRaises(frozendict_error,
+                          exec, code, {'__builtins__': frozen_builtins})
+
+        # read-only globals
+        namespace = frozendict({})
+        code = compile("x=1", "test", "exec")
+        self.assertRaises(frozendict_error,
+                          exec, code, namespace)
+
     def test_exec_redirected(self):
         savestdout = sys.stdout
         sys.stdout = None # Whatever that cannot flush()
@@ -556,6 +626,11 @@ class BuiltinTest(unittest.TestCase):
         self.assertEqual(list(filter(lambda x: x>=3, (1, 2, 3, 4))), [3, 4])
         self.assertRaises(TypeError, list, filter(42, (1, 2)))
 
+    def test_filter_pickle(self):
+        f1 = filter(filter_char, "abcdeabcde")
+        f2 = filter(filter_char, "abcdeabcde")
+        self.check_iter_pickle(f1, list(f2))
+
     def test_getattr(self):
         self.assertTrue(getattr(sys, 'stdout') is sys.stdout)
         self.assertRaises(TypeError, getattr, sys, 1)
@@ -749,6 +824,11 @@ class BuiltinTest(unittest.TestCase):
             raise RuntimeError
         self.assertRaises(RuntimeError, list, map(badfunc, range(5)))
 
+    def test_map_pickle(self):
+        m1 = map(map_char, "Is this the real life?")
+        m2 = map(map_char, "Is this the real life?")
+        self.check_iter_pickle(m1, list(m2))
+
     def test_max(self):
         self.assertEqual(max('123123'), '3')
         self.assertEqual(max(1, 2, 3), 3)
@@ -882,7 +962,29 @@ class BuiltinTest(unittest.TestCase):
             self.assertEqual(fp.read(1000), 'YYY'*100)
         finally:
             fp.close()
-        unlink(TESTFN)
+            unlink(TESTFN)
+
+    def test_open_default_encoding(self):
+        old_environ = dict(os.environ)
+        try:
+            # try to get a user preferred encoding different than the current
+            # locale encoding to check that open() uses the current locale
+            # encoding and not the user preferred encoding
+            for key in ('LC_ALL', 'LANG', 'LC_CTYPE'):
+                if key in os.environ:
+                    del os.environ[key]
+
+            self.write_testfile()
+            current_locale_encoding = locale.getpreferredencoding(False)
+            fp = open(TESTFN, 'w')
+            try:
+                self.assertEqual(fp.encoding, current_locale_encoding)
+            finally:
+                fp.close()
+                unlink(TESTFN)
+        finally:
+            os.environ.clear()
+            os.environ.update(old_environ)
 
     def test_ord(self):
         self.assertEqual(ord(' '), 32)
@@ -1072,6 +1174,8 @@ class BuiltinTest(unittest.TestCase):
         # Check stdin/stdout error handler is used when invoking GNU readline
         self.check_input_tty("prompté", b"quux\xe9", "ascii")
 
+    # test_int(): see test_int.py for tests of built-in function int().
+
     def test_repr(self):
         self.assertEqual(repr(''), '\'\'')
         self.assertEqual(repr(0), '0')
@@ -1200,6 +1304,9 @@ class BuiltinTest(unittest.TestCase):
         self.assertRaises(TypeError, sum, 42)
         self.assertRaises(TypeError, sum, ['a', 'b', 'c'])
         self.assertRaises(TypeError, sum, ['a', 'b', 'c'], '')
+        self.assertRaises(TypeError, sum, [b'a', b'c'], b'')
+        values = [bytearray(b'a'), bytearray(b'b')]
+        self.assertRaises(TypeError, sum, values, bytearray(b''))
         self.assertRaises(TypeError, sum, [[1], [2], [3]])
         self.assertRaises(TypeError, sum, [{2:3}])
         self.assertRaises(TypeError, sum, [{2:3}]*2, {2:3})
@@ -1288,6 +1395,13 @@ class BuiltinTest(unittest.TestCase):
                     return i
         self.assertRaises(ValueError, list, zip(BadSeq(), BadSeq()))
 
+    def test_zip_pickle(self):
+        a = (1, 2, 3)
+        b = (4, 5, 6)
+        t = [(1, 4), (2, 5), (3, 6)]
+        z1 = zip(a, b)
+        self.check_iter_pickle(z1, t)
+
     def test_format(self):
         # Test the basic machinery of the format() builtin.  Don't test
         #  the specifics of the various formatters
@@ -1361,14 +1475,14 @@ class BuiltinTest(unittest.TestCase):
 
         # --------------------------------------------------------------------
         # Issue #7994: object.__format__ with a non-empty format string is
-        #  pending deprecated
+        #  deprecated
         def test_deprecated_format_string(obj, fmt_str, should_raise_warning):
             with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter("always", PendingDeprecationWarning)
+                warnings.simplefilter("always", DeprecationWarning)
                 format(obj, fmt_str)
             if should_raise_warning:
                 self.assertEqual(len(w), 1)
-                self.assertIsInstance(w[0].message, PendingDeprecationWarning)
+                self.assertIsInstance(w[0].message, DeprecationWarning)
                 self.assertIn('object.__format__ with a non-empty format '
                               'string', str(w[0].message))
             else:
@@ -1412,6 +1526,13 @@ class BuiltinTest(unittest.TestCase):
         self.assertRaises(ValueError, x.translate, b"1", 1)
         self.assertRaises(TypeError, x.translate, b"1"*256, 1)
 
+    def test_construct_singletons(self):
+        for const in None, Ellipsis, NotImplemented:
+            tp = type(const)
+            self.assertIs(tp(), const)
+            self.assertRaises(TypeError, tp, 1, 2)
+            self.assertRaises(TypeError, tp, a=1, b=2)
+
 class TestSorted(unittest.TestCase):
 
     def test_basic(self):
diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index ba40915f3e..3520e837a1 100644
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -38,7 +38,7 @@ class Indexable:
         return self.value
 
 
-class BaseBytesTest(unittest.TestCase):
+class BaseBytesTest:
 
     def test_basics(self):
         b = self.type2test()
@@ -188,24 +188,26 @@ class BaseBytesTest(unittest.TestCase):
 
     def test_encoding(self):
         sample = "Hello world\n\u1234\u5678\u9abc"
-        for enc in ("utf8", "utf16"):
+        for enc in ("utf-8", "utf-16"):
             b = self.type2test(sample, enc)
             self.assertEqual(b, self.type2test(sample.encode(enc)))
-        self.assertRaises(UnicodeEncodeError, self.type2test, sample, "latin1")
-        b = self.type2test(sample, "latin1", "ignore")
+        self.assertRaises(UnicodeEncodeError, self.type2test, sample, "latin-1")
+        b = self.type2test(sample, "latin-1", "ignore")
         self.assertEqual(b, self.type2test(sample[:-3], "utf-8"))
 
     def test_decode(self):
         sample = "Hello world\n\u1234\u5678\u9abc\def0\def0"
-        for enc in ("utf8", "utf16"):
+        for enc in ("utf-8", "utf-16"):
             b = self.type2test(sample, enc)
             self.assertEqual(b.decode(enc), sample)
         sample = "Hello world\n\x80\x81\xfe\xff"
-        b = self.type2test(sample, "latin1")
-        self.assertRaises(UnicodeDecodeError, b.decode, "utf8")
-        self.assertEqual(b.decode("utf8", "ignore"), "Hello world\n")
-        self.assertEqual(b.decode(errors="ignore", encoding="utf8"),
+        b = self.type2test(sample, "latin-1")
+        self.assertRaises(UnicodeDecodeError, b.decode, "utf-8")
+        self.assertEqual(b.decode("utf-8", "ignore"), "Hello world\n")
+        self.assertEqual(b.decode(errors="ignore", encoding="utf-8"),
                          "Hello world\n")
+        # Default encoding is utf-8
+        self.assertEqual(self.type2test(b'\xe2\x98\x83').decode(), '\u2603')
 
     def test_from_int(self):
         b = self.type2test(0)
@@ -291,10 +293,27 @@ class BaseBytesTest(unittest.TestCase):
 
     def test_count(self):
         b = self.type2test(b'mississippi')
+        i = 105
+        p = 112
+        w = 119
+
         self.assertEqual(b.count(b'i'), 4)
         self.assertEqual(b.count(b'ss'), 2)
         self.assertEqual(b.count(b'w'), 0)
 
+        self.assertEqual(b.count(i), 4)
+        self.assertEqual(b.count(w), 0)
+
+        self.assertEqual(b.count(b'i', 6), 2)
+        self.assertEqual(b.count(b'p', 6), 2)
+        self.assertEqual(b.count(b'i', 1, 3), 1)
+        self.assertEqual(b.count(b'p', 7, 9), 1)
+
+        self.assertEqual(b.count(i, 6), 2)
+        self.assertEqual(b.count(p, 6), 2)
+        self.assertEqual(b.count(i, 1, 3), 1)
+        self.assertEqual(b.count(p, 7, 9), 1)
+
     def test_startswith(self):
         b = self.type2test(b'hello')
         self.assertFalse(self.type2test().startswith(b"anything"))
@@ -325,35 +344,86 @@ class BaseBytesTest(unittest.TestCase):
 
     def test_find(self):
         b = self.type2test(b'mississippi')
+        i = 105
+        w = 119
+
         self.assertEqual(b.find(b'ss'), 2)
+        self.assertEqual(b.find(b'w'), -1)
+        self.assertEqual(b.find(b'mississippian'), -1)
+
+        self.assertEqual(b.find(i), 1)
+        self.assertEqual(b.find(w), -1)
+
         self.assertEqual(b.find(b'ss', 3), 5)
         self.assertEqual(b.find(b'ss', 1, 7), 2)
         self.assertEqual(b.find(b'ss', 1, 3), -1)
-        self.assertEqual(b.find(b'w'), -1)
-        self.assertEqual(b.find(b'mississippian'), -1)
+
+        self.assertEqual(b.find(i, 6), 7)
+        self.assertEqual(b.find(i, 1, 3), 1)
+        self.assertEqual(b.find(w, 1, 3), -1)
+
+        for index in (-1, 256, sys.maxsize + 1):
+            self.assertRaisesRegex(
+                ValueError, r'byte must be in range\(0, 256\)',
+                b.find, index)
 
     def test_rfind(self):
         b = self.type2test(b'mississippi')
+        i = 105
+        w = 119
+
         self.assertEqual(b.rfind(b'ss'), 5)
-        self.assertEqual(b.rfind(b'ss', 3), 5)
-        self.assertEqual(b.rfind(b'ss', 0, 6), 2)
         self.assertEqual(b.rfind(b'w'), -1)
         self.assertEqual(b.rfind(b'mississippian'), -1)
 
+        self.assertEqual(b.rfind(i), 10)
+        self.assertEqual(b.rfind(w), -1)
+
+        self.assertEqual(b.rfind(b'ss', 3), 5)
+        self.assertEqual(b.rfind(b'ss', 0, 6), 2)
+
+        self.assertEqual(b.rfind(i, 1, 3), 1)
+        self.assertEqual(b.rfind(i, 3, 9), 7)
+        self.assertEqual(b.rfind(w, 1, 3), -1)
+
     def test_index(self):
-        b = self.type2test(b'world')
-        self.assertEqual(b.index(b'w'), 0)
-        self.assertEqual(b.index(b'orl'), 1)
-        self.assertRaises(ValueError, b.index, b'worm')
-        self.assertRaises(ValueError, b.index, b'ldo')
+        b = self.type2test(b'mississippi')
+        i = 105
+        w = 119
+
+        self.assertEqual(b.index(b'ss'), 2)
+        self.assertRaises(ValueError, b.index, b'w')
+        self.assertRaises(ValueError, b.index, b'mississippian')
+
+        self.assertEqual(b.index(i), 1)
+        self.assertRaises(ValueError, b.index, w)
+
+        self.assertEqual(b.index(b'ss', 3), 5)
+        self.assertEqual(b.index(b'ss', 1, 7), 2)
+        self.assertRaises(ValueError, b.index, b'ss', 1, 3)
+
+        self.assertEqual(b.index(i, 6), 7)
+        self.assertEqual(b.index(i, 1, 3), 1)
+        self.assertRaises(ValueError, b.index, w, 1, 3)
 
     def test_rindex(self):
-        # XXX could be more rigorous
-        b = self.type2test(b'world')
-        self.assertEqual(b.rindex(b'w'), 0)
-        self.assertEqual(b.rindex(b'orl'), 1)
-        self.assertRaises(ValueError, b.rindex, b'worm')
-        self.assertRaises(ValueError, b.rindex, b'ldo')
+        b = self.type2test(b'mississippi')
+        i = 105
+        w = 119
+
+        self.assertEqual(b.rindex(b'ss'), 5)
+        self.assertRaises(ValueError, b.rindex, b'w')
+        self.assertRaises(ValueError, b.rindex, b'mississippian')
+
+        self.assertEqual(b.rindex(i), 10)
+        self.assertRaises(ValueError, b.rindex, w)
+
+        self.assertEqual(b.rindex(b'ss', 3), 5)
+        self.assertEqual(b.rindex(b'ss', 0, 6), 2)
+
+        self.assertEqual(b.rindex(i, 1, 3), 1)
+        self.assertEqual(b.rindex(i, 3, 9), 7)
+        self.assertRaises(ValueError, b.rindex, w, 1, 3)
 
     def test_replace(self):
         b = self.type2test(b'mississippi')
@@ -365,6 +435,14 @@ class BaseBytesTest(unittest.TestCase):
         self.assertEqual(b.split(b'i'), [b'm', b'ss', b'ss', b'pp', b''])
         self.assertEqual(b.split(b'ss'), [b'mi', b'i', b'ippi'])
         self.assertEqual(b.split(b'w'), [b])
+        # with keyword args
+        b = self.type2test(b'a|b|c|d')
+        self.assertEqual(b.split(sep=b'|'), [b'a', b'b', b'c', b'd'])
+        self.assertEqual(b.split(b'|', maxsplit=1), [b'a', b'b|c|d'])
+        self.assertEqual(b.split(sep=b'|', maxsplit=1), [b'a', b'b|c|d'])
+        self.assertEqual(b.split(maxsplit=1, sep=b'|'), [b'a', b'b|c|d'])
+        b = self.type2test(b'a b c d')
+        self.assertEqual(b.split(maxsplit=1), [b'a', b'b c d'])
 
     def test_split_whitespace(self):
         for b in (b'  arf  barf  ', b'arf\tbarf', b'arf\nbarf', b'arf\rbarf',
@@ -393,6 +471,14 @@ class BaseBytesTest(unittest.TestCase):
         self.assertEqual(b.rsplit(b'i'), [b'm', b'ss', b'ss', b'pp', b''])
         self.assertEqual(b.rsplit(b'ss'), [b'mi', b'i', b'ippi'])
         self.assertEqual(b.rsplit(b'w'), [b])
+        # with keyword args
+        b = self.type2test(b'a|b|c|d')
+        self.assertEqual(b.rsplit(sep=b'|'), [b'a', b'b', b'c', b'd'])
+        self.assertEqual(b.rsplit(b'|', maxsplit=1), [b'a|b|c', b'd'])
+        self.assertEqual(b.rsplit(sep=b'|', maxsplit=1), [b'a|b|c', b'd'])
+        self.assertEqual(b.rsplit(maxsplit=1, sep=b'|'), [b'a|b|c', b'd'])
+        b = self.type2test(b'a b c d')
+        self.assertEqual(b.rsplit(maxsplit=1), [b'a b c', b'd'])
 
     def test_rsplit_whitespace(self):
         for b in (b'  arf  barf  ', b'arf\tbarf', b'arf\nbarf', b'arf\rbarf',
@@ -432,6 +518,24 @@ class BaseBytesTest(unittest.TestCase):
                 q = pickle.loads(ps)
                 self.assertEqual(b, q)
 
+    def test_iterator_pickling(self):
+        for b in b"", b"a", b"abc", b"\xffab\x80", b"\0\0\377\0\0":
+            it = itorg = iter(self.type2test(b))
+            data = list(self.type2test(b))
+            d = pickle.dumps(it)
+            it = pickle.loads(d)
+            self.assertEqual(type(itorg), type(it))
+            self.assertEqual(list(it), data)
+
+            it = pickle.loads(d)
+            try:
+                next(it)
+            except StopIteration:
+                continue
+            d = pickle.dumps(it)
+            it = pickle.loads(d)
+            self.assertEqual(list(it), data[1:])
+
     def test_strip(self):
         b = self.type2test(b'mississippi')
         self.assertEqual(b.strip(b'i'), b'mississipp')
@@ -473,6 +577,27 @@ class BaseBytesTest(unittest.TestCase):
         self.assertRaises(TypeError, self.type2test(b'abc').lstrip, 'b')
         self.assertRaises(TypeError, self.type2test(b'abc').rstrip, 'b')
 
+    def test_center(self):
+        # Fill character can be either bytes or bytearray (issue 12380)
+        b = self.type2test(b'abc')
+        for fill_type in (bytes, bytearray):
+            self.assertEqual(b.center(7, fill_type(b'-')),
+                             self.type2test(b'--abc--'))
+
+    def test_ljust(self):
+        # Fill character can be either bytes or bytearray (issue 12380)
+        b = self.type2test(b'abc')
+        for fill_type in (bytes, bytearray):
+            self.assertEqual(b.ljust(7, fill_type(b'-')),
+                             self.type2test(b'abc----'))
+
+    def test_rjust(self):
+        # Fill character can be either bytes or bytearray (issue 12380)
+        b = self.type2test(b'abc')
+        for fill_type in (bytes, bytearray):
+            self.assertEqual(b.rjust(7, fill_type(b'-')),
+                             self.type2test(b'----abc'))
+
     def test_ord(self):
         b = self.type2test(b'\0A\x7f\x80\xff')
         self.assertEqual([ord(b[i:i+1]) for i in range(len(b))],
@@ -529,6 +654,14 @@ class BaseBytesTest(unittest.TestCase):
         self.assertEqual(True, b.startswith(h, None, -2))
         self.assertEqual(False, b.startswith(x, None, None))
 
+    def test_integer_arguments_out_of_byte_range(self):
+        b = self.type2test(b'hello')
+
+        for method in (b.count, b.find, b.index, b.rfind, b.rindex):
+            self.assertRaises(ValueError, method, -1)
+            self.assertRaises(ValueError, method, 256)
+            self.assertRaises(ValueError, method, 9999)
+
     def test_find_etc_raise_correct_error_messages(self):
         # issue 11828
         b = self.type2test(b'hello')
@@ -549,7 +682,7 @@ class BaseBytesTest(unittest.TestCase):
                                 x, None, None, None)
 
 
-class BytesTest(BaseBytesTest):
+class BytesTest(BaseBytesTest, unittest.TestCase):
     type2test = bytes
 
     def test_buffer_is_readonly(self):
@@ -568,6 +701,12 @@ class BytesTest(BaseBytesTest):
             def __bytes__(self):
                 return None
         self.assertRaises(TypeError, bytes, A())
+        class A:
+            def __bytes__(self):
+                return b'a'
+            def __index__(self):
+                return 42
+        self.assertEqual(bytes(A()), b'a')
 
     # Test PyBytes_FromFormat()
     def test_from_format(self):
@@ -591,7 +730,7 @@ class BytesTest(BaseBytesTest):
                          b's:cstr')
 
 
-class ByteArrayTest(BaseBytesTest):
+class ByteArrayTest(BaseBytesTest, unittest.TestCase):
     type2test = bytearray
 
     def test_nohash(self):
@@ -634,6 +773,39 @@ class ByteArrayTest(BaseBytesTest):
         b.reverse()
         self.assertFalse(b)
 
+    def test_clear(self):
+        b = bytearray(b'python')
+        b.clear()
+        self.assertEqual(b, b'')
+
+        b = bytearray(b'')
+        b.clear()
+        self.assertEqual(b, b'')
+
+        b = bytearray(b'')
+        b.append(ord('r'))
+        b.clear()
+        b.append(ord('p'))
+        self.assertEqual(b, b'p')
+
+    def test_copy(self):
+        b = bytearray(b'abc')
+        bb = b.copy()
+        self.assertEqual(bb, b'abc')
+
+        b = bytearray(b'')
+        bb = b.copy()
+        self.assertEqual(bb, b'')
+
+        # test that it's indeed a copy and not a reference
+        b = bytearray(b'abc')
+        bb = b.copy()
+        self.assertEqual(b, bb)
+        self.assertIsNot(b, bb)
+        bb.append(ord('d'))
+        self.assertEqual(bb, b'abcd')
+        self.assertEqual(b, b'abc')
+
     def test_regexps(self):
         def by(s):
             return bytearray(map(ord, s))
@@ -1122,14 +1294,16 @@ class FixedStringTest(test.string_tests.BaseTest):
     def test_lower(self):
         pass
 
-class ByteArrayAsStringTest(FixedStringTest):
+class ByteArrayAsStringTest(FixedStringTest, unittest.TestCase):
     type2test = bytearray
+    contains_bytes = True
 
-class BytesAsStringTest(FixedStringTest):
+class BytesAsStringTest(FixedStringTest, unittest.TestCase):
     type2test = bytes
+    contains_bytes = True
 
 
-class SubclassTest(unittest.TestCase):
+class SubclassTest:
 
     def test_basic(self):
         self.assertTrue(issubclass(self.subclass2test, self.type2test))
@@ -1201,7 +1375,7 @@ class ByteArraySubclass(bytearray):
 class BytesSubclass(bytes):
     pass
 
-class ByteArraySubclassTest(SubclassTest):
+class ByteArraySubclassTest(SubclassTest, unittest.TestCase):
     type2test = bytearray
     subclass2test = ByteArraySubclass
 
@@ -1216,16 +1390,10 @@ class ByteArraySubclassTest(SubclassTest):
         self.assertEqual(x, b"abcd")
 
 
-class BytesSubclassTest(SubclassTest):
+class BytesSubclassTest(SubclassTest, unittest.TestCase):
     type2test = bytes
     subclass2test = BytesSubclass
 
 
-def test_main():
-    test.support.run_unittest(
-        BytesTest, AssortedBytesTest, BytesAsStringTest,
-        ByteArrayTest, ByteArrayAsStringTest, BytesSubclassTest,
-        ByteArraySubclassTest, BytearrayPEP3137Test)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py
index fb104d71ba..df7e18cbaf 100644
--- a/Lib/test/test_bz2.py
+++ b/Lib/test/test_bz2.py
@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 from test import support
-from test.support import TESTFN, _4G, bigmemtest, findfile
+from test.support import TESTFN, bigmemtest, _4G
 
 import unittest
 from io import BytesIO
 import os
+import random
 import subprocess
 import sys
 
@@ -21,13 +22,39 @@ has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx")
 
 class BaseTest(unittest.TestCase):
     "Base for other testcases."
-    TEXT = b'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n'
+    TEXT_LINES = [
+        b'root:x:0:0:root:/root:/bin/bash\n',
+        b'bin:x:1:1:bin:/bin:\n',
+        b'daemon:x:2:2:daemon:/sbin:\n',
+        b'adm:x:3:4:adm:/var/adm:\n',
+        b'lp:x:4:7:lp:/var/spool/lpd:\n',
+        b'sync:x:5:0:sync:/sbin:/bin/sync\n',
+        b'shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\n',
+        b'halt:x:7:0:halt:/sbin:/sbin/halt\n',
+        b'mail:x:8:12:mail:/var/spool/mail:\n',
+        b'news:x:9:13:news:/var/spool/news:\n',
+        b'uucp:x:10:14:uucp:/var/spool/uucp:\n',
+        b'operator:x:11:0:operator:/root:\n',
+        b'games:x:12:100:games:/usr/games:\n',
+        b'gopher:x:13:30:gopher:/usr/lib/gopher-data:\n',
+        b'ftp:x:14:50:FTP User:/var/ftp:/bin/bash\n',
+        b'nobody:x:65534:65534:Nobody:/home:\n',
+        b'postfix:x:100:101:postfix:/var/spool/postfix:\n',
+        b'niemeyer:x:500:500::/home/niemeyer:/bin/bash\n',
+        b'postgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\n',
+        b'mysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\n',
+        b'www:x:103:104::/var/www:/bin/false\n',
+        ]
+    TEXT = b''.join(TEXT_LINES)
     DATA = b'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2<Q\xb5\x0fH\xd3\xd4\xdd\xd5\x87\xbb\xf8\x94\r\x8f\xafI\x12\xe1\xc9\xf8/E\x00pu\x89\x12]\xc9\xbbDL\nQ\x0e\t1\x12\xdf\xa0\xc0\x97\xac2O9\x89\x13\x94\x0e\x1c7\x0ed\x95I\x0c\xaaJ\xa4\x18L\x10\x05#\x9c\xaf\xba\xbc/\x97\x8a#C\xc8\xe1\x8cW\xf9\xe2\xd0\xd6M\xa7\x8bXa<e\x84t\xcbL\xb3\xa7\xd9\xcd\xd1\xcb\x84.\xaf\xb3\xab\xab\xad`n}\xa0lh\tE,\x8eZ\x15\x17VH>\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`'
-    DATA_CRLF = b'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1<l\xba\xcb_\xc00xY\x17r\x17\x88\x08\x08@\xa0\ry@\x10\x04$)`\xf2\xce\x89z\xb0s\xec\x9b.iW\x9d\x81\xb5-+t\x9f\x1a\'\x97dB\xf5x\xb5\xbe.[.\xd7\x0e\x81\xe7\x08\x1cN`\x88\x10\xca\x87\xc3!"\x80\x92R\xa1/\xd1\xc0\xe6mf\xac\xbd\x99\xcca\xb3\x8780>\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80'
     EMPTY_DATA = b'BZh9\x17rE8P\x90\x00\x00\x00\x00'
 
-    with open(findfile("testbz2_bigmem.bz2"), "rb") as f:
-        DATA_BIGMEM = f.read()
+    def setUp(self):
+        self.filename = TESTFN
+
+    def tearDown(self):
+        if os.path.isfile(self.filename):
+            os.unlink(self.filename)
 
     if has_cmdline_bunzip2:
         def decompress(self, data):
@@ -48,94 +75,152 @@ class BaseTest(unittest.TestCase):
         def decompress(self, data):
             return bz2.decompress(data)
 
-
 class BZ2FileTest(BaseTest):
     "Test BZ2File type miscellaneous methods."
 
-    def setUp(self):
-        self.filename = TESTFN
-
-    def tearDown(self):
-        if os.path.isfile(self.filename):
-            os.unlink(self.filename)
-
-    def createTempFile(self, crlf=0):
+    def createTempFile(self, streams=1):
         with open(self.filename, "wb") as f:
-            if crlf:
-                data = self.DATA_CRLF
-            else:
-                data = self.DATA
-            f.write(data)
+            f.write(self.DATA * streams)
+
+    def testBadArgs(self):
+        with self.assertRaises(TypeError):
+            BZ2File(123.456)
+        with self.assertRaises(ValueError):
+            BZ2File("/dev/null", "z")
+        with self.assertRaises(ValueError):
+            BZ2File("/dev/null", "rx")
+        with self.assertRaises(ValueError):
+            BZ2File("/dev/null", "rbt")
+        with self.assertRaises(ValueError):
+            BZ2File("/dev/null", compresslevel=0)
+        with self.assertRaises(ValueError):
+            BZ2File("/dev/null", compresslevel=10)
 
     def testRead(self):
-        # "Test BZ2File.read()"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             self.assertRaises(TypeError, bz2f.read, None)
             self.assertEqual(bz2f.read(), self.TEXT)
 
+    def testReadMultiStream(self):
+        self.createTempFile(streams=5)
+        with BZ2File(self.filename) as bz2f:
+            self.assertRaises(TypeError, bz2f.read, None)
+            self.assertEqual(bz2f.read(), self.TEXT * 5)
+
+    def testReadMonkeyMultiStream(self):
+        # Test BZ2File.read() on a multi-stream archive where a stream
+        # boundary coincides with the end of the raw read buffer.
+        buffer_size = bz2._BUFFER_SIZE
+        bz2._BUFFER_SIZE = len(self.DATA)
+        try:
+            self.createTempFile(streams=5)
+            with BZ2File(self.filename) as bz2f:
+                self.assertRaises(TypeError, bz2f.read, None)
+                self.assertEqual(bz2f.read(), self.TEXT * 5)
+        finally:
+            bz2._BUFFER_SIZE = buffer_size
+
     def testRead0(self):
-        # Test BBZ2File.read(0)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             self.assertRaises(TypeError, bz2f.read, None)
             self.assertEqual(bz2f.read(0), b"")
 
     def testReadChunk10(self):
-        # "Test BZ2File.read() in chunks of 10 bytes"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             text = b''
-            while 1:
+            while True:
                 str = bz2f.read(10)
                 if not str:
                     break
                 text += str
             self.assertEqual(text, self.TEXT)
 
+    def testReadChunk10MultiStream(self):
+        self.createTempFile(streams=5)
+        with BZ2File(self.filename) as bz2f:
+            text = b''
+            while True:
+                str = bz2f.read(10)
+                if not str:
+                    break
+                text += str
+            self.assertEqual(text, self.TEXT * 5)
+
     def testRead100(self):
-        # "Test BZ2File.read(100)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             self.assertEqual(bz2f.read(100), self.TEXT[:100])
 
+    def testPeek(self):
+        self.createTempFile()
+        with BZ2File(self.filename) as bz2f:
+            pdata = bz2f.peek()
+            self.assertNotEqual(len(pdata), 0)
+            self.assertTrue(self.TEXT.startswith(pdata))
+            self.assertEqual(bz2f.read(), self.TEXT)
+
+    def testReadInto(self):
+        self.createTempFile()
+        with BZ2File(self.filename) as bz2f:
+            n = 128
+            b = bytearray(n)
+            self.assertEqual(bz2f.readinto(b), n)
+            self.assertEqual(b, self.TEXT[:n])
+            n = len(self.TEXT) - n
+            b = bytearray(len(self.TEXT))
+            self.assertEqual(bz2f.readinto(b), n)
+            self.assertEqual(b[:n], self.TEXT[-n:])
+
     def testReadLine(self):
-        # "Test BZ2File.readline()"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             self.assertRaises(TypeError, bz2f.readline, None)
-            sio = BytesIO(self.TEXT)
-            for line in sio.readlines():
+            for line in self.TEXT_LINES:
+                self.assertEqual(bz2f.readline(), line)
+
+    def testReadLineMultiStream(self):
+        self.createTempFile(streams=5)
+        with BZ2File(self.filename) as bz2f:
+            self.assertRaises(TypeError, bz2f.readline, None)
+            for line in self.TEXT_LINES * 5:
                 self.assertEqual(bz2f.readline(), line)
 
     def testReadLines(self):
-        # "Test BZ2File.readlines()"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             self.assertRaises(TypeError, bz2f.readlines, None)
-            sio = BytesIO(self.TEXT)
-            self.assertEqual(bz2f.readlines(), sio.readlines())
+            self.assertEqual(bz2f.readlines(), self.TEXT_LINES)
+
+    def testReadLinesMultiStream(self):
+        self.createTempFile(streams=5)
+        with BZ2File(self.filename) as bz2f:
+            self.assertRaises(TypeError, bz2f.readlines, None)
+            self.assertEqual(bz2f.readlines(), self.TEXT_LINES * 5)
 
     def testIterator(self):
-        # "Test iter(BZ2File)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
-            sio = BytesIO(self.TEXT)
-            self.assertEqual(list(iter(bz2f)), sio.readlines())
+            self.assertEqual(list(iter(bz2f)), self.TEXT_LINES)
+
+    def testIteratorMultiStream(self):
+        self.createTempFile(streams=5)
+        with BZ2File(self.filename) as bz2f:
+            self.assertEqual(list(iter(bz2f)), self.TEXT_LINES * 5)
 
     def testClosedIteratorDeadlock(self):
-        # "Test that iteration on a closed bz2file releases the lock."
-        # http://bugs.python.org/issue3309
+        # Issue #3309: Iteration on a closed BZ2File should release the lock.
         self.createTempFile()
         bz2f = BZ2File(self.filename)
         bz2f.close()
         self.assertRaises(ValueError, bz2f.__next__)
-        # This call will deadlock of the above .__next__ call failed to
+        # This call will deadlock if the above .__next__ call failed to
         # release the lock.
         self.assertRaises(ValueError, bz2f.readlines)
 
     def testWrite(self):
-        # "Test BZ2File.write()"
         with BZ2File(self.filename, "w") as bz2f:
             self.assertRaises(TypeError, bz2f.write)
             bz2f.write(self.TEXT)
@@ -143,10 +228,9 @@ class BZ2FileTest(BaseTest):
             self.assertEqual(self.decompress(f.read()), self.TEXT)
 
     def testWriteChunks10(self):
-        # "Test BZ2File.write() with chunks of 10 bytes"
         with BZ2File(self.filename, "w") as bz2f:
             n = 0
-            while 1:
+            while True:
                 str = self.TEXT[n*10:(n+1)*10]
                 if not str:
                     break
@@ -155,13 +239,19 @@ class BZ2FileTest(BaseTest):
         with open(self.filename, 'rb') as f:
             self.assertEqual(self.decompress(f.read()), self.TEXT)
 
+    def testWriteNonDefaultCompressLevel(self):
+        expected = bz2.compress(self.TEXT, compresslevel=5)
+        with BZ2File(self.filename, "w", compresslevel=5) as bz2f:
+            bz2f.write(self.TEXT)
+        with open(self.filename, "rb") as f:
+            self.assertEqual(f.read(), expected)
+
     def testWriteLines(self):
-        # "Test BZ2File.writelines()"
         with BZ2File(self.filename, "w") as bz2f:
             self.assertRaises(TypeError, bz2f.writelines)
-            sio = BytesIO(self.TEXT)
-            bz2f.writelines(sio.readlines())
-        # patch #1535500
+            bz2f.writelines(self.TEXT_LINES)
+        # Issue #1535500: Calling writelines() on a closed BZ2File
+        # should raise an exception.
         self.assertRaises(ValueError, bz2f.writelines, ["a"])
         with open(self.filename, 'rb') as f:
             self.assertEqual(self.decompress(f.read()), self.TEXT)
@@ -174,39 +264,73 @@ class BZ2FileTest(BaseTest):
             self.assertRaises(IOError, bz2f.write, b"a")
             self.assertRaises(IOError, bz2f.writelines, [b"a"])
 
+    def testAppend(self):
+        with BZ2File(self.filename, "w") as bz2f:
+            self.assertRaises(TypeError, bz2f.write)
+            bz2f.write(self.TEXT)
+        with BZ2File(self.filename, "a") as bz2f:
+            self.assertRaises(TypeError, bz2f.write)
+            bz2f.write(self.TEXT)
+        with open(self.filename, 'rb') as f:
+            self.assertEqual(self.decompress(f.read()), self.TEXT * 2)
+
     def testSeekForward(self):
-        # "Test BZ2File.seek(150, 0)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             self.assertRaises(TypeError, bz2f.seek)
             bz2f.seek(150)
             self.assertEqual(bz2f.read(), self.TEXT[150:])
 
+    def testSeekForwardAcrossStreams(self):
+        self.createTempFile(streams=2)
+        with BZ2File(self.filename) as bz2f:
+            self.assertRaises(TypeError, bz2f.seek)
+            bz2f.seek(len(self.TEXT) + 150)
+            self.assertEqual(bz2f.read(), self.TEXT[150:])
+
     def testSeekBackwards(self):
-        # "Test BZ2File.seek(-150, 1)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             bz2f.read(500)
             bz2f.seek(-150, 1)
             self.assertEqual(bz2f.read(), self.TEXT[500-150:])
 
+    def testSeekBackwardsAcrossStreams(self):
+        self.createTempFile(streams=2)
+        with BZ2File(self.filename) as bz2f:
+            readto = len(self.TEXT) + 100
+            while readto > 0:
+                readto -= len(bz2f.read(readto))
+            bz2f.seek(-150, 1)
+            self.assertEqual(bz2f.read(), self.TEXT[100-150:] + self.TEXT)
+
     def testSeekBackwardsFromEnd(self):
-        # "Test BZ2File.seek(-150, 2)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             bz2f.seek(-150, 2)
             self.assertEqual(bz2f.read(), self.TEXT[len(self.TEXT)-150:])
 
+    def testSeekBackwardsFromEndAcrossStreams(self):
+        self.createTempFile(streams=2)
+        with BZ2File(self.filename) as bz2f:
+            bz2f.seek(-1000, 2)
+            self.assertEqual(bz2f.read(), (self.TEXT * 2)[-1000:])
+
     def testSeekPostEnd(self):
-        # "Test BZ2File.seek(150000)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             bz2f.seek(150000)
             self.assertEqual(bz2f.tell(), len(self.TEXT))
             self.assertEqual(bz2f.read(), b"")
 
+    def testSeekPostEndMultiStream(self):
+        self.createTempFile(streams=5)
+        with BZ2File(self.filename) as bz2f:
+            bz2f.seek(150000)
+            self.assertEqual(bz2f.tell(), len(self.TEXT) * 5)
+            self.assertEqual(bz2f.read(), b"")
+
     def testSeekPostEndTwice(self):
-        # "Test BZ2File.seek(150000) twice"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             bz2f.seek(150000)
@@ -214,27 +338,109 @@ class BZ2FileTest(BaseTest):
             self.assertEqual(bz2f.tell(), len(self.TEXT))
             self.assertEqual(bz2f.read(), b"")
 
+    def testSeekPostEndTwiceMultiStream(self):
+        self.createTempFile(streams=5)
+        with BZ2File(self.filename) as bz2f:
+            bz2f.seek(150000)
+            bz2f.seek(150000)
+            self.assertEqual(bz2f.tell(), len(self.TEXT) * 5)
+            self.assertEqual(bz2f.read(), b"")
+
     def testSeekPreStart(self):
-        # "Test BZ2File.seek(-150, 0)"
         self.createTempFile()
         with BZ2File(self.filename) as bz2f:
             bz2f.seek(-150)
             self.assertEqual(bz2f.tell(), 0)
             self.assertEqual(bz2f.read(), self.TEXT)
 
+    def testSeekPreStartMultiStream(self):
+        self.createTempFile(streams=2)
+        with BZ2File(self.filename) as bz2f:
+            bz2f.seek(-150)
+            self.assertEqual(bz2f.tell(), 0)
+            self.assertEqual(bz2f.read(), self.TEXT * 2)
+
+    def testFileno(self):
+        self.createTempFile()
+        with open(self.filename, 'rb') as rawf:
+            bz2f = BZ2File(rawf)
+            try:
+                self.assertEqual(bz2f.fileno(), rawf.fileno())
+            finally:
+                bz2f.close()
+        self.assertRaises(ValueError, bz2f.fileno)
+
+    def testSeekable(self):
+        bz2f = BZ2File(BytesIO(self.DATA))
+        try:
+            self.assertTrue(bz2f.seekable())
+            bz2f.read()
+            self.assertTrue(bz2f.seekable())
+        finally:
+            bz2f.close()
+        self.assertRaises(ValueError, bz2f.seekable)
+
+        bz2f = BZ2File(BytesIO(), mode="w")
+        try:
+            self.assertFalse(bz2f.seekable())
+        finally:
+            bz2f.close()
+        self.assertRaises(ValueError, bz2f.seekable)
+
+        src = BytesIO(self.DATA)
+        src.seekable = lambda: False
+        bz2f = BZ2File(src)
+        try:
+            self.assertFalse(bz2f.seekable())
+        finally:
+            bz2f.close()
+        self.assertRaises(ValueError, bz2f.seekable)
+
+    def testReadable(self):
+        bz2f = BZ2File(BytesIO(self.DATA))
+        try:
+            self.assertTrue(bz2f.readable())
+            bz2f.read()
+            self.assertTrue(bz2f.readable())
+        finally:
+            bz2f.close()
+        self.assertRaises(ValueError, bz2f.readable)
+
+        bz2f = BZ2File(BytesIO(), mode="w")
+        try:
+            self.assertFalse(bz2f.readable())
+        finally:
+            bz2f.close()
+        self.assertRaises(ValueError, bz2f.readable)
+
+    def testWritable(self):
+        bz2f = BZ2File(BytesIO(self.DATA))
+        try:
+            self.assertFalse(bz2f.writable())
+            bz2f.read()
+            self.assertFalse(bz2f.writable())
+        finally:
+            bz2f.close()
+        self.assertRaises(ValueError, bz2f.writable)
+
+        bz2f = BZ2File(BytesIO(), mode="w")
+        try:
+            self.assertTrue(bz2f.writable())
+        finally:
+            bz2f.close()
+        self.assertRaises(ValueError, bz2f.writable)
+
     def testOpenDel(self):
-        # "Test opening and deleting a file many times"
         self.createTempFile()
         for i in range(10000):
             o = BZ2File(self.filename)
             del o
 
     def testOpenNonexistent(self):
-        # "Test opening a nonexistent file"
         self.assertRaises(IOError, BZ2File, "/non/existent")
 
-    def testBug1191043(self):
-        # readlines() for files containing no newline
+    def testReadlinesNoNewline(self):
+        # Issue #1191043: readlines() fails on a file containing no newline.
         data = b'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t'
         with open(self.filename, "wb") as f:
             f.write(data)
@@ -246,7 +452,6 @@ class BZ2FileTest(BaseTest):
         self.assertEqual(xlines, [b'Test'])
 
     def testContextProtocol(self):
-        # BZ2File supports the context management protocol
         f = None
         with BZ2File(self.filename, "wb") as f:
             f.write(b"xxx")
@@ -269,7 +474,7 @@ class BZ2FileTest(BaseTest):
 
     @unittest.skipUnless(threading, 'Threading required for this test.')
     def testThreading(self):
-        # Using a BZ2File from several threads doesn't deadlock (issue #7205).
+        # Issue #7205: Using a BZ2File from several threads shouldn't deadlock.
         data = b"1" * 2**20
         nthreads = 10
         with bz2.BZ2File(self.filename, 'wb') as f:
@@ -282,40 +487,112 @@ class BZ2FileTest(BaseTest):
             for t in threads:
                 t.join()
 
-    def testMixedIterationReads(self):
-        # Issue #8397: mixed iteration and reads should be forbidden.
-        with bz2.BZ2File(self.filename, 'wb') as f:
-            # The internal buffer size is hard-wired to 8192 bytes, we must
-            # write out more than that for the test to stop half through
-            # the buffer.
-            f.write(self.TEXT * 100)
-        with bz2.BZ2File(self.filename, 'rb') as f:
-            next(f)
-            self.assertRaises(ValueError, f.read)
-            self.assertRaises(ValueError, f.readline)
-            self.assertRaises(ValueError, f.readlines)
+    def testWithoutThreading(self):
+        bz2 = support.import_fresh_module("bz2", blocked=("threading",))
+        with bz2.BZ2File(self.filename, "wb") as f:
+            f.write(b"abc")
+        with bz2.BZ2File(self.filename, "rb") as f:
+            self.assertEqual(f.read(), b"abc")
+
+    def testMixedIterationAndReads(self):
+        self.createTempFile()
+        linelen = len(self.TEXT_LINES[0])
+        halflen = linelen // 2
+        with bz2.BZ2File(self.filename) as bz2f:
+            bz2f.read(halflen)
+            self.assertEqual(next(bz2f), self.TEXT_LINES[0][halflen:])
+            self.assertEqual(bz2f.read(), self.TEXT[linelen:])
+        with bz2.BZ2File(self.filename) as bz2f:
+            bz2f.readline()
+            self.assertEqual(next(bz2f), self.TEXT_LINES[1])
+            self.assertEqual(bz2f.readline(), self.TEXT_LINES[2])
+        with bz2.BZ2File(self.filename) as bz2f:
+            bz2f.readlines()
+            with self.assertRaises(StopIteration):
+                next(bz2f)
+            self.assertEqual(bz2f.readlines(), [])
+
+    def testMultiStreamOrdering(self):
+        # Test the ordering of streams when reading a multi-stream archive.
+        data1 = b"foo" * 1000
+        data2 = b"bar" * 1000
+        with BZ2File(self.filename, "w") as bz2f:
+            bz2f.write(data1)
+        with BZ2File(self.filename, "a") as bz2f:
+            bz2f.write(data2)
+        with BZ2File(self.filename) as bz2f:
+            self.assertEqual(bz2f.read(), data1 + data2)
+
+    def testOpenBytesFilename(self):
+        str_filename = self.filename
+        try:
+            bytes_filename = str_filename.encode("ascii")
+        except UnicodeEncodeError:
+            self.skipTest("Temporary file name needs to be ASCII")
+        with BZ2File(bytes_filename, "wb") as f:
+            f.write(self.DATA)
+        with BZ2File(bytes_filename, "rb") as f:
+            self.assertEqual(f.read(), self.DATA)
+        # Sanity check that we are actually operating on the right file.
+        with BZ2File(str_filename, "rb") as f:
+            self.assertEqual(f.read(), self.DATA)
+
+
+    # Tests for a BZ2File wrapping another file object:
+
+    def testReadBytesIO(self):
+        with BytesIO(self.DATA) as bio:
+            with BZ2File(bio) as bz2f:
+                self.assertRaises(TypeError, bz2f.read, None)
+                self.assertEqual(bz2f.read(), self.TEXT)
+            self.assertFalse(bio.closed)
+
+    def testPeekBytesIO(self):
+        with BytesIO(self.DATA) as bio:
+            with BZ2File(bio) as bz2f:
+                pdata = bz2f.peek()
+                self.assertNotEqual(len(pdata), 0)
+                self.assertTrue(self.TEXT.startswith(pdata))
+                self.assertEqual(bz2f.read(), self.TEXT)
+
+    def testWriteBytesIO(self):
+        with BytesIO() as bio:
+            with BZ2File(bio, "w") as bz2f:
+                self.assertRaises(TypeError, bz2f.write)
+                bz2f.write(self.TEXT)
+            self.assertEqual(self.decompress(bio.getvalue()), self.TEXT)
+            self.assertFalse(bio.closed)
+
+    def testSeekForwardBytesIO(self):
+        with BytesIO(self.DATA) as bio:
+            with BZ2File(bio) as bz2f:
+                self.assertRaises(TypeError, bz2f.seek)
+                bz2f.seek(150)
+                self.assertEqual(bz2f.read(), self.TEXT[150:])
+
+    def testSeekBackwardsBytesIO(self):
+        with BytesIO(self.DATA) as bio:
+            with BZ2File(bio) as bz2f:
+                bz2f.read(500)
+                bz2f.seek(-150, 1)
+                self.assertEqual(bz2f.read(), self.TEXT[500-150:])
 
     def test_read_truncated(self):
         # Drop the eos_magic field (6 bytes) and CRC (4 bytes).
         truncated = self.DATA[:-10]
-        with open(self.filename, 'wb') as f:
-            f.write(truncated)
-        with BZ2File(self.filename) as f:
+        with BZ2File(BytesIO(truncated)) as f:
             self.assertRaises(EOFError, f.read)
-        with BZ2File(self.filename) as f:
+        with BZ2File(BytesIO(truncated)) as f:
             self.assertEqual(f.read(len(self.TEXT)), self.TEXT)
             self.assertRaises(EOFError, f.read, 1)
         # Incomplete 4-byte file header, and block header of at least 146 bits.
         for i in range(22):
-            with open(self.filename, 'wb') as f:
-                f.write(truncated[:i])
-            with BZ2File(self.filename) as f:
+            with BZ2File(BytesIO(truncated[:i])) as f:
                 self.assertRaises(EOFError, f.read, 1)
 
 
 class BZ2CompressorTest(BaseTest):
     def testCompress(self):
-        # "Test BZ2Compressor.compress()/flush()"
         bz2c = BZ2Compressor()
         self.assertRaises(TypeError, bz2c.compress)
         data = bz2c.compress(self.TEXT)
@@ -329,11 +606,10 @@ class BZ2CompressorTest(BaseTest):
         self.assertEqual(data, self.EMPTY_DATA)
 
     def testCompressChunks10(self):
-        # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes"
         bz2c = BZ2Compressor()
         n = 0
         data = b''
-        while 1:
+        while True:
             str = self.TEXT[n*10:(n+1)*10]
             if not str:
                 break
@@ -342,34 +618,38 @@ class BZ2CompressorTest(BaseTest):
         data += bz2c.flush()
         self.assertEqual(self.decompress(data), self.TEXT)
 
-    @bigmemtest(size=_4G, memuse=1.25)
-    def testBigmem(self, size):
-        text = b"a" * size
-        bz2c = bz2.BZ2Compressor()
-        data = bz2c.compress(text) + bz2c.flush()
-        del text
-        text = self.decompress(data)
-        self.assertEqual(len(text), size)
-        self.assertEqual(text.strip(b"a"), b"")
-
+    @bigmemtest(size=_4G + 100, memuse=2)
+    def testCompress4G(self, size):
+        # "Test BZ2Compressor.compress()/flush() with >4GiB input"
+        bz2c = BZ2Compressor()
+        data = b"x" * size
+        try:
+            compressed = bz2c.compress(data)
+            compressed += bz2c.flush()
+        finally:
+            data = None  # Release memory
+        data = bz2.decompress(compressed)
+        try:
+            self.assertEqual(len(data), size)
+            self.assertEqual(len(data.strip(b"x")), 0)
+        finally:
+            data = None
 
 class BZ2DecompressorTest(BaseTest):
     def test_Constructor(self):
         self.assertRaises(TypeError, BZ2Decompressor, 42)
 
     def testDecompress(self):
-        # "Test BZ2Decompressor.decompress()"
         bz2d = BZ2Decompressor()
         self.assertRaises(TypeError, bz2d.decompress)
         text = bz2d.decompress(self.DATA)
         self.assertEqual(text, self.TEXT)
 
     def testDecompressChunks10(self):
-        # "Test BZ2Decompressor.decompress() with chunks of 10 bytes"
         bz2d = BZ2Decompressor()
         text = b''
         n = 0
-        while 1:
+        while True:
             str = self.DATA[n*10:(n+1)*10]
             if not str:
                 break
@@ -378,7 +658,6 @@ class BZ2DecompressorTest(BaseTest):
         self.assertEqual(text, self.TEXT)
 
     def testDecompressUnusedData(self):
-        # "Test BZ2Decompressor.decompress() with unused data"
         bz2d = BZ2Decompressor()
         unused_data = b"this is unused data"
         text = bz2d.decompress(self.DATA+unused_data)
@@ -386,25 +665,30 @@ class BZ2DecompressorTest(BaseTest):
         self.assertEqual(bz2d.unused_data, unused_data)
 
     def testEOFError(self):
-        # "Calling BZ2Decompressor.decompress() after EOS must raise EOFError"
         bz2d = BZ2Decompressor()
         text = bz2d.decompress(self.DATA)
         self.assertRaises(EOFError, bz2d.decompress, b"anything")
         self.assertRaises(EOFError, bz2d.decompress, b"")
 
-    @bigmemtest(size=_4G, memuse=1.25, dry_run=False)
-    def testBigmem(self, unused_size):
-        # Issue #14398: decompression fails when output data is >=2GB.
-        text = bz2.BZ2Decompressor().decompress(self.DATA_BIGMEM)
-        self.assertEqual(len(text), _4G)
-        self.assertEqual(text.strip(b"\0"), b"")
-
-
-class FuncTest(BaseTest):
-    "Test module functions"
-
+    @bigmemtest(size=_4G + 100, memuse=3)
+    def testDecompress4G(self, size):
+        # "Test BZ2Decompressor.decompress() with >4GiB input"
+        blocksize = 10 * 1024 * 1024
+        block = random.getrandbits(blocksize * 8).to_bytes(blocksize, 'little')
+        try:
+            data = block * (size // blocksize + 1)
+            compressed = bz2.compress(data)
+            bz2d = BZ2Decompressor()
+            decompressed = bz2d.decompress(compressed)
+            self.assertTrue(decompressed == data)
+        finally:
+            data = None
+            compressed = None
+            decompressed = None
+
+
+class CompressDecompressTest(BaseTest):
     def testCompress(self):
-        # "Test compress() function"
         data = bz2.compress(self.TEXT)
         self.assertEqual(self.decompress(data), self.TEXT)
 
@@ -413,12 +697,10 @@ class FuncTest(BaseTest):
         self.assertEqual(text, self.EMPTY_DATA)
 
     def testDecompress(self):
-        # "Test decompress() function"
         text = bz2.decompress(self.DATA)
         self.assertEqual(text, self.TEXT)
 
     def testDecompressEmpty(self):
-        # "Test decompress() function with empty string"
         text = bz2.decompress(b"")
         self.assertEqual(text, b"")
 
@@ -427,35 +709,117 @@ class FuncTest(BaseTest):
         self.assertEqual(text, b'')
 
     def testDecompressIncomplete(self):
-        # "Test decompress() function with incomplete data"
         self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10])
 
-    @bigmemtest(size=_4G, memuse=1.25)
-    def testCompressBigmem(self, size):
-        text = b"a" * size
-        data = bz2.compress(text)
-        del text
-        text = self.decompress(data)
-        self.assertEqual(len(text), size)
-        self.assertEqual(text.strip(b"a"), b"")
-
-    @bigmemtest(size=_4G, memuse=1.25, dry_run=False)
-    def testDecompressBigmem(self, unused_size):
-        # Issue #14398: decompression fails when output data is >=2GB.
-        text = bz2.decompress(self.DATA_BIGMEM)
-        self.assertEqual(len(text), _4G)
-        self.assertEqual(text.strip(b"\0"), b"")
+    def testDecompressMultiStream(self):
+        text = bz2.decompress(self.DATA * 5)
+        self.assertEqual(text, self.TEXT * 5)
+
+
+class OpenTest(BaseTest):
+    def test_binary_modes(self):
+        with bz2.open(self.filename, "wb") as f:
+            f.write(self.TEXT)
+        with open(self.filename, "rb") as f:
+            file_data = bz2.decompress(f.read())
+            self.assertEqual(file_data, self.TEXT)
+        with bz2.open(self.filename, "rb") as f:
+            self.assertEqual(f.read(), self.TEXT)
+        with bz2.open(self.filename, "ab") as f:
+            f.write(self.TEXT)
+        with open(self.filename, "rb") as f:
+            file_data = bz2.decompress(f.read())
+            self.assertEqual(file_data, self.TEXT * 2)
+
+    def test_implicit_binary_modes(self):
+        # Test implicit binary modes (no "b" or "t" in mode string).
+        with bz2.open(self.filename, "w") as f:
+            f.write(self.TEXT)
+        with open(self.filename, "rb") as f:
+            file_data = bz2.decompress(f.read())
+            self.assertEqual(file_data, self.TEXT)
+        with bz2.open(self.filename, "r") as f:
+            self.assertEqual(f.read(), self.TEXT)
+        with bz2.open(self.filename, "a") as f:
+            f.write(self.TEXT)
+        with open(self.filename, "rb") as f:
+            file_data = bz2.decompress(f.read())
+            self.assertEqual(file_data, self.TEXT * 2)
+
+    def test_text_modes(self):
+        text = self.TEXT.decode("ascii")
+        text_native_eol = text.replace("\n", os.linesep)
+        with bz2.open(self.filename, "wt") as f:
+            f.write(text)
+        with open(self.filename, "rb") as f:
+            file_data = bz2.decompress(f.read()).decode("ascii")
+            self.assertEqual(file_data, text_native_eol)
+        with bz2.open(self.filename, "rt") as f:
+            self.assertEqual(f.read(), text)
+        with bz2.open(self.filename, "at") as f:
+            f.write(text)
+        with open(self.filename, "rb") as f:
+            file_data = bz2.decompress(f.read()).decode("ascii")
+            self.assertEqual(file_data, text_native_eol * 2)
+
+    def test_fileobj(self):
+        with bz2.open(BytesIO(self.DATA), "r") as f:
+            self.assertEqual(f.read(), self.TEXT)
+        with bz2.open(BytesIO(self.DATA), "rb") as f:
+            self.assertEqual(f.read(), self.TEXT)
+        text = self.TEXT.decode("ascii")
+        with bz2.open(BytesIO(self.DATA), "rt") as f:
+            self.assertEqual(f.read(), text)
+
+    def test_bad_params(self):
+        # Test invalid parameter combinations.
+        with self.assertRaises(ValueError):
+            bz2.open(self.filename, "wbt")
+        with self.assertRaises(ValueError):
+            bz2.open(self.filename, "rb", encoding="utf-8")
+        with self.assertRaises(ValueError):
+            bz2.open(self.filename, "rb", errors="ignore")
+        with self.assertRaises(ValueError):
+            bz2.open(self.filename, "rb", newline="\n")
+
+    def test_encoding(self):
+        # Test non-default encoding.
+        text = self.TEXT.decode("ascii")
+        text_native_eol = text.replace("\n", os.linesep)
+        with bz2.open(self.filename, "wt", encoding="utf-16-le") as f:
+            f.write(text)
+        with open(self.filename, "rb") as f:
+            file_data = bz2.decompress(f.read()).decode("utf-16-le")
+            self.assertEqual(file_data, text_native_eol)
+        with bz2.open(self.filename, "rt", encoding="utf-16-le") as f:
+            self.assertEqual(f.read(), text)
+
+    def test_encoding_error_handler(self):
+        # Test with non-default encoding error handler.
+        with bz2.open(self.filename, "wb") as f:
+            f.write(b"foo\xffbar")
+        with bz2.open(self.filename, "rt", encoding="ascii", errors="ignore") \
+                as f:
+            self.assertEqual(f.read(), "foobar")
+
+    def test_newline(self):
+        # Test with explicit newline (universal newline mode disabled).
+        text = self.TEXT.decode("ascii")
+        with bz2.open(self.filename, "wt", newline="\n") as f:
+            f.write(text)
+        with bz2.open(self.filename, "rt", newline="\r") as f:
+            self.assertEqual(f.readlines(), [text])
+
 
 def test_main():
     support.run_unittest(
         BZ2FileTest,
         BZ2CompressorTest,
         BZ2DecompressorTest,
-        FuncTest
+        CompressDecompressTest,
+        OpenTest,
     )
     support.reap_children()
 
 if __name__ == '__main__':
     test_main()
-
-# vim:ts=4:sw=4
diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py
index 37b28188f8..8adf5bd14b 100644
--- a/Lib/test/test_calendar.py
+++ b/Lib/test/test_calendar.py
@@ -5,8 +5,19 @@ from test import support
 from test.script_helper import assert_python_ok
 import time
 import locale
+import sys
 import datetime
 
+result_2004_01_text = """
+    January 2004
+Mo Tu We Th Fr Sa Su
+          1  2  3  4
+ 5  6  7  8  9 10 11
+12 13 14 15 16 17 18
+19 20 21 22 23 24 25
+26 27 28 29 30 31
+"""
+
 result_2004_text = """
                                   2004
 
@@ -46,11 +57,11 @@ Mo Tu We Th Fr Sa Su      Mo Tu We Th Fr Sa Su      Mo Tu We Th Fr Sa Su
 """
 
 result_2004_html = """
-<?xml version="1.0" encoding="ascii"?>
+<?xml version="1.0" encoding="%(e)s"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
 <head>
-<meta http-equiv="Content-Type" content="text/html; charset=ascii" />
+<meta http-equiv="Content-Type" content="text/html; charset=%(e)s" />
 <link rel="stylesheet" type="text/css" href="calendar.css" />
 <title>Calendar for 2004</title>
 </head>
@@ -170,6 +181,135 @@ result_2004_html = """
 </html>
 """
 
+result_2004_days = [
+    [[[0, 0, 0, 1, 2, 3, 4],
+      [5, 6, 7, 8, 9, 10, 11],
+      [12, 13, 14, 15, 16, 17, 18],
+      [19, 20, 21, 22, 23, 24, 25],
+      [26, 27, 28, 29, 30, 31, 0]],
+     [[0, 0, 0, 0, 0, 0, 1],
+      [2, 3, 4, 5, 6, 7, 8],
+      [9, 10, 11, 12, 13, 14, 15],
+      [16, 17, 18, 19, 20, 21, 22],
+      [23, 24, 25, 26, 27, 28, 29]],
+     [[1, 2, 3, 4, 5, 6, 7],
+      [8, 9, 10, 11, 12, 13, 14],
+      [15, 16, 17, 18, 19, 20, 21],
+      [22, 23, 24, 25, 26, 27, 28],
+      [29, 30, 31, 0, 0, 0, 0]]],
+    [[[0, 0, 0, 1, 2, 3, 4],
+      [5, 6, 7, 8, 9, 10, 11],
+      [12, 13, 14, 15, 16, 17, 18],
+      [19, 20, 21, 22, 23, 24, 25],
+      [26, 27, 28, 29, 30, 0, 0]],
+     [[0, 0, 0, 0, 0, 1, 2],
+      [3, 4, 5, 6, 7, 8, 9],
+      [10, 11, 12, 13, 14, 15, 16],
+      [17, 18, 19, 20, 21, 22, 23],
+      [24, 25, 26, 27, 28, 29, 30],
+      [31, 0, 0, 0, 0, 0, 0]],
+     [[0, 1, 2, 3, 4, 5, 6],
+      [7, 8, 9, 10, 11, 12, 13],
+      [14, 15, 16, 17, 18, 19, 20],
+      [21, 22, 23, 24, 25, 26, 27],
+      [28, 29, 30, 0, 0, 0, 0]]],
+    [[[0, 0, 0, 1, 2, 3, 4],
+      [5, 6, 7, 8, 9, 10, 11],
+      [12, 13, 14, 15, 16, 17, 18],
+      [19, 20, 21, 22, 23, 24, 25],
+      [26, 27, 28, 29, 30, 31, 0]],
+     [[0, 0, 0, 0, 0, 0, 1],
+      [2, 3, 4, 5, 6, 7, 8],
+      [9, 10, 11, 12, 13, 14, 15],
+      [16, 17, 18, 19, 20, 21, 22],
+      [23, 24, 25, 26, 27, 28, 29],
+      [30, 31, 0, 0, 0, 0, 0]],
+     [[0, 0, 1, 2, 3, 4, 5],
+      [6, 7, 8, 9, 10, 11, 12],
+      [13, 14, 15, 16, 17, 18, 19],
+      [20, 21, 22, 23, 24, 25, 26],
+      [27, 28, 29, 30, 0, 0, 0]]],
+    [[[0, 0, 0, 0, 1, 2, 3],
+      [4, 5, 6, 7, 8, 9, 10],
+      [11, 12, 13, 14, 15, 16, 17],
+      [18, 19, 20, 21, 22, 23, 24],
+      [25, 26, 27, 28, 29, 30, 31]],
+     [[1, 2, 3, 4, 5, 6, 7],
+      [8, 9, 10, 11, 12, 13, 14],
+      [15, 16, 17, 18, 19, 20, 21],
+      [22, 23, 24, 25, 26, 27, 28],
+      [29, 30, 0, 0, 0, 0, 0]],
+     [[0, 0, 1, 2, 3, 4, 5],
+      [6, 7, 8, 9, 10, 11, 12],
+      [13, 14, 15, 16, 17, 18, 19],
+      [20, 21, 22, 23, 24, 25, 26],
+      [27, 28, 29, 30, 31, 0, 0]]]
+]
+
+result_2004_dates = \
+    [[['12/29/03 12/30/03 12/31/03 01/01/04 01/02/04 01/03/04 01/04/04',
+       '01/05/04 01/06/04 01/07/04 01/08/04 01/09/04 01/10/04 01/11/04',
+       '01/12/04 01/13/04 01/14/04 01/15/04 01/16/04 01/17/04 01/18/04',
+       '01/19/04 01/20/04 01/21/04 01/22/04 01/23/04 01/24/04 01/25/04',
+       '01/26/04 01/27/04 01/28/04 01/29/04 01/30/04 01/31/04 02/01/04'],
+      ['01/26/04 01/27/04 01/28/04 01/29/04 01/30/04 01/31/04 02/01/04',
+       '02/02/04 02/03/04 02/04/04 02/05/04 02/06/04 02/07/04 02/08/04',
+       '02/09/04 02/10/04 02/11/04 02/12/04 02/13/04 02/14/04 02/15/04',
+       '02/16/04 02/17/04 02/18/04 02/19/04 02/20/04 02/21/04 02/22/04',
+       '02/23/04 02/24/04 02/25/04 02/26/04 02/27/04 02/28/04 02/29/04'],
+      ['03/01/04 03/02/04 03/03/04 03/04/04 03/05/04 03/06/04 03/07/04',
+       '03/08/04 03/09/04 03/10/04 03/11/04 03/12/04 03/13/04 03/14/04',
+       '03/15/04 03/16/04 03/17/04 03/18/04 03/19/04 03/20/04 03/21/04',
+       '03/22/04 03/23/04 03/24/04 03/25/04 03/26/04 03/27/04 03/28/04',
+       '03/29/04 03/30/04 03/31/04 04/01/04 04/02/04 04/03/04 04/04/04']],
+     [['03/29/04 03/30/04 03/31/04 04/01/04 04/02/04 04/03/04 04/04/04',
+       '04/05/04 04/06/04 04/07/04 04/08/04 04/09/04 04/10/04 04/11/04',
+       '04/12/04 04/13/04 04/14/04 04/15/04 04/16/04 04/17/04 04/18/04',
+       '04/19/04 04/20/04 04/21/04 04/22/04 04/23/04 04/24/04 04/25/04',
+       '04/26/04 04/27/04 04/28/04 04/29/04 04/30/04 05/01/04 05/02/04'],
+      ['04/26/04 04/27/04 04/28/04 04/29/04 04/30/04 05/01/04 05/02/04',
+       '05/03/04 05/04/04 05/05/04 05/06/04 05/07/04 05/08/04 05/09/04',
+       '05/10/04 05/11/04 05/12/04 05/13/04 05/14/04 05/15/04 05/16/04',
+       '05/17/04 05/18/04 05/19/04 05/20/04 05/21/04 05/22/04 05/23/04',
+       '05/24/04 05/25/04 05/26/04 05/27/04 05/28/04 05/29/04 05/30/04',
+       '05/31/04 06/01/04 06/02/04 06/03/04 06/04/04 06/05/04 06/06/04'],
+      ['05/31/04 06/01/04 06/02/04 06/03/04 06/04/04 06/05/04 06/06/04',
+       '06/07/04 06/08/04 06/09/04 06/10/04 06/11/04 06/12/04 06/13/04',
+       '06/14/04 06/15/04 06/16/04 06/17/04 06/18/04 06/19/04 06/20/04',
+       '06/21/04 06/22/04 06/23/04 06/24/04 06/25/04 06/26/04 06/27/04',
+       '06/28/04 06/29/04 06/30/04 07/01/04 07/02/04 07/03/04 07/04/04']],
+     [['06/28/04 06/29/04 06/30/04 07/01/04 07/02/04 07/03/04 07/04/04',
+       '07/05/04 07/06/04 07/07/04 07/08/04 07/09/04 07/10/04 07/11/04',
+       '07/12/04 07/13/04 07/14/04 07/15/04 07/16/04 07/17/04 07/18/04',
+       '07/19/04 07/20/04 07/21/04 07/22/04 07/23/04 07/24/04 07/25/04',
+       '07/26/04 07/27/04 07/28/04 07/29/04 07/30/04 07/31/04 08/01/04'],
+      ['07/26/04 07/27/04 07/28/04 07/29/04 07/30/04 07/31/04 08/01/04',
+       '08/02/04 08/03/04 08/04/04 08/05/04 08/06/04 08/07/04 08/08/04',
+       '08/09/04 08/10/04 08/11/04 08/12/04 08/13/04 08/14/04 08/15/04',
+       '08/16/04 08/17/04 08/18/04 08/19/04 08/20/04 08/21/04 08/22/04',
+       '08/23/04 08/24/04 08/25/04 08/26/04 08/27/04 08/28/04 08/29/04',
+       '08/30/04 08/31/04 09/01/04 09/02/04 09/03/04 09/04/04 09/05/04'],
+      ['08/30/04 08/31/04 09/01/04 09/02/04 09/03/04 09/04/04 09/05/04',
+       '09/06/04 09/07/04 09/08/04 09/09/04 09/10/04 09/11/04 09/12/04',
+       '09/13/04 09/14/04 09/15/04 09/16/04 09/17/04 09/18/04 09/19/04',
+       '09/20/04 09/21/04 09/22/04 09/23/04 09/24/04 09/25/04 09/26/04',
+       '09/27/04 09/28/04 09/29/04 09/30/04 10/01/04 10/02/04 10/03/04']],
+     [['09/27/04 09/28/04 09/29/04 09/30/04 10/01/04 10/02/04 10/03/04',
+       '10/04/04 10/05/04 10/06/04 10/07/04 10/08/04 10/09/04 10/10/04',
+       '10/11/04 10/12/04 10/13/04 10/14/04 10/15/04 10/16/04 10/17/04',
+       '10/18/04 10/19/04 10/20/04 10/21/04 10/22/04 10/23/04 10/24/04',
+       '10/25/04 10/26/04 10/27/04 10/28/04 10/29/04 10/30/04 10/31/04'],
+      ['11/01/04 11/02/04 11/03/04 11/04/04 11/05/04 11/06/04 11/07/04',
+       '11/08/04 11/09/04 11/10/04 11/11/04 11/12/04 11/13/04 11/14/04',
+       '11/15/04 11/16/04 11/17/04 11/18/04 11/19/04 11/20/04 11/21/04',
+       '11/22/04 11/23/04 11/24/04 11/25/04 11/26/04 11/27/04 11/28/04',
+       '11/29/04 11/30/04 12/01/04 12/02/04 12/03/04 12/04/04 12/05/04'],
+      ['11/29/04 11/30/04 12/01/04 12/02/04 12/03/04 12/04/04 12/05/04',
+       '12/06/04 12/07/04 12/08/04 12/09/04 12/10/04 12/11/04 12/12/04',
+       '12/13/04 12/14/04 12/15/04 12/16/04 12/17/04 12/18/04 12/19/04',
+       '12/20/04 12/21/04 12/22/04 12/23/04 12/24/04 12/25/04 12/26/04',
+       '12/27/04 12/28/04 12/29/04 12/30/04 12/31/04 01/01/05 01/02/05']]]
+
 
 class OutputTestCase(unittest.TestCase):
     def normalize_calendar(self, s):
@@ -178,12 +318,19 @@ class OutputTestCase(unittest.TestCase):
             return not c.isspace() and not c.isdigit()
 
         lines = []
-        for line in s.splitlines(False):
+        for line in s.splitlines(keepends=False):
             # Drop texts, as they are locale dependent
             if line and not filter(neitherspacenordigit, line):
                 lines.append(line)
         return lines
 
+    def check_htmlcalendar_encoding(self, req, res):
+        cal = calendar.HTMLCalendar()
+        self.assertEqual(
+            cal.formatyearpage(2004, encoding=req).strip(b' \t\n'),
+            (result_2004_html % {'e': res}).strip(' \t\n').encode(res)
+        )
+
     def test_output(self):
         self.assertEqual(
             self.normalize_calendar(calendar.calendar(2004)),
@@ -196,12 +343,60 @@ class OutputTestCase(unittest.TestCase):
             result_2004_text.strip()
         )
 
-    def test_output_htmlcalendar(self):
-        encoding = 'ascii'
-        cal = calendar.HTMLCalendar()
+    def test_output_htmlcalendar_encoding_ascii(self):
+        self.check_htmlcalendar_encoding('ascii', 'ascii')
+
+    def test_output_htmlcalendar_encoding_utf8(self):
+        self.check_htmlcalendar_encoding('utf-8', 'utf-8')
+
+    def test_output_htmlcalendar_encoding_default(self):
+        self.check_htmlcalendar_encoding(None, sys.getdefaultencoding())
+
+    def test_yeardatescalendar(self):
+        def shrink(cal):
+            return [[[' '.join('{:02d}/{:02d}/{}'.format(
+                                d.month, d.day, str(d.year)[-2:]) for d in z)
+                            for z in y] for y in x] for x in cal]
+        self.assertEqual(
+            shrink(calendar.Calendar().yeardatescalendar(2004)),
+            result_2004_dates
+        )
+
+    def test_yeardayscalendar(self):
         self.assertEqual(
-            cal.formatyearpage(2004, encoding=encoding).strip(b' \t\n'),
-            result_2004_html.strip(' \t\n').encode(encoding)
+            calendar.Calendar().yeardayscalendar(2004),
+            result_2004_days
+        )
+
+    def test_formatweekheader_short(self):
+        self.assertEqual(
+            calendar.TextCalendar().formatweekheader(2),
+            'Mo Tu We Th Fr Sa Su'
+        )
+
+    def test_formatweekheader_long(self):
+        self.assertEqual(
+            calendar.TextCalendar().formatweekheader(9),
+            '  Monday   Tuesday  Wednesday  Thursday '
+            '  Friday   Saturday   Sunday '
+        )
+
+    def test_formatmonth(self):
+        self.assertEqual(
+            calendar.TextCalendar().formatmonth(2004, 1).strip(),
+            result_2004_01_text.strip()
+        )
+
+    def test_formatmonthname_with_year(self):
+        self.assertEqual(
+            calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=True),
+            '<tr><th colspan="7" class="month">January 2004</th></tr>'
+        )
+
+    def test_formatmonthname_without_year(self):
+        self.assertEqual(
+            calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=False),
+            '<tr><th colspan="7" class="month">January</th></tr>'
         )
 
 
@@ -227,7 +422,11 @@ class CalendarTestCase(unittest.TestCase):
         self.assertEqual(calendar.firstweekday(), calendar.MONDAY)
         calendar.setfirstweekday(orig)
 
-    def test_enumerateweekdays(self):
+    def test_illegal_weekday_reported(self):
+        with self.assertRaisesRegex(calendar.IllegalWeekdayError, '123'):
+            calendar.setfirstweekday(123)
+
+    def test_enumerate_weekdays(self):
         self.assertRaises(IndexError, calendar.day_abbr.__getitem__, -10)
         self.assertRaises(IndexError, calendar.day_name.__getitem__, 10)
         self.assertEqual(len([d for d in calendar.day_abbr]), 7)
@@ -253,7 +452,7 @@ class CalendarTestCase(unittest.TestCase):
             # verify it "acts like a sequence" in two forms of iteration
             self.assertEqual(value[::-1], list(reversed(value)))
 
-    def test_localecalendars(self):
+    def test_locale_calendars(self):
         # ensure that Locale{Text,HTML}Calendar resets the locale properly
         # (it is still not thread-safe though)
         old_october = calendar.TextCalendar().formatmonthname(2010, 10, 10)
@@ -447,6 +646,10 @@ class MonthRangeTestCase(unittest.TestCase):
         with self.assertRaises(calendar.IllegalMonthError):
             calendar.monthrange(2004, 13)
 
+    def test_illegal_month_reported(self):
+        with self.assertRaisesRegex(calendar.IllegalMonthError, '65'):
+            calendar.monthrange(2004, 65)
+
 class LeapdaysTestCase(unittest.TestCase):
     def test_no_range(self):
         # test when no range i.e. two identical years as args
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 1c4c0f8c46..f1ea5a9fde 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -15,10 +15,8 @@ try:
 except ImportError:
     _posixsubprocess = None
 try:
-    import _thread
     import threading
 except ImportError:
-    _thread = None
     threading = None
 import _testcapi
 
@@ -55,13 +53,36 @@ class CAPITest(unittest.TestCase):
         (out, err) = p.communicate()
         self.assertEqual(out, b'')
         # This used to cause an infinite loop.
-        self.assertEqual(err.rstrip(),
+        self.assertTrue(err.rstrip().startswith(
                          b'Fatal Python error:'
-                         b' PyThreadState_Get: no current thread')
+                         b' PyThreadState_Get: no current thread'))
 
     def test_memoryview_from_NULL_pointer(self):
         self.assertRaises(ValueError, _testcapi.make_memoryview_from_NULL_pointer)
 
+    def test_exc_info(self):
+        raised_exception = ValueError("5")
+        new_exc = TypeError("TEST")
+        try:
+            raise raised_exception
+        except ValueError as e:
+            tb = e.__traceback__
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = _testcapi.set_exc_info(new_exc.__class__, new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = _testcapi.set_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            self.assertEqual(orig_exc_info[1], e)
+
+            self.assertSequenceEqual(orig_exc_info, (raised_exception.__class__, raised_exception, tb))
+            self.assertSequenceEqual(orig_sys_exc_info, orig_exc_info)
+            self.assertSequenceEqual(reset_sys_exc_info, orig_exc_info)
+            self.assertSequenceEqual(new_exc_info, (new_exc.__class__, new_exc, None))
+            self.assertSequenceEqual(new_sys_exc_info, new_exc_info)
+        else:
+            self.assertTrue(False)
+
     @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.')
     def test_seq_bytes_to_charp_array(self):
         # Issue #15732: crash in _PySequence_BytesToCharpArray()
@@ -224,8 +245,89 @@ class EmbeddingTest(unittest.TestCase):
         finally:
             os.chdir(oldcwd)
 
+class SkipitemTest(unittest.TestCase):
+
+    def test_skipitem(self):
+        """
+        If this test failed, you probably added a new "format unit"
+        in Python/getargs.c, but neglected to update our poor friend
+        skipitem() in the same file.  (If so, shame on you!)
+
+        With a few exceptions**, this function brute-force tests all
+        printable ASCII*** characters (32 to 126 inclusive) as format units,
+        checking to see that PyArg_ParseTupleAndKeywords() return consistent
+        errors both when the unit is attempted to be used and when it is
+        skipped.  If the format unit doesn't exist, we'll get one of two
+        specific error messages (one for used, one for skipped); if it does
+        exist we *won't* get that error--we'll get either no error or some
+        other error.  If we get the specific "does not exist" error for one
+        test and not for the other, there's a mismatch, and the test fails.
+
+           ** Some format units have special funny semantics and it would
+              be difficult to accomodate them here.  Since these are all
+              well-established and properly skipped in skipitem() we can
+              get away with not testing them--this test is really intended
+              to catch *new* format units.
+
+          *** Python C source files must be ASCII.  Therefore it's impossible
+              to have non-ASCII format units.
+
+        """
+        empty_tuple = ()
+        tuple_1 = (0,)
+        dict_b = {'b':1}
+        keywords = ["a", "b"]
+
+        for i in range(32, 127):
+            c = chr(i)
+
+            # skip parentheses, the error reporting is inconsistent about them
+            # skip 'e', it's always a two-character code
+            # skip '|' and '$', they don't represent arguments anyway
+            if c in '()e|$':
+                continue
 
-@unittest.skipUnless(threading and _thread, 'Threading required for this test.')
+            # test the format unit when not skipped
+            format = c + "i"
+            try:
+                # (note: the format string must be bytes!)
+                _testcapi.parse_tuple_and_keywords(tuple_1, dict_b,
+                    format.encode("ascii"), keywords)
+                when_not_skipped = False
+            except TypeError as e:
+                s = "argument 1 must be impossible<bad format char>, not int"
+                when_not_skipped = (str(e) == s)
+            except RuntimeError as e:
+                when_not_skipped = False
+
+            # test the format unit when skipped
+            optional_format = "|" + format
+            try:
+                _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b,
+                    optional_format.encode("ascii"), keywords)
+                when_skipped = False
+            except RuntimeError as e:
+                s = "impossible<bad format char>: '{}'".format(format)
+                when_skipped = (str(e) == s)
+
+            message = ("test_skipitem_parity: "
+                "detected mismatch between convertsimple and skipitem "
+                "for format unit '{}' ({}), not skipped {}, skipped {}".format(
+                    c, i, when_skipped, when_not_skipped))
+            self.assertIs(when_skipped, when_not_skipped, message)
+
+    def test_parse_tuple_and_keywords(self):
+        # parse_tuple_and_keywords error handling tests
+        self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, 42, [])
+        self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, b'', 42)
+        self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, b'', [''] * 42)
+        self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords,
+                          (), {}, b'', [42])
+
+@unittest.skipUnless(threading, 'Threading required for this test.')
 class TestThreadState(unittest.TestCase):
 
     @support.reap_threads
@@ -235,13 +337,13 @@ class TestThreadState(unittest.TestCase):
             idents = []
 
             def callback():
-                idents.append(_thread.get_ident())
+                idents.append(threading.get_ident())
 
             _testcapi._test_thread_state(callback)
             a = b = callback
             time.sleep(1)
             # Check our main thread is in the list exactly 3 times.
-            self.assertEqual(idents.count(_thread.get_ident()), 3,
+            self.assertEqual(idents.count(threading.get_ident()), 3,
                              "Couldn't find main thread correctly in the list")
 
         target()
@@ -252,7 +354,7 @@ class TestThreadState(unittest.TestCase):
 
 def test_main():
     support.run_unittest(CAPITest, TestPendingCalls, Test6012,
-                         EmbeddingTest, TestThreadState)
+                         EmbeddingTest, SkipitemTest, TestThreadState)
 
     for name in dir(_testcapi):
         if name.startswith('test_'):
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
index 07e760b8e3..cb28aa8712 100644
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -4,6 +4,7 @@ import os
 import sys
 import tempfile
 import unittest
+import warnings
 from collections import namedtuple
 from io import StringIO, BytesIO
 
@@ -137,9 +138,13 @@ class CgiTests(unittest.TestCase):
         self.assertTrue(fs)
 
     def test_escape(self):
-        self.assertEqual("test &amp; string", cgi.escape("test & string"))
-        self.assertEqual("&lt;test string&gt;", cgi.escape("<test string>"))
-        self.assertEqual("&quot;test string&quot;", cgi.escape('"test string"', True))
+        # cgi.escape() is deprecated.
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore', 'cgi\.escape',
+                                     DeprecationWarning)
+            self.assertEqual("test &amp; string", cgi.escape("test & string"))
+            self.assertEqual("&lt;test string&gt;", cgi.escape("<test string>"))
+            self.assertEqual("&quot;test string&quot;", cgi.escape('"test string"', True))
 
     def test_strict(self):
         for orig, expect in parse_strict_test_cases:
@@ -169,8 +174,7 @@ class CgiTests(unittest.TestCase):
 
     def test_log(self):
         cgi.log("Testing")
-        cgi.logfile = "fail/"
-        cgi.initlog("%s", "Testing initlog")
+
         cgi.logfp = StringIO()
         cgi.initlog("%s", "Testing initlog 1")
         cgi.log("%s", "Testing log 2")
@@ -179,13 +183,7 @@ class CgiTests(unittest.TestCase):
             cgi.logfp = None
             cgi.logfile = "/dev/null"
             cgi.initlog("%s", "Testing log 3")
-            def log_cleanup():
-                """Restore the global state of the log vars."""
-                cgi.logfile = ''
-                cgi.logfp.close()
-                cgi.logfp = None
-                cgi.log = cgi.initlog
-            self.addCleanup(log_cleanup)
+            self.addCleanup(cgi.closelog)
             cgi.log("Testing log 4")
 
     def test_fieldstorage_readline(self):
diff --git a/Lib/test/test_cgitb.py b/Lib/test/test_cgitb.py
new file mode 100644
index 0000000000..2e072a9f2a
--- /dev/null
+++ b/Lib/test/test_cgitb.py
@@ -0,0 +1,70 @@
+from test.support import run_unittest
+from test.script_helper import assert_python_failure, temp_dir
+import unittest
+import sys
+import cgitb
+
+class TestCgitb(unittest.TestCase):
+
+    def test_fonts(self):
+        text = "Hello Robbie!"
+        self.assertEqual(cgitb.small(text), "<small>{}</small>".format(text))
+        self.assertEqual(cgitb.strong(text), "<strong>{}</strong>".format(text))
+        self.assertEqual(cgitb.grey(text),
+                         '<font color="#909090">{}</font>'.format(text))
+
+    def test_blanks(self):
+        self.assertEqual(cgitb.small(""), "")
+        self.assertEqual(cgitb.strong(""), "")
+        self.assertEqual(cgitb.grey(""), "")
+
+    def test_html(self):
+        try:
+            raise ValueError("Hello World")
+        except ValueError as err:
+            # If the html was templated we could do a bit more here.
+            # At least check that we get details on what we just raised.
+            html = cgitb.html(sys.exc_info())
+            self.assertIn("ValueError", html)
+            self.assertIn(str(err), html)
+
+    def test_text(self):
+        try:
+            raise ValueError("Hello World")
+        except ValueError as err:
+            text = cgitb.text(sys.exc_info())
+            self.assertIn("ValueError", text)
+            self.assertIn("Hello World", text)
+
+    def test_syshook_no_logdir_default_format(self):
+        with temp_dir() as tracedir:
+            rc, out, err = assert_python_failure(
+                  '-c',
+                  ('import cgitb; cgitb.enable(logdir=%s); '
+                   'raise ValueError("Hello World")') % repr(tracedir))
+        out = out.decode(sys.getfilesystemencoding())
+        self.assertIn("ValueError", out)
+        self.assertIn("Hello World", out)
+        # By default we emit HTML markup.
+        self.assertIn('<p>', out)
+        self.assertIn('</p>', out)
+
+    def test_syshook_no_logdir_text_format(self):
+        # Issue 12890: we were emitting the <p> tag in text mode.
+        with temp_dir() as tracedir:
+            rc, out, err = assert_python_failure(
+                  '-c',
+                  ('import cgitb; cgitb.enable(format="text", logdir=%s); '
+                   'raise ValueError("Hello World")') % repr(tracedir))
+        out = out.decode(sys.getfilesystemencoding())
+        self.assertIn("ValueError", out)
+        self.assertIn("Hello World", out)
+        self.assertNotIn('<p>', out)
+        self.assertNotIn('</p>', out)
+
+
+def test_main():
+    run_unittest(TestCgitb)
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py
index 3a463558fb..6618535823 100644
--- a/Lib/test/test_cmd.py
+++ b/Lib/test/test_cmd.py
@@ -228,7 +228,7 @@ def test_main(verbose=None):
 
 def test_coverage(coverdir):
     trace = support.import_module('trace')
-    tracer=trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix,],
+    tracer=trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,],
                         trace=0, count=1)
     tracer.run('reload(cmd);test_main()')
     r=tracer.results()
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index 67375cd12a..a89d7e4f10 100644
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -32,12 +32,6 @@ class CmdLineTest(unittest.TestCase):
         self.verify_valid_flag('-O')
         self.verify_valid_flag('-OO')
 
-    def test_q(self):
-        self.verify_valid_flag('-Qold')
-        self.verify_valid_flag('-Qnew')
-        self.verify_valid_flag('-Qwarn')
-        self.verify_valid_flag('-Qwarnall')
-
     def test_site_flag(self):
         self.verify_valid_flag('-S')
 
@@ -148,7 +142,7 @@ class CmdLineTest(unittest.TestCase):
     @unittest.skipUnless(sys.platform == 'darwin', 'test specific to Mac OS X')
     def test_osx_utf8(self):
         def check_output(text):
-            decoded = text.decode('utf8', 'surrogateescape')
+            decoded = text.decode('utf-8', 'surrogateescape')
             expected = ascii(decoded).encode('ascii') + b'\n'
 
             env = os.environ.copy()
@@ -220,7 +214,7 @@ class CmdLineTest(unittest.TestCase):
         self.assertIn(path2.encode('ascii'), out)
 
     def test_displayhook_unencodable(self):
-        for encoding in ('ascii', 'latin1', 'utf8'):
+        for encoding in ('ascii', 'latin-1', 'utf-8'):
             env = os.environ.copy()
             env['PYTHONIOENCODING'] = encoding
             p = subprocess.Popen(
@@ -296,7 +290,7 @@ class CmdLineTest(unittest.TestCase):
         rc, out, err = assert_python_ok('-c', code)
         self.assertEqual(b'', out)
         self.assertRegex(err.decode('ascii', 'ignore'),
-                         'Exception IOError: .* ignored')
+                         'Exception OSError: .* ignored')
 
     def test_closed_stdout(self):
         # Issue #13444: if stdout has been explicitly closed, we should
@@ -350,14 +344,14 @@ class CmdLineTest(unittest.TestCase):
         hashes = []
         for i in range(2):
             code = 'print(hash("spam"))'
-            rc, out, err = assert_python_ok('-R', '-c', code)
+            rc, out, err = assert_python_ok('-c', code)
             self.assertEqual(rc, 0)
             hashes.append(out)
         self.assertNotEqual(hashes[0], hashes[1])
 
         # Verify that sys.flags contains hash_randomization
         code = 'import sys; print("random is", sys.flags.hash_randomization)'
-        rc, out, err = assert_python_ok('-R', '-c', code)
+        rc, out, err = assert_python_ok('-c', code)
         self.assertEqual(rc, 0)
         self.assertIn(b'random is 1', out)
 
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index 70f7d1ebcc..6051e18edd 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -1,15 +1,20 @@
 # tests command line execution of scripts
 
+import importlib
+import importlib.machinery
+import zipimport
 import unittest
 import sys
 import os
 import os.path
 import py_compile
 
+import textwrap
 from test import support
 from test.script_helper import (
     make_pkg, make_script, make_zip_pkg, make_zip_script,
-    assert_python_ok, assert_python_failure, temp_dir)
+    assert_python_ok, assert_python_failure, temp_dir,
+    spawn_python, kill_python)
 
 verbose = support.verbose
 
@@ -32,6 +37,9 @@ f()
 assertEqual(result, ['Top level assignment', 'Lower level reference'])
 # Check population of magic variables
 assertEqual(__name__, '__main__')
+from importlib.machinery import BuiltinImporter
+_loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__)
+print('__loader__==%a' % _loader)
 print('__file__==%a' % __file__)
 assertEqual(__cached__, None)
 print('__package__==%r' % __package__)
@@ -49,12 +57,16 @@ print('cwd==%a' % os.getcwd())
 """
 
 def _make_test_script(script_dir, script_basename, source=test_source):
-    return make_script(script_dir, script_basename, source)
+    to_return = make_script(script_dir, script_basename, source)
+    importlib.invalidate_caches()
+    return to_return
 
 def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
                        source=test_source, depth=1):
-    return make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
-                        source, depth)
+    to_return = make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
+                             source, depth)
+    importlib.invalidate_caches()
+    return to_return
 
 # There's no easy way to pass the script directory in to get
 # -m to work (avoiding that is the whole point of making
@@ -72,16 +84,20 @@ def _make_launch_script(script_dir, script_basename, module_name, path=None):
     else:
         path = repr(path)
     source = launch_source % (path, module_name)
-    return make_script(script_dir, script_basename, source)
+    to_return = make_script(script_dir, script_basename, source)
+    importlib.invalidate_caches()
+    return to_return
 
 class CmdLineTest(unittest.TestCase):
     def _check_output(self, script_name, exit_code, data,
                              expected_file, expected_argv0,
-                             expected_path0, expected_package):
+                             expected_path0, expected_package,
+                             expected_loader):
         if verbose > 1:
             print("Output from test script %r:" % script_name)
             print(data)
         self.assertEqual(exit_code, 0)
+        printed_loader = '__loader__==%a' % expected_loader
         printed_file = '__file__==%a' % expected_file
         printed_package = '__package__==%r' % expected_package
         printed_argv0 = 'sys.argv[0]==%a' % expected_argv0
@@ -93,6 +109,7 @@ class CmdLineTest(unittest.TestCase):
             print(printed_package)
             print(printed_argv0)
             print(printed_cwd)
+        self.assertIn(printed_loader.encode('utf-8'), data)
         self.assertIn(printed_file.encode('utf-8'), data)
         self.assertIn(printed_package.encode('utf-8'), data)
         self.assertIn(printed_argv0.encode('utf-8'), data)
@@ -101,14 +118,15 @@ class CmdLineTest(unittest.TestCase):
 
     def _check_script(self, script_name, expected_file,
                             expected_argv0, expected_path0,
-                            expected_package,
+                            expected_package, expected_loader,
                             *cmd_line_switches):
         if not __debug__:
             cmd_line_switches += ('-' + 'O' * sys.flags.optimize,)
         run_args = cmd_line_switches + (script_name,) + tuple(example_args)
         rc, out, err = assert_python_ok(*run_args)
         self._check_output(script_name, rc, out + err, expected_file,
-                           expected_argv0, expected_path0, expected_package)
+                           expected_argv0, expected_path0,
+                           expected_package, expected_loader)
 
     def _check_import_error(self, script_name, expected_msg,
                             *cmd_line_switches):
@@ -120,11 +138,30 @@ class CmdLineTest(unittest.TestCase):
             print('Expected output: %r' % expected_msg)
         self.assertIn(expected_msg.encode('utf-8'), err)
 
+    def test_dash_c_loader(self):
+        rc, out, err = assert_python_ok("-c", "print(__loader__)")
+        expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8")
+        self.assertIn(expected, out)
+
+    def test_stdin_loader(self):
+        # Unfortunately, there's no way to automatically test the fully
+        # interactive REPL, since that code path only gets executed when
+        # stdin is an interactive tty.
+        p = spawn_python()
+        try:
+            p.stdin.write(b"print(__loader__)\n")
+            p.stdin.flush()
+        finally:
+            out = kill_python(p)
+        expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8")
+        self.assertIn(expected, out)
+
     def test_basic_script(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, 'script')
             self._check_script(script_name, script_name, script_name,
-                               script_dir, None)
+                               script_dir, None,
+                               importlib.machinery.SourceFileLoader)
 
     def test_script_compiled(self):
         with temp_dir() as script_dir:
@@ -133,13 +170,15 @@ class CmdLineTest(unittest.TestCase):
             os.remove(script_name)
             pyc_file = support.make_legacy_pyc(script_name)
             self._check_script(pyc_file, pyc_file,
-                               pyc_file, script_dir, None)
+                               pyc_file, script_dir, None,
+                               importlib.machinery.SourcelessFileLoader)
 
     def test_directory(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             self._check_script(script_dir, script_name, script_dir,
-                               script_dir, '')
+                               script_dir, '',
+                               importlib.machinery.SourceFileLoader)
 
     def test_directory_compiled(self):
         with temp_dir() as script_dir:
@@ -148,7 +187,8 @@ class CmdLineTest(unittest.TestCase):
             os.remove(script_name)
             pyc_file = support.make_legacy_pyc(script_name)
             self._check_script(script_dir, pyc_file, script_dir,
-                               script_dir, '')
+                               script_dir, '',
+                               importlib.machinery.SourcelessFileLoader)
 
     def test_directory_error(self):
         with temp_dir() as script_dir:
@@ -159,14 +199,16 @@ class CmdLineTest(unittest.TestCase):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name)
-            self._check_script(zip_name, run_name, zip_name, zip_name, '')
+            self._check_script(zip_name, run_name, zip_name, zip_name, '',
+                               zipimport.zipimporter)
 
     def test_zipfile_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             compiled_name = py_compile.compile(script_name, doraise=True)
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
-            self._check_script(zip_name, run_name, zip_name, zip_name, '')
+            self._check_script(zip_name, run_name, zip_name, zip_name, '',
+                               zipimport.zipimporter)
 
     def test_zipfile_error(self):
         with temp_dir() as script_dir:
@@ -181,19 +223,24 @@ class CmdLineTest(unittest.TestCase):
             make_pkg(pkg_dir)
             script_name = _make_test_script(pkg_dir, 'script')
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script')
-            self._check_script(launch_name, script_name, script_name, script_dir, 'test_pkg')
+            self._check_script(launch_name, script_name, script_name,
+                               script_dir, 'test_pkg',
+                               importlib.machinery.SourceFileLoader)
 
     def test_module_in_package_in_zipfile(self):
         with temp_dir() as script_dir:
             zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script')
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name)
-            self._check_script(launch_name, run_name, run_name, zip_name, 'test_pkg')
+            self._check_script(launch_name, run_name, run_name,
+                               zip_name, 'test_pkg', zipimport.zipimporter)
 
     def test_module_in_subpackage_in_zipfile(self):
         with temp_dir() as script_dir:
             zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2)
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name)
-            self._check_script(launch_name, run_name, run_name, zip_name, 'test_pkg.test_pkg')
+            self._check_script(launch_name, run_name, run_name,
+                               zip_name, 'test_pkg.test_pkg',
+                               zipimport.zipimporter)
 
     def test_package(self):
         with temp_dir() as script_dir:
@@ -202,7 +249,8 @@ class CmdLineTest(unittest.TestCase):
             script_name = _make_test_script(pkg_dir, '__main__')
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
             self._check_script(launch_name, script_name,
-                               script_name, script_dir, 'test_pkg')
+                               script_name, script_dir, 'test_pkg',
+                               importlib.machinery.SourceFileLoader)
 
     def test_package_compiled(self):
         with temp_dir() as script_dir:
@@ -214,7 +262,8 @@ class CmdLineTest(unittest.TestCase):
             pyc_file = support.make_legacy_pyc(script_name)
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
             self._check_script(launch_name, pyc_file,
-                               pyc_file, script_dir, 'test_pkg')
+                               pyc_file, script_dir, 'test_pkg',
+                               importlib.machinery.SourcelessFileLoader)
 
     def test_package_error(self):
         with temp_dir() as script_dir:
@@ -251,7 +300,8 @@ class CmdLineTest(unittest.TestCase):
                 expected = "init_argv0==%r" % '-m'
                 self.assertIn(expected.encode('utf-8'), out)
                 self._check_output(script_name, rc, out,
-                                   script_name, script_name, '', 'test_pkg')
+                                   script_name, script_name, '', 'test_pkg',
+                                   importlib.machinery.SourceFileLoader)
 
     def test_issue8202_dash_c_file_ignored(self):
         # Make sure a "-c" file in the current directory
@@ -277,7 +327,8 @@ class CmdLineTest(unittest.TestCase):
                     f.write("data")
                     rc, out, err = assert_python_ok('-m', 'other', *example_args)
                     self._check_output(script_name, rc, out,
-                                      script_name, script_name, '', '')
+                                      script_name, script_name, '', '',
+                                      importlib.machinery.SourceFileLoader)
 
     def test_dash_m_error_code_is_one(self):
         # If a module is invoked with the -m command line flag
@@ -294,6 +345,24 @@ class CmdLineTest(unittest.TestCase):
                     print(out)
                 self.assertEqual(rc, 1)
 
+    def test_pep_409_verbiage(self):
+        # Make sure PEP 409 syntax properly suppresses
+        # the context of an exception
+        script = textwrap.dedent("""\
+            try:
+                raise ValueError
+            except:
+                raise NameError from None
+            """)
+        with temp_dir() as script_dir:
+            script_name = _make_test_script(script_dir, 'script', script)
+            exitcode, stdout, stderr = assert_python_failure(script_name)
+            text = stderr.decode('ascii').split('\n')
+            self.assertEqual(len(text), 4)
+            self.assertTrue(text[0].startswith('Traceback'))
+            self.assertTrue(text[1].startswith('  File '))
+            self.assertTrue(text[3].startswith('NameError'))
+
     def test_non_ascii(self):
         # Mac OS X denies the creation of a file with an invalid UTF-8 name.
         # Windows allows to create a name with an arbitrary bytes name, but
diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
index e1c7a783de..3377a7b07a 100644
--- a/Lib/test/test_code.py
+++ b/Lib/test/test_code.py
@@ -16,7 +16,7 @@ cellvars: ('x',)
 freevars: ()
 nlocals: 2
 flags: 3
-consts: ('None', '<code object g>')
+consts: ('None', '<code object g>', "'f.<locals>.g'")
 
 >>> dump(f(4).__code__)
 name: g
diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py
new file mode 100644
index 0000000000..adef1701d4
--- /dev/null
+++ b/Lib/test/test_code_module.py
@@ -0,0 +1,72 @@
+"Test InteractiveConsole and InteractiveInterpreter from code module"
+import sys
+import unittest
+from contextlib import ExitStack
+from unittest import mock
+from test import support
+
+code = support.import_module('code')
+
+
+class TestInteractiveConsole(unittest.TestCase):
+
+    def setUp(self):
+        self.console = code.InteractiveConsole()
+        self.mock_sys()
+
+    def mock_sys(self):
+        "Mock system environment for InteractiveConsole"
+        # use exit stack to match patch context managers to addCleanup
+        stack = ExitStack()
+        self.addCleanup(stack.close)
+        self.infunc = stack.enter_context(mock.patch('code.input',
+                                          create=True))
+        self.stdout = stack.enter_context(mock.patch('code.sys.stdout'))
+        self.stderr = stack.enter_context(mock.patch('code.sys.stderr'))
+        prepatch = mock.patch('code.sys', wraps=code.sys, spec=code.sys)
+        self.sysmod = stack.enter_context(prepatch)
+        if sys.excepthook is sys.__excepthook__:
+            self.sysmod.excepthook = self.sysmod.__excepthook__
+
+    def test_ps1(self):
+        self.infunc.side_effect = EOFError('Finished')
+        self.console.interact()
+        self.assertEqual(self.sysmod.ps1, '>>> ')
+
+    def test_ps2(self):
+        self.infunc.side_effect = EOFError('Finished')
+        self.console.interact()
+        self.assertEqual(self.sysmod.ps2, '... ')
+
+    def test_console_stderr(self):
+        self.infunc.side_effect = ["'antioch'", "", EOFError('Finished')]
+        self.console.interact()
+        for call in list(self.stdout.method_calls):
+            if 'antioch' in ''.join(call[1]):
+                break
+        else:
+            raise AssertionError("no console stdout")
+
+    def test_syntax_error(self):
+        self.infunc.side_effect = ["undefined", EOFError('Finished')]
+        self.console.interact()
+        for call in self.stderr.method_calls:
+            if 'NameError:' in ''.join(call[1]):
+                break
+        else:
+            raise AssertionError("No syntax error from console")
+
+    def test_sysexcepthook(self):
+        self.infunc.side_effect = ["raise ValueError('')",
+                                    EOFError('Finished')]
+        hook = mock.Mock()
+        self.sysmod.excepthook = hook
+        self.console.interact()
+        self.assertTrue(hook.called)
+
+
+def test_main():
+    support.run_unittest(TestInteractiveConsole)
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py
index 3b78b4af52..fd88505081 100644
--- a/Lib/test/test_codeccallbacks.py
+++ b/Lib/test/test_codeccallbacks.py
@@ -1,5 +1,18 @@
-import test.support, unittest
-import sys, codecs, html.entities, unicodedata
+import codecs
+import html.entities
+import sys
+import test.support
+import unicodedata
+import unittest
+import warnings
+
+try:
+    import ctypes
+except ImportError:
+    ctypes = None
+    SIZEOF_WCHAR_T = -1
+else:
+    SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar)
 
 class PosReturn:
     # this can be used for configurable callbacks
@@ -135,22 +148,14 @@ class CodecCallbackTest(unittest.TestCase):
     def test_backslashescape(self):
         # Does the same as the "unicode-escape" encoding, but with different
         # base encodings.
-        sin = "a\xac\u1234\u20ac\u8000"
-        if sys.maxunicode > 0xffff:
-            sin += chr(sys.maxunicode)
-        sout = b"a\\xac\\u1234\\u20ac\\u8000"
-        if sys.maxunicode > 0xffff:
-            sout += bytes("\\U%08x" % sys.maxunicode, "ascii")
+        sin = "a\xac\u1234\u20ac\u8000\U0010ffff"
+        sout = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff"
         self.assertEqual(sin.encode("ascii", "backslashreplace"), sout)
 
-        sout = b"a\xac\\u1234\\u20ac\\u8000"
-        if sys.maxunicode > 0xffff:
-            sout += bytes("\\U%08x" % sys.maxunicode, "ascii")
+        sout = b"a\xac\\u1234\\u20ac\\u8000\\U0010ffff"
         self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout)
 
-        sout = b"a\xac\\u1234\xa4\\u8000"
-        if sys.maxunicode > 0xffff:
-            sout += bytes("\\U%08x" % sys.maxunicode, "ascii")
+        sout = b"a\xac\\u1234\xa4\\u8000\\U0010ffff"
         self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout)
 
     def test_decoding_callbacks(self):
@@ -200,33 +205,37 @@ class CodecCallbackTest(unittest.TestCase):
         self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
 
     def test_decodeunicodeinternal(self):
-        self.assertRaises(
-            UnicodeDecodeError,
-            b"\x00\x00\x00\x00\x00".decode,
-            "unicode-internal",
-        )
-        if sys.maxunicode > 0xffff:
+        with test.support.check_warnings(('unicode_internal codec has been '
+                                          'deprecated', DeprecationWarning)):
+            self.assertRaises(
+                UnicodeDecodeError,
+                b"\x00\x00\x00\x00\x00".decode,
+                "unicode-internal",
+            )
+        if SIZEOF_WCHAR_T == 4:
             def handler_unicodeinternal(exc):
                 if not isinstance(exc, UnicodeDecodeError):
                     raise TypeError("don't know how to handle %r" % exc)
                 return ("\x01", 1)
 
-            self.assertEqual(
-                b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"),
-                "\u0000"
-            )
+            with test.support.check_warnings(('unicode_internal codec has been '
+                                              'deprecated', DeprecationWarning)):
+                self.assertEqual(
+                    b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"),
+                    "\u0000"
+                )
 
-            self.assertEqual(
-                b"\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"),
-                "\u0000\ufffd"
-            )
+                self.assertEqual(
+                    b"\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"),
+                    "\u0000\ufffd"
+                )
 
-            codecs.register_error("test.hui", handler_unicodeinternal)
+                codecs.register_error("test.hui", handler_unicodeinternal)
 
-            self.assertEqual(
-                b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"),
-                "\u0000\u0001\u0000"
-            )
+                self.assertEqual(
+                    b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"),
+                    "\u0000\u0001\u0000"
+                )
 
     def test_callbacks(self):
         def handler1(exc):
@@ -355,7 +364,7 @@ class CodecCallbackTest(unittest.TestCase):
             ["ascii", "\uffffx", 0, 1, "ouch"],
             "'ascii' codec can't encode character '\\uffff' in position 0: ouch"
         )
-        if sys.maxunicode > 0xffff:
+        if SIZEOF_WCHAR_T == 4:
             self.check_exceptionobjectargs(
                 UnicodeEncodeError,
                 ["ascii", "\U00010000x", 0, 1, "ouch"],
@@ -390,7 +399,7 @@ class CodecCallbackTest(unittest.TestCase):
             ["g\uffffrk", 1, 2, "ouch"],
             "can't translate character '\\uffff' in position 1: ouch"
         )
-        if sys.maxunicode > 0xffff:
+        if SIZEOF_WCHAR_T == 4:
             self.check_exceptionobjectargs(
                 UnicodeTranslateError,
                 ["g\U00010000rk", 1, 2, "ouch"],
@@ -577,31 +586,30 @@ class CodecCallbackTest(unittest.TestCase):
                 UnicodeEncodeError("ascii", "\uffff", 0, 1, "ouch")),
             ("\\uffff", 1)
         )
-        # 1 on UCS-4 builds, 2 on UCS-2
-        len_wide = len("\U00010000")
-        self.assertEqual(
-            codecs.backslashreplace_errors(
-                UnicodeEncodeError("ascii", "\U00010000",
-                                   0, len_wide, "ouch")),
-            ("\\U00010000", len_wide)
-        )
-        self.assertEqual(
-            codecs.backslashreplace_errors(
-                UnicodeEncodeError("ascii", "\U0010ffff",
-                                   0, len_wide, "ouch")),
-            ("\\U0010ffff", len_wide)
-        )
-        # Lone surrogates (regardless of unicode width)
-        self.assertEqual(
-            codecs.backslashreplace_errors(
-                UnicodeEncodeError("ascii", "\ud800", 0, 1, "ouch")),
-            ("\\ud800", 1)
-        )
-        self.assertEqual(
-            codecs.backslashreplace_errors(
-                UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")),
-            ("\\udfff", 1)
-        )
+        if SIZEOF_WCHAR_T > 0:
+            self.assertEqual(
+                codecs.backslashreplace_errors(
+                    UnicodeEncodeError("ascii", "\U00010000",
+                                       0, 1, "ouch")),
+                ("\\U00010000", 1)
+            )
+            self.assertEqual(
+                codecs.backslashreplace_errors(
+                    UnicodeEncodeError("ascii", "\U0010ffff",
+                                       0, 1, "ouch")),
+                ("\\U0010ffff", 1)
+            )
+            # Lone surrogates (regardless of unicode width)
+            self.assertEqual(
+                codecs.backslashreplace_errors(
+                    UnicodeEncodeError("ascii", "\ud800", 0, 1, "ouch")),
+                ("\\ud800", 1)
+            )
+            self.assertEqual(
+                codecs.backslashreplace_errors(
+                    UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")),
+                ("\\udfff", 1)
+            )
 
     def test_badhandlerresults(self):
         results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
@@ -622,12 +630,14 @@ class CodecCallbackTest(unittest.TestCase):
                 ("utf-7", b"+x-"),
                 ("unicode-internal", b"\x00"),
             ):
-                self.assertRaises(
-                    TypeError,
-                    bytes.decode,
-                    enc,
-                    "test.badhandler"
-                )
+                with test.support.check_warnings():
+                    # unicode-internal has been deprecated
+                    self.assertRaises(
+                        TypeError,
+                        bytes.decode,
+                        enc,
+                        "test.badhandler"
+                    )
 
     def test_lookup(self):
         self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict"))
@@ -679,7 +689,7 @@ class CodecCallbackTest(unittest.TestCase):
         # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors()
         # and inline implementations
         v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000)
-        if sys.maxunicode>=100000:
+        if SIZEOF_WCHAR_T == 4:
             v += (100000, 500000, 1000000)
         s = "".join([chr(x) for x in v])
         codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors)
@@ -744,7 +754,7 @@ class CodecCallbackTest(unittest.TestCase):
                 raise ValueError
         self.assertRaises(UnicodeError, codecs.charmap_decode, b"\xff", "strict", {0xff: None})
         self.assertRaises(ValueError, codecs.charmap_decode, b"\xff", "strict", D())
-        self.assertRaises(TypeError, codecs.charmap_decode, b"\xff", "strict", {0xff: 0x110000})
+        self.assertRaises(TypeError, codecs.charmap_decode, b"\xff", "strict", {0xff: sys.maxunicode+1})
 
     def test_encodehelper(self):
         # enhance coverage of:
@@ -843,8 +853,12 @@ class CodecCallbackTest(unittest.TestCase):
             else:
                 raise TypeError("don't know how to handle %r" % exc)
         codecs.register_error("test.replacing", replacing)
-        for (encoding, data) in baddata:
-            self.assertRaises(TypeError, data.decode, encoding, "test.replacing")
+
+        with test.support.check_warnings():
+            # unicode-internal has been deprecated
+            for (encoding, data) in baddata:
+                with self.assertRaises(TypeError):
+                    data.decode(encoding, "test.replacing")
 
         def mutating(exc):
             if isinstance(exc, UnicodeDecodeError):
@@ -855,8 +869,11 @@ class CodecCallbackTest(unittest.TestCase):
         codecs.register_error("test.mutating", mutating)
         # If the decoder doesn't pick up the modified input the following
         # will lead to an endless loop
-        for (encoding, data) in baddata:
-            self.assertRaises(TypeError, data.decode, encoding, "test.replacing")
+        with test.support.check_warnings():
+            # unicode-internal has been deprecated
+            for (encoding, data) in baddata:
+                with self.assertRaises(TypeError):
+                    data.decode(encoding, "test.replacing")
 
 def test_main():
     test.support.run_unittest(CodecCallbackTest)
diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py
index dca9f10b83..b08c5fcb1a 100644
--- a/Lib/test/test_codecencodings_cn.py
+++ b/Lib/test/test_codecencodings_cn.py
@@ -5,54 +5,57 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class Test_GB2312(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'gb2312'
-    tstring = test_multibytecodec_support.load_teststring('gb2312')
+    tstring = multibytecodec_support.load_teststring('gb2312')
     codectests = (
         # invalid bytes
         (b"abc\x81\x81\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\u804a"),
-        (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"),
+        (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"),
+        (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"),
         (b"abc\x81\x81\xc1\xc4", "ignore",  "abc\u804a"),
         (b"\xc1\x64", "strict", None),
     )
 
-class Test_GBK(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'gbk'
-    tstring = test_multibytecodec_support.load_teststring('gbk')
+    tstring = multibytecodec_support.load_teststring('gbk')
     codectests = (
         # invalid bytes
         (b"abc\x80\x80\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u804a"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"),
+        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"),
+        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\u804a"),
         (b"\x83\x34\x83\x31", "strict", None),
         ("\u30fb", "strict", None),
     )
 
-class Test_GB18030(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'gb18030'
-    tstring = test_multibytecodec_support.load_teststring('gb18030')
+    tstring = multibytecodec_support.load_teststring('gb18030')
     codectests = (
         # invalid bytes
         (b"abc\x80\x80\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u804a"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"),
+        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"),
+        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\u804a"),
-        (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd\u804a"),
+        (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd9\ufffd9\u804a"),
         ("\u30fb", "strict", b"\x819\xa79"),
+        (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'),
+        (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'),
+        (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'),
     )
     has_iso10646 = True
 
-class Test_HZ(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'hz'
-    tstring = test_multibytecodec_support.load_teststring('hz')
+    tstring = multibytecodec_support.load_teststring('hz')
     codectests = (
         # test '~\n' (3 lines)
         (b'This sentence is in ASCII.\n'
@@ -74,9 +77,11 @@ class Test_HZ(test_multibytecodec_support.TestBase, unittest.TestCase):
          '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002'
          'Bye.\n'),
         # invalid bytes
-        (b'ab~cd', 'replace', 'ab\uFFFDd'),
+        (b'ab~cd', 'replace', 'ab\uFFFDcd'),
         (b'ab\xffcd', 'replace', 'ab\uFFFDcd'),
         (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'),
+        (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'),
+        (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"),
     )
 
 def test_main():
diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py
index ccdc0b4c55..31363f4bea 100644
--- a/Lib/test/test_codecencodings_hk.py
+++ b/Lib/test/test_codecencodings_hk.py
@@ -5,18 +5,18 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class Test_Big5HKSCS(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_Big5HKSCS(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'big5hkscs'
-    tstring = test_multibytecodec_support.load_teststring('big5hkscs')
+    tstring = multibytecodec_support.load_teststring('big5hkscs')
     codectests = (
         # invalid bytes
         (b"abc\x80\x80\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u8b10"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u8b10\ufffd"),
+        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"),
+        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\u8b10"),
     )
 
diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py
index 8c6e8a5965..e4c1839b42 100644
--- a/Lib/test/test_codecencodings_iso2022.py
+++ b/Lib/test/test_codecencodings_iso2022.py
@@ -3,7 +3,7 @@
 # Codec encoding tests for ISO 2022 encodings.
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
 COMMON_CODEC_TESTS = (
@@ -13,23 +13,23 @@ COMMON_CODEC_TESTS = (
         (b'ab\x1B$def', 'replace', 'ab\uFFFD'),
     )
 
-class Test_ISO2022_JP(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'iso2022_jp'
-    tstring = test_multibytecodec_support.load_teststring('iso2022_jp')
+    tstring = multibytecodec_support.load_teststring('iso2022_jp')
     codectests = COMMON_CODEC_TESTS + (
         (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
     )
 
-class Test_ISO2022_JP2(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'iso2022_jp_2'
-    tstring = test_multibytecodec_support.load_teststring('iso2022_jp')
+    tstring = multibytecodec_support.load_teststring('iso2022_jp')
     codectests = COMMON_CODEC_TESTS + (
         (b'ab\x1BNdef', 'replace', 'abdef'),
     )
 
-class Test_ISO2022_KR(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'iso2022_kr'
-    tstring = test_multibytecodec_support.load_teststring('iso2022_kr')
+    tstring = multibytecodec_support.load_teststring('iso2022_kr')
     codectests = COMMON_CODEC_TESTS + (
         (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
     )
diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py
index f56a373896..30c9e195f3 100644
--- a/Lib/test/test_codecencodings_jp.py
+++ b/Lib/test/test_codecencodings_jp.py
@@ -5,60 +5,67 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class Test_CP932(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_CP932(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'cp932'
-    tstring = test_multibytecodec_support.load_teststring('shift_jis')
+    tstring = multibytecodec_support.load_teststring('shift_jis')
     codectests = (
         # invalid bytes
         (b"abc\x81\x00\x81\x00\x82\x84", "strict",  None),
         (b"abc\xf8", "strict",  None),
-        (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\uff44"),
-        (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"),
-        (b"abc\x81\x00\x82\x84", "ignore",  "abc\uff44"),
+        (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\x00\uff44"),
+        (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\x00\uff44\ufffd"),
+        (b"abc\x81\x00\x82\x84", "ignore",  "abc\x00\uff44"),
+        (b"ab\xEBxy", "replace", "ab\uFFFDxy"),
+        (b"ab\xF0\x39xy", "replace", "ab\uFFFD9xy"),
+        (b"ab\xEA\xF0xy", "replace", 'ab\ufffd\ue038y'),
         # sjis vs cp932
         (b"\\\x7e", "replace", "\\\x7e"),
         (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\uff3c\u2225\uff0d"),
     )
 
-class Test_EUC_JISX0213(test_multibytecodec_support.TestBase,
+euc_commontests = (
+    # invalid bytes
+    (b"abc\x80\x80\xc1\xc4", "strict",  None),
+    (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u7956"),
+    (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u7956\ufffd"),
+    (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\u7956"),
+    (b"abc\xc8", "strict",  None),
+    (b"abc\x8f\x83\x83", "replace", "abc\ufffd\ufffd\ufffd"),
+    (b"\x82\xFCxy", "replace", "\ufffd\ufffdxy"),
+    (b"\xc1\x64", "strict", None),
+    (b"\xa1\xc0", "strict", "\uff3c"),
+    (b"\xa1\xc0\\", "strict", "\uff3c\\"),
+    (b"\x8eXY", "replace", "\ufffdXY"),
+)
+
+class Test_EUC_JIS_2004(multibytecodec_support.TestBase,
                         unittest.TestCase):
-    encoding = 'euc_jisx0213'
-    tstring = test_multibytecodec_support.load_teststring('euc_jisx0213')
-    codectests = (
-        # invalid bytes
-        (b"abc\x80\x80\xc1\xc4", "strict",  None),
-        (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u7956"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u7956\ufffd"),
-        (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\u7956"),
-        (b"abc\x8f\x83\x83", "replace", "abc\ufffd"),
-        (b"\xc1\x64", "strict", None),
-        (b"\xa1\xc0", "strict", "\uff3c"),
-    )
+    encoding = 'euc_jis_2004'
+    tstring = multibytecodec_support.load_teststring('euc_jisx0213')
+    codectests = euc_commontests
     xmlcharnametest = (
         "\xab\u211c\xbb = \u2329\u1234\u232a",
         b"\xa9\xa8&real;\xa9\xb2 = &lang;&#4660;&rang;"
     )
 
-eucjp_commontests = (
-    (b"abc\x80\x80\xc1\xc4", "strict",  None),
-    (b"abc\xc8", "strict",  None),
-    (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u7956"),
-    (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u7956\ufffd"),
-    (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\u7956"),
-    (b"abc\x8f\x83\x83", "replace", "abc\ufffd"),
-    (b"\xc1\x64", "strict", None),
-)
+class Test_EUC_JISX0213(multibytecodec_support.TestBase,
+                        unittest.TestCase):
+    encoding = 'euc_jisx0213'
+    tstring = multibytecodec_support.load_teststring('euc_jisx0213')
+    codectests = euc_commontests
+    xmlcharnametest = (
+        "\xab\u211c\xbb = \u2329\u1234\u232a",
+        b"\xa9\xa8&real;\xa9\xb2 = &lang;&#4660;&rang;"
+    )
 
-class Test_EUC_JP_COMPAT(test_multibytecodec_support.TestBase,
+class Test_EUC_JP_COMPAT(multibytecodec_support.TestBase,
                          unittest.TestCase):
     encoding = 'euc_jp'
-    tstring = test_multibytecodec_support.load_teststring('euc_jp')
-    codectests = eucjp_commontests + (
-        (b"\xa1\xc0\\", "strict", "\uff3c\\"),
+    tstring = multibytecodec_support.load_teststring('euc_jp')
+    codectests = euc_commontests + (
         ("\xa5", "strict", b"\x5c"),
         ("\u203e", "strict", b"\x7e"),
     )
@@ -66,29 +73,48 @@ class Test_EUC_JP_COMPAT(test_multibytecodec_support.TestBase,
 shiftjis_commonenctests = (
     (b"abc\x80\x80\x82\x84", "strict",  None),
     (b"abc\xf8", "strict",  None),
-    (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\uff44"),
-    (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"),
     (b"abc\x80\x80\x82\x84def", "ignore",  "abc\uff44def"),
 )
 
-class Test_SJIS_COMPAT(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'shift_jis'
-    tstring = test_multibytecodec_support.load_teststring('shift_jis')
+    tstring = multibytecodec_support.load_teststring('shift_jis')
     codectests = shiftjis_commonenctests + (
+        (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"),
+        (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"),
+
         (b"\\\x7e", "strict", "\\\x7e"),
         (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\uff3c\u2016\u2212"),
+        (b"abc\x81\x39", "replace",  "abc\ufffd9"),
+        (b"abc\xEA\xFC", "replace",  "abc\ufffd\ufffd"),
+        (b"abc\xFF\x58", "replace",  "abc\ufffdX"),
     )
 
-class Test_SJISX0213(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase):
+    encoding = 'shift_jis_2004'
+    tstring = multibytecodec_support.load_teststring('shift_jis')
+    codectests = shiftjis_commonenctests + (
+        (b"\\\x7e", "strict", "\xa5\u203e"),
+        (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\\\u2016\u2212"),
+        (b"abc\xEA\xFC", "strict",  "abc\u64bf"),
+        (b"\x81\x39xy", "replace",  "\ufffd9xy"),
+        (b"\xFF\x58xy", "replace",  "\ufffdXxy"),
+        (b"\x80\x80\x82\x84xy", "replace", "\ufffd\ufffd\uff44xy"),
+        (b"\x80\x80\x82\x84\x88xy", "replace", "\ufffd\ufffd\uff44\u5864y"),
+        (b"\xFC\xFBxy", "replace", '\ufffd\u95b4y'),
+    )
+    xmlcharnametest = (
+        "\xab\u211c\xbb = \u2329\u1234\u232a",
+        b"\x85G&real;\x85Q = &lang;&#4660;&rang;"
+    )
+
+class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'shift_jisx0213'
-    tstring = test_multibytecodec_support.load_teststring('shift_jisx0213')
-    codectests = (
-        # invalid bytes
-        (b"abc\x80\x80\x82\x84", "strict",  None),
-        (b"abc\xf8", "strict",  None),
-        (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\uff44"),
-        (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"),
-        (b"abc\x80\x80\x82\x84def", "ignore",  "abc\uff44def"),
+    tstring = multibytecodec_support.load_teststring('shift_jisx0213')
+    codectests = shiftjis_commonenctests + (
+        (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"),
+        (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"),
+
         # sjis vs cp932
         (b"\\\x7e", "replace", "\xa5\u203e"),
         (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\x5c\u2016\u2212"),
diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py
index de4da7f5b6..4dd60499c8 100644
--- a/Lib/test/test_codecencodings_kr.py
+++ b/Lib/test/test_codecencodings_kr.py
@@ -5,30 +5,30 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class Test_CP949(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'cp949'
-    tstring = test_multibytecodec_support.load_teststring('cp949')
+    tstring = multibytecodec_support.load_teststring('cp949')
     codectests = (
         # invalid bytes
         (b"abc\x80\x80\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\uc894"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\uc894\ufffd"),
+        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\uc894"),
+        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\uc894"),
     )
 
-class Test_EUCKR(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'euc_kr'
-    tstring = test_multibytecodec_support.load_teststring('euc_kr')
+    tstring = multibytecodec_support.load_teststring('euc_kr')
     codectests = (
         # invalid bytes
         (b"abc\x80\x80\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\uc894"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\uc894\ufffd"),
+        (b"abc\x80\x80\xc1\xc4", "replace", 'abc\ufffd\ufffd\uc894'),
+        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\uc894"),
 
         # composed make-up sequence errors
@@ -40,26 +40,31 @@ class Test_EUCKR(test_multibytecodec_support.TestBase, unittest.TestCase):
         (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4", "strict", None),
         (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "strict", "\uc4d4"),
         (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4x", "strict", "\uc4d4x"),
-        (b"a\xa4\xd4\xa4\xb6\xa4", "replace", "a\ufffd"),
+        (b"a\xa4\xd4\xa4\xb6\xa4", "replace", 'a\ufffd'),
         (b"\xa4\xd4\xa3\xb6\xa4\xd0\xa4\xd4", "strict", None),
         (b"\xa4\xd4\xa4\xb6\xa3\xd0\xa4\xd4", "strict", None),
         (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa3\xd4", "strict", None),
-        (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", "\ufffd"),
-        (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", "\ufffd"),
-        (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", "\ufffd"),
+        (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", '\ufffd\u6e21\ufffd\u3160\ufffd'),
+        (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", '\ufffd\u6e21\ub544\ufffd\ufffd'),
+        (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", '\ufffd\u6e21\ub544\u572d\ufffd'),
+        (b"\xa4\xd4\xff\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "replace", '\ufffd\ufffd\ufffd\uc4d4'),
         (b"\xc1\xc4", "strict", "\uc894"),
     )
 
-class Test_JOHAB(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_JOHAB(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'johab'
-    tstring = test_multibytecodec_support.load_teststring('johab')
+    tstring = multibytecodec_support.load_teststring('johab')
     codectests = (
         # invalid bytes
         (b"abc\x80\x80\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ucd27"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ucd27\ufffd"),
+        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\ucd27"),
+        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\ucd27\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\ucd27"),
+        (b"\xD8abc", "replace",  "\uFFFDabc"),
+        (b"\xD8\xFFabc", "replace",  "\uFFFD\uFFFDabc"),
+        (b"\x84bxy", "replace",  "\uFFFDbxy"),
+        (b"\x8CBxy", "replace",  "\uFFFDBxy"),
     )
 
 def test_main():
diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py
index 12d3c9fa04..96245b74ed 100644
--- a/Lib/test/test_codecencodings_tw.py
+++ b/Lib/test/test_codecencodings_tw.py
@@ -5,18 +5,18 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class Test_Big5(test_multibytecodec_support.TestBase, unittest.TestCase):
+class Test_Big5(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'big5'
-    tstring = test_multibytecodec_support.load_teststring('big5')
+    tstring = multibytecodec_support.load_teststring('big5')
     codectests = (
         # invalid bytes
         (b"abc\x80\x80\xc1\xc4", "strict",  None),
         (b"abc\xc8", "strict",  None),
-        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u8b10"),
-        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u8b10\ufffd"),
+        (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"),
+        (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\u8b10"),
     )
 
diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py
index 063919d6d8..1a761cffc7 100644
--- a/Lib/test/test_codecmaps_cn.py
+++ b/Lib/test/test_codecmaps_cn.py
@@ -5,21 +5,21 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class TestGB2312Map(test_multibytecodec_support.TestBase_Mapping,
+class TestGB2312Map(multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'gb2312'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-CN.TXT'
 
-class TestGBKMap(test_multibytecodec_support.TestBase_Mapping,
+class TestGBKMap(multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'gbk'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/' \
                  'MICSFT/WINDOWS/CP936.TXT'
 
-class TestGB18030Map(test_multibytecodec_support.TestBase_Mapping,
+class TestGB18030Map(multibytecodec_support.TestBase_Mapping,
                      unittest.TestCase):
     encoding = 'gb18030'
     mapfileurl = 'http://source.icu-project.org/repos/icu/data/' \
diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py
index bbe1f2f905..5f4e7c7e02 100644
--- a/Lib/test/test_codecmaps_hk.py
+++ b/Lib/test/test_codecmaps_hk.py
@@ -5,10 +5,10 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class TestBig5HKSCSMap(test_multibytecodec_support.TestBase_Mapping,
+class TestBig5HKSCSMap(multibytecodec_support.TestBase_Mapping,
                        unittest.TestCase):
     encoding = 'big5hkscs'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/BIG5HKSCS-2004.TXT'
diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py
index 652bd81515..1fdbf634e3 100644
--- a/Lib/test/test_codecmaps_jp.py
+++ b/Lib/test/test_codecmaps_jp.py
@@ -5,10 +5,10 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class TestCP932Map(test_multibytecodec_support.TestBase_Mapping,
+class TestCP932Map(multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp932'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/' \
@@ -24,14 +24,14 @@ class TestCP932Map(test_multibytecodec_support.TestBase_Mapping,
         supmaps.append((bytes([i]), chr(i+0xfec0)))
 
 
-class TestEUCJPCOMPATMap(test_multibytecodec_support.TestBase_Mapping,
+class TestEUCJPCOMPATMap(multibytecodec_support.TestBase_Mapping,
                          unittest.TestCase):
     encoding = 'euc_jp'
     mapfilename = 'EUC-JP.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-JP.TXT'
 
 
-class TestSJISCOMPATMap(test_multibytecodec_support.TestBase_Mapping,
+class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping,
                         unittest.TestCase):
     encoding = 'shift_jis'
     mapfilename = 'SHIFTJIS.TXT'
@@ -46,14 +46,14 @@ class TestSJISCOMPATMap(test_multibytecodec_support.TestBase_Mapping,
         (b'\x81_', '\\'),
     ]
 
-class TestEUCJISX0213Map(test_multibytecodec_support.TestBase_Mapping,
+class TestEUCJISX0213Map(multibytecodec_support.TestBase_Mapping,
                          unittest.TestCase):
     encoding = 'euc_jisx0213'
     mapfilename = 'EUC-JISX0213.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-JISX0213.TXT'
 
 
-class TestSJISX0213Map(test_multibytecodec_support.TestBase_Mapping,
+class TestSJISX0213Map(multibytecodec_support.TestBase_Mapping,
                        unittest.TestCase):
     encoding = 'shift_jisx0213'
     mapfilename = 'SHIFT_JISX0213.TXT'
diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py
index d909c8bd58..03564025ed 100644
--- a/Lib/test/test_codecmaps_kr.py
+++ b/Lib/test/test_codecmaps_kr.py
@@ -5,17 +5,17 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class TestCP949Map(test_multibytecodec_support.TestBase_Mapping,
+class TestCP949Map(multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp949'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT' \
                  '/WINDOWS/CP949.TXT'
 
 
-class TestEUCKRMap(test_multibytecodec_support.TestBase_Mapping,
+class TestEUCKRMap(multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'euc_kr'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-KR.TXT'
@@ -25,7 +25,7 @@ class TestEUCKRMap(test_multibytecodec_support.TestBase_Mapping,
     pass_dectest = [(b'\xa4\xd4', '\u3164')]
 
 
-class TestJOHABMap(test_multibytecodec_support.TestBase_Mapping,
+class TestJOHABMap(multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'johab'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/' \
diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py
index 6db5091fc3..44467e378a 100644
--- a/Lib/test/test_codecmaps_tw.py
+++ b/Lib/test/test_codecmaps_tw.py
@@ -5,16 +5,16 @@
 #
 
 from test import support
-from test import test_multibytecodec_support
+from test import multibytecodec_support
 import unittest
 
-class TestBIG5Map(test_multibytecodec_support.TestBase_Mapping,
+class TestBIG5Map(multibytecodec_support.TestBase_Mapping,
                   unittest.TestCase):
     encoding = 'big5'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE/' \
                  'EASTASIA/OTHER/BIG5.TXT'
 
-class TestCP950Map(test_multibytecodec_support.TestBase_Mapping,
+class TestCP950Map(multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp950'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/' \
@@ -23,6 +23,9 @@ class TestCP950Map(test_multibytecodec_support.TestBase_Mapping,
         (b'\xa2\xcc', '\u5341'),
         (b'\xa2\xce', '\u5345'),
     ]
+    codectests = (
+        (b"\xFFxy", "replace",  "\ufffdxy"),
+    )
 
 def test_main():
     support.run_unittest(__name__)
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 43886fc446..c68088ed75 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1,8 +1,25 @@
-from test import support
-import unittest
+import _testcapi
 import codecs
+import io
 import locale
-import sys, _testcapi, io
+import sys
+import unittest
+import warnings
+
+from test import support
+
+if sys.platform == 'win32':
+    VISTA_OR_LATER = (sys.getwindowsversion().major >= 6)
+else:
+    VISTA_OR_LATER = False
+
+try:
+    import ctypes
+except ImportError:
+    ctypes = None
+    SIZEOF_WCHAR_T = -1
+else:
+    SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar)
 
 def coding_checker(self, coder):
     def check(input, expect):
@@ -62,7 +79,7 @@ class MixInCheckStateHandling:
             part2 = d.encode(u[i:], True)
             self.assertEqual(s, part1+part2)
 
-class ReadTest(unittest.TestCase, MixInCheckStateHandling):
+class ReadTest(MixInCheckStateHandling):
     def check_partial(self, input, partialresults):
         # get a StreamReader for the encoding and feed the bytestring version
         # of input to the reader byte by byte. Read everything available from
@@ -282,7 +299,7 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling):
         self.assertEqual(reader.readline(), s5)
         self.assertEqual(reader.readline(), "")
 
-class UTF32Test(ReadTest):
+class UTF32Test(ReadTest, unittest.TestCase):
     encoding = "utf-32"
 
     spamle = (b'\xff\xfe\x00\x00'
@@ -373,7 +390,7 @@ class UTF32Test(ReadTest):
         self.assertEqual('\U00010000' * 1024,
                          codecs.utf_32_decode(encoded_be)[0])
 
-class UTF32LETest(ReadTest):
+class UTF32LETest(ReadTest, unittest.TestCase):
     encoding = "utf-32-le"
 
     def test_partial(self):
@@ -417,7 +434,7 @@ class UTF32LETest(ReadTest):
         self.assertEqual('\U00010000' * 1024,
                          codecs.utf_32_le_decode(encoded)[0])
 
-class UTF32BETest(ReadTest):
+class UTF32BETest(ReadTest, unittest.TestCase):
     encoding = "utf-32-be"
 
     def test_partial(self):
@@ -462,7 +479,7 @@ class UTF32BETest(ReadTest):
                          codecs.utf_32_be_decode(encoded)[0])
 
 
-class UTF16Test(ReadTest):
+class UTF16Test(ReadTest, unittest.TestCase):
     encoding = "utf-16"
 
     spamle = b'\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00'
@@ -542,7 +559,7 @@ class UTF16Test(ReadTest):
         with codecs.open(support.TESTFN, 'U', encoding=self.encoding) as reader:
             self.assertEqual(reader.read(), s1)
 
-class UTF16LETest(ReadTest):
+class UTF16LETest(ReadTest, unittest.TestCase):
     encoding = "utf-16-le"
 
     def test_partial(self):
@@ -585,7 +602,7 @@ class UTF16LETest(ReadTest):
         self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding),
                          "\U00010203")
 
-class UTF16BETest(ReadTest):
+class UTF16BETest(ReadTest, unittest.TestCase):
     encoding = "utf-16-be"
 
     def test_partial(self):
@@ -628,7 +645,7 @@ class UTF16BETest(ReadTest):
         self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding),
                          "\U00010203")
 
-class UTF8Test(ReadTest):
+class UTF8Test(ReadTest, unittest.TestCase):
     encoding = "utf-8"
 
     def test_partial(self):
@@ -677,13 +694,118 @@ class UTF8Test(ReadTest):
                          b"abc\xed\xa0\x80def")
         self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"),
                          "abc\ud800def")
+        self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"),
+                         b"\xf0\x90\xbf\xbf\xed\xa0\x80")
+        self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"),
+                         "\U00010fff\uD800")
         self.assertTrue(codecs.lookup_error("surrogatepass"))
         with self.assertRaises(UnicodeDecodeError):
             b"abc\xed\xa0".decode("utf-8", "surrogatepass")
         with self.assertRaises(UnicodeDecodeError):
             b"abc\xed\xa0z".decode("utf-8", "surrogatepass")
 
-class UTF7Test(ReadTest):
+@unittest.skipUnless(sys.platform == 'win32',
+                     'cp65001 is a Windows-only codec')
+class CP65001Test(ReadTest, unittest.TestCase):
+    encoding = "cp65001"
+
+    def test_encode(self):
+        tests = [
+            ('abc', 'strict', b'abc'),
+            ('\xe9\u20ac', 'strict',  b'\xc3\xa9\xe2\x82\xac'),
+            ('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'),
+        ]
+        if VISTA_OR_LATER:
+            tests.extend((
+                ('\udc80', 'strict', None),
+                ('\udc80', 'ignore', b''),
+                ('\udc80', 'replace', b'?'),
+                ('\udc80', 'backslashreplace', b'\\udc80'),
+                ('\udc80', 'surrogatepass', b'\xed\xb2\x80'),
+            ))
+        else:
+            tests.append(('\udc80', 'strict', b'\xed\xb2\x80'))
+        for text, errors, expected in tests:
+            if expected is not None:
+                try:
+                    encoded = text.encode('cp65001', errors)
+                except UnicodeEncodeError as err:
+                    self.fail('Unable to encode %a to cp65001 with '
+                              'errors=%r: %s' % (text, errors, err))
+                self.assertEqual(encoded, expected,
+                    '%a.encode("cp65001", %r)=%a != %a'
+                    % (text, errors, encoded, expected))
+            else:
+                self.assertRaises(UnicodeEncodeError,
+                    text.encode, "cp65001", errors)
+
+    def test_decode(self):
+        tests = [
+            (b'abc', 'strict', 'abc'),
+            (b'\xc3\xa9\xe2\x82\xac', 'strict', '\xe9\u20ac'),
+            (b'\xf4\x8f\xbf\xbf', 'strict', '\U0010ffff'),
+            (b'\xef\xbf\xbd', 'strict', '\ufffd'),
+            (b'[\xc3\xa9]', 'strict', '[\xe9]'),
+            # invalid bytes
+            (b'[\xff]', 'strict', None),
+            (b'[\xff]', 'ignore', '[]'),
+            (b'[\xff]', 'replace', '[\ufffd]'),
+            (b'[\xff]', 'surrogateescape', '[\udcff]'),
+        ]
+        if VISTA_OR_LATER:
+            tests.extend((
+                (b'[\xed\xb2\x80]', 'strict', None),
+                (b'[\xed\xb2\x80]', 'ignore', '[]'),
+                (b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'),
+            ))
+        else:
+            tests.extend((
+                (b'[\xed\xb2\x80]', 'strict', '[\udc80]'),
+            ))
+        for raw, errors, expected in tests:
+            if expected is not None:
+                try:
+                    decoded = raw.decode('cp65001', errors)
+                except UnicodeDecodeError as err:
+                    self.fail('Unable to decode %a from cp65001 with '
+                              'errors=%r: %s' % (raw, errors, err))
+                self.assertEqual(decoded, expected,
+                    '%a.decode("cp65001", %r)=%a != %a'
+                    % (raw, errors, decoded, expected))
+            else:
+                self.assertRaises(UnicodeDecodeError,
+                    raw.decode, 'cp65001', errors)
+
+    @unittest.skipUnless(VISTA_OR_LATER, 'require Windows Vista or later')
+    def test_lone_surrogates(self):
+        self.assertRaises(UnicodeEncodeError, "\ud800".encode, "cp65001")
+        self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "cp65001")
+        self.assertEqual("[\uDC80]".encode("cp65001", "backslashreplace"),
+                         b'[\\udc80]')
+        self.assertEqual("[\uDC80]".encode("cp65001", "xmlcharrefreplace"),
+                         b'[&#56448;]')
+        self.assertEqual("[\uDC80]".encode("cp65001", "surrogateescape"),
+                         b'[\x80]')
+        self.assertEqual("[\uDC80]".encode("cp65001", "ignore"),
+                         b'[]')
+        self.assertEqual("[\uDC80]".encode("cp65001", "replace"),
+                         b'[?]')
+
+    @unittest.skipUnless(VISTA_OR_LATER, 'require Windows Vista or later')
+    def test_surrogatepass_handler(self):
+        self.assertEqual("abc\ud800def".encode("cp65001", "surrogatepass"),
+                         b"abc\xed\xa0\x80def")
+        self.assertEqual(b"abc\xed\xa0\x80def".decode("cp65001", "surrogatepass"),
+                         "abc\ud800def")
+        self.assertEqual("\U00010fff\uD800".encode("cp65001", "surrogatepass"),
+                         b"\xf0\x90\xbf\xbf\xed\xa0\x80")
+        self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("cp65001", "surrogatepass"),
+                         "\U00010fff\uD800")
+        self.assertTrue(codecs.lookup_error("surrogatepass"))
+
+
+
+class UTF7Test(ReadTest, unittest.TestCase):
     encoding = "utf-7"
 
     def test_partial(self):
@@ -722,7 +844,7 @@ class ReadBufferTest(unittest.TestCase):
         self.assertRaises(TypeError, codecs.readbuffer_encode)
         self.assertRaises(TypeError, codecs.readbuffer_encode, 42)
 
-class UTF8SigTest(ReadTest):
+class UTF8SigTest(ReadTest, unittest.TestCase):
     encoding = "utf-8-sig"
 
     def test_partial(self):
@@ -995,61 +1117,80 @@ class PunycodeTest(unittest.TestCase):
             self.assertEqual(uni, puny.decode("punycode"))
 
 class UnicodeInternalTest(unittest.TestCase):
+    @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
     def test_bug1251300(self):
         # Decoding with unicode_internal used to not correctly handle "code
         # points" above 0x10ffff on UCS-4 builds.
-        if sys.maxunicode > 0xffff:
-            ok = [
-                (b"\x00\x10\xff\xff", "\U0010ffff"),
-                (b"\x00\x00\x01\x01", "\U00000101"),
-                (b"", ""),
-            ]
-            not_ok = [
-                b"\x7f\xff\xff\xff",
-                b"\x80\x00\x00\x00",
-                b"\x81\x00\x00\x00",
-                b"\x00",
-                b"\x00\x00\x00\x00\x00",
-            ]
-            for internal, uni in ok:
-                if sys.byteorder == "little":
-                    internal = bytes(reversed(internal))
+        ok = [
+            (b"\x00\x10\xff\xff", "\U0010ffff"),
+            (b"\x00\x00\x01\x01", "\U00000101"),
+            (b"", ""),
+        ]
+        not_ok = [
+            b"\x7f\xff\xff\xff",
+            b"\x80\x00\x00\x00",
+            b"\x81\x00\x00\x00",
+            b"\x00",
+            b"\x00\x00\x00\x00\x00",
+        ]
+        for internal, uni in ok:
+            if sys.byteorder == "little":
+                internal = bytes(reversed(internal))
+            with support.check_warnings():
                 self.assertEqual(uni, internal.decode("unicode_internal"))
-            for internal in not_ok:
-                if sys.byteorder == "little":
-                    internal = bytes(reversed(internal))
+        for internal in not_ok:
+            if sys.byteorder == "little":
+                internal = bytes(reversed(internal))
+            with support.check_warnings(('unicode_internal codec has been '
+                                         'deprecated', DeprecationWarning)):
                 self.assertRaises(UnicodeDecodeError, internal.decode,
-                    "unicode_internal")
-
+                                  "unicode_internal")
+        if sys.byteorder == "little":
+            invalid = b"\x00\x00\x11\x00"
+        else:
+            invalid = b"\x00\x11\x00\x00"
+        with support.check_warnings():
+            self.assertRaises(UnicodeDecodeError,
+                              invalid.decode, "unicode_internal")
+        with support.check_warnings():
+            self.assertEqual(invalid.decode("unicode_internal", "replace"),
+                             '\ufffd')
+
+    @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
     def test_decode_error_attributes(self):
-        if sys.maxunicode > 0xffff:
-            try:
+        try:
+            with support.check_warnings(('unicode_internal codec has been '
+                                         'deprecated', DeprecationWarning)):
                 b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal")
-            except UnicodeDecodeError as ex:
-                self.assertEqual("unicode_internal", ex.encoding)
-                self.assertEqual(b"\x00\x00\x00\x00\x00\x11\x11\x00", ex.object)
-                self.assertEqual(4, ex.start)
-                self.assertEqual(8, ex.end)
-            else:
-                self.fail()
+        except UnicodeDecodeError as ex:
+            self.assertEqual("unicode_internal", ex.encoding)
+            self.assertEqual(b"\x00\x00\x00\x00\x00\x11\x11\x00", ex.object)
+            self.assertEqual(4, ex.start)
+            self.assertEqual(8, ex.end)
+        else:
+            self.fail()
 
+    @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
     def test_decode_callback(self):
-        if sys.maxunicode > 0xffff:
-            codecs.register_error("UnicodeInternalTest", codecs.ignore_errors)
-            decoder = codecs.getdecoder("unicode_internal")
+        codecs.register_error("UnicodeInternalTest", codecs.ignore_errors)
+        decoder = codecs.getdecoder("unicode_internal")
+        with support.check_warnings(('unicode_internal codec has been '
+                                     'deprecated', DeprecationWarning)):
             ab = "ab".encode("unicode_internal").decode()
             ignored = decoder(bytes("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]),
                                     "ascii"),
                               "UnicodeInternalTest")
-            self.assertEqual(("ab", 12), ignored)
+        self.assertEqual(("ab", 12), ignored)
 
     def test_encode_length(self):
-        # Issue 3739
-        encoder = codecs.getencoder("unicode_internal")
-        self.assertEqual(encoder("a")[1], 1)
-        self.assertEqual(encoder("\xe9\u0142")[1], 2)
+        with support.check_warnings(('unicode_internal codec has been '
+                                     'deprecated', DeprecationWarning)):
+            # Issue 3739
+            encoder = codecs.getencoder("unicode_internal")
+            self.assertEqual(encoder("a")[1], 1)
+            self.assertEqual(encoder("\xe9\u0142")[1], 2)
 
-        self.assertEqual(codecs.escape_encode(br'\x00')[1], 4)
+            self.assertEqual(codecs.escape_encode(br'\x00')[1], 4)
 
 # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
 nameprep_tests = [
@@ -1373,7 +1514,7 @@ class EncodedFileTest(unittest.TestCase):
         self.assertEqual(ef.read(), b'\\\xd5\n\x00\x00\xae')
 
         f = io.BytesIO()
-        ef = codecs.EncodedFile(f, 'utf-8', 'latin1')
+        ef = codecs.EncodedFile(f, 'utf-8', 'latin-1')
         ef.write(b'\xc3\xbc')
         self.assertEqual(f.getvalue(), b'\xfc')
 
@@ -1505,10 +1646,13 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
             elif encoding == "latin_1":
                 name = "latin_1"
             self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-"))
-            (b, size) = codecs.getencoder(encoding)(s)
-            self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
-            (chars, size) = codecs.getdecoder(encoding)(b)
-            self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
+
+            with support.check_warnings():
+                # unicode-internal has been deprecated
+                (b, size) = codecs.getencoder(encoding)(s)
+                self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
+                (chars, size) = codecs.getdecoder(encoding)(b)
+                self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
 
             if encoding not in broken_unicode_with_streams:
                 # check stream reader/writer
@@ -1612,7 +1756,9 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
     def test_bad_encode_args(self):
         for encoding in all_unicode_encodings:
             encoder = codecs.getencoder(encoding)
-            self.assertRaises(TypeError, encoder)
+            with support.check_warnings():
+                # unicode-internal has been deprecated
+                self.assertRaises(TypeError, encoder)
 
     def test_encoding_map_type_initialized(self):
         from encodings import cp1140
@@ -1635,6 +1781,11 @@ class CharmapTest(unittest.TestCase):
             ("abc", 3)
         )
 
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict", "\U0010FFFFbc"),
+            ("\U0010FFFFbc", 3)
+        )
+
         self.assertRaises(UnicodeDecodeError,
             codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
         )
@@ -1772,9 +1923,15 @@ class CharmapTest(unittest.TestCase):
             ("\U0010FFFFbc", 3)
         )
 
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: sys.maxunicode, 1: b, 2: c}),
+            (chr(sys.maxunicode) + "bc", 3)
+        )
+
         self.assertRaises(TypeError,
             codecs.charmap_decode, b"\x00\x01\x02", "strict",
-                                   {0: 0x110000, 1: b, 2: c}
+                                   {0: sys.maxunicode + 1, 1: b, 2: c}
         )
 
         self.assertRaises(UnicodeDecodeError,
@@ -1855,6 +2012,12 @@ class TypesTest(unittest.TestCase):
         self.assertEqual(codecs.raw_unicode_escape_decode(r"\u1234"), ("\u1234", 6))
         self.assertEqual(codecs.raw_unicode_escape_decode(br"\u1234"), ("\u1234", 6))
 
+        self.assertRaises(UnicodeDecodeError, codecs.unicode_escape_decode, br"\U00110000")
+        self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
+
+        self.assertRaises(UnicodeDecodeError, codecs.raw_unicode_escape_decode, br"\U00110000")
+        self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
+
 
 class UnicodeEscapeTest(unittest.TestCase):
     def test_empty(self):
@@ -2014,7 +2177,7 @@ class SurrogateEscapeTest(unittest.TestCase):
 
     def test_latin1(self):
         # Issue6373
-        self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin1", "surrogateescape"),
+        self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape"),
                          b"\xe4\xeb\xef\xf6\xfc")
 
 
@@ -2123,39 +2286,154 @@ class TransformCodecTest(unittest.TestCase):
             self.assertEqual(sout, b"\x80")
 
 
-def test_main():
-    support.run_unittest(
-        UTF32Test,
-        UTF32LETest,
-        UTF32BETest,
-        UTF16Test,
-        UTF16LETest,
-        UTF16BETest,
-        UTF8Test,
-        UTF8SigTest,
-        EscapeDecodeTest,
-        UTF7Test,
-        UTF16ExTest,
-        ReadBufferTest,
-        RecodingTest,
-        PunycodeTest,
-        UnicodeInternalTest,
-        NameprepTest,
-        IDNACodecTest,
-        CodecsModuleTest,
-        StreamReaderTest,
-        EncodedFileTest,
-        BasicUnicodeTest,
-        CharmapTest,
-        WithStmtTest,
-        TypesTest,
-        UnicodeEscapeTest,
-        RawUnicodeEscapeTest,
-        SurrogateEscapeTest,
-        BomTest,
-        TransformCodecTest,
-    )
+@unittest.skipUnless(sys.platform == 'win32',
+                     'code pages are specific to Windows')
+class CodePageTest(unittest.TestCase):
+    # CP_UTF8 is already tested by CP65001Test
+    CP_UTF8 = 65001
+
+    def test_invalid_code_page(self):
+        self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a')
+        self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a')
+        self.assertRaises(WindowsError, codecs.code_page_encode, 123, 'a')
+        self.assertRaises(WindowsError, codecs.code_page_decode, 123, b'a')
+
+    def test_code_page_name(self):
+        self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
+            codecs.code_page_encode, 932, '\xff')
+        self.assertRaisesRegex(UnicodeDecodeError, 'cp932',
+            codecs.code_page_decode, 932, b'\x81\x00')
+        self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8',
+            codecs.code_page_decode, self.CP_UTF8, b'\xff')
+
+    def check_decode(self, cp, tests):
+        for raw, errors, expected in tests:
+            if expected is not None:
+                try:
+                    decoded = codecs.code_page_decode(cp, raw, errors)
+                except UnicodeDecodeError as err:
+                    self.fail('Unable to decode %a from "cp%s" with '
+                              'errors=%r: %s' % (raw, cp, errors, err))
+                self.assertEqual(decoded[0], expected,
+                    '%a.decode("cp%s", %r)=%a != %a'
+                    % (raw, cp, errors, decoded[0], expected))
+                # assert 0 <= decoded[1] <= len(raw)
+                self.assertGreaterEqual(decoded[1], 0)
+                self.assertLessEqual(decoded[1], len(raw))
+            else:
+                self.assertRaises(UnicodeDecodeError,
+                    codecs.code_page_decode, cp, raw, errors)
+
+    def check_encode(self, cp, tests):
+        for text, errors, expected in tests:
+            if expected is not None:
+                try:
+                    encoded = codecs.code_page_encode(cp, text, errors)
+                except UnicodeEncodeError as err:
+                    self.fail('Unable to encode %a to "cp%s" with '
+                              'errors=%r: %s' % (text, cp, errors, err))
+                self.assertEqual(encoded[0], expected,
+                    '%a.encode("cp%s", %r)=%a != %a'
+                    % (text, cp, errors, encoded[0], expected))
+                self.assertEqual(encoded[1], len(text))
+            else:
+                self.assertRaises(UnicodeEncodeError,
+                    codecs.code_page_encode, cp, text, errors)
+
+    def test_cp932(self):
+        self.check_encode(932, (
+            ('abc', 'strict', b'abc'),
+            ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'),
+            # test error handlers
+            ('\xff', 'strict', None),
+            ('[\xff]', 'ignore', b'[]'),
+            ('[\xff]', 'replace', b'[y]'),
+            ('[\u20ac]', 'replace', b'[?]'),
+            ('[\xff]', 'backslashreplace', b'[\\xff]'),
+            ('[\xff]', 'xmlcharrefreplace', b'[&#255;]'),
+        ))
+        self.check_decode(932, (
+            (b'abc', 'strict', 'abc'),
+            (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'),
+            # invalid bytes
+            (b'[\xff]', 'strict', None),
+            (b'[\xff]', 'ignore', '[]'),
+            (b'[\xff]', 'replace', '[\ufffd]'),
+            (b'[\xff]', 'surrogateescape', '[\udcff]'),
+            (b'\x81\x00abc', 'strict', None),
+            (b'\x81\x00abc', 'ignore', '\x00abc'),
+            (b'\x81\x00abc', 'replace', '\ufffd\x00abc'),
+        ))
+
+    def test_cp1252(self):
+        self.check_encode(1252, (
+            ('abc', 'strict', b'abc'),
+            ('\xe9\u20ac', 'strict',  b'\xe9\x80'),
+            ('\xff', 'strict', b'\xff'),
+            ('\u0141', 'strict', None),
+            ('\u0141', 'ignore', b''),
+            ('\u0141', 'replace', b'L'),
+        ))
+        self.check_decode(1252, (
+            (b'abc', 'strict', 'abc'),
+            (b'\xe9\x80', 'strict', '\xe9\u20ac'),
+            (b'\xff', 'strict', '\xff'),
+        ))
+
+    def test_cp_utf7(self):
+        cp = 65000
+        self.check_encode(cp, (
+            ('abc', 'strict', b'abc'),
+            ('\xe9\u20ac', 'strict',  b'+AOkgrA-'),
+            ('\U0010ffff', 'strict',  b'+2//f/w-'),
+            ('\udc80', 'strict', b'+3IA-'),
+            ('\ufffd', 'strict', b'+//0-'),
+        ))
+        self.check_decode(cp, (
+            (b'abc', 'strict', 'abc'),
+            (b'+AOkgrA-', 'strict', '\xe9\u20ac'),
+            (b'+2//f/w-', 'strict', '\U0010ffff'),
+            (b'+3IA-', 'strict', '\udc80'),
+            (b'+//0-', 'strict', '\ufffd'),
+            # invalid bytes
+            (b'[+/]', 'strict', '[]'),
+            (b'[\xff]', 'strict', '[\xff]'),
+        ))
+
+    def test_multibyte_encoding(self):
+        self.check_decode(932, (
+            (b'\x84\xe9\x80', 'ignore', '\u9a3e'),
+            (b'\x84\xe9\x80', 'replace', '\ufffd\u9a3e'),
+        ))
+        self.check_decode(self.CP_UTF8, (
+            (b'\xff\xf4\x8f\xbf\xbf', 'ignore', '\U0010ffff'),
+            (b'\xff\xf4\x8f\xbf\xbf', 'replace', '\ufffd\U0010ffff'),
+        ))
+        if VISTA_OR_LATER:
+            self.check_encode(self.CP_UTF8, (
+                ('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'),
+                ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'),
+            ))
+
+    def test_incremental(self):
+        decoded = codecs.code_page_decode(932, b'\x82', 'strict', False)
+        self.assertEqual(decoded, ('', 0))
+
+        decoded = codecs.code_page_decode(932,
+                                          b'\xe9\x80\xe9', 'strict',
+                                          False)
+        self.assertEqual(decoded, ('\u9a3e', 2))
+
+        decoded = codecs.code_page_decode(932,
+                                          b'\xe9\x80\xe9\x80', 'strict',
+                                          False)
+        self.assertEqual(decoded, ('\u9a3e\u9a3e', 4))
+
+        decoded = codecs.code_page_decode(932,
+                                          b'abc', 'strict',
+                                          False)
+        self.assertEqual(decoded, ('abc', 3))
 
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_coding.py b/Lib/test/test_coding.py
index f9db0b40f0..dfd5431dee 100644
--- a/Lib/test/test_coding.py
+++ b/Lib/test/test_coding.py
@@ -1,7 +1,6 @@
-
 import test.support, unittest
 from test.support import TESTFN, unlink, unload
-import os, sys
+import importlib, os, sys
 
 class CodingTest(unittest.TestCase):
     def test_bad_coding(self):
@@ -40,6 +39,7 @@ class CodingTest(unittest.TestCase):
             f.write("'A very long string %s'\n" % ("X" * 1000))
             f.close()
 
+            importlib.invalidate_caches()
             __import__(TESTFN)
         finally:
             f.close()
diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py
index b2a5f05260..8850e8bc13 100644
--- a/Lib/test/test_collections.py
+++ b/Lib/test/test_collections.py
@@ -1,6 +1,7 @@
 """Unit tests for collections.py."""
 
 import unittest, doctest, operator
+from test.support import TESTFN, forget, unlink
 import inspect
 from test import support
 from collections import namedtuple, Counter, OrderedDict, _count_elements
@@ -10,21 +11,20 @@ from random import randrange, shuffle
 import keyword
 import re
 import sys
-from collections import _ChainMap
-from collections import Hashable, Iterable, Iterator
-from collections import Sized, Container, Callable
-from collections import Set, MutableSet
-from collections import Mapping, MutableMapping, KeysView, ItemsView, UserDict
-from collections import Sequence, MutableSequence
-from collections import ByteString
+from collections import UserDict
+from collections import ChainMap
+from collections.abc import Hashable, Iterable, Iterator
+from collections.abc import Sized, Container, Callable
+from collections.abc import Set, MutableSet
+from collections.abc import Mapping, MutableMapping, KeysView, ItemsView
+from collections.abc import Sequence, MutableSequence
+from collections.abc import ByteString
 
 
 ################################################################################
-### _ChainMap (helper class for configparser)
+### ChainMap (helper class for configparser and the string module)
 ################################################################################
 
-ChainMap = _ChainMap  # rename to keep test code in sync with 3.3 version
-
 class TestChainMap(unittest.TestCase):
 
     def test_basics(self):
@@ -128,6 +128,7 @@ class TestNamedTuple(unittest.TestCase):
         self.assertEqual(Point.__module__, __name__)
         self.assertEqual(Point.__getitem__, tuple.__getitem__)
         self.assertEqual(Point._fields, ('x', 'y'))
+        self.assertIn('class Point(tuple)', Point._source)
 
         self.assertRaises(ValueError, namedtuple, 'abc%', 'efg ghi')       # type has non-alpha char
         self.assertRaises(ValueError, namedtuple, 'class', 'efg ghi')      # type has keyword
@@ -327,6 +328,17 @@ class TestNamedTuple(unittest.TestCase):
             pass
         self.assertEqual(repr(B(1)), 'B(x=1)')
 
+    def test_source(self):
+        # verify that _source can be run through exec()
+        tmp = namedtuple('NTColor', 'red green blue')
+        globals().pop('NTColor', None)          # remove artifacts from other tests
+        exec(tmp._source, globals())
+        self.assertIn('NTColor', globals())
+        c = NTColor(10, 20, 30)
+        self.assertEqual((c.red, c.green, c.blue), (10, 20, 30))
+        self.assertEqual(NTColor._fields, ('red', 'green', 'blue'))
+        globals().pop('NTColor', None)          # clean-up after this test
+
 
 ################################################################################
 ### Abstract Base Classes
@@ -762,6 +774,44 @@ class TestCollectionABCs(ABCTestCase):
         self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__',
             '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert')
 
+    def test_MutableSequence_mixins(self):
+        # Test the mixins of MutableSequence by creating a miminal concrete
+        # class inherited from it.
+        class MutableSequenceSubclass(MutableSequence):
+            def __init__(self):
+                self.lst = []
+
+            def __setitem__(self, index, value):
+                self.lst[index] = value
+
+            def __getitem__(self, index):
+                return self.lst[index]
+
+            def __len__(self):
+                return len(self.lst)
+
+            def __delitem__(self, index):
+                del self.lst[index]
+
+            def insert(self, index, value):
+                self.lst.insert(index, value)
+
+        mss = MutableSequenceSubclass()
+        mss.append(0)
+        mss.extend((1, 2, 3, 4))
+        self.assertEqual(len(mss), 5)
+        self.assertEqual(mss[3], 3)
+        mss.reverse()
+        self.assertEqual(mss[3], 1)
+        mss.pop()
+        self.assertEqual(len(mss), 4)
+        mss.remove(3)
+        self.assertEqual(len(mss), 3)
+        mss += (10, 20, 30)
+        self.assertEqual(len(mss), 6)
+        self.assertEqual(mss[-1], 30)
+        mss.clear()
+        self.assertEqual(len(mss), 0)
 
 ################################################################################
 ### Counter
@@ -915,6 +965,27 @@ class TestCounter(unittest.TestCase):
                 set_result = setop(set(p.elements()), set(q.elements()))
                 self.assertEqual(counter_result, dict.fromkeys(set_result, 1))
 
+    def test_inplace_operations(self):
+        elements = 'abcd'
+        for i in range(1000):
+            # test random pairs of multisets
+            p = Counter(dict((elem, randrange(-2,4)) for elem in elements))
+            p.update(e=1, f=-1, g=0)
+            q = Counter(dict((elem, randrange(-2,4)) for elem in elements))
+            q.update(h=1, i=-1, j=0)
+            for inplace_op, regular_op in [
+                (Counter.__iadd__, Counter.__add__),
+                (Counter.__isub__, Counter.__sub__),
+                (Counter.__ior__, Counter.__or__),
+                (Counter.__iand__, Counter.__and__),
+            ]:
+                c = p.copy()
+                c_id = id(c)
+                regular_result = regular_op(c, q)
+                inplace_result = inplace_op(c, q)
+                self.assertEqual(inplace_result, regular_result)
+                self.assertEqual(id(inplace_result), c_id)
+
     def test_subtract(self):
         c = Counter(a=-5, b=0, c=5, d=10, e=15,g=40)
         c.subtract(a=1, b=2, c=-3, d=10, e=20, f=30, h=-50)
@@ -926,6 +997,11 @@ class TestCounter(unittest.TestCase):
         c.subtract('aaaabbcce')
         self.assertEqual(c, Counter(a=-1, b=0, c=-1, d=1, e=-1))
 
+    def test_unary(self):
+        c = Counter(a=-5, b=0, c=5, d=10, e=15,g=40)
+        self.assertEqual(dict(+c), dict(c=5, d=10, e=15, g=40))
+        self.assertEqual(dict(-c), dict(a=5))
+
     def test_repr_nonsortable(self):
         c = Counter(a=2, b=None)
         r = repr(c)
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 58ef2979a0..1071f4ab1a 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -1,10 +1,17 @@
 import unittest
 import sys
 import _ast
+import types
 from test import support
 
 class TestSpecifics(unittest.TestCase):
 
+    def compile_single(self, source):
+        compile(source, "<single>", "single")
+
+    def assertInvalidSingle(self, source):
+        self.assertRaises(SyntaxError, self.compile_single, source)
+
     def test_no_ending_newline(self):
         compile("hi", "<test>", "exec")
         compile("hi\r", "<test>", "exec")
@@ -433,6 +440,66 @@ if 1:
         ast.body = [_ast.BoolOp()]
         self.assertRaises(TypeError, compile, ast, '<ast>', 'exec')
 
+    @support.cpython_only
+    def test_same_filename_used(self):
+        s = """def f(): pass\ndef g(): pass"""
+        c = compile(s, "myfile", "exec")
+        for obj in c.co_consts:
+            if isinstance(obj, types.CodeType):
+                self.assertIs(obj.co_filename, c.co_filename)
+
+    def test_single_statement(self):
+        self.compile_single("1 + 2")
+        self.compile_single("\n1 + 2")
+        self.compile_single("1 + 2\n")
+        self.compile_single("1 + 2\n\n")
+        self.compile_single("1 + 2\t\t\n")
+        self.compile_single("1 + 2\t\t\n        ")
+        self.compile_single("1 + 2 # one plus two")
+        self.compile_single("1; 2")
+        self.compile_single("import sys; sys")
+        self.compile_single("def f():\n   pass")
+        self.compile_single("while False:\n   pass")
+        self.compile_single("if x:\n   f(x)")
+        self.compile_single("if x:\n   f(x)\nelse:\n   g(x)")
+        self.compile_single("class T:\n   pass")
+
+    def test_bad_single_statement(self):
+        self.assertInvalidSingle('1\n2')
+        self.assertInvalidSingle('def f(): pass')
+        self.assertInvalidSingle('a = 13\nb = 187')
+        self.assertInvalidSingle('del x\ndel y')
+        self.assertInvalidSingle('f()\ng()')
+        self.assertInvalidSingle('f()\n# blah\nblah()')
+        self.assertInvalidSingle('f()\nxy # blah\nblah()')
+        self.assertInvalidSingle('x = 5 # comment\nx = 6\n')
+
+    @support.cpython_only
+    def test_compiler_recursion_limit(self):
+        # Expected limit is sys.getrecursionlimit() * the scaling factor
+        # in symtable.c (currently 3)
+        # We expect to fail *at* that limit, because we use up some of
+        # the stack depth limit in the test suite code
+        # So we check the expected limit and 75% of that
+        # XXX (ncoghlan): duplicating the scaling factor here is a little
+        # ugly. Perhaps it should be exposed somewhere...
+        fail_depth = sys.getrecursionlimit() * 3
+        success_depth = int(fail_depth * 0.75)
+
+        def check_limit(prefix, repeated):
+            expect_ok = prefix + repeated * success_depth
+            self.compile_single(expect_ok)
+            broken = prefix + repeated * fail_depth
+            details = "Compiling ({!r} + {!r} * {})".format(
+                         prefix, repeated, fail_depth)
+            with self.assertRaises(RuntimeError, msg=details):
+                self.compile_single(broken)
+
+        check_limit("a", "()")
+        check_limit("a", ".b")
+        check_limit("a", "[0]")
+        check_limit("a", "*a")
+
 
 def test_main():
     support.run_unittest(TestSpecifics)
diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py
index 2afa93802e..6ae450df06 100644
--- a/Lib/test/test_concurrent_futures.py
+++ b/Lib/test/test_concurrent_futures.py
@@ -19,7 +19,7 @@ import unittest
 from concurrent import futures
 from concurrent.futures._base import (
     PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future)
-import concurrent.futures.process
+from concurrent.futures.process import BrokenProcessPool
 
 
 def create_future(state=PENDING, exception=None, result=None):
@@ -34,7 +34,7 @@ PENDING_FUTURE = create_future(state=PENDING)
 RUNNING_FUTURE = create_future(state=RUNNING)
 CANCELLED_FUTURE = create_future(state=CANCELLED)
 CANCELLED_AND_NOTIFIED_FUTURE = create_future(state=CANCELLED_AND_NOTIFIED)
-EXCEPTION_FUTURE = create_future(state=FINISHED, exception=IOError())
+EXCEPTION_FUTURE = create_future(state=FINISHED, exception=OSError())
 SUCCESSFUL_FUTURE = create_future(state=FINISHED, result=42)
 
 
@@ -160,7 +160,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest):
         processes = self.executor._processes
         self.executor.shutdown()
 
-        for p in processes:
+        for p in processes.values():
             p.join()
 
     def test_context_manager_shutdown(self):
@@ -169,7 +169,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest):
             self.assertEqual(list(e.map(abs, range(-5, 5))),
                              [5, 4, 3, 2, 1, 0, 1, 2, 3, 4])
 
-        for p in processes:
+        for p in processes.values():
             p.join()
 
     def test_del_shutdown(self):
@@ -180,7 +180,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest):
         del executor
 
         queue_management_thread.join()
-        for p in processes:
+        for p in processes.values():
             p.join()
 
 
@@ -268,14 +268,14 @@ class WaitTests(unittest.TestCase):
 
     def test_timeout(self):
         future1 = self.executor.submit(mul, 6, 7)
-        future2 = self.executor.submit(time.sleep, 3)
+        future2 = self.executor.submit(time.sleep, 6)
 
         finished, pending = futures.wait(
                 [CANCELLED_AND_NOTIFIED_FUTURE,
                  EXCEPTION_FUTURE,
                  SUCCESSFUL_FUTURE,
                  future1, future2],
-                timeout=1.5,
+                timeout=5,
                 return_when=futures.ALL_COMPLETED)
 
         self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
@@ -379,8 +379,8 @@ class ExecutorTest(unittest.TestCase):
         results = []
         try:
             for i in self.executor.map(time.sleep,
-                                       [0, 0, 3],
-                                       timeout=1.5):
+                                       [0, 0, 6],
+                                       timeout=5):
                 results.append(i)
         except futures.TimeoutError:
             pass
@@ -389,13 +389,38 @@ class ExecutorTest(unittest.TestCase):
 
         self.assertEqual([None, None], results)
 
+    def test_shutdown_race_issue12456(self):
+        # Issue #12456: race condition at shutdown where trying to post a
+        # sentinel in the call queue blocks (the queue is full while processes
+        # have exited).
+        self.executor.map(str, [2] * (self.worker_count + 1))
+        self.executor.shutdown()
+
 
 class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest):
-    pass
+    def test_map_submits_without_iteration(self):
+        """Tests verifying issue 11777."""
+        finished = []
+        def record_finished(n):
+            finished.append(n)
+
+        self.executor.map(record_finished, range(10))
+        self.executor.shutdown(wait=True)
+        self.assertCountEqual(finished, range(10))
 
 
 class ProcessPoolExecutorTest(ProcessPoolMixin, ExecutorTest):
-    pass
+    def test_killed_child(self):
+        # When a child process is abruptly terminated, the whole pool gets
+        # "broken".
+        futures = [self.executor.submit(time.sleep, 3)]
+        # Get one of the processes, and terminate (kill) it
+        p = next(iter(self.executor._processes.values()))
+        p.terminate()
+        for fut in futures:
+            self.assertRaises(BrokenProcessPool, fut.result)
+        # Submitting other jobs fails as well.
+        self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8)
 
 
 class FutureTests(unittest.TestCase):
@@ -498,7 +523,7 @@ class FutureTests(unittest.TestCase):
                          '<Future at 0x[0-9a-f]+ state=cancelled>')
         self.assertRegex(
                 repr(EXCEPTION_FUTURE),
-                '<Future at 0x[0-9a-f]+ state=finished raised IOError>')
+                '<Future at 0x[0-9a-f]+ state=finished raised OSError>')
         self.assertRegex(
                 repr(SUCCESSFUL_FUTURE),
                 '<Future at 0x[0-9a-f]+ state=finished returned int>')
@@ -509,7 +534,7 @@ class FutureTests(unittest.TestCase):
         f2 = create_future(state=RUNNING)
         f3 = create_future(state=CANCELLED)
         f4 = create_future(state=CANCELLED_AND_NOTIFIED)
-        f5 = create_future(state=FINISHED, exception=IOError())
+        f5 = create_future(state=FINISHED, exception=OSError())
         f6 = create_future(state=FINISHED, result=5)
 
         self.assertTrue(f1.cancel())
@@ -563,7 +588,7 @@ class FutureTests(unittest.TestCase):
                           CANCELLED_FUTURE.result, timeout=0)
         self.assertRaises(futures.CancelledError,
                           CANCELLED_AND_NOTIFIED_FUTURE.result, timeout=0)
-        self.assertRaises(IOError, EXCEPTION_FUTURE.result, timeout=0)
+        self.assertRaises(OSError, EXCEPTION_FUTURE.result, timeout=0)
         self.assertEqual(SUCCESSFUL_FUTURE.result(timeout=0), 42)
 
     def test_result_with_success(self):
@@ -602,7 +627,7 @@ class FutureTests(unittest.TestCase):
         self.assertRaises(futures.CancelledError,
                           CANCELLED_AND_NOTIFIED_FUTURE.exception, timeout=0)
         self.assertTrue(isinstance(EXCEPTION_FUTURE.exception(timeout=0),
-                                   IOError))
+                                   OSError))
         self.assertEqual(SUCCESSFUL_FUTURE.exception(timeout=0), None)
 
     def test_exception_with_success(self):
@@ -611,14 +636,14 @@ class FutureTests(unittest.TestCase):
             time.sleep(1)
             with f1._condition:
                 f1._state = FINISHED
-                f1._exception = IOError()
+                f1._exception = OSError()
                 f1._condition.notify_all()
 
         f1 = create_future(state=PENDING)
         t = threading.Thread(target=notification)
         t.start()
 
-        self.assertTrue(isinstance(f1.exception(timeout=5), IOError))
+        self.assertTrue(isinstance(f1.exception(timeout=5), OSError))
 
 @test.support.reap_threads
 def test_main():
@@ -631,7 +656,8 @@ def test_main():
                                   ThreadPoolAsCompletedTests,
                                   FutureTests,
                                   ProcessPoolShutdownTest,
-                                  ThreadPoolShutdownTest)
+                                  ThreadPoolShutdownTest,
+                                  )
     finally:
         test.support.reap_children()
 
diff --git a/Lib/test/test_cfgparser.py b/Lib/test/test_configparser.py
index cec9b44e19..fc56550644 100644
--- a/Lib/test/test_cfgparser.py
+++ b/Lib/test/test_configparser.py
@@ -32,7 +32,7 @@ class SortedDict(collections.UserDict):
     __iter__ = iterkeys
 
 
-class CfgParserTestCaseClass(unittest.TestCase):
+class CfgParserTestCaseClass:
     allow_no_value = False
     delimiters = ('=', ':')
     comment_prefixes = (';', '#')
@@ -822,14 +822,20 @@ boolean {0[0]} NO
         self.assertEqual(set(cf['section3'].keys()), {'named'})
         self.assertNotIn('name3', cf['section3'])
         self.assertEqual(cf.sections(), ['section1', 'section2', 'section3'])
+        cf[self.default_section] = {}
+        self.assertEqual(set(cf[self.default_section].keys()), set())
+        self.assertEqual(set(cf['section1'].keys()), {'name1'})
+        self.assertEqual(set(cf['section2'].keys()), {'name22'})
+        self.assertEqual(set(cf['section3'].keys()), set())
+        self.assertEqual(cf.sections(), ['section1', 'section2', 'section3'])
 
 
-class StrictTestCase(BasicTestCase):
+class StrictTestCase(BasicTestCase, unittest.TestCase):
     config_class = configparser.RawConfigParser
     strict = True
 
 
-class ConfigParserTestCase(BasicTestCase):
+class ConfigParserTestCase(BasicTestCase, unittest.TestCase):
     config_class = configparser.ConfigParser
 
     def test_interpolation(self):
@@ -918,7 +924,7 @@ class ConfigParserTestCase(BasicTestCase):
         self.assertRaises(ValueError, cf.add_section, self.default_section)
 
 
-class ConfigParserTestCaseNoInterpolation(BasicTestCase):
+class ConfigParserTestCaseNoInterpolation(BasicTestCase, unittest.TestCase):
     config_class = configparser.ConfigParser
     interpolation = None
     ini = textwrap.dedent("""
@@ -983,7 +989,7 @@ class ConfigParserTestCaseNonStandardDelimiters(ConfigParserTestCase):
 class ConfigParserTestCaseNonStandardDefaultSection(ConfigParserTestCase):
     default_section = 'general'
 
-class MultilineValuesTestCase(BasicTestCase):
+class MultilineValuesTestCase(BasicTestCase, unittest.TestCase):
     config_class = configparser.ConfigParser
     wonderful_spam = ("I'm having spam spam spam spam "
                       "spam spam spam beaked beans spam "
@@ -1011,7 +1017,7 @@ class MultilineValuesTestCase(BasicTestCase):
         self.assertEqual(cf_from_file.get('section8', 'lovely_spam4'),
                          self.wonderful_spam.replace('\t\n', '\n'))
 
-class RawConfigParserTestCase(BasicTestCase):
+class RawConfigParserTestCase(BasicTestCase, unittest.TestCase):
     config_class = configparser.RawConfigParser
 
     def test_interpolation(self):
@@ -1058,7 +1064,7 @@ class RawConfigParserTestCaseNonStandardDelimiters(RawConfigParserTestCase):
     comment_prefixes = ('//', '"')
     inline_comment_prefixes = ('//', '"')
 
-class RawConfigParserTestSambaConf(CfgParserTestCaseClass):
+class RawConfigParserTestSambaConf(CfgParserTestCaseClass, unittest.TestCase):
     config_class = configparser.RawConfigParser
     comment_prefixes = ('#', ';', '----')
     inline_comment_prefixes = ('//',)
@@ -1078,7 +1084,7 @@ class RawConfigParserTestSambaConf(CfgParserTestCaseClass):
         self.assertEqual(cf.get("global", "hosts allow"), "127.")
         self.assertEqual(cf.get("tmp", "echo command"), "cat %s; rm %s")
 
-class ConfigParserTestCaseExtendedInterpolation(BasicTestCase):
+class ConfigParserTestCaseExtendedInterpolation(BasicTestCase, unittest.TestCase):
     config_class = configparser.ConfigParser
     interpolation = configparser.ExtendedInterpolation()
     default_section = 'common'
@@ -1252,7 +1258,7 @@ class ConfigParserTestCaseExtendedInterpolation(BasicTestCase):
 class ConfigParserTestCaseNoValue(ConfigParserTestCase):
     allow_no_value = True
 
-class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass):
+class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase):
     config_class = configparser.ConfigParser
     delimiters = {'='}
     comment_prefixes = {'#'}
@@ -1349,7 +1355,7 @@ class SortedTestCase(RawConfigParserTestCase):
                          "o4 = 1\n\n")
 
 
-class CompatibleTestCase(CfgParserTestCaseClass):
+class CompatibleTestCase(CfgParserTestCaseClass, unittest.TestCase):
     config_class = configparser.RawConfigParser
     comment_prefixes = '#;'
     inline_comment_prefixes = ';'
@@ -1371,7 +1377,7 @@ class CompatibleTestCase(CfgParserTestCaseClass):
         self.assertEqual(cf.get('Commented Bar', 'quirk'),
                          'this;is not a comment')
 
-class CopyTestCase(BasicTestCase):
+class CopyTestCase(BasicTestCase, unittest.TestCase):
     config_class = configparser.ConfigParser
 
     def fromstring(self, string, defaults=None):
@@ -1671,26 +1677,41 @@ class ExceptionPicklingTestCase(unittest.TestCase):
         self.assertEqual(repr(e1), repr(e2))
 
 
-def test_main():
-    support.run_unittest(
-        ConfigParserTestCase,
-        ConfigParserTestCaseNonStandardDelimiters,
-        ConfigParserTestCaseNoValue,
-        ConfigParserTestCaseExtendedInterpolation,
-        ConfigParserTestCaseLegacyInterpolation,
-        ConfigParserTestCaseNoInterpolation,
-        ConfigParserTestCaseTrickyFile,
-        MultilineValuesTestCase,
-        RawConfigParserTestCase,
-        RawConfigParserTestCaseNonStandardDelimiters,
-        RawConfigParserTestSambaConf,
-        SortedTestCase,
-        Issue7005TestCase,
-        StrictTestCase,
-        CompatibleTestCase,
-        CopyTestCase,
-        ConfigParserTestCaseNonStandardDefaultSection,
-        ReadFileTestCase,
-        CoverageOneHundredTestCase,
-        ExceptionPicklingTestCase,
-        )
+class InlineCommentStrippingTestCase(unittest.TestCase):
+    """Tests for issue #14590: ConfigParser doesn't strip inline comment when
+    delimiter occurs earlier without preceding space.."""
+
+    def test_stripping(self):
+        cfg = configparser.ConfigParser(inline_comment_prefixes=(';', '#',
+                '//'))
+        cfg.read_string("""
+        [section]
+        k1 = v1;still v1
+        k2 = v2 ;a comment
+        k3 = v3 ; also a comment
+        k4 = v4;still v4 ;a comment
+        k5 = v5;still v5 ; also a comment
+        k6 = v6;still v6; and still v6 ;a comment
+        k7 = v7;still v7; and still v7 ; also a comment
+
+        [multiprefix]
+        k1 = v1;still v1 #a comment ; yeah, pretty much
+        k2 = v2 // this already is a comment ; continued
+        k3 = v3;#//still v3# and still v3 ; a comment
+        """)
+        s = cfg['section']
+        self.assertEqual(s['k1'], 'v1;still v1')
+        self.assertEqual(s['k2'], 'v2')
+        self.assertEqual(s['k3'], 'v3')
+        self.assertEqual(s['k4'], 'v4;still v4')
+        self.assertEqual(s['k5'], 'v5;still v5')
+        self.assertEqual(s['k6'], 'v6;still v6; and still v6')
+        self.assertEqual(s['k7'], 'v7;still v7; and still v7')
+        s = cfg['multiprefix']
+        self.assertEqual(s['k1'], 'v1;still v1')
+        self.assertEqual(s['k2'], 'v2')
+        self.assertEqual(s['k3'], 'v3;#//still v3# and still v3')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py
index 6e38305123..e52ed91a58 100644
--- a/Lib/test/test_contextlib.py
+++ b/Lib/test/test_contextlib.py
@@ -370,6 +370,231 @@ class TestContextDecorator(unittest.TestCase):
         self.assertEqual(state, [1, 'something else', 999])
 
 
+class TestExitStack(unittest.TestCase):
+
+    def test_no_resources(self):
+        with ExitStack():
+            pass
+
+    def test_callback(self):
+        expected = [
+            ((), {}),
+            ((1,), {}),
+            ((1,2), {}),
+            ((), dict(example=1)),
+            ((1,), dict(example=1)),
+            ((1,2), dict(example=1)),
+        ]
+        result = []
+        def _exit(*args, **kwds):
+            """Test metadata propagation"""
+            result.append((args, kwds))
+        with ExitStack() as stack:
+            for args, kwds in reversed(expected):
+                if args and kwds:
+                    f = stack.callback(_exit, *args, **kwds)
+                elif args:
+                    f = stack.callback(_exit, *args)
+                elif kwds:
+                    f = stack.callback(_exit, **kwds)
+                else:
+                    f = stack.callback(_exit)
+                self.assertIs(f, _exit)
+            for wrapper in stack._exit_callbacks:
+                self.assertIs(wrapper.__wrapped__, _exit)
+                self.assertNotEqual(wrapper.__name__, _exit.__name__)
+                self.assertIsNone(wrapper.__doc__, _exit.__doc__)
+        self.assertEqual(result, expected)
+
+    def test_push(self):
+        exc_raised = ZeroDivisionError
+        def _expect_exc(exc_type, exc, exc_tb):
+            self.assertIs(exc_type, exc_raised)
+        def _suppress_exc(*exc_details):
+            return True
+        def _expect_ok(exc_type, exc, exc_tb):
+            self.assertIsNone(exc_type)
+            self.assertIsNone(exc)
+            self.assertIsNone(exc_tb)
+        class ExitCM(object):
+            def __init__(self, check_exc):
+                self.check_exc = check_exc
+            def __enter__(self):
+                self.fail("Should not be called!")
+            def __exit__(self, *exc_details):
+                self.check_exc(*exc_details)
+        with ExitStack() as stack:
+            stack.push(_expect_ok)
+            self.assertIs(stack._exit_callbacks[-1], _expect_ok)
+            cm = ExitCM(_expect_ok)
+            stack.push(cm)
+            self.assertIs(stack._exit_callbacks[-1].__self__, cm)
+            stack.push(_suppress_exc)
+            self.assertIs(stack._exit_callbacks[-1], _suppress_exc)
+            cm = ExitCM(_expect_exc)
+            stack.push(cm)
+            self.assertIs(stack._exit_callbacks[-1].__self__, cm)
+            stack.push(_expect_exc)
+            self.assertIs(stack._exit_callbacks[-1], _expect_exc)
+            stack.push(_expect_exc)
+            self.assertIs(stack._exit_callbacks[-1], _expect_exc)
+            1/0
+
+    def test_enter_context(self):
+        class TestCM(object):
+            def __enter__(self):
+                result.append(1)
+            def __exit__(self, *exc_details):
+                result.append(3)
+
+        result = []
+        cm = TestCM()
+        with ExitStack() as stack:
+            @stack.callback  # Registered first => cleaned up last
+            def _exit():
+                result.append(4)
+            self.assertIsNotNone(_exit)
+            stack.enter_context(cm)
+            self.assertIs(stack._exit_callbacks[-1].__self__, cm)
+            result.append(2)
+        self.assertEqual(result, [1, 2, 3, 4])
+
+    def test_close(self):
+        result = []
+        with ExitStack() as stack:
+            @stack.callback
+            def _exit():
+                result.append(1)
+            self.assertIsNotNone(_exit)
+            stack.close()
+            result.append(2)
+        self.assertEqual(result, [1, 2])
+
+    def test_pop_all(self):
+        result = []
+        with ExitStack() as stack:
+            @stack.callback
+            def _exit():
+                result.append(3)
+            self.assertIsNotNone(_exit)
+            new_stack = stack.pop_all()
+            result.append(1)
+        result.append(2)
+        new_stack.close()
+        self.assertEqual(result, [1, 2, 3])
+
+    def test_exit_raise(self):
+        with self.assertRaises(ZeroDivisionError):
+            with ExitStack() as stack:
+                stack.push(lambda *exc: False)
+                1/0
+
+    def test_exit_suppress(self):
+        with ExitStack() as stack:
+            stack.push(lambda *exc: True)
+            1/0
+
+    def test_exit_exception_chaining_reference(self):
+        # Sanity check to make sure that ExitStack chaining matches
+        # actual nested with statements
+        class RaiseExc:
+            def __init__(self, exc):
+                self.exc = exc
+            def __enter__(self):
+                return self
+            def __exit__(self, *exc_details):
+                raise self.exc
+
+        class RaiseExcWithContext:
+            def __init__(self, outer, inner):
+                self.outer = outer
+                self.inner = inner
+            def __enter__(self):
+                return self
+            def __exit__(self, *exc_details):
+                try:
+                    raise self.inner
+                except:
+                    raise self.outer
+
+        class SuppressExc:
+            def __enter__(self):
+                return self
+            def __exit__(self, *exc_details):
+                type(self).saved_details = exc_details
+                return True
+
+        try:
+            with RaiseExc(IndexError):
+                with RaiseExcWithContext(KeyError, AttributeError):
+                    with SuppressExc():
+                        with RaiseExc(ValueError):
+                            1 / 0
+        except IndexError as exc:
+            self.assertIsInstance(exc.__context__, KeyError)
+            self.assertIsInstance(exc.__context__.__context__, AttributeError)
+            # Inner exceptions were suppressed
+            self.assertIsNone(exc.__context__.__context__.__context__)
+        else:
+            self.fail("Expected IndexError, but no exception was raised")
+        # Check the inner exceptions
+        inner_exc = SuppressExc.saved_details[1]
+        self.assertIsInstance(inner_exc, ValueError)
+        self.assertIsInstance(inner_exc.__context__, ZeroDivisionError)
+
+    def test_exit_exception_chaining(self):
+        # Ensure exception chaining matches the reference behaviour
+        def raise_exc(exc):
+            raise exc
+
+        saved_details = None
+        def suppress_exc(*exc_details):
+            nonlocal saved_details
+            saved_details = exc_details
+            return True
+
+        try:
+            with ExitStack() as stack:
+                stack.callback(raise_exc, IndexError)
+                stack.callback(raise_exc, KeyError)
+                stack.callback(raise_exc, AttributeError)
+                stack.push(suppress_exc)
+                stack.callback(raise_exc, ValueError)
+                1 / 0
+        except IndexError as exc:
+            self.assertIsInstance(exc.__context__, KeyError)
+            self.assertIsInstance(exc.__context__.__context__, AttributeError)
+            # Inner exceptions were suppressed
+            self.assertIsNone(exc.__context__.__context__.__context__)
+        else:
+            self.fail("Expected IndexError, but no exception was raised")
+        # Check the inner exceptions
+        inner_exc = saved_details[1]
+        self.assertIsInstance(inner_exc, ValueError)
+        self.assertIsInstance(inner_exc.__context__, ZeroDivisionError)
+
+    def test_exit_exception_chaining_suppress(self):
+        with ExitStack() as stack:
+            stack.push(lambda *exc: True)
+            stack.push(lambda *exc: 1/0)
+            stack.push(lambda *exc: {}[1])
+
+    def test_excessive_nesting(self):
+        # The original implementation would die with RecursionError here
+        with ExitStack() as stack:
+            for i in range(10000):
+                stack.callback(int)
+
+    def test_instance_bypass(self):
+        class Example(object): pass
+        cm = Example()
+        cm.__exit__ = object()
+        stack = ExitStack()
+        self.assertRaises(AttributeError, stack.enter_context, cm)
+        stack.push(cm)
+        self.assertIs(stack._exit_callbacks[-1], cm)
+
+
 # This is needed to make the test actually run under regrtest.py!
 def test_main():
     support.run_unittest(__name__)
diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py
index a84c109b87..c4baae4861 100644
--- a/Lib/test/test_copy.py
+++ b/Lib/test/test_copy.py
@@ -17,7 +17,7 @@ class TestCopy(unittest.TestCase):
     # Attempt full line coverage of copy.py from top to bottom
 
     def test_exceptions(self):
-        self.assertTrue(copy.Error is copy.error)
+        self.assertIs(copy.Error, copy.error)
         self.assertTrue(issubclass(copy.Error, Exception))
 
     # The copy() method
@@ -54,20 +54,26 @@ class TestCopy(unittest.TestCase):
     def test_copy_reduce_ex(self):
         class C(object):
             def __reduce_ex__(self, proto):
+                c.append(1)
                 return ""
             def __reduce__(self):
-                raise support.TestFailed("shouldn't call this")
+                self.fail("shouldn't call this")
+        c = []
         x = C()
         y = copy.copy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
+        self.assertEqual(c, [1])
 
     def test_copy_reduce(self):
         class C(object):
             def __reduce__(self):
+                c.append(1)
                 return ""
+        c = []
         x = C()
         y = copy.copy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
+        self.assertEqual(c, [1])
 
     def test_copy_cant(self):
         class C(object):
@@ -91,7 +97,7 @@ class TestCopy(unittest.TestCase):
                  "hello", "hello\u1234", f.__code__,
                  NewStyle, range(10), Classic, max]
         for x in tests:
-            self.assertTrue(copy.copy(x) is x, repr(x))
+            self.assertIs(copy.copy(x), x)
 
     def test_copy_list(self):
         x = [1, 2, 3]
@@ -185,9 +191,9 @@ class TestCopy(unittest.TestCase):
         x = [x, x]
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y[0] is not x[0])
-        self.assertTrue(y[0] is y[1])
+        self.assertIsNot(y, x)
+        self.assertIsNot(y[0], x[0])
+        self.assertIs(y[0], y[1])
 
     def test_deepcopy_issubclass(self):
         # XXX Note: there's no way to test the TypeError coming out of
@@ -227,20 +233,26 @@ class TestCopy(unittest.TestCase):
     def test_deepcopy_reduce_ex(self):
         class C(object):
             def __reduce_ex__(self, proto):
+                c.append(1)
                 return ""
             def __reduce__(self):
-                raise support.TestFailed("shouldn't call this")
+                self.fail("shouldn't call this")
+        c = []
         x = C()
         y = copy.deepcopy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
+        self.assertEqual(c, [1])
 
     def test_deepcopy_reduce(self):
         class C(object):
             def __reduce__(self):
+                c.append(1)
                 return ""
+        c = []
         x = C()
         y = copy.deepcopy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
+        self.assertEqual(c, [1])
 
     def test_deepcopy_cant(self):
         class C(object):
@@ -264,14 +276,14 @@ class TestCopy(unittest.TestCase):
                  "hello", "hello\u1234", f.__code__,
                  NewStyle, range(10), Classic, max]
         for x in tests:
-            self.assertTrue(copy.deepcopy(x) is x, repr(x))
+            self.assertIs(copy.deepcopy(x), x)
 
     def test_deepcopy_list(self):
         x = [[1, 2], 3]
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(x is not y)
-        self.assertTrue(x[0] is not y[0])
+        self.assertIsNot(x, y)
+        self.assertIsNot(x[0], y[0])
 
     def test_deepcopy_reflexive_list(self):
         x = []
@@ -279,16 +291,26 @@ class TestCopy(unittest.TestCase):
         y = copy.deepcopy(x)
         for op in comparisons:
             self.assertRaises(RuntimeError, op, y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y[0] is y)
+        self.assertIsNot(y, x)
+        self.assertIs(y[0], y)
         self.assertEqual(len(y), 1)
 
+    def test_deepcopy_empty_tuple(self):
+        x = ()
+        y = copy.deepcopy(x)
+        self.assertIs(x, y)
+
     def test_deepcopy_tuple(self):
         x = ([1, 2], 3)
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(x is not y)
-        self.assertTrue(x[0] is not y[0])
+        self.assertIsNot(x, y)
+        self.assertIsNot(x[0], y[0])
+
+    def test_deepcopy_tuple_of_immutables(self):
+        x = ((1, 2), 3)
+        y = copy.deepcopy(x)
+        self.assertIs(x, y)
 
     def test_deepcopy_reflexive_tuple(self):
         x = ([],)
@@ -296,16 +318,16 @@ class TestCopy(unittest.TestCase):
         y = copy.deepcopy(x)
         for op in comparisons:
             self.assertRaises(RuntimeError, op, y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y[0] is not x[0])
-        self.assertTrue(y[0][0] is y)
+        self.assertIsNot(y, x)
+        self.assertIsNot(y[0], x[0])
+        self.assertIs(y[0][0], y)
 
     def test_deepcopy_dict(self):
         x = {"foo": [1, 2], "bar": 3}
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(x is not y)
-        self.assertTrue(x["foo"] is not y["foo"])
+        self.assertIsNot(x, y)
+        self.assertIsNot(x["foo"], y["foo"])
 
     def test_deepcopy_reflexive_dict(self):
         x = {}
@@ -315,15 +337,30 @@ class TestCopy(unittest.TestCase):
             self.assertRaises(TypeError, op, y, x)
         for op in equality_comparisons:
             self.assertRaises(RuntimeError, op, y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y['foo'] is y)
+        self.assertIsNot(y, x)
+        self.assertIs(y['foo'], y)
         self.assertEqual(len(y), 1)
 
     def test_deepcopy_keepalive(self):
         memo = {}
-        x = 42
+        x = []
+        y = copy.deepcopy(x, memo)
+        self.assertIs(memo[id(memo)][0], x)
+
+    def test_deepcopy_dont_memo_immutable(self):
+        memo = {}
+        x = [1, 2, 3, 4]
         y = copy.deepcopy(x, memo)
-        self.assertTrue(memo[id(x)] is x)
+        self.assertEqual(y, x)
+        # There's the entry for the new list, and the keep alive.
+        self.assertEqual(len(memo), 2)
+
+        memo = {}
+        x = [(1, 2)]
+        y = copy.deepcopy(x, memo)
+        self.assertEqual(y, x)
+        # Tuples with immutable contents are immutable for deepcopy.
+        self.assertEqual(len(memo), 2)
 
     def test_deepcopy_inst_vanilla(self):
         class C:
@@ -334,7 +371,7 @@ class TestCopy(unittest.TestCase):
         x = C([42])
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_deepcopy_inst_deepcopy(self):
         class C:
@@ -347,8 +384,8 @@ class TestCopy(unittest.TestCase):
         x = C([42])
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y, x)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_deepcopy_inst_getinitargs(self):
         class C:
@@ -361,8 +398,8 @@ class TestCopy(unittest.TestCase):
         x = C([42])
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y, x)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_deepcopy_inst_getstate(self):
         class C:
@@ -375,8 +412,8 @@ class TestCopy(unittest.TestCase):
         x = C([42])
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y, x)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_deepcopy_inst_setstate(self):
         class C:
@@ -389,8 +426,8 @@ class TestCopy(unittest.TestCase):
         x = C([42])
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y, x)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_deepcopy_inst_getstate_setstate(self):
         class C:
@@ -405,8 +442,8 @@ class TestCopy(unittest.TestCase):
         x = C([42])
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y, x)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_deepcopy_reflexive_inst(self):
         class C:
@@ -414,8 +451,8 @@ class TestCopy(unittest.TestCase):
         x = C()
         x.foo = x
         y = copy.deepcopy(x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y.foo is y)
+        self.assertIsNot(y, x)
+        self.assertIs(y.foo, y)
 
     # _reconstruct()
 
@@ -425,9 +462,9 @@ class TestCopy(unittest.TestCase):
                 return ""
         x = C()
         y = copy.copy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
         y = copy.deepcopy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
 
     def test_reconstruct_nostate(self):
         class C(object):
@@ -436,9 +473,9 @@ class TestCopy(unittest.TestCase):
         x = C()
         x.foo = 42
         y = copy.copy(x)
-        self.assertTrue(y.__class__ is x.__class__)
+        self.assertIs(y.__class__, x.__class__)
         y = copy.deepcopy(x)
-        self.assertTrue(y.__class__ is x.__class__)
+        self.assertIs(y.__class__, x.__class__)
 
     def test_reconstruct_state(self):
         class C(object):
@@ -452,7 +489,7 @@ class TestCopy(unittest.TestCase):
         self.assertEqual(y, x)
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_reconstruct_state_setstate(self):
         class C(object):
@@ -468,7 +505,7 @@ class TestCopy(unittest.TestCase):
         self.assertEqual(y, x)
         y = copy.deepcopy(x)
         self.assertEqual(y, x)
-        self.assertTrue(y.foo is not x.foo)
+        self.assertIsNot(y.foo, x.foo)
 
     def test_reconstruct_reflexive(self):
         class C(object):
@@ -476,8 +513,8 @@ class TestCopy(unittest.TestCase):
         x = C()
         x.foo = x
         y = copy.deepcopy(x)
-        self.assertTrue(y is not x)
-        self.assertTrue(y.foo is y)
+        self.assertIsNot(y, x)
+        self.assertIs(y.foo, y)
 
     # Additions for Python 2.3 and pickle protocol 2
 
@@ -491,12 +528,12 @@ class TestCopy(unittest.TestCase):
         x = C([[1, 2], 3])
         y = copy.copy(x)
         self.assertEqual(x, y)
-        self.assertTrue(x is not y)
-        self.assertTrue(x[0] is y[0])
+        self.assertIsNot(x, y)
+        self.assertIs(x[0], y[0])
         y = copy.deepcopy(x)
         self.assertEqual(x, y)
-        self.assertTrue(x is not y)
-        self.assertTrue(x[0] is not y[0])
+        self.assertIsNot(x, y)
+        self.assertIsNot(x[0], y[0])
 
     def test_reduce_5tuple(self):
         class C(dict):
@@ -508,12 +545,12 @@ class TestCopy(unittest.TestCase):
         x = C([("foo", [1, 2]), ("bar", 3)])
         y = copy.copy(x)
         self.assertEqual(x, y)
-        self.assertTrue(x is not y)
-        self.assertTrue(x["foo"] is y["foo"])
+        self.assertIsNot(x, y)
+        self.assertIs(x["foo"], y["foo"])
         y = copy.deepcopy(x)
         self.assertEqual(x, y)
-        self.assertTrue(x is not y)
-        self.assertTrue(x["foo"] is not y["foo"])
+        self.assertIsNot(x, y)
+        self.assertIsNot(x["foo"], y["foo"])
 
     def test_copy_slots(self):
         class C(object):
@@ -521,7 +558,7 @@ class TestCopy(unittest.TestCase):
         x = C()
         x.foo = [42]
         y = copy.copy(x)
-        self.assertTrue(x.foo is y.foo)
+        self.assertIs(x.foo, y.foo)
 
     def test_deepcopy_slots(self):
         class C(object):
@@ -530,7 +567,7 @@ class TestCopy(unittest.TestCase):
         x.foo = [42]
         y = copy.deepcopy(x)
         self.assertEqual(x.foo, y.foo)
-        self.assertTrue(x.foo is not y.foo)
+        self.assertIsNot(x.foo, y.foo)
 
     def test_deepcopy_dict_subclass(self):
         class C(dict):
@@ -547,7 +584,7 @@ class TestCopy(unittest.TestCase):
         y = copy.deepcopy(x)
         self.assertEqual(x, y)
         self.assertEqual(x._keys, y._keys)
-        self.assertTrue(x is not y)
+        self.assertIsNot(x, y)
         x['bar'] = 1
         self.assertNotEqual(x, y)
         self.assertNotEqual(x._keys, y._keys)
@@ -560,8 +597,8 @@ class TestCopy(unittest.TestCase):
         y = copy.copy(x)
         self.assertEqual(list(x), list(y))
         self.assertEqual(x.foo, y.foo)
-        self.assertTrue(x[0] is y[0])
-        self.assertTrue(x.foo is y.foo)
+        self.assertIs(x[0], y[0])
+        self.assertIs(x.foo, y.foo)
 
     def test_deepcopy_list_subclass(self):
         class C(list):
@@ -571,8 +608,8 @@ class TestCopy(unittest.TestCase):
         y = copy.deepcopy(x)
         self.assertEqual(list(x), list(y))
         self.assertEqual(x.foo, y.foo)
-        self.assertTrue(x[0] is not y[0])
-        self.assertTrue(x.foo is not y.foo)
+        self.assertIsNot(x[0], y[0])
+        self.assertIsNot(x.foo, y.foo)
 
     def test_copy_tuple_subclass(self):
         class C(tuple):
@@ -589,8 +626,8 @@ class TestCopy(unittest.TestCase):
         self.assertEqual(tuple(x), ([1, 2], 3))
         y = copy.deepcopy(x)
         self.assertEqual(tuple(y), ([1, 2], 3))
-        self.assertTrue(x is not y)
-        self.assertTrue(x[0] is not y[0])
+        self.assertIsNot(x, y)
+        self.assertIsNot(x[0], y[0])
 
     def test_getstate_exc(self):
         class EvilState(object):
@@ -618,10 +655,10 @@ class TestCopy(unittest.TestCase):
         obj = C()
         x = weakref.ref(obj)
         y = _copy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
         del obj
         y = _copy(x)
-        self.assertTrue(y is x)
+        self.assertIs(y, x)
 
     def test_copy_weakref(self):
         self._check_weakref(copy.copy)
@@ -637,7 +674,7 @@ class TestCopy(unittest.TestCase):
         u[a] = b
         u[c] = d
         v = copy.copy(u)
-        self.assertFalse(v is u)
+        self.assertIsNot(v, u)
         self.assertEqual(v, u)
         self.assertEqual(v[a], b)
         self.assertEqual(v[c], d)
@@ -667,8 +704,8 @@ class TestCopy(unittest.TestCase):
         v = copy.deepcopy(u)
         self.assertNotEqual(v, u)
         self.assertEqual(len(v), 2)
-        self.assertFalse(v[a] is b)
-        self.assertFalse(v[c] is d)
+        self.assertIsNot(v[a], b)
+        self.assertIsNot(v[c], d)
         self.assertEqual(v[a].i, b.i)
         self.assertEqual(v[c].i, d.i)
         del c
@@ -687,12 +724,12 @@ class TestCopy(unittest.TestCase):
         self.assertNotEqual(v, u)
         self.assertEqual(len(v), 2)
         (x, y), (z, t) = sorted(v.items(), key=lambda pair: pair[0].i)
-        self.assertFalse(x is a)
+        self.assertIsNot(x, a)
         self.assertEqual(x.i, a.i)
-        self.assertTrue(y is b)
-        self.assertFalse(z is c)
+        self.assertIs(y, b)
+        self.assertIsNot(z, c)
         self.assertEqual(z.i, c.i)
-        self.assertTrue(t is d)
+        self.assertIs(t, d)
         del x, y, z, t
         del d
         self.assertEqual(len(v), 1)
@@ -705,7 +742,7 @@ class TestCopy(unittest.TestCase):
         f.b = f.m
         g = copy.deepcopy(f)
         self.assertEqual(g.m, g.b)
-        self.assertTrue(g.b.__self__ is g)
+        self.assertIs(g.b.__self__, g)
         g.b()
 
 
diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py
index ae17c2b694..56766682b3 100644
--- a/Lib/test/test_cprofile.py
+++ b/Lib/test/test_cprofile.py
@@ -18,16 +18,19 @@ class CProfileTest(ProfileTest):
     def test_bad_counter_during_dealloc(self):
         import _lsprof
         # Must use a file as StringIO doesn't trigger the bug.
-        with open(TESTFN, 'w') as file:
-            sys.stderr = file
-            try:
-                obj = _lsprof.Profiler(lambda: int)
-                obj.enable()
-                obj = _lsprof.Profiler(1)
-                obj.disable()
-            finally:
-                sys.stderr = sys.__stderr__
-        unlink(TESTFN)
+        orig_stderr = sys.stderr
+        try:
+            with open(TESTFN, 'w') as file:
+                sys.stderr = file
+                try:
+                    obj = _lsprof.Profiler(lambda: int)
+                    obj.enable()
+                    obj = _lsprof.Profiler(1)
+                    obj.disable()
+                finally:
+                    sys.stderr = orig_stderr
+        finally:
+            unlink(TESTFN)
 
 
 def test_main():
diff --git a/Lib/test/test_crashers.py b/Lib/test/test_crashers.py
new file mode 100644
index 0000000000..336ccbeaff
--- /dev/null
+++ b/Lib/test/test_crashers.py
@@ -0,0 +1,38 @@
+# Tests that the crashers in the Lib/test/crashers directory actually
+# do crash the interpreter as expected
+#
+# If a crasher is fixed, it should be moved elsewhere in the test suite to
+# ensure it continues to work correctly.
+
+import unittest
+import glob
+import os.path
+import test.support
+from test.script_helper import assert_python_failure
+
+CRASHER_DIR = os.path.join(os.path.dirname(__file__), "crashers")
+CRASHER_FILES = os.path.join(CRASHER_DIR, "*.py")
+
+infinite_loops = ["infinite_loop_re.py", "nasty_eq_vs_dict.py"]
+
+class CrasherTest(unittest.TestCase):
+
+    @unittest.skip("these tests are too fragile")
+    @test.support.cpython_only
+    def test_crashers_crash(self):
+        for fname in glob.glob(CRASHER_FILES):
+            if os.path.basename(fname) in infinite_loops:
+                continue
+            # Some "crashers" only trigger an exception rather than a
+            # segfault. Consider that an acceptable outcome.
+            if test.support.verbose:
+                print("Checking crasher:", fname)
+            assert_python_failure(fname)
+
+
+def test_main():
+    test.support.run_unittest(CrasherTest)
+    test.support.reap_children()
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py
index 2adb28d7ce..cfb7341e20 100755..100644
--- a/Lib/test/test_crypt.py
+++ b/Lib/test/test_crypt.py
@@ -1,7 +1,11 @@
 from test import support
 import unittest
 
-crypt = support.import_module('crypt')
+def setUpModule():
+    # this import will raise unittest.SkipTest if _crypt doesn't exist,
+    # so it has to be done in setUpModule for test discovery to work
+    global crypt
+    crypt = support.import_module('crypt')
 
 class CryptTestCase(unittest.TestCase):
 
@@ -10,8 +14,24 @@ class CryptTestCase(unittest.TestCase):
         if support.verbose:
             print('Test encryption: ', c)
 
-def test_main():
-    support.run_unittest(CryptTestCase)
+    def test_salt(self):
+        self.assertEqual(len(crypt._saltchars), 64)
+        for method in crypt.methods:
+            salt = crypt.mksalt(method)
+            self.assertEqual(len(salt),
+                    method.salt_chars + (3 if method.ident else 0))
+
+    def test_saltedcrypt(self):
+        for method in crypt.methods:
+            pw = crypt.crypt('assword', method)
+            self.assertEqual(len(pw), method.total_size)
+            pw = crypt.crypt('assword', crypt.mksalt(method))
+            self.assertEqual(len(pw), method.total_size)
+
+    def test_methods(self):
+        # Gurantee that METHOD_CRYPT is the last method in crypt.methods.
+        self.assertTrue(len(crypt.methods) >= 1)
+        self.assertEqual(crypt.METHOD_CRYPT, crypt.methods[-1])
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py
index 55796a204a..96f8aa7ee1 100644
--- a/Lib/test/test_csv.py
+++ b/Lib/test/test_csv.py
@@ -197,6 +197,17 @@ class Test_Csv(unittest.TestCase):
             fileobj.seek(0)
             self.assertEqual(fileobj.read(), "a,b\r\nc,d\r\n")
 
+    @support.cpython_only
+    def test_writerows_legacy_strings(self):
+        import _testcapi
+
+        c = _testcapi.unicode_legacy_string('a')
+        with TemporaryFile("w+", newline='') as fileobj:
+            writer = csv.writer(fileobj)
+            writer.writerows([[c]])
+            fileobj.seek(0)
+            self.assertEqual(fileobj.read(), "a\r\n")
+
     def _read_test(self, input, expect, **kwargs):
         reader = csv.reader(input, **kwargs)
         result = list(reader)
diff --git a/Lib/test/test_ctypes.py b/Lib/test/test_ctypes.py
index 7d9abdcba7..496355e61d 100644
--- a/Lib/test/test_ctypes.py
+++ b/Lib/test/test_ctypes.py
@@ -1,16 +1,16 @@
 import unittest
 
-from test.support import run_unittest, import_module
+from test.support import import_module
 
 # Skip tests if _ctypes module was not built.
 import_module('_ctypes')
 
 import ctypes.test
 
-def test_main():
+def load_tests(*args):
     skipped, testcases = ctypes.test.get_tests(ctypes.test, "test_*.py", verbosity=0)
     suites = [unittest.makeSuite(t) for t in testcases]
-    run_unittest(unittest.TestSuite(suites))
+    return unittest.TestSuite(suites)
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py
index 58121477b1..e959622c73 100644
--- a/Lib/test/test_curses.py
+++ b/Lib/test/test_curses.py
@@ -264,11 +264,53 @@ def test_issue6243(stdscr):
     curses.ungetch(1025)
     stdscr.getkey()
 
+def test_unget_wch(stdscr):
+    if not hasattr(curses, 'unget_wch'):
+        return
+    encoding = stdscr.encoding
+    for ch in ('a', '\xe9', '\u20ac', '\U0010FFFF'):
+        try:
+            ch.encode(encoding)
+        except UnicodeEncodeError:
+            continue
+        try:
+            curses.unget_wch(ch)
+        except Exception as err:
+            raise Exception("unget_wch(%a) failed with encoding %s: %s"
+                            % (ch, stdscr.encoding, err))
+        read = stdscr.get_wch()
+        if read != ch:
+            raise AssertionError("%r != %r" % (read, ch))
+
+        code = ord(ch)
+        curses.unget_wch(code)
+        read = stdscr.get_wch()
+        if read != ch:
+            raise AssertionError("%r != %r" % (read, ch))
+
 def test_issue10570():
     b = curses.tparm(curses.tigetstr("cup"), 5, 3)
     assert type(b) is bytes
     curses.putp(b)
 
+def test_encoding(stdscr):
+    import codecs
+    encoding = stdscr.encoding
+    codecs.lookup(encoding)
+    try:
+        stdscr.encoding = 10
+    except TypeError:
+        pass
+    else:
+        raise AssertionError("TypeError not raised")
+    stdscr.encoding = encoding
+    try:
+        del stdscr.encoding
+    except TypeError:
+        pass
+    else:
+        raise AssertionError("TypeError not raised")
+
 def main(stdscr):
     curses.savetty()
     try:
@@ -277,16 +319,18 @@ def main(stdscr):
         test_userptr_without_set(stdscr)
         test_resize_term(stdscr)
         test_issue6243(stdscr)
+        test_unget_wch(stdscr)
         test_issue10570()
+        test_encoding(stdscr)
     finally:
         curses.resetty()
 
 def test_main():
-    if not sys.stdout.isatty():
-        raise unittest.SkipTest("sys.stdout is not a tty")
+    if not sys.__stdout__.isatty():
+        raise unittest.SkipTest("sys.__stdout__ is not a tty")
     # testing setupterm() inside initscr/endwin
     # causes terminal breakage
-    curses.setupterm(fd=sys.stdout.fileno())
+    curses.setupterm(fd=sys.__stdout__.fileno())
     try:
         stdscr = curses.initscr()
         main(stdscr)
diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py
index 26d4c14623..752e0b77a6 100644
--- a/Lib/test/test_dbm.py
+++ b/Lib/test/test_dbm.py
@@ -34,7 +34,7 @@ def delete_files():
         test.support.unlink(f)
 
 
-class AnyDBMTestCase(unittest.TestCase):
+class AnyDBMTestCase:
     _dict = {'0': b'',
              'a': b'Python:',
              'b': b'Programming',
@@ -71,8 +71,8 @@ class AnyDBMTestCase(unittest.TestCase):
         f.close()
 
     def test_anydbm_creation_n_file_exists_with_invalid_contents(self):
-        with open(_fname, "w") as w:
-            pass # create an empty file
+        # create an empty file
+        test.support.create_empty_file(_fname)
 
         f = dbm.open(_fname, 'n')
         self.addCleanup(f.close)
@@ -119,10 +119,6 @@ class AnyDBMTestCase(unittest.TestCase):
 
 
 class WhichDBTestCase(unittest.TestCase):
-    # Actual test methods are added to namespace after class definition.
-    def __init__(self, *args):
-        unittest.TestCase.__init__(self, *args)
-
     def test_whichdb(self):
         for module in dbm_iterator():
             # Check whether whichdb correctly guesses module name
@@ -169,12 +165,16 @@ class WhichDBTestCase(unittest.TestCase):
         self.d.close()
 
 
-def test_main():
-    classes = [WhichDBTestCase]
+def load_tests(loader, tests, pattern):
+    classes = []
     for mod in dbm_iterator():
-        classes.append(type("TestCase-" + mod.__name__, (AnyDBMTestCase,),
+        classes.append(type("TestCase-" + mod.__name__,
+                            (AnyDBMTestCase, unittest.TestCase),
                             {'module': mod}))
-    test.support.run_unittest(*classes)
+    suites = [unittest.makeSuite(c) for c in classes]
+
+    tests.addTests(suites)
+    return tests
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py
index 6b981c429d..e23392959a 100644
--- a/Lib/test/test_dbm_dumb.py
+++ b/Lib/test/test_dbm_dumb.py
@@ -29,9 +29,6 @@ class DumbDBMTestCase(unittest.TestCase):
              '\u00fc'.encode('utf-8') : b'!',
              }
 
-    def __init__(self, *args):
-        unittest.TestCase.__init__(self, *args)
-
     def test_dumbdbm_creation(self):
         f = dumbdbm.open(_fname, 'c')
         self.assertEqual(list(f.keys()), [])
@@ -195,11 +192,6 @@ class DumbDBMTestCase(unittest.TestCase):
     def setUp(self):
         _delete_files()
 
-def test_main():
-    try:
-        support.run_unittest(DumbDBMTestCase)
-    finally:
-        _delete_files()
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py
index 30a39f7b89..bf6294685a 100755
--- a/Lib/test/test_dbm_gnu.py
+++ b/Lib/test/test_dbm_gnu.py
@@ -2,7 +2,7 @@ from test import support
 gdbm = support.import_module("dbm.gnu") #skip if not supported
 import unittest
 import os
-from test.support import verbose, TESTFN, run_unittest, unlink
+from test.support import verbose, TESTFN, unlink
 
 
 filename = TESTFN
@@ -81,8 +81,5 @@ class TestGdbm(unittest.TestCase):
         self.assertTrue(size1 > size2 >= size0)
 
 
-def test_main():
-    run_unittest(TestGdbm)
-
 if __name__ == '__main__':
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py
index 00dcbd2160..f9d3befde9 100755
--- a/Lib/test/test_dbm_ndbm.py
+++ b/Lib/test/test_dbm_ndbm.py
@@ -36,8 +36,5 @@ class DbmTestCase(unittest.TestCase):
             except error:
                 self.fail()
 
-def test_main():
-    support.run_unittest(DbmTestCase)
-
 if __name__ == '__main__':
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
index 30d397178d..69a2fafffb 100644
--- a/Lib/test/test_decimal.py
+++ b/Lib/test/test_decimal.py
@@ -16,7 +16,7 @@ test the pythonic behaviour according to PEP 327.
 
 Cowlishaw's tests can be downloaded from:
 
-   www2.hursley.ibm.com/decimal/dectest.zip
+   http://speleotrove.com/decimal/dectest.zip
 
 This test module can be called from command line with one parameter (Arithmetic
 or Behaviour) to test each part, or without parameter to test both parts. If
@@ -30,37 +30,81 @@ import operator
 import warnings
 import pickle, copy
 import unittest
-from decimal import *
 import numbers
+import locale
 from test.support import (run_unittest, run_doctest, is_resource_enabled,
                           requires_IEEE_754)
-from test.support import check_warnings
+from test.support import (check_warnings, import_fresh_module, TestFailed,
+                          run_with_locale, cpython_only)
 import random
+import time
+import warnings
 try:
     import threading
 except ImportError:
     threading = None
 
-# Useful Test Constant
-Signals = tuple(getcontext().flags.keys())
 
+C = import_fresh_module('decimal', fresh=['_decimal'])
+P = import_fresh_module('decimal', blocked=['_decimal'])
+orig_sys_decimal = sys.modules['decimal']
+
+# fractions module must import the correct decimal module.
+cfractions = import_fresh_module('fractions', fresh=['fractions'])
+sys.modules['decimal'] = P
+pfractions = import_fresh_module('fractions', fresh=['fractions'])
+sys.modules['decimal'] = C
+fractions = {C:cfractions, P:pfractions}
+sys.modules['decimal'] = orig_sys_decimal
+
+
+# Useful Test Constant
+Signals = {
+  C: tuple(C.getcontext().flags.keys()) if C else None,
+  P: tuple(P.getcontext().flags.keys())
+}
 # Signals ordered with respect to precedence: when an operation
 # produces multiple signals, signals occurring later in the list
 # should be handled before those occurring earlier in the list.
-OrderedSignals = (Clamped, Rounded, Inexact, Subnormal,
-                  Underflow, Overflow, DivisionByZero, InvalidOperation)
+OrderedSignals = {
+  C: [C.Clamped, C.Rounded, C.Inexact, C.Subnormal, C.Underflow,
+      C.Overflow, C.DivisionByZero, C.InvalidOperation,
+      C.FloatOperation] if C else None,
+  P: [P.Clamped, P.Rounded, P.Inexact, P.Subnormal, P.Underflow,
+      P.Overflow, P.DivisionByZero, P.InvalidOperation,
+      P.FloatOperation]
+}
+def assert_signals(cls, context, attr, expected):
+    d = getattr(context, attr)
+    cls.assertTrue(all(d[s] if s in expected else not d[s] for s in d))
+
+ROUND_UP = P.ROUND_UP
+ROUND_DOWN = P.ROUND_DOWN
+ROUND_CEILING = P.ROUND_CEILING
+ROUND_FLOOR = P.ROUND_FLOOR
+ROUND_HALF_UP = P.ROUND_HALF_UP
+ROUND_HALF_DOWN = P.ROUND_HALF_DOWN
+ROUND_HALF_EVEN = P.ROUND_HALF_EVEN
+ROUND_05UP = P.ROUND_05UP
+
+RoundingModes = [
+  ROUND_UP, ROUND_DOWN, ROUND_CEILING, ROUND_FLOOR,
+  ROUND_HALF_UP, ROUND_HALF_DOWN, ROUND_HALF_EVEN,
+  ROUND_05UP
+]
 
 # Tests are built around these assumed context defaults.
 # test_main() restores the original context.
-def init():
-    global ORIGINAL_CONTEXT
-    ORIGINAL_CONTEXT = getcontext().copy()
-    DefaultTestContext = Context(
-        prec = 9,
-        rounding = ROUND_HALF_EVEN,
-        traps = dict.fromkeys(Signals, 0)
-        )
-    setcontext(DefaultTestContext)
+ORIGINAL_CONTEXT = {
+  C: C.getcontext().copy() if C else None,
+  P: P.getcontext().copy()
+}
+def init(m):
+    if not m: return
+    DefaultTestContext = m.Context(
+       prec=9, rounding=ROUND_HALF_EVEN, traps=dict.fromkeys(Signals[m], 0)
+    )
+    m.setcontext(DefaultTestContext)
 
 TESTDATADIR = 'decimaltestdata'
 if __name__ == '__main__':
@@ -72,149 +116,175 @@ directory = testdir + os.sep + TESTDATADIR + os.sep
 
 skip_expected = not os.path.isdir(directory)
 
-# list of individual .decTest test ids that correspond to tests that
-# we're skipping for one reason or another.
-skipped_test_ids = set([
-    # Skip implementation-specific scaleb tests.
-    'scbx164',
-    'scbx165',
-
-    # For some operations (currently exp, ln, log10, power), the decNumber
-    # reference implementation imposes additional restrictions on the context
-    # and operands.  These restrictions are not part of the specification;
-    # however, the effect of these restrictions does show up in some of the
-    # testcases.  We skip testcases that violate these restrictions, since
-    # Decimal behaves differently from decNumber for these testcases so these
-    # testcases would otherwise fail.
-    'expx901',
-    'expx902',
-    'expx903',
-    'expx905',
-    'lnx901',
-    'lnx902',
-    'lnx903',
-    'lnx905',
-    'logx901',
-    'logx902',
-    'logx903',
-    'logx905',
-    'powx1183',
-    'powx1184',
-    'powx4001',
-    'powx4002',
-    'powx4003',
-    'powx4005',
-    'powx4008',
-    'powx4010',
-    'powx4012',
-    'powx4014',
-    ])
-
 # Make sure it actually raises errors when not expected and caught in flags
 # Slower, since it runs some things several times.
 EXTENDEDERRORTEST = False
 
-#Map the test cases' error names to the actual errors
-ErrorNames = {'clamped' : Clamped,
-              'conversion_syntax' : InvalidOperation,
-              'division_by_zero' : DivisionByZero,
-              'division_impossible' : InvalidOperation,
-              'division_undefined' : InvalidOperation,
-              'inexact' : Inexact,
-              'invalid_context' : InvalidOperation,
-              'invalid_operation' : InvalidOperation,
-              'overflow' : Overflow,
-              'rounded' : Rounded,
-              'subnormal' : Subnormal,
-              'underflow' : Underflow}
-
-
-def Nonfunction(*args):
-    """Doesn't do anything."""
-    return None
-
-RoundingDict = {'ceiling' : ROUND_CEILING, #Maps test-case names to roundings.
-                'down' : ROUND_DOWN,
-                'floor' : ROUND_FLOOR,
-                'half_down' : ROUND_HALF_DOWN,
-                'half_even' : ROUND_HALF_EVEN,
-                'half_up' : ROUND_HALF_UP,
-                'up' : ROUND_UP,
-                '05up' : ROUND_05UP}
-
-# Name adapter to be able to change the Decimal and Context
-# interface without changing the test files from Cowlishaw
-nameAdapter = {'and':'logical_and',
-               'apply':'_apply',
-               'class':'number_class',
-               'comparesig':'compare_signal',
-               'comparetotal':'compare_total',
-               'comparetotmag':'compare_total_mag',
-               'copy':'copy_decimal',
-               'copyabs':'copy_abs',
-               'copynegate':'copy_negate',
-               'copysign':'copy_sign',
-               'divideint':'divide_int',
-               'invert':'logical_invert',
-               'iscanonical':'is_canonical',
-               'isfinite':'is_finite',
-               'isinfinite':'is_infinite',
-               'isnan':'is_nan',
-               'isnormal':'is_normal',
-               'isqnan':'is_qnan',
-               'issigned':'is_signed',
-               'issnan':'is_snan',
-               'issubnormal':'is_subnormal',
-               'iszero':'is_zero',
-               'maxmag':'max_mag',
-               'minmag':'min_mag',
-               'nextminus':'next_minus',
-               'nextplus':'next_plus',
-               'nexttoward':'next_toward',
-               'or':'logical_or',
-               'reduce':'normalize',
-               'remaindernear':'remainder_near',
-               'samequantum':'same_quantum',
-               'squareroot':'sqrt',
-               'toeng':'to_eng_string',
-               'tointegral':'to_integral_value',
-               'tointegralx':'to_integral_exact',
-               'tosci':'to_sci_string',
-               'xor':'logical_xor',
-              }
-
-# The following functions return True/False rather than a Decimal instance
-
-LOGICAL_FUNCTIONS = (
-    'is_canonical',
-    'is_finite',
-    'is_infinite',
-    'is_nan',
-    'is_normal',
-    'is_qnan',
-    'is_signed',
-    'is_snan',
-    'is_subnormal',
-    'is_zero',
-    'same_quantum',
-    )
+# Test extra functionality in the C version (-DEXTRA_FUNCTIONALITY).
+EXTRA_FUNCTIONALITY = True if hasattr(C, 'DecClamped') else False
+requires_extra_functionality = unittest.skipUnless(
+  EXTRA_FUNCTIONALITY, "test requires build with -DEXTRA_FUNCTIONALITY")
+skip_if_extra_functionality = unittest.skipIf(
+  EXTRA_FUNCTIONALITY, "test requires regular build")
 
-class DecimalTest(unittest.TestCase):
-    """Class which tests the Decimal class against the test cases.
 
-    Changed for unittest.
-    """
+class IBMTestCases(unittest.TestCase):
+    """Class which tests the Decimal class against the IBM test cases."""
+
     def setUp(self):
-        self.context = Context()
+        self.context = self.decimal.Context()
+        self.readcontext = self.decimal.Context()
         self.ignore_list = ['#']
-        # Basically, a # means return NaN InvalidOperation.
-        # Different from a sNaN in trim
 
+        # List of individual .decTest test ids that correspond to tests that
+        # we're skipping for one reason or another.
+        self.skipped_test_ids = set([
+            # Skip implementation-specific scaleb tests.
+            'scbx164',
+            'scbx165',
+
+            # For some operations (currently exp, ln, log10, power), the decNumber
+            # reference implementation imposes additional restrictions on the context
+            # and operands.  These restrictions are not part of the specification;
+            # however, the effect of these restrictions does show up in some of the
+            # testcases.  We skip testcases that violate these restrictions, since
+            # Decimal behaves differently from decNumber for these testcases so these
+            # testcases would otherwise fail.
+            'expx901',
+            'expx902',
+            'expx903',
+            'expx905',
+            'lnx901',
+            'lnx902',
+            'lnx903',
+            'lnx905',
+            'logx901',
+            'logx902',
+            'logx903',
+            'logx905',
+            'powx1183',
+            'powx1184',
+            'powx4001',
+            'powx4002',
+            'powx4003',
+            'powx4005',
+            'powx4008',
+            'powx4010',
+            'powx4012',
+            'powx4014',
+            ])
+
+        if self.decimal == C:
+            # status has additional Subnormal, Underflow
+            self.skipped_test_ids.add('pwsx803')
+            self.skipped_test_ids.add('pwsx805')
+            # Correct rounding (skipped for decNumber, too)
+            self.skipped_test_ids.add('powx4302')
+            self.skipped_test_ids.add('powx4303')
+            self.skipped_test_ids.add('powx4342')
+            self.skipped_test_ids.add('powx4343')
+            # http://bugs.python.org/issue7049
+            self.skipped_test_ids.add('pwmx325')
+            self.skipped_test_ids.add('pwmx326')
+
+        # Map test directives to setter functions.
         self.ChangeDict = {'precision' : self.change_precision,
-                      'rounding' : self.change_rounding_method,
-                      'maxexponent' : self.change_max_exponent,
-                      'minexponent' : self.change_min_exponent,
-                      'clamp' : self.change_clamp}
+                           'rounding' : self.change_rounding_method,
+                           'maxexponent' : self.change_max_exponent,
+                           'minexponent' : self.change_min_exponent,
+                           'clamp' : self.change_clamp}
+
+        # Name adapter to be able to change the Decimal and Context
+        # interface without changing the test files from Cowlishaw.
+        self.NameAdapter = {'and':'logical_and',
+                            'apply':'_apply',
+                            'class':'number_class',
+                            'comparesig':'compare_signal',
+                            'comparetotal':'compare_total',
+                            'comparetotmag':'compare_total_mag',
+                            'copy':'copy_decimal',
+                            'copyabs':'copy_abs',
+                            'copynegate':'copy_negate',
+                            'copysign':'copy_sign',
+                            'divideint':'divide_int',
+                            'invert':'logical_invert',
+                            'iscanonical':'is_canonical',
+                            'isfinite':'is_finite',
+                            'isinfinite':'is_infinite',
+                            'isnan':'is_nan',
+                            'isnormal':'is_normal',
+                            'isqnan':'is_qnan',
+                            'issigned':'is_signed',
+                            'issnan':'is_snan',
+                            'issubnormal':'is_subnormal',
+                            'iszero':'is_zero',
+                            'maxmag':'max_mag',
+                            'minmag':'min_mag',
+                            'nextminus':'next_minus',
+                            'nextplus':'next_plus',
+                            'nexttoward':'next_toward',
+                            'or':'logical_or',
+                            'reduce':'normalize',
+                            'remaindernear':'remainder_near',
+                            'samequantum':'same_quantum',
+                            'squareroot':'sqrt',
+                            'toeng':'to_eng_string',
+                            'tointegral':'to_integral_value',
+                            'tointegralx':'to_integral_exact',
+                            'tosci':'to_sci_string',
+                            'xor':'logical_xor'}
+
+        # Map test-case names to roundings.
+        self.RoundingDict = {'ceiling' : ROUND_CEILING,
+                             'down' : ROUND_DOWN,
+                             'floor' : ROUND_FLOOR,
+                             'half_down' : ROUND_HALF_DOWN,
+                             'half_even' : ROUND_HALF_EVEN,
+                             'half_up' : ROUND_HALF_UP,
+                             'up' : ROUND_UP,
+                             '05up' : ROUND_05UP}
+
+        # Map the test cases' error names to the actual errors.
+        self.ErrorNames = {'clamped' : self.decimal.Clamped,
+                           'conversion_syntax' : self.decimal.InvalidOperation,
+                           'division_by_zero' : self.decimal.DivisionByZero,
+                           'division_impossible' : self.decimal.InvalidOperation,
+                           'division_undefined' : self.decimal.InvalidOperation,
+                           'inexact' : self.decimal.Inexact,
+                           'invalid_context' : self.decimal.InvalidOperation,
+                           'invalid_operation' : self.decimal.InvalidOperation,
+                           'overflow' : self.decimal.Overflow,
+                           'rounded' : self.decimal.Rounded,
+                           'subnormal' : self.decimal.Subnormal,
+                           'underflow' : self.decimal.Underflow}
+
+        # The following functions return True/False rather than a
+        # Decimal instance.
+        self.LogicalFunctions = ('is_canonical',
+                                 'is_finite',
+                                 'is_infinite',
+                                 'is_nan',
+                                 'is_normal',
+                                 'is_qnan',
+                                 'is_signed',
+                                 'is_snan',
+                                 'is_subnormal',
+                                 'is_zero',
+                                 'same_quantum')
+
+    def read_unlimited(self, v, context):
+        """Work around the limitations of the 32-bit _decimal version. The
+           guaranteed maximum values for prec, Emax etc. are 425000000,
+           but higher values usually work, except for rare corner cases.
+           In particular, all of the IBM tests pass with maximum values
+           of 1070000000."""
+        if self.decimal == C and self.decimal.MAX_EMAX == 425000000:
+            self.readcontext._unsafe_setprec(1070000000)
+            self.readcontext._unsafe_setemax(1070000000)
+            self.readcontext._unsafe_setemin(-1070000000)
+            return self.readcontext.create_decimal(v)
+        else:
+            return self.decimal.Decimal(v, context)
 
     def eval_file(self, file):
         global skip_expected
@@ -227,7 +297,7 @@ class DecimalTest(unittest.TestCase):
                 #print line
                 try:
                     t = self.eval_line(line)
-                except DecimalException as exception:
+                except self.decimal.DecimalException as exception:
                     #Exception raised where there shouldn't have been one.
                     self.fail('Exception "'+exception.__class__.__name__ + '" raised on line '+line)
 
@@ -254,23 +324,23 @@ class DecimalTest(unittest.TestCase):
     def eval_directive(self, s):
         funct, value = (x.strip().lower() for x in s.split(':'))
         if funct == 'rounding':
-            value = RoundingDict[value]
+            value = self.RoundingDict[value]
         else:
             try:
                 value = int(value)
             except ValueError:
                 pass
 
-        funct = self.ChangeDict.get(funct, Nonfunction)
+        funct = self.ChangeDict.get(funct, (lambda *args: None))
         funct(value)
 
     def eval_equation(self, s):
-        #global DEFAULT_PRECISION
-        #print DEFAULT_PRECISION
 
         if not TEST_ALL and random.random() < 0.90:
             return
 
+        self.context.clear_flags()
+
         try:
             Sides = s.split('->')
             L = Sides[0].strip().split()
@@ -283,26 +353,26 @@ class DecimalTest(unittest.TestCase):
             ans = L[0]
             exceptions = L[1:]
         except (TypeError, AttributeError, IndexError):
-            raise InvalidOperation
+            raise self.decimal.InvalidOperation
         def FixQuotes(val):
             val = val.replace("''", 'SingleQuote').replace('""', 'DoubleQuote')
             val = val.replace("'", '').replace('"', '')
             val = val.replace('SingleQuote', "'").replace('DoubleQuote', '"')
             return val
 
-        if id in skipped_test_ids:
+        if id in self.skipped_test_ids:
             return
 
-        fname = nameAdapter.get(funct, funct)
+        fname = self.NameAdapter.get(funct, funct)
         if fname == 'rescale':
             return
         funct = getattr(self.context, fname)
         vals = []
         conglomerate = ''
         quote = 0
-        theirexceptions = [ErrorNames[x.lower()] for x in exceptions]
+        theirexceptions = [self.ErrorNames[x.lower()] for x in exceptions]
 
-        for exception in Signals:
+        for exception in Signals[self.decimal]:
             self.context.traps[exception] = 1 #Catch these bugs...
         for exception in theirexceptions:
             self.context.traps[exception] = 0
@@ -324,7 +394,7 @@ class DecimalTest(unittest.TestCase):
                             funct(self.context.create_decimal(v))
                         except error:
                             pass
-                        except Signals as e:
+                        except Signals[self.decimal] as e:
                             self.fail("Raised %s in %s when %s disabled" % \
                                       (e, s, error))
                         else:
@@ -332,7 +402,7 @@ class DecimalTest(unittest.TestCase):
                         self.context.traps[error] = 0
                 v = self.context.create_decimal(v)
             else:
-                v = Decimal(v, self.context)
+                v = self.read_unlimited(v, self.context)
             vals.append(v)
 
         ans = FixQuotes(ans)
@@ -344,7 +414,7 @@ class DecimalTest(unittest.TestCase):
                     funct(*vals)
                 except error:
                     pass
-                except Signals as e:
+                except Signals[self.decimal] as e:
                     self.fail("Raised %s in %s when %s disabled" % \
                               (e, s, error))
                 else:
@@ -352,14 +422,14 @@ class DecimalTest(unittest.TestCase):
                 self.context.traps[error] = 0
 
             # as above, but add traps cumulatively, to check precedence
-            ordered_errors = [e for e in OrderedSignals if e in theirexceptions]
+            ordered_errors = [e for e in OrderedSignals[self.decimal] if e in theirexceptions]
             for error in ordered_errors:
                 self.context.traps[error] = 1
                 try:
                     funct(*vals)
                 except error:
                     pass
-                except Signals as e:
+                except Signals[self.decimal] as e:
                     self.fail("Raised %s in %s; expected %s" %
                               (type(e), s, error))
                 else:
@@ -373,54 +443,69 @@ class DecimalTest(unittest.TestCase):
             print("--", self.context)
         try:
             result = str(funct(*vals))
-            if fname in LOGICAL_FUNCTIONS:
+            if fname in self.LogicalFunctions:
                 result = str(int(eval(result))) # 'True', 'False' -> '1', '0'
-        except Signals as error:
+        except Signals[self.decimal] as error:
             self.fail("Raised %s in %s" % (error, s))
         except: #Catch any error long enough to state the test case.
             print("ERROR:", s)
             raise
 
         myexceptions = self.getexceptions()
-        self.context.clear_flags()
 
         myexceptions.sort(key=repr)
         theirexceptions.sort(key=repr)
 
         self.assertEqual(result, ans,
                          'Incorrect answer for ' + s + ' -- got ' + result)
+
         self.assertEqual(myexceptions, theirexceptions,
               'Incorrect flags set in ' + s + ' -- got ' + str(myexceptions))
         return
 
     def getexceptions(self):
-        return [e for e in Signals if self.context.flags[e]]
+        return [e for e in Signals[self.decimal] if self.context.flags[e]]
 
     def change_precision(self, prec):
-        self.context.prec = prec
+        if self.decimal == C and self.decimal.MAX_PREC == 425000000:
+            self.context._unsafe_setprec(prec)
+        else:
+            self.context.prec = prec
     def change_rounding_method(self, rounding):
         self.context.rounding = rounding
     def change_min_exponent(self, exp):
-        self.context.Emin = exp
+        if self.decimal == C and self.decimal.MAX_PREC == 425000000:
+            self.context._unsafe_setemin(exp)
+        else:
+            self.context.Emin = exp
     def change_max_exponent(self, exp):
-        self.context.Emax = exp
+        if self.decimal == C and self.decimal.MAX_PREC == 425000000:
+            self.context._unsafe_setemax(exp)
+        else:
+            self.context.Emax = exp
     def change_clamp(self, clamp):
         self.context.clamp = clamp
 
-
+class CIBMTestCases(IBMTestCases):
+    decimal = C
+class PyIBMTestCases(IBMTestCases):
+    decimal = P
 
 # The following classes test the behaviour of Decimal according to PEP 327
 
-class DecimalExplicitConstructionTest(unittest.TestCase):
+class ExplicitConstructionTest(unittest.TestCase):
     '''Unit tests for Explicit Construction cases of Decimal.'''
 
     def test_explicit_empty(self):
+        Decimal = self.decimal.Decimal
         self.assertEqual(Decimal(), Decimal("0"))
 
     def test_explicit_from_None(self):
+        Decimal = self.decimal.Decimal
         self.assertRaises(TypeError, Decimal, None)
 
     def test_explicit_from_int(self):
+        Decimal = self.decimal.Decimal
 
         #positive
         d = Decimal(45)
@@ -438,7 +523,18 @@ class DecimalExplicitConstructionTest(unittest.TestCase):
         d = Decimal(0)
         self.assertEqual(str(d), '0')
 
+        # single word longs
+        for n in range(0, 32):
+            for sign in (-1, 1):
+                for x in range(-5, 5):
+                    i = sign * (2**n + x)
+                    d = Decimal(i)
+                    self.assertEqual(str(d), str(i))
+
     def test_explicit_from_string(self):
+        Decimal = self.decimal.Decimal
+        InvalidOperation = self.decimal.InvalidOperation
+        localcontext = self.decimal.localcontext
 
         #empty
         self.assertEqual(str(Decimal('')), 'NaN')
@@ -458,8 +554,44 @@ class DecimalExplicitConstructionTest(unittest.TestCase):
         #leading and trailing whitespace permitted
         self.assertEqual(str(Decimal('1.3E4 \n')), '1.3E+4')
         self.assertEqual(str(Decimal('  -7.89')), '-7.89')
+        self.assertEqual(str(Decimal("  3.45679  ")), '3.45679')
+
+        # unicode whitespace
+        for lead in ["", ' ', '\u00a0', '\u205f']:
+            for trail in ["", ' ', '\u00a0', '\u205f']:
+                self.assertEqual(str(Decimal(lead + '9.311E+28' + trail)),
+                                 '9.311E+28')
+
+        with localcontext() as c:
+            c.traps[InvalidOperation] = True
+            # Invalid string
+            self.assertRaises(InvalidOperation, Decimal, "xyz")
+            # Two arguments max
+            self.assertRaises(TypeError, Decimal, "1234", "x", "y")
+
+            # space within the numeric part
+            self.assertRaises(InvalidOperation, Decimal, "1\u00a02\u00a03")
+            self.assertRaises(InvalidOperation, Decimal, "\u00a01\u00a02\u00a0")
+
+            # unicode whitespace
+            self.assertRaises(InvalidOperation, Decimal, "\u00a0")
+            self.assertRaises(InvalidOperation, Decimal, "\u00a0\u00a0")
+
+            # embedded NUL
+            self.assertRaises(InvalidOperation, Decimal, "12\u00003")
+
+    @cpython_only
+    def test_from_legacy_strings(self):
+        import _testcapi
+        Decimal = self.decimal.Decimal
+        context = self.decimal.Context()
+
+        s = _testcapi.unicode_legacy_string('9.999999')
+        self.assertEqual(str(Decimal(s)), '9.999999')
+        self.assertEqual(str(context.create_decimal(s)), '9.999999')
 
     def test_explicit_from_tuples(self):
+        Decimal = self.decimal.Decimal
 
         #zero
         d = Decimal( (0, (0,), 0) )
@@ -477,6 +609,10 @@ class DecimalExplicitConstructionTest(unittest.TestCase):
         d = Decimal( (1, (4, 3, 4, 9, 1, 3, 5, 3, 4), -25) )
         self.assertEqual(str(d), '-4.34913534E-17')
 
+        #inf
+        d = Decimal( (0, (), "F") )
+        self.assertEqual(str(d), 'Infinity')
+
         #wrong number of items
         self.assertRaises(ValueError, Decimal, (1, (4, 3, 4, 9, 1)) )
 
@@ -491,45 +627,63 @@ class DecimalExplicitConstructionTest(unittest.TestCase):
         self.assertRaises(ValueError, Decimal, (1, (4, 3, 4, 9, 1), '1') )
 
         #bad coefficients
+        self.assertRaises(ValueError, Decimal, (1, "xyz", 2) )
         self.assertRaises(ValueError, Decimal, (1, (4, 3, 4, None, 1), 2) )
         self.assertRaises(ValueError, Decimal, (1, (4, -3, 4, 9, 1), 2) )
         self.assertRaises(ValueError, Decimal, (1, (4, 10, 4, 9, 1), 2) )
         self.assertRaises(ValueError, Decimal, (1, (4, 3, 4, 'a', 1), 2) )
 
+    def test_explicit_from_list(self):
+        Decimal = self.decimal.Decimal
+
+        d = Decimal([0, [0], 0])
+        self.assertEqual(str(d), '0')
+
+        d = Decimal([1, [4, 3, 4, 9, 1, 3, 5, 3, 4], -25])
+        self.assertEqual(str(d), '-4.34913534E-17')
+
+        d = Decimal([1, (4, 3, 4, 9, 1, 3, 5, 3, 4), -25])
+        self.assertEqual(str(d), '-4.34913534E-17')
+
+        d = Decimal((1, [4, 3, 4, 9, 1, 3, 5, 3, 4], -25))
+        self.assertEqual(str(d), '-4.34913534E-17')
+
     def test_explicit_from_bool(self):
+        Decimal = self.decimal.Decimal
+
         self.assertIs(bool(Decimal(0)), False)
         self.assertIs(bool(Decimal(1)), True)
         self.assertEqual(Decimal(False), Decimal(0))
         self.assertEqual(Decimal(True), Decimal(1))
 
     def test_explicit_from_Decimal(self):
+        Decimal = self.decimal.Decimal
 
         #positive
         d = Decimal(45)
         e = Decimal(d)
         self.assertEqual(str(e), '45')
-        self.assertNotEqual(id(d), id(e))
 
         #very large positive
         d = Decimal(500000123)
         e = Decimal(d)
         self.assertEqual(str(e), '500000123')
-        self.assertNotEqual(id(d), id(e))
 
         #negative
         d = Decimal(-45)
         e = Decimal(d)
         self.assertEqual(str(e), '-45')
-        self.assertNotEqual(id(d), id(e))
 
         #zero
         d = Decimal(0)
         e = Decimal(d)
         self.assertEqual(str(e), '0')
-        self.assertNotEqual(id(d), id(e))
 
     @requires_IEEE_754
     def test_explicit_from_float(self):
+
+        Decimal = self.decimal.Decimal
+
         r = Decimal(0.1)
         self.assertEqual(type(r), Decimal)
         self.assertEqual(str(r),
@@ -550,8 +704,11 @@ class DecimalExplicitConstructionTest(unittest.TestCase):
             self.assertEqual(x, float(Decimal(x))) # roundtrip
 
     def test_explicit_context_create_decimal(self):
+        Decimal = self.decimal.Decimal
+        InvalidOperation = self.decimal.InvalidOperation
+        Rounded = self.decimal.Rounded
 
-        nc = copy.copy(getcontext())
+        nc = copy.copy(self.decimal.getcontext())
         nc.prec = 3
 
         # empty
@@ -592,7 +749,73 @@ class DecimalExplicitConstructionTest(unittest.TestCase):
         d = nc.create_decimal(prevdec)
         self.assertEqual(str(d), '5.00E+8')
 
+        # more integers
+        nc.prec = 28
+        nc.traps[InvalidOperation] = True
+
+        for v in [-2**63-1, -2**63, -2**31-1, -2**31, 0,
+                   2**31-1, 2**31, 2**63-1, 2**63]:
+            d = nc.create_decimal(v)
+            self.assertTrue(isinstance(d, Decimal))
+            self.assertEqual(int(d), v)
+
+        nc.prec = 3
+        nc.traps[Rounded] = True
+        self.assertRaises(Rounded, nc.create_decimal, 1234)
+
+        # from string
+        nc.prec = 28
+        self.assertEqual(str(nc.create_decimal('0E-017')), '0E-17')
+        self.assertEqual(str(nc.create_decimal('45')), '45')
+        self.assertEqual(str(nc.create_decimal('-Inf')), '-Infinity')
+        self.assertEqual(str(nc.create_decimal('NaN123')), 'NaN123')
+
+        # invalid arguments
+        self.assertRaises(InvalidOperation, nc.create_decimal, "xyz")
+        self.assertRaises(ValueError, nc.create_decimal, (1, "xyz", -25))
+        self.assertRaises(TypeError, nc.create_decimal, "1234", "5678")
+
+        # too many NaN payload digits
+        nc.prec = 3
+        self.assertRaises(InvalidOperation, nc.create_decimal, 'NaN12345')
+        self.assertRaises(InvalidOperation, nc.create_decimal,
+                          Decimal('NaN12345'))
+
+        nc.traps[InvalidOperation] = False
+        self.assertEqual(str(nc.create_decimal('NaN12345')), 'NaN')
+        self.assertTrue(nc.flags[InvalidOperation])
+
+        nc.flags[InvalidOperation] = False
+        self.assertEqual(str(nc.create_decimal(Decimal('NaN12345'))), 'NaN')
+        self.assertTrue(nc.flags[InvalidOperation])
+
+    def test_explicit_context_create_from_float(self):
+
+        Decimal = self.decimal.Decimal
+
+        nc = self.decimal.Context()
+        r = nc.create_decimal(0.1)
+        self.assertEqual(type(r), Decimal)
+        self.assertEqual(str(r), '0.1000000000000000055511151231')
+        self.assertTrue(nc.create_decimal(float('nan')).is_qnan())
+        self.assertTrue(nc.create_decimal(float('inf')).is_infinite())
+        self.assertTrue(nc.create_decimal(float('-inf')).is_infinite())
+        self.assertEqual(str(nc.create_decimal(float('nan'))),
+                         str(nc.create_decimal('NaN')))
+        self.assertEqual(str(nc.create_decimal(float('inf'))),
+                         str(nc.create_decimal('Infinity')))
+        self.assertEqual(str(nc.create_decimal(float('-inf'))),
+                         str(nc.create_decimal('-Infinity')))
+        self.assertEqual(str(nc.create_decimal(float('-0.0'))),
+                         str(nc.create_decimal('-0')))
+        nc.prec = 100
+        for i in range(200):
+            x = random.expovariate(0.01) * (random.random() * 2.0 - 1.0)
+            self.assertEqual(x, float(nc.create_decimal(x))) # roundtrip
+
     def test_unicode_digits(self):
+        Decimal = self.decimal.Decimal
+
         test_values = {
             '\uff11': '1',
             '\u0660.\u0660\u0663\u0667\u0662e-\u0663' : '0.0000372',
@@ -601,29 +824,41 @@ class DecimalExplicitConstructionTest(unittest.TestCase):
         for input, expected in test_values.items():
             self.assertEqual(str(Decimal(input)), expected)
 
+class CExplicitConstructionTest(ExplicitConstructionTest):
+    decimal = C
+class PyExplicitConstructionTest(ExplicitConstructionTest):
+    decimal = P
 
-class DecimalImplicitConstructionTest(unittest.TestCase):
+class ImplicitConstructionTest(unittest.TestCase):
     '''Unit tests for Implicit Construction cases of Decimal.'''
 
     def test_implicit_from_None(self):
-        self.assertRaises(TypeError, eval, 'Decimal(5) + None', globals())
+        Decimal = self.decimal.Decimal
+        self.assertRaises(TypeError, eval, 'Decimal(5) + None', locals())
 
     def test_implicit_from_int(self):
+        Decimal = self.decimal.Decimal
+
         #normal
         self.assertEqual(str(Decimal(5) + 45), '50')
         #exceeding precision
         self.assertEqual(Decimal(5) + 123456789000, Decimal(123456789000))
 
     def test_implicit_from_string(self):
-        self.assertRaises(TypeError, eval, 'Decimal(5) + "3"', globals())
+        Decimal = self.decimal.Decimal
+        self.assertRaises(TypeError, eval, 'Decimal(5) + "3"', locals())
 
     def test_implicit_from_float(self):
-        self.assertRaises(TypeError, eval, 'Decimal(5) + 2.2', globals())
+        Decimal = self.decimal.Decimal
+        self.assertRaises(TypeError, eval, 'Decimal(5) + 2.2', locals())
 
     def test_implicit_from_Decimal(self):
+        Decimal = self.decimal.Decimal
         self.assertEqual(Decimal(5) + Decimal(45), Decimal(50))
 
     def test_rop(self):
+        Decimal = self.decimal.Decimal
+
         # Allow other classes to be trained to interact with Decimals
         class E:
             def __divmod__(self, other):
@@ -671,10 +906,16 @@ class DecimalImplicitConstructionTest(unittest.TestCase):
             self.assertEqual(eval('Decimal(10)' + sym + 'E()'),
                              '10' + rop + 'str')
 
+class CImplicitConstructionTest(ImplicitConstructionTest):
+    decimal = C
+class PyImplicitConstructionTest(ImplicitConstructionTest):
+    decimal = P
 
-class DecimalFormatTest(unittest.TestCase):
+class FormatTest(unittest.TestCase):
     '''Unit tests for the format function.'''
     def test_formatting(self):
+        Decimal = self.decimal.Decimal
+
         # triples giving a format, a Decimal, and the expected result
         test_values = [
             ('e', '0E-15', '0e-15'),
@@ -730,6 +971,7 @@ class DecimalFormatTest(unittest.TestCase):
             ('g', '0E-7', '0e-7'),
             ('g', '-0E2', '-0e+2'),
             ('.0g', '3.14159265', '3'),  # 0 sig fig -> 1 sig fig
+            ('.0n', '3.14159265', '3'),  # same for 'n'
             ('.1g', '3.14159265', '3'),
             ('.2g', '3.14159265', '3.1'),
             ('.5g', '3.14159265', '3.1416'),
@@ -814,56 +1056,60 @@ class DecimalFormatTest(unittest.TestCase):
 
             # issue 6850
             ('a=-7.0', '0.12345', 'aaaa0.1'),
-
-            # Issue 7094: Alternate formatting (specified by #)
-            ('.0e', '1.0', '1e+0'),
-            ('#.0e', '1.0', '1.e+0'),
-            ('.0f', '1.0', '1'),
-            ('#.0f', '1.0', '1.'),
-            ('g', '1.1', '1.1'),
-            ('#g', '1.1', '1.1'),
-            ('.0g', '1', '1'),
-            ('#.0g', '1', '1.'),
-            ('.0%', '1.0', '100%'),
-            ('#.0%', '1.0', '100.%'),
             ]
         for fmt, d, result in test_values:
             self.assertEqual(format(Decimal(d), fmt), result)
 
+        # bytes format argument
+        self.assertRaises(TypeError, Decimal(1).__format__, b'-020')
+
     def test_n_format(self):
+        Decimal = self.decimal.Decimal
+
         try:
             from locale import CHAR_MAX
         except ImportError:
             return
 
+        def make_grouping(lst):
+            return ''.join([chr(x) for x in lst]) if self.decimal == C else lst
+
+        def get_fmt(x, override=None, fmt='n'):
+            if self.decimal == C:
+                return Decimal(x).__format__(fmt, override)
+            else:
+                return Decimal(x).__format__(fmt, _localeconv=override)
+
         # Set up some localeconv-like dictionaries
         en_US = {
             'decimal_point' : '.',
-            'grouping' : [3, 3, 0],
-            'thousands_sep': ','
+            'grouping' : make_grouping([3, 3, 0]),
+            'thousands_sep' : ','
             }
 
         fr_FR = {
             'decimal_point' : ',',
-            'grouping' : [CHAR_MAX],
+            'grouping' : make_grouping([CHAR_MAX]),
             'thousands_sep' : ''
             }
 
         ru_RU = {
             'decimal_point' : ',',
-            'grouping' : [3, 3, 0],
+            'grouping': make_grouping([3, 3, 0]),
             'thousands_sep' : ' '
             }
 
         crazy = {
             'decimal_point' : '&',
-            'grouping' : [1, 4, 2, CHAR_MAX],
+            'grouping': make_grouping([1, 4, 2, CHAR_MAX]),
             'thousands_sep' : '-'
             }
 
-
-        def get_fmt(x, locale, fmt='n'):
-            return Decimal.__format__(Decimal(x), fmt, _localeconv=locale)
+        dotsep_wide = {
+            'decimal_point' : b'\xc2\xbf'.decode('utf-8'),
+            'grouping': make_grouping([3, 3, 0]),
+            'thousands_sep' : b'\xc2\xb4'.decode('utf-8')
+            }
 
         self.assertEqual(get_fmt(Decimal('12.7'), en_US), '12.7')
         self.assertEqual(get_fmt(Decimal('12.7'), fr_FR), '12,7')
@@ -902,11 +1148,34 @@ class DecimalFormatTest(unittest.TestCase):
         self.assertEqual(get_fmt(123456, crazy, '012n'), '00-01-2345-6')
         self.assertEqual(get_fmt(123456, crazy, '013n'), '000-01-2345-6')
 
+        # wide char separator and decimal point
+        self.assertEqual(get_fmt(Decimal('-1.5'), dotsep_wide, '020n'),
+                         '-0\u00b4000\u00b4000\u00b4000\u00b4001\u00bf5')
+
+    @run_with_locale('LC_ALL', 'ps_AF')
+    def test_wide_char_separator_decimal_point(self):
+        # locale with wide char separator and decimal point
+        import locale
+        Decimal = self.decimal.Decimal
 
-class DecimalArithmeticOperatorsTest(unittest.TestCase):
+        decimal_point = locale.localeconv()['decimal_point']
+        thousands_sep = locale.localeconv()['thousands_sep']
+        if decimal_point != '\u066b' or thousands_sep != '\u066c':
+            return
+
+        self.assertEqual(format(Decimal('100000000.123'), 'n'),
+                         '100\u066c000\u066c000\u066b123')
+
+class CFormatTest(FormatTest):
+    decimal = C
+class PyFormatTest(FormatTest):
+    decimal = P
+
+class ArithmeticOperatorsTest(unittest.TestCase):
     '''Unit tests for all arithmetic operators, binary and unary.'''
 
     def test_addition(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('-11.1')
         d2 = Decimal('22.2')
@@ -934,6 +1203,7 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(d1, Decimal('16.1'))
 
     def test_subtraction(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('-11.1')
         d2 = Decimal('22.2')
@@ -961,6 +1231,7 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(d1, Decimal('-38.3'))
 
     def test_multiplication(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('-5')
         d2 = Decimal('3')
@@ -988,6 +1259,7 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(d1, Decimal('-75'))
 
     def test_division(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('-5')
         d2 = Decimal('2')
@@ -1015,6 +1287,7 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(d1, Decimal('-0.625'))
 
     def test_floor_division(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('5')
         d2 = Decimal('2')
@@ -1042,6 +1315,7 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(d1, Decimal('1'))
 
     def test_powering(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('5')
         d2 = Decimal('2')
@@ -1069,6 +1343,7 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(d1, Decimal('390625'))
 
     def test_module(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('5')
         d2 = Decimal('2')
@@ -1096,6 +1371,7 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(d1, Decimal('1'))
 
     def test_floor_div_module(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('5')
         d2 = Decimal('2')
@@ -1122,6 +1398,8 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
         self.assertEqual(type(q), type(d1))
 
     def test_unary_operators(self):
+        Decimal = self.decimal.Decimal
+
         self.assertEqual(+Decimal(45), Decimal(+45))           #  +
         self.assertEqual(-Decimal(45), Decimal(-45))           #  -
         self.assertEqual(abs(Decimal(45)), abs(Decimal(-45)))  # abs
@@ -1134,6 +1412,9 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
 
         # equality comparisons (==, !=) involving only quiet nans
         # don't signal, but return False or True respectively.
+        Decimal = self.decimal.Decimal
+        InvalidOperation = self.decimal.InvalidOperation
+        localcontext = self.decimal.localcontext
 
         n = Decimal('NaN')
         s = Decimal('sNaN')
@@ -1179,53 +1460,124 @@ class DecimalArithmeticOperatorsTest(unittest.TestCase):
                     self.assertRaises(InvalidOperation, op, x, y)
 
     def test_copy_sign(self):
-        d = Decimal(1).copy_sign(Decimal(-2))
+        Decimal = self.decimal.Decimal
 
+        d = Decimal(1).copy_sign(Decimal(-2))
         self.assertEqual(Decimal(1).copy_sign(-2), d)
         self.assertRaises(TypeError, Decimal(1).copy_sign, '-2')
 
+class CArithmeticOperatorsTest(ArithmeticOperatorsTest):
+    decimal = C
+class PyArithmeticOperatorsTest(ArithmeticOperatorsTest):
+    decimal = P
+
 # The following are two functions used to test threading in the next class
 
 def thfunc1(cls):
+    Decimal = cls.decimal.Decimal
+    InvalidOperation = cls.decimal.InvalidOperation
+    DivisionByZero = cls.decimal.DivisionByZero
+    Overflow = cls.decimal.Overflow
+    Underflow = cls.decimal.Underflow
+    Inexact = cls.decimal.Inexact
+    getcontext = cls.decimal.getcontext
+    localcontext = cls.decimal.localcontext
+
     d1 = Decimal(1)
     d3 = Decimal(3)
     test1 = d1/d3
-    cls.synchro.wait()
-    test2 = d1/d3
+
     cls.finish1.set()
+    cls.synchro.wait()
 
-    cls.assertEqual(test1, Decimal('0.3333333333333333333333333333'))
-    cls.assertEqual(test2, Decimal('0.3333333333333333333333333333'))
+    test2 = d1/d3
+    with localcontext() as c2:
+        cls.assertTrue(c2.flags[Inexact])
+        cls.assertRaises(DivisionByZero, c2.divide, d1, 0)
+        cls.assertTrue(c2.flags[DivisionByZero])
+        with localcontext() as c3:
+            cls.assertTrue(c3.flags[Inexact])
+            cls.assertTrue(c3.flags[DivisionByZero])
+            cls.assertRaises(InvalidOperation, c3.compare, d1, Decimal('sNaN'))
+            cls.assertTrue(c3.flags[InvalidOperation])
+            del c3
+        cls.assertFalse(c2.flags[InvalidOperation])
+        del c2
+
+    cls.assertEqual(test1, Decimal('0.333333333333333333333333'))
+    cls.assertEqual(test2, Decimal('0.333333333333333333333333'))
+
+    c1 = getcontext()
+    cls.assertTrue(c1.flags[Inexact])
+    for sig in Overflow, Underflow, DivisionByZero, InvalidOperation:
+        cls.assertFalse(c1.flags[sig])
     return
 
 def thfunc2(cls):
+    Decimal = cls.decimal.Decimal
+    InvalidOperation = cls.decimal.InvalidOperation
+    DivisionByZero = cls.decimal.DivisionByZero
+    Overflow = cls.decimal.Overflow
+    Underflow = cls.decimal.Underflow
+    Inexact = cls.decimal.Inexact
+    getcontext = cls.decimal.getcontext
+    localcontext = cls.decimal.localcontext
+
     d1 = Decimal(1)
     d3 = Decimal(3)
     test1 = d1/d3
+
     thiscontext = getcontext()
     thiscontext.prec = 18
     test2 = d1/d3
+
+    with localcontext() as c2:
+        cls.assertTrue(c2.flags[Inexact])
+        cls.assertRaises(Overflow, c2.multiply, Decimal('1e425000000'), 999)
+        cls.assertTrue(c2.flags[Overflow])
+        with localcontext(thiscontext) as c3:
+            cls.assertTrue(c3.flags[Inexact])
+            cls.assertFalse(c3.flags[Overflow])
+            c3.traps[Underflow] = True
+            cls.assertRaises(Underflow, c3.divide, Decimal('1e-425000000'), 999)
+            cls.assertTrue(c3.flags[Underflow])
+            del c3
+        cls.assertFalse(c2.flags[Underflow])
+        cls.assertFalse(c2.traps[Underflow])
+        del c2
+
     cls.synchro.set()
     cls.finish2.set()
 
-    cls.assertEqual(test1, Decimal('0.3333333333333333333333333333'))
+    cls.assertEqual(test1, Decimal('0.333333333333333333333333'))
     cls.assertEqual(test2, Decimal('0.333333333333333333'))
-    return
-
 
-class DecimalUseOfContextTest(unittest.TestCase):
-    '''Unit tests for Use of Context cases in Decimal.'''
+    cls.assertFalse(thiscontext.traps[Underflow])
+    cls.assertTrue(thiscontext.flags[Inexact])
+    for sig in Overflow, Underflow, DivisionByZero, InvalidOperation:
+        cls.assertFalse(thiscontext.flags[sig])
+    return
 
-    try:
-        import threading
-    except ImportError:
-        threading = None
+class ThreadingTest(unittest.TestCase):
+    '''Unit tests for thread local contexts in Decimal.'''
 
     # Take care executing this test from IDLE, there's an issue in threading
     # that hangs IDLE and I couldn't find it
 
     def test_threading(self):
-        #Test the "threading isolation" of a Context.
+        DefaultContext = self.decimal.DefaultContext
+
+        if self.decimal == C and not self.decimal.HAVE_THREADS:
+            self.skipTest("compiled without threading")
+        # Test the "threading isolation" of a Context. Also test changing
+        # the DefaultContext, which acts as a template for the thread-local
+        # contexts.
+        save_prec = DefaultContext.prec
+        save_emax = DefaultContext.Emax
+        save_emin = DefaultContext.Emin
+        DefaultContext.prec = 24
+        DefaultContext.Emax = 425000000
+        DefaultContext.Emin = -425000000
 
         self.synchro = threading.Event()
         self.finish1 = threading.Event()
@@ -1239,17 +1591,29 @@ class DecimalUseOfContextTest(unittest.TestCase):
 
         self.finish1.wait()
         self.finish2.wait()
-        return
 
-    if threading is None:
-        del test_threading
+        for sig in Signals[self.decimal]:
+            self.assertFalse(DefaultContext.flags[sig])
+
+        DefaultContext.prec = save_prec
+        DefaultContext.Emax = save_emax
+        DefaultContext.Emin = save_emin
+        return
 
+@unittest.skipUnless(threading, 'threading required')
+class CThreadingTest(ThreadingTest):
+    decimal = C
+@unittest.skipUnless(threading, 'threading required')
+class PyThreadingTest(ThreadingTest):
+    decimal = P
 
-class DecimalUsabilityTest(unittest.TestCase):
+class UsabilityTest(unittest.TestCase):
     '''Unit tests for Usability cases of Decimal.'''
 
     def test_comparison_operators(self):
 
+        Decimal = self.decimal.Decimal
+
         da = Decimal('23.42')
         db = Decimal('23.42')
         dc = Decimal('45')
@@ -1283,6 +1647,8 @@ class DecimalUsabilityTest(unittest.TestCase):
         self.assertEqual(a, b)
 
     def test_decimal_float_comparison(self):
+        Decimal = self.decimal.Decimal
+
         da = Decimal('0.25')
         db = Decimal('3.0')
         self.assertLess(da, 3.0)
@@ -1299,7 +1665,71 @@ class DecimalUsabilityTest(unittest.TestCase):
         self.assertEqual(3.0, db)
         self.assertNotEqual(0.1, Decimal('0.1'))
 
+    def test_decimal_complex_comparison(self):
+        Decimal = self.decimal.Decimal
+
+        da = Decimal('0.25')
+        db = Decimal('3.0')
+        self.assertNotEqual(da, (1.5+0j))
+        self.assertNotEqual((1.5+0j), da)
+        self.assertEqual(da, (0.25+0j))
+        self.assertEqual((0.25+0j), da)
+        self.assertEqual((3.0+0j), db)
+        self.assertEqual(db, (3.0+0j))
+
+        self.assertNotEqual(db, (3.0+1j))
+        self.assertNotEqual((3.0+1j), db)
+
+        self.assertIs(db.__lt__(3.0+0j), NotImplemented)
+        self.assertIs(db.__le__(3.0+0j), NotImplemented)
+        self.assertIs(db.__gt__(3.0+0j), NotImplemented)
+        self.assertIs(db.__le__(3.0+0j), NotImplemented)
+
+    def test_decimal_fraction_comparison(self):
+        D = self.decimal.Decimal
+        F = fractions[self.decimal].Fraction
+        Context = self.decimal.Context
+        localcontext = self.decimal.localcontext
+        InvalidOperation = self.decimal.InvalidOperation
+
+
+        emax = C.MAX_EMAX if C else 999999999
+        emin = C.MIN_EMIN if C else -999999999
+        etiny = C.MIN_ETINY if C else -1999999997
+        c = Context(Emax=emax, Emin=emin)
+
+        with localcontext(c):
+            c.prec = emax
+            self.assertLess(D(0), F(1,9999999999999999999999999999999999999))
+            self.assertLess(F(-1,9999999999999999999999999999999999999), D(0))
+            self.assertLess(F(0,1), D("1e" + str(etiny)))
+            self.assertLess(D("-1e" + str(etiny)), F(0,1))
+            self.assertLess(F(0,9999999999999999999999999), D("1e" + str(etiny)))
+            self.assertLess(D("-1e" + str(etiny)), F(0,9999999999999999999999999))
+
+            self.assertEqual(D("0.1"), F(1,10))
+            self.assertEqual(F(1,10), D("0.1"))
+
+            c.prec = 300
+            self.assertNotEqual(D(1)/3, F(1,3))
+            self.assertNotEqual(F(1,3), D(1)/3)
+
+            self.assertLessEqual(F(120984237, 9999999999), D("9e" + str(emax)))
+            self.assertGreaterEqual(D("9e" + str(emax)), F(120984237, 9999999999))
+
+            self.assertGreater(D('inf'), F(99999999999,123))
+            self.assertGreater(D('inf'), F(-99999999999,123))
+            self.assertLess(D('-inf'), F(99999999999,123))
+            self.assertLess(D('-inf'), F(-99999999999,123))
+
+            self.assertRaises(InvalidOperation, D('nan').__gt__, F(-9,123))
+            self.assertIs(NotImplemented, F(-9,123).__lt__(D('nan')))
+            self.assertNotEqual(D('nan'), F(-9,123))
+            self.assertNotEqual(F(-9,123), D('nan'))
+
     def test_copy_and_deepcopy_methods(self):
+        Decimal = self.decimal.Decimal
+
         d = Decimal('43.24')
         c = copy.copy(d)
         self.assertEqual(id(c), id(d))
@@ -1307,6 +1737,10 @@ class DecimalUsabilityTest(unittest.TestCase):
         self.assertEqual(id(dc), id(d))
 
     def test_hash_method(self):
+
+        Decimal = self.decimal.Decimal
+        localcontext = self.decimal.localcontext
+
         def hashit(d):
             a = hash(d)
             b = d.__hash__()
@@ -1367,24 +1801,27 @@ class DecimalUsabilityTest(unittest.TestCase):
             d = Decimal(s)
             self.assertEqual(hashit(f), hashit(d))
 
-        # check that the value of the hash doesn't depend on the
-        # current context (issue #1757)
-        c = getcontext()
-        old_precision = c.prec
-        x = Decimal("123456789.1")
+        with localcontext() as c:
+            # check that the value of the hash doesn't depend on the
+            # current context (issue #1757)
+            x = Decimal("123456789.1")
 
-        c.prec = 6
-        h1 = hashit(x)
-        c.prec = 10
-        h2 = hashit(x)
-        c.prec = 16
-        h3 = hashit(x)
+            c.prec = 6
+            h1 = hashit(x)
+            c.prec = 10
+            h2 = hashit(x)
+            c.prec = 16
+            h3 = hashit(x)
 
-        self.assertEqual(h1, h2)
-        self.assertEqual(h1, h3)
-        c.prec = old_precision
+            self.assertEqual(h1, h2)
+            self.assertEqual(h1, h3)
+
+            c.prec = 10000
+            x = 1100 ** 1248
+            self.assertEqual(hashit(Decimal(x)), hashit(x))
 
     def test_min_and_max_methods(self):
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('15.32')
         d2 = Decimal('28.5')
@@ -1404,6 +1841,8 @@ class DecimalUsabilityTest(unittest.TestCase):
         self.assertIs(max(d2,l1), d2)
 
     def test_as_nonzero(self):
+        Decimal = self.decimal.Decimal
+
         #as false
         self.assertFalse(Decimal(0))
         #as true
@@ -1411,6 +1850,7 @@ class DecimalUsabilityTest(unittest.TestCase):
 
     def test_tostring_methods(self):
         #Test str and repr methods.
+        Decimal = self.decimal.Decimal
 
         d = Decimal('15.32')
         self.assertEqual(str(d), '15.32')               # str
@@ -1418,6 +1858,7 @@ class DecimalUsabilityTest(unittest.TestCase):
 
     def test_tonum_methods(self):
         #Test float and int methods.
+        Decimal = self.decimal.Decimal
 
         d1 = Decimal('66')
         d2 = Decimal('15.32')
@@ -1440,6 +1881,7 @@ class DecimalUsabilityTest(unittest.TestCase):
             ('-11.0', -11),
             ('0.0', 0),
             ('-0E3', 0),
+            ('89891211712379812736.1', 89891211712379812736),
             ]
         for d, i in test_pairs:
             self.assertEqual(math.floor(Decimal(d)), i)
@@ -1459,6 +1901,7 @@ class DecimalUsabilityTest(unittest.TestCase):
             ('-11.0', -11),
             ('0.0', 0),
             ('-0E3', 0),
+            ('89891211712379812736.1', 89891211712379812737),
             ]
         for d, i in test_pairs:
             self.assertEqual(math.ceil(Decimal(d)), i)
@@ -1519,16 +1962,21 @@ class DecimalUsabilityTest(unittest.TestCase):
     def test_nan_to_float(self):
         # Test conversions of decimal NANs to float.
         # See http://bugs.python.org/issue15544
+        Decimal = self.decimal.Decimal
         for s in ('nan', 'nan1234', '-nan', '-nan2468'):
             f = float(Decimal(s))
             self.assertTrue(math.isnan(f))
+            sign = math.copysign(1.0, f)
+            self.assertEqual(sign, -1.0 if s.startswith('-') else 1.0)
 
     def test_snan_to_float(self):
+        Decimal = self.decimal.Decimal
         for s in ('snan', '-snan', 'snan1357', '-snan1234'):
             d = Decimal(s)
             self.assertRaises(ValueError, float, d)
 
     def test_eval_round_trip(self):
+        Decimal = self.decimal.Decimal
 
         #with zero
         d = Decimal( (0, (0,), 0) )
@@ -1547,6 +1995,7 @@ class DecimalUsabilityTest(unittest.TestCase):
         self.assertEqual(d, eval(repr(d)))
 
     def test_as_tuple(self):
+        Decimal = self.decimal.Decimal
 
         #with zero
         d = Decimal(0)
@@ -1560,7 +2009,8 @@ class DecimalUsabilityTest(unittest.TestCase):
         d = Decimal("-4.34913534E-17")
         self.assertEqual(d.as_tuple(), (1, (4, 3, 4, 9, 1, 3, 5, 3, 4), -25) )
 
-        #inf
+        # The '0' coefficient is implementation specific to decimal.py.
+        # It has no meaning in the C-version and is ignored there.
         d = Decimal("Infinity")
         self.assertEqual(d.as_tuple(), (0, (0,), 'F') )
 
@@ -1580,90 +2030,21 @@ class DecimalUsabilityTest(unittest.TestCase):
         d = Decimal( (1, (), 'n') )
         self.assertEqual(d.as_tuple(), (1, (), 'n') )
 
-        #coefficient in infinity should be ignored
+        # For infinities, decimal.py has always silently accepted any
+        # coefficient tuple.
+        d = Decimal( (0, (0,), 'F') )
+        self.assertEqual(d.as_tuple(), (0, (0,), 'F'))
         d = Decimal( (0, (4, 5, 3, 4), 'F') )
         self.assertEqual(d.as_tuple(), (0, (0,), 'F'))
         d = Decimal( (1, (0, 2, 7, 1), 'F') )
         self.assertEqual(d.as_tuple(), (1, (0,), 'F'))
 
-    def test_immutability_operations(self):
-        # Do operations and check that it didn't change change internal objects.
-
-        d1 = Decimal('-25e55')
-        b1 = Decimal('-25e55')
-        d2 = Decimal('33e+33')
-        b2 = Decimal('33e+33')
-
-        def checkSameDec(operation, useOther=False):
-            if useOther:
-                eval("d1." + operation + "(d2)")
-                self.assertEqual(d1._sign, b1._sign)
-                self.assertEqual(d1._int, b1._int)
-                self.assertEqual(d1._exp, b1._exp)
-                self.assertEqual(d2._sign, b2._sign)
-                self.assertEqual(d2._int, b2._int)
-                self.assertEqual(d2._exp, b2._exp)
-            else:
-                eval("d1." + operation + "()")
-                self.assertEqual(d1._sign, b1._sign)
-                self.assertEqual(d1._int, b1._int)
-                self.assertEqual(d1._exp, b1._exp)
-            return
-
-        Decimal(d1)
-        self.assertEqual(d1._sign, b1._sign)
-        self.assertEqual(d1._int, b1._int)
-        self.assertEqual(d1._exp, b1._exp)
-
-        checkSameDec("__abs__")
-        checkSameDec("__add__", True)
-        checkSameDec("__divmod__", True)
-        checkSameDec("__eq__", True)
-        checkSameDec("__ne__", True)
-        checkSameDec("__le__", True)
-        checkSameDec("__lt__", True)
-        checkSameDec("__ge__", True)
-        checkSameDec("__gt__", True)
-        checkSameDec("__float__")
-        checkSameDec("__floordiv__", True)
-        checkSameDec("__hash__")
-        checkSameDec("__int__")
-        checkSameDec("__trunc__")
-        checkSameDec("__mod__", True)
-        checkSameDec("__mul__", True)
-        checkSameDec("__neg__")
-        checkSameDec("__bool__")
-        checkSameDec("__pos__")
-        checkSameDec("__pow__", True)
-        checkSameDec("__radd__", True)
-        checkSameDec("__rdivmod__", True)
-        checkSameDec("__repr__")
-        checkSameDec("__rfloordiv__", True)
-        checkSameDec("__rmod__", True)
-        checkSameDec("__rmul__", True)
-        checkSameDec("__rpow__", True)
-        checkSameDec("__rsub__", True)
-        checkSameDec("__str__")
-        checkSameDec("__sub__", True)
-        checkSameDec("__truediv__", True)
-        checkSameDec("adjusted")
-        checkSameDec("as_tuple")
-        checkSameDec("compare", True)
-        checkSameDec("max", True)
-        checkSameDec("min", True)
-        checkSameDec("normalize")
-        checkSameDec("quantize", True)
-        checkSameDec("remainder_near", True)
-        checkSameDec("same_quantum", True)
-        checkSameDec("sqrt")
-        checkSameDec("to_eng_string")
-        checkSameDec("to_integral")
-
     def test_subclassing(self):
         # Different behaviours when subclassing Decimal
+        Decimal = self.decimal.Decimal
 
         class MyDecimal(Decimal):
-            pass
+            y = None
 
         d1 = MyDecimal(1)
         d2 = MyDecimal(2)
@@ -1673,15 +2054,291 @@ class DecimalUsabilityTest(unittest.TestCase):
         d = d1.max(d2)
         self.assertIs(type(d), Decimal)
 
+        d = copy.copy(d1)
+        self.assertIs(type(d), MyDecimal)
+        self.assertEqual(d, d1)
+
+        d = copy.deepcopy(d1)
+        self.assertIs(type(d), MyDecimal)
+        self.assertEqual(d, d1)
+
+        # Decimal(Decimal)
+        d = Decimal('1.0')
+        x = Decimal(d)
+        self.assertIs(type(x), Decimal)
+        self.assertEqual(x, d)
+
+        # MyDecimal(Decimal)
+        m = MyDecimal(d)
+        self.assertIs(type(m), MyDecimal)
+        self.assertEqual(m, d)
+        self.assertIs(m.y, None)
+
+        # Decimal(MyDecimal)
+        x = Decimal(m)
+        self.assertIs(type(x), Decimal)
+        self.assertEqual(x, d)
+
+        # MyDecimal(MyDecimal)
+        m.y = 9
+        x = MyDecimal(m)
+        self.assertIs(type(x), MyDecimal)
+        self.assertEqual(x, d)
+        self.assertIs(x.y, None)
+
     def test_implicit_context(self):
+        Decimal = self.decimal.Decimal
+        getcontext = self.decimal.getcontext
+
         # Check results when context given implicitly.  (Issue 2478)
         c = getcontext()
         self.assertEqual(str(Decimal(0).sqrt()),
                          str(c.sqrt(Decimal(0))))
 
+    def test_none_args(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+        localcontext = self.decimal.localcontext
+        InvalidOperation = self.decimal.InvalidOperation
+        DivisionByZero = self.decimal.DivisionByZero
+        Overflow = self.decimal.Overflow
+        Underflow = self.decimal.Underflow
+        Subnormal = self.decimal.Subnormal
+        Inexact = self.decimal.Inexact
+        Rounded = self.decimal.Rounded
+        Clamped = self.decimal.Clamped
+
+        with localcontext(Context()) as c:
+            c.prec = 7
+            c.Emax = 999
+            c.Emin = -999
+
+            x = Decimal("111")
+            y = Decimal("1e9999")
+            z = Decimal("1e-9999")
+
+            ##### Unary functions
+            c.clear_flags()
+            self.assertEqual(str(x.exp(context=None)), '1.609487E+48')
+            self.assertTrue(c.flags[Inexact])
+            self.assertTrue(c.flags[Rounded])
+            c.clear_flags()
+            self.assertRaises(Overflow, y.exp, context=None)
+            self.assertTrue(c.flags[Overflow])
+
+            self.assertIs(z.is_normal(context=None), False)
+            self.assertIs(z.is_subnormal(context=None), True)
+
+            c.clear_flags()
+            self.assertEqual(str(x.ln(context=None)), '4.709530')
+            self.assertTrue(c.flags[Inexact])
+            self.assertTrue(c.flags[Rounded])
+            c.clear_flags()
+            self.assertRaises(InvalidOperation, Decimal(-1).ln, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            self.assertEqual(str(x.log10(context=None)), '2.045323')
+            self.assertTrue(c.flags[Inexact])
+            self.assertTrue(c.flags[Rounded])
+            c.clear_flags()
+            self.assertRaises(InvalidOperation, Decimal(-1).log10, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            self.assertEqual(str(x.logb(context=None)), '2')
+            self.assertRaises(DivisionByZero, Decimal(0).logb, context=None)
+            self.assertTrue(c.flags[DivisionByZero])
+
+            c.clear_flags()
+            self.assertEqual(str(x.logical_invert(context=None)), '1111000')
+            self.assertRaises(InvalidOperation, y.logical_invert, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            self.assertEqual(str(y.next_minus(context=None)), '9.999999E+999')
+            self.assertRaises(InvalidOperation, Decimal('sNaN').next_minus, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            self.assertEqual(str(y.next_plus(context=None)), 'Infinity')
+            self.assertRaises(InvalidOperation, Decimal('sNaN').next_plus, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            self.assertEqual(str(z.normalize(context=None)), '0')
+            self.assertRaises(Overflow, y.normalize, context=None)
+            self.assertTrue(c.flags[Overflow])
+
+            self.assertEqual(str(z.number_class(context=None)), '+Subnormal')
+
+            c.clear_flags()
+            self.assertEqual(str(z.sqrt(context=None)), '0E-1005')
+            self.assertTrue(c.flags[Clamped])
+            self.assertTrue(c.flags[Inexact])
+            self.assertTrue(c.flags[Rounded])
+            self.assertTrue(c.flags[Subnormal])
+            self.assertTrue(c.flags[Underflow])
+            c.clear_flags()
+            self.assertRaises(Overflow, y.sqrt, context=None)
+            self.assertTrue(c.flags[Overflow])
+
+            c.capitals = 0
+            self.assertEqual(str(z.to_eng_string(context=None)), '1e-9999')
+            c.capitals = 1
+
+
+            ##### Binary functions
+            c.clear_flags()
+            ans = str(x.compare(Decimal('Nan891287828'), context=None))
+            self.assertEqual(ans, 'NaN1287828')
+            self.assertRaises(InvalidOperation, x.compare, Decimal('sNaN'), context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.compare_signal(8224, context=None))
+            self.assertEqual(ans, '-1')
+            self.assertRaises(InvalidOperation, x.compare_signal, Decimal('NaN'), context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.logical_and(101, context=None))
+            self.assertEqual(ans, '101')
+            self.assertRaises(InvalidOperation, x.logical_and, 123, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.logical_or(101, context=None))
+            self.assertEqual(ans, '111')
+            self.assertRaises(InvalidOperation, x.logical_or, 123, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.logical_xor(101, context=None))
+            self.assertEqual(ans, '10')
+            self.assertRaises(InvalidOperation, x.logical_xor, 123, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.max(101, context=None))
+            self.assertEqual(ans, '111')
+            self.assertRaises(InvalidOperation, x.max, Decimal('sNaN'), context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.max_mag(101, context=None))
+            self.assertEqual(ans, '111')
+            self.assertRaises(InvalidOperation, x.max_mag, Decimal('sNaN'), context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.min(101, context=None))
+            self.assertEqual(ans, '101')
+            self.assertRaises(InvalidOperation, x.min, Decimal('sNaN'), context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.min_mag(101, context=None))
+            self.assertEqual(ans, '101')
+            self.assertRaises(InvalidOperation, x.min_mag, Decimal('sNaN'), context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.remainder_near(101, context=None))
+            self.assertEqual(ans, '10')
+            self.assertRaises(InvalidOperation, y.remainder_near, 101, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.rotate(2, context=None))
+            self.assertEqual(ans, '11100')
+            self.assertRaises(InvalidOperation, x.rotate, 101, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.scaleb(7, context=None))
+            self.assertEqual(ans, '1.11E+9')
+            self.assertRaises(InvalidOperation, x.scaleb, 10000, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            ans = str(x.shift(2, context=None))
+            self.assertEqual(ans, '11100')
+            self.assertRaises(InvalidOperation, x.shift, 10000, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+
+            ##### Ternary functions
+            c.clear_flags()
+            ans = str(x.fma(2, 3, context=None))
+            self.assertEqual(ans, '225')
+            self.assertRaises(Overflow, x.fma, Decimal('1e9999'), 3, context=None)
+            self.assertTrue(c.flags[Overflow])
+
+
+            ##### Special cases
+            c.rounding = ROUND_HALF_EVEN
+            ans = str(Decimal('1.5').to_integral(rounding=None, context=None))
+            self.assertEqual(ans, '2')
+            c.rounding = ROUND_DOWN
+            ans = str(Decimal('1.5').to_integral(rounding=None, context=None))
+            self.assertEqual(ans, '1')
+            ans = str(Decimal('1.5').to_integral(rounding=ROUND_UP, context=None))
+            self.assertEqual(ans, '2')
+            c.clear_flags()
+            self.assertRaises(InvalidOperation, Decimal('sNaN').to_integral, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.rounding = ROUND_HALF_EVEN
+            ans = str(Decimal('1.5').to_integral_value(rounding=None, context=None))
+            self.assertEqual(ans, '2')
+            c.rounding = ROUND_DOWN
+            ans = str(Decimal('1.5').to_integral_value(rounding=None, context=None))
+            self.assertEqual(ans, '1')
+            ans = str(Decimal('1.5').to_integral_value(rounding=ROUND_UP, context=None))
+            self.assertEqual(ans, '2')
+            c.clear_flags()
+            self.assertRaises(InvalidOperation, Decimal('sNaN').to_integral_value, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.rounding = ROUND_HALF_EVEN
+            ans = str(Decimal('1.5').to_integral_exact(rounding=None, context=None))
+            self.assertEqual(ans, '2')
+            c.rounding = ROUND_DOWN
+            ans = str(Decimal('1.5').to_integral_exact(rounding=None, context=None))
+            self.assertEqual(ans, '1')
+            ans = str(Decimal('1.5').to_integral_exact(rounding=ROUND_UP, context=None))
+            self.assertEqual(ans, '2')
+            c.clear_flags()
+            self.assertRaises(InvalidOperation, Decimal('sNaN').to_integral_exact, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.rounding = ROUND_UP
+            ans = str(Decimal('1.50001').quantize(exp=Decimal('1e-3'), rounding=None, context=None))
+            self.assertEqual(ans, '1.501')
+            c.rounding = ROUND_DOWN
+            ans = str(Decimal('1.50001').quantize(exp=Decimal('1e-3'), rounding=None, context=None))
+            self.assertEqual(ans, '1.500')
+            ans = str(Decimal('1.50001').quantize(exp=Decimal('1e-3'), rounding=ROUND_UP, context=None))
+            self.assertEqual(ans, '1.501')
+            c.clear_flags()
+            self.assertRaises(InvalidOperation, y.quantize, Decimal('1e-10'), rounding=ROUND_UP, context=None)
+            self.assertTrue(c.flags[InvalidOperation])
+
+        with localcontext(Context()) as context:
+            context.prec = 7
+            context.Emax = 999
+            context.Emin = -999
+            with localcontext(ctx=None) as c:
+                self.assertEqual(c.prec, 7)
+                self.assertEqual(c.Emax, 999)
+                self.assertEqual(c.Emin, -999)
+
     def test_conversions_from_int(self):
         # Check that methods taking a second Decimal argument will
         # always accept an integer in place of a Decimal.
+        Decimal = self.decimal.Decimal
+
         self.assertEqual(Decimal(4).compare(3),
                          Decimal(4).compare(Decimal(3)))
         self.assertEqual(Decimal(4).compare_signal(3),
@@ -1726,22 +2383,57 @@ class DecimalUsabilityTest(unittest.TestCase):
         self.assertEqual(Decimal(-12).fma(45, Decimal(67)),
                          Decimal(-12).fma(Decimal(45), Decimal(67)))
 
+class CUsabilityTest(UsabilityTest):
+    decimal = C
+class PyUsabilityTest(UsabilityTest):
+    decimal = P
 
-class DecimalPythonAPItests(unittest.TestCase):
+class PythonAPItests(unittest.TestCase):
 
     def test_abc(self):
+        Decimal = self.decimal.Decimal
+
         self.assertTrue(issubclass(Decimal, numbers.Number))
         self.assertFalse(issubclass(Decimal, numbers.Real))
         self.assertIsInstance(Decimal(0), numbers.Number)
         self.assertNotIsInstance(Decimal(0), numbers.Real)
 
     def test_pickle(self):
+        Decimal = self.decimal.Decimal
+
+        savedecimal = sys.modules['decimal']
+
+        # Round trip
+        sys.modules['decimal'] = self.decimal
         d = Decimal('-3.141590000')
         p = pickle.dumps(d)
         e = pickle.loads(p)
         self.assertEqual(d, e)
 
+        if C:
+            # Test interchangeability
+            x = C.Decimal('-3.123e81723')
+            y = P.Decimal('-3.123e81723')
+
+            sys.modules['decimal'] = C
+            sx = pickle.dumps(x)
+            sys.modules['decimal'] = P
+            r = pickle.loads(sx)
+            self.assertIsInstance(r, P.Decimal)
+            self.assertEqual(r, y)
+
+            sys.modules['decimal'] = P
+            sy = pickle.dumps(y)
+            sys.modules['decimal'] = C
+            r = pickle.loads(sy)
+            self.assertIsInstance(r, C.Decimal)
+            self.assertEqual(r, x)
+
+        sys.modules['decimal'] = savedecimal
+
     def test_int(self):
+        Decimal = self.decimal.Decimal
+
         for x in range(-250, 250):
             s = '%0.2f' % (x / 100.0)
             # should work the same as for floats
@@ -1757,6 +2449,8 @@ class DecimalPythonAPItests(unittest.TestCase):
         self.assertRaises(OverflowError, int, Decimal('-inf'))
 
     def test_trunc(self):
+        Decimal = self.decimal.Decimal
+
         for x in range(-250, 250):
             s = '%0.2f' % (x / 100.0)
             # should work the same as for floats
@@ -1768,9 +2462,13 @@ class DecimalPythonAPItests(unittest.TestCase):
 
     def test_from_float(self):
 
-        class  MyDecimal(Decimal):
+        Decimal = self.decimal.Decimal
+
+        class MyDecimal(Decimal):
             pass
 
+        self.assertTrue(issubclass(MyDecimal, Decimal))
+
         r = MyDecimal.from_float(0.1)
         self.assertEqual(type(r), MyDecimal)
         self.assertEqual(str(r),
@@ -1792,6 +2490,10 @@ class DecimalPythonAPItests(unittest.TestCase):
             self.assertEqual(x, float(MyDecimal.from_float(x))) # roundtrip
 
     def test_create_decimal_from_float(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+        Inexact = self.decimal.Inexact
+
         context = Context(prec=5, rounding=ROUND_DOWN)
         self.assertEqual(
             context.create_decimal_from_float(math.pi),
@@ -1815,27 +2517,320 @@ class DecimalPythonAPItests(unittest.TestCase):
         self.assertEqual(repr(context.create_decimal_from_float(10)),
                          "Decimal('10')")
 
+    def test_quantize(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+        InvalidOperation = self.decimal.InvalidOperation
+
+        c = Context(Emax=99999, Emin=-99999)
+        self.assertEqual(
+            Decimal('7.335').quantize(Decimal('.01')),
+            Decimal('7.34')
+        )
+        self.assertEqual(
+            Decimal('7.335').quantize(Decimal('.01'), rounding=ROUND_DOWN),
+            Decimal('7.33')
+        )
+        self.assertRaises(
+            InvalidOperation,
+            Decimal("10e99999").quantize, Decimal('1e100000'), context=c
+        )
+
+        c = Context()
+        d = Decimal("0.871831e800")
+        x = d.quantize(context=c, exp=Decimal("1e797"), rounding=ROUND_DOWN)
+        self.assertEqual(x, Decimal('8.71E+799'))
+
+    def test_complex(self):
+        Decimal = self.decimal.Decimal
+
+        x = Decimal("9.8182731e181273")
+        self.assertEqual(x.real, x)
+        self.assertEqual(x.imag, 0)
+        self.assertEqual(x.conjugate(), x)
+
+        x = Decimal("1")
+        self.assertEqual(complex(x), complex(float(1)))
+
+        self.assertRaises(AttributeError, setattr, x, 'real', 100)
+        self.assertRaises(AttributeError, setattr, x, 'imag', 100)
+        self.assertRaises(AttributeError, setattr, x, 'conjugate', 100)
+        self.assertRaises(AttributeError, setattr, x, '__complex__', 100)
+
+    def test_named_parameters(self):
+        D = self.decimal.Decimal
+        Context = self.decimal.Context
+        localcontext = self.decimal.localcontext
+        InvalidOperation = self.decimal.InvalidOperation
+        Overflow = self.decimal.Overflow
+
+        xc = Context()
+        xc.prec = 1
+        xc.Emax = 1
+        xc.Emin = -1
+
+        with localcontext() as c:
+            c.clear_flags()
+
+            self.assertEqual(D(9, xc), 9)
+            self.assertEqual(D(9, context=xc), 9)
+            self.assertEqual(D(context=xc, value=9), 9)
+            self.assertEqual(D(context=xc), 0)
+            xc.clear_flags()
+            self.assertRaises(InvalidOperation, D, "xyz", context=xc)
+            self.assertTrue(xc.flags[InvalidOperation])
+            self.assertFalse(c.flags[InvalidOperation])
+
+            xc.clear_flags()
+            self.assertEqual(D(2).exp(context=xc), 7)
+            self.assertRaises(Overflow, D(8).exp, context=xc)
+            self.assertTrue(xc.flags[Overflow])
+            self.assertFalse(c.flags[Overflow])
+
+            xc.clear_flags()
+            self.assertEqual(D(2).ln(context=xc), D('0.7'))
+            self.assertRaises(InvalidOperation, D(-1).ln, context=xc)
+            self.assertTrue(xc.flags[InvalidOperation])
+            self.assertFalse(c.flags[InvalidOperation])
+
+            self.assertEqual(D(0).log10(context=xc), D('-inf'))
+            self.assertEqual(D(-1).next_minus(context=xc), -2)
+            self.assertEqual(D(-1).next_plus(context=xc), D('-0.9'))
+            self.assertEqual(D("9.73").normalize(context=xc), D('1E+1'))
+            self.assertEqual(D("9999").to_integral(context=xc), 9999)
+            self.assertEqual(D("-2000").to_integral_exact(context=xc), -2000)
+            self.assertEqual(D("123").to_integral_value(context=xc), 123)
+            self.assertEqual(D("0.0625").sqrt(context=xc), D('0.2'))
+
+            self.assertEqual(D("0.0625").compare(context=xc, other=3), -1)
+            xc.clear_flags()
+            self.assertRaises(InvalidOperation,
+                              D("0").compare_signal, D('nan'), context=xc)
+            self.assertTrue(xc.flags[InvalidOperation])
+            self.assertFalse(c.flags[InvalidOperation])
+            self.assertEqual(D("0.01").max(D('0.0101'), context=xc), D('0.0'))
+            self.assertEqual(D("0.01").max(D('0.0101'), context=xc), D('0.0'))
+            self.assertEqual(D("0.2").max_mag(D('-0.3'), context=xc),
+                             D('-0.3'))
+            self.assertEqual(D("0.02").min(D('-0.03'), context=xc), D('-0.0'))
+            self.assertEqual(D("0.02").min_mag(D('-0.03'), context=xc),
+                             D('0.0'))
+            self.assertEqual(D("0.2").next_toward(D('-1'), context=xc), D('0.1'))
+            xc.clear_flags()
+            self.assertRaises(InvalidOperation,
+                              D("0.2").quantize, D('1e10'), context=xc)
+            self.assertTrue(xc.flags[InvalidOperation])
+            self.assertFalse(c.flags[InvalidOperation])
+            self.assertEqual(D("9.99").remainder_near(D('1.5'), context=xc),
+                             D('-0.5'))
+
+            self.assertEqual(D("9.9").fma(third=D('0.9'), context=xc, other=7),
+                             D('7E+1'))
+
+            self.assertRaises(TypeError, D(1).is_canonical, context=xc)
+            self.assertRaises(TypeError, D(1).is_finite, context=xc)
+            self.assertRaises(TypeError, D(1).is_infinite, context=xc)
+            self.assertRaises(TypeError, D(1).is_nan, context=xc)
+            self.assertRaises(TypeError, D(1).is_qnan, context=xc)
+            self.assertRaises(TypeError, D(1).is_snan, context=xc)
+            self.assertRaises(TypeError, D(1).is_signed, context=xc)
+            self.assertRaises(TypeError, D(1).is_zero, context=xc)
+
+            self.assertFalse(D("0.01").is_normal(context=xc))
+            self.assertTrue(D("0.01").is_subnormal(context=xc))
+
+            self.assertRaises(TypeError, D(1).adjusted, context=xc)
+            self.assertRaises(TypeError, D(1).conjugate, context=xc)
+            self.assertRaises(TypeError, D(1).radix, context=xc)
+
+            self.assertEqual(D(-111).logb(context=xc), 2)
+            self.assertEqual(D(0).logical_invert(context=xc), 1)
+            self.assertEqual(D('0.01').number_class(context=xc), '+Subnormal')
+            self.assertEqual(D('0.21').to_eng_string(context=xc), '0.21')
+
+            self.assertEqual(D('11').logical_and(D('10'), context=xc), 0)
+            self.assertEqual(D('11').logical_or(D('10'), context=xc), 1)
+            self.assertEqual(D('01').logical_xor(D('10'), context=xc), 1)
+            self.assertEqual(D('23').rotate(1, context=xc), 3)
+            self.assertEqual(D('23').rotate(1, context=xc), 3)
+            xc.clear_flags()
+            self.assertRaises(Overflow,
+                              D('23').scaleb, 1, context=xc)
+            self.assertTrue(xc.flags[Overflow])
+            self.assertFalse(c.flags[Overflow])
+            self.assertEqual(D('23').shift(-1, context=xc), 0)
+
+            self.assertRaises(TypeError, D.from_float, 1.1, context=xc)
+            self.assertRaises(TypeError, D(0).as_tuple, context=xc)
+
+            self.assertEqual(D(1).canonical(), 1)
+            self.assertRaises(TypeError, D("-1").copy_abs, context=xc)
+            self.assertRaises(TypeError, D("-1").copy_negate, context=xc)
+            self.assertRaises(TypeError, D(1).canonical, context="x")
+            self.assertRaises(TypeError, D(1).canonical, xyz="x")
+
+    def test_exception_hierarchy(self):
+
+        decimal = self.decimal
+        DecimalException = decimal.DecimalException
+        InvalidOperation = decimal.InvalidOperation
+        FloatOperation = decimal.FloatOperation
+        DivisionByZero = decimal.DivisionByZero
+        Overflow = decimal.Overflow
+        Underflow = decimal.Underflow
+        Subnormal = decimal.Subnormal
+        Inexact = decimal.Inexact
+        Rounded = decimal.Rounded
+        Clamped = decimal.Clamped
+
+        self.assertTrue(issubclass(DecimalException, ArithmeticError))
+
+        self.assertTrue(issubclass(InvalidOperation, DecimalException))
+        self.assertTrue(issubclass(FloatOperation, DecimalException))
+        self.assertTrue(issubclass(FloatOperation, TypeError))
+        self.assertTrue(issubclass(DivisionByZero, DecimalException))
+        self.assertTrue(issubclass(DivisionByZero, ZeroDivisionError))
+        self.assertTrue(issubclass(Overflow, Rounded))
+        self.assertTrue(issubclass(Overflow, Inexact))
+        self.assertTrue(issubclass(Overflow, DecimalException))
+        self.assertTrue(issubclass(Underflow, Inexact))
+        self.assertTrue(issubclass(Underflow, Rounded))
+        self.assertTrue(issubclass(Underflow, Subnormal))
+        self.assertTrue(issubclass(Underflow, DecimalException))
+
+        self.assertTrue(issubclass(Subnormal, DecimalException))
+        self.assertTrue(issubclass(Inexact, DecimalException))
+        self.assertTrue(issubclass(Rounded, DecimalException))
+        self.assertTrue(issubclass(Clamped, DecimalException))
+
+        self.assertTrue(issubclass(decimal.ConversionSyntax, InvalidOperation))
+        self.assertTrue(issubclass(decimal.DivisionImpossible, InvalidOperation))
+        self.assertTrue(issubclass(decimal.DivisionUndefined, InvalidOperation))
+        self.assertTrue(issubclass(decimal.DivisionUndefined, ZeroDivisionError))
+        self.assertTrue(issubclass(decimal.InvalidContext, InvalidOperation))
+
+class CPythonAPItests(PythonAPItests):
+    decimal = C
+class PyPythonAPItests(PythonAPItests):
+    decimal = P
+
 class ContextAPItests(unittest.TestCase):
 
+    def test_none_args(self):
+        Context = self.decimal.Context
+        InvalidOperation = self.decimal.InvalidOperation
+        DivisionByZero = self.decimal.DivisionByZero
+        Overflow = self.decimal.Overflow
+
+        c1 = Context()
+        c2 = Context(prec=None, rounding=None, Emax=None, Emin=None,
+                     capitals=None, clamp=None, flags=None, traps=None)
+        for c in [c1, c2]:
+            self.assertEqual(c.prec, 28)
+            self.assertEqual(c.rounding, ROUND_HALF_EVEN)
+            self.assertEqual(c.Emax, 999999)
+            self.assertEqual(c.Emin, -999999)
+            self.assertEqual(c.capitals, 1)
+            self.assertEqual(c.clamp, 0)
+            assert_signals(self, c, 'flags', [])
+            assert_signals(self, c, 'traps', [InvalidOperation, DivisionByZero,
+                                              Overflow])
+
+    @cpython_only
+    def test_from_legacy_strings(self):
+        import _testcapi
+        c = self.decimal.Context()
+
+        for rnd in RoundingModes:
+            c.rounding = _testcapi.unicode_legacy_string(rnd)
+            self.assertEqual(c.rounding, rnd)
+
+        s = _testcapi.unicode_legacy_string('')
+        self.assertRaises(TypeError, setattr, c, 'rounding', s)
+
+        s = _testcapi.unicode_legacy_string('ROUND_\x00UP')
+        self.assertRaises(TypeError, setattr, c, 'rounding', s)
+
     def test_pickle(self):
+
+        Context = self.decimal.Context
+
+        savedecimal = sys.modules['decimal']
+
+        # Round trip
+        sys.modules['decimal'] = self.decimal
         c = Context()
         e = pickle.loads(pickle.dumps(c))
-        for k in vars(c):
-            v1 = vars(c)[k]
-            v2 = vars(e)[k]
-            self.assertEqual(v1, v2)
+
+        self.assertEqual(c.prec, e.prec)
+        self.assertEqual(c.Emin, e.Emin)
+        self.assertEqual(c.Emax, e.Emax)
+        self.assertEqual(c.rounding, e.rounding)
+        self.assertEqual(c.capitals, e.capitals)
+        self.assertEqual(c.clamp, e.clamp)
+        self.assertEqual(c.flags, e.flags)
+        self.assertEqual(c.traps, e.traps)
+
+        # Test interchangeability
+        combinations = [(C, P), (P, C)] if C else [(P, P)]
+        for dumper, loader in combinations:
+            for ri, _ in enumerate(RoundingModes):
+                for fi, _ in enumerate(OrderedSignals[dumper]):
+                    for ti, _ in enumerate(OrderedSignals[dumper]):
+
+                        prec = random.randrange(1, 100)
+                        emin = random.randrange(-100, 0)
+                        emax = random.randrange(1, 100)
+                        caps = random.randrange(2)
+                        clamp = random.randrange(2)
+
+                        # One module dumps
+                        sys.modules['decimal'] = dumper
+                        c = dumper.Context(
+                              prec=prec, Emin=emin, Emax=emax,
+                              rounding=RoundingModes[ri],
+                              capitals=caps, clamp=clamp,
+                              flags=OrderedSignals[dumper][:fi],
+                              traps=OrderedSignals[dumper][:ti]
+                        )
+                        s = pickle.dumps(c)
+
+                        # The other module loads
+                        sys.modules['decimal'] = loader
+                        d = pickle.loads(s)
+                        self.assertIsInstance(d, loader.Context)
+
+                        self.assertEqual(d.prec, prec)
+                        self.assertEqual(d.Emin, emin)
+                        self.assertEqual(d.Emax, emax)
+                        self.assertEqual(d.rounding, RoundingModes[ri])
+                        self.assertEqual(d.capitals, caps)
+                        self.assertEqual(d.clamp, clamp)
+                        assert_signals(self, d, 'flags', OrderedSignals[loader][:fi])
+                        assert_signals(self, d, 'traps', OrderedSignals[loader][:ti])
+
+        sys.modules['decimal'] = savedecimal
 
     def test_equality_with_other_types(self):
+        Decimal = self.decimal.Decimal
+
         self.assertIn(Decimal(10), ['a', 1.0, Decimal(10), (1,2), {}])
         self.assertNotIn(Decimal(10), ['a', 1.0, (1,2), {}])
 
     def test_copy(self):
         # All copies should be deep
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.copy()
         self.assertNotEqual(id(c), id(d))
         self.assertNotEqual(id(c.flags), id(d.flags))
         self.assertNotEqual(id(c.traps), id(d.traps))
+        k1 = set(c.flags.keys())
+        k2 = set(d.flags.keys())
+        self.assertEqual(k1, k2)
+        self.assertEqual(c.flags, d.flags)
 
     def test__clamp(self):
         # In Python 3.2, the private attribute `_clamp` was made
@@ -1844,26 +2839,23 @@ class ContextAPItests(unittest.TestCase):
         # only, the attribute should be gettable/settable via both
         # `clamp` and `_clamp`; in Python 3.3, `_clamp` should be
         # removed.
-        c = Context(clamp = 0)
-        self.assertEqual(c.clamp, 0)
-
-        with check_warnings(("", DeprecationWarning)):
-            c._clamp = 1
-        self.assertEqual(c.clamp, 1)
-        with check_warnings(("", DeprecationWarning)):
-            self.assertEqual(c._clamp, 1)
-        c.clamp = 0
-        self.assertEqual(c.clamp, 0)
-        with check_warnings(("", DeprecationWarning)):
-            self.assertEqual(c._clamp, 0)
+        Context = self.decimal.Context
+        c = Context()
+        self.assertRaises(AttributeError, getattr, c, '_clamp')
 
     def test_abs(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.abs(Decimal(-1))
         self.assertEqual(c.abs(-1), d)
         self.assertRaises(TypeError, c.abs, '-1')
 
     def test_add(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.add(Decimal(1), Decimal(1))
         self.assertEqual(c.add(1, 1), d)
@@ -1873,6 +2865,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.add, 1, '1')
 
     def test_compare(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.compare(Decimal(1), Decimal(1))
         self.assertEqual(c.compare(1, 1), d)
@@ -1882,6 +2877,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.compare, 1, '1')
 
     def test_compare_signal(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.compare_signal(Decimal(1), Decimal(1))
         self.assertEqual(c.compare_signal(1, 1), d)
@@ -1891,6 +2889,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.compare_signal, 1, '1')
 
     def test_compare_total(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.compare_total(Decimal(1), Decimal(1))
         self.assertEqual(c.compare_total(1, 1), d)
@@ -1900,6 +2901,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.compare_total, 1, '1')
 
     def test_compare_total_mag(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.compare_total_mag(Decimal(1), Decimal(1))
         self.assertEqual(c.compare_total_mag(1, 1), d)
@@ -1909,24 +2913,36 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.compare_total_mag, 1, '1')
 
     def test_copy_abs(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.copy_abs(Decimal(-1))
         self.assertEqual(c.copy_abs(-1), d)
         self.assertRaises(TypeError, c.copy_abs, '-1')
 
     def test_copy_decimal(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.copy_decimal(Decimal(-1))
         self.assertEqual(c.copy_decimal(-1), d)
         self.assertRaises(TypeError, c.copy_decimal, '-1')
 
     def test_copy_negate(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.copy_negate(Decimal(-1))
         self.assertEqual(c.copy_negate(-1), d)
         self.assertRaises(TypeError, c.copy_negate, '-1')
 
     def test_copy_sign(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.copy_sign(Decimal(1), Decimal(-2))
         self.assertEqual(c.copy_sign(1, -2), d)
@@ -1936,6 +2952,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.copy_sign, 1, '-2')
 
     def test_divide(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.divide(Decimal(1), Decimal(2))
         self.assertEqual(c.divide(1, 2), d)
@@ -1945,6 +2964,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.divide, 1, '2')
 
     def test_divide_int(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.divide_int(Decimal(1), Decimal(2))
         self.assertEqual(c.divide_int(1, 2), d)
@@ -1954,6 +2976,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.divide_int, 1, '2')
 
     def test_divmod(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.divmod(Decimal(1), Decimal(2))
         self.assertEqual(c.divmod(1, 2), d)
@@ -1963,12 +2988,18 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.divmod, 1, '2')
 
     def test_exp(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.exp(Decimal(10))
         self.assertEqual(c.exp(10), d)
         self.assertRaises(TypeError, c.exp, '10')
 
     def test_fma(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.fma(Decimal(2), Decimal(3), Decimal(4))
         self.assertEqual(c.fma(2, 3, 4), d)
@@ -1980,79 +3011,129 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.fma, 2, '3', 4)
         self.assertRaises(TypeError, c.fma, 2, 3, '4')
 
+        # Issue 12079 for Context.fma ...
+        self.assertRaises(TypeError, c.fma,
+                          Decimal('Infinity'), Decimal(0), "not a decimal")
+        self.assertRaises(TypeError, c.fma,
+                          Decimal(1), Decimal('snan'), 1.222)
+        # ... and for Decimal.fma.
+        self.assertRaises(TypeError, Decimal('Infinity').fma,
+                          Decimal(0), "not a decimal")
+        self.assertRaises(TypeError, Decimal(1).fma,
+                          Decimal('snan'), 1.222)
+
     def test_is_finite(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_finite(Decimal(10))
         self.assertEqual(c.is_finite(10), d)
         self.assertRaises(TypeError, c.is_finite, '10')
 
     def test_is_infinite(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_infinite(Decimal(10))
         self.assertEqual(c.is_infinite(10), d)
         self.assertRaises(TypeError, c.is_infinite, '10')
 
     def test_is_nan(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_nan(Decimal(10))
         self.assertEqual(c.is_nan(10), d)
         self.assertRaises(TypeError, c.is_nan, '10')
 
     def test_is_normal(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_normal(Decimal(10))
         self.assertEqual(c.is_normal(10), d)
         self.assertRaises(TypeError, c.is_normal, '10')
 
     def test_is_qnan(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_qnan(Decimal(10))
         self.assertEqual(c.is_qnan(10), d)
         self.assertRaises(TypeError, c.is_qnan, '10')
 
     def test_is_signed(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_signed(Decimal(10))
         self.assertEqual(c.is_signed(10), d)
         self.assertRaises(TypeError, c.is_signed, '10')
 
     def test_is_snan(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_snan(Decimal(10))
         self.assertEqual(c.is_snan(10), d)
         self.assertRaises(TypeError, c.is_snan, '10')
 
     def test_is_subnormal(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_subnormal(Decimal(10))
         self.assertEqual(c.is_subnormal(10), d)
         self.assertRaises(TypeError, c.is_subnormal, '10')
 
     def test_is_zero(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.is_zero(Decimal(10))
         self.assertEqual(c.is_zero(10), d)
         self.assertRaises(TypeError, c.is_zero, '10')
 
     def test_ln(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.ln(Decimal(10))
         self.assertEqual(c.ln(10), d)
         self.assertRaises(TypeError, c.ln, '10')
 
     def test_log10(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.log10(Decimal(10))
         self.assertEqual(c.log10(10), d)
         self.assertRaises(TypeError, c.log10, '10')
 
     def test_logb(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.logb(Decimal(10))
         self.assertEqual(c.logb(10), d)
         self.assertRaises(TypeError, c.logb, '10')
 
     def test_logical_and(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.logical_and(Decimal(1), Decimal(1))
         self.assertEqual(c.logical_and(1, 1), d)
@@ -2062,12 +3143,18 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.logical_and, 1, '1')
 
     def test_logical_invert(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.logical_invert(Decimal(1000))
         self.assertEqual(c.logical_invert(1000), d)
         self.assertRaises(TypeError, c.logical_invert, '1000')
 
     def test_logical_or(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.logical_or(Decimal(1), Decimal(1))
         self.assertEqual(c.logical_or(1, 1), d)
@@ -2077,6 +3164,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.logical_or, 1, '1')
 
     def test_logical_xor(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.logical_xor(Decimal(1), Decimal(1))
         self.assertEqual(c.logical_xor(1, 1), d)
@@ -2086,6 +3176,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.logical_xor, 1, '1')
 
     def test_max(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.max(Decimal(1), Decimal(2))
         self.assertEqual(c.max(1, 2), d)
@@ -2095,6 +3188,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.max, 1, '2')
 
     def test_max_mag(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.max_mag(Decimal(1), Decimal(2))
         self.assertEqual(c.max_mag(1, 2), d)
@@ -2104,6 +3200,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.max_mag, 1, '2')
 
     def test_min(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.min(Decimal(1), Decimal(2))
         self.assertEqual(c.min(1, 2), d)
@@ -2113,6 +3212,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.min, 1, '2')
 
     def test_min_mag(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.min_mag(Decimal(1), Decimal(2))
         self.assertEqual(c.min_mag(1, 2), d)
@@ -2122,12 +3224,18 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.min_mag, 1, '2')
 
     def test_minus(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.minus(Decimal(10))
         self.assertEqual(c.minus(10), d)
         self.assertRaises(TypeError, c.minus, '10')
 
     def test_multiply(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.multiply(Decimal(1), Decimal(2))
         self.assertEqual(c.multiply(1, 2), d)
@@ -2137,18 +3245,27 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.multiply, 1, '2')
 
     def test_next_minus(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.next_minus(Decimal(10))
         self.assertEqual(c.next_minus(10), d)
         self.assertRaises(TypeError, c.next_minus, '10')
 
     def test_next_plus(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.next_plus(Decimal(10))
         self.assertEqual(c.next_plus(10), d)
         self.assertRaises(TypeError, c.next_plus, '10')
 
     def test_next_toward(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.next_toward(Decimal(1), Decimal(2))
         self.assertEqual(c.next_toward(1, 2), d)
@@ -2158,36 +3275,50 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.next_toward, 1, '2')
 
     def test_normalize(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.normalize(Decimal(10))
         self.assertEqual(c.normalize(10), d)
         self.assertRaises(TypeError, c.normalize, '10')
 
     def test_number_class(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         self.assertEqual(c.number_class(123), c.number_class(Decimal(123)))
         self.assertEqual(c.number_class(0), c.number_class(Decimal(0)))
         self.assertEqual(c.number_class(-45), c.number_class(Decimal(-45)))
 
-    def test_power(self):
-        c = Context()
-        d = c.power(Decimal(1), Decimal(4), Decimal(2))
-        self.assertEqual(c.power(1, 4, 2), d)
-        self.assertEqual(c.power(Decimal(1), 4, 2), d)
-        self.assertEqual(c.power(1, Decimal(4), 2), d)
-        self.assertEqual(c.power(1, 4, Decimal(2)), d)
-        self.assertEqual(c.power(Decimal(1), Decimal(4), 2), d)
-        self.assertRaises(TypeError, c.power, '1', 4, 2)
-        self.assertRaises(TypeError, c.power, 1, '4', 2)
-        self.assertRaises(TypeError, c.power, 1, 4, '2')
-
     def test_plus(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.plus(Decimal(10))
         self.assertEqual(c.plus(10), d)
         self.assertRaises(TypeError, c.plus, '10')
 
+    def test_power(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
+        c = Context()
+        d = c.power(Decimal(1), Decimal(4))
+        self.assertEqual(c.power(1, 4), d)
+        self.assertEqual(c.power(Decimal(1), 4), d)
+        self.assertEqual(c.power(1, Decimal(4)), d)
+        self.assertEqual(c.power(Decimal(1), Decimal(4)), d)
+        self.assertRaises(TypeError, c.power, '1', 4)
+        self.assertRaises(TypeError, c.power, 1, '4')
+        self.assertEqual(c.power(modulo=5, b=8, a=2), 1)
+
     def test_quantize(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.quantize(Decimal(1), Decimal(2))
         self.assertEqual(c.quantize(1, 2), d)
@@ -2197,6 +3328,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.quantize, 1, '2')
 
     def test_remainder(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.remainder(Decimal(1), Decimal(2))
         self.assertEqual(c.remainder(1, 2), d)
@@ -2206,6 +3340,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.remainder, 1, '2')
 
     def test_remainder_near(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.remainder_near(Decimal(1), Decimal(2))
         self.assertEqual(c.remainder_near(1, 2), d)
@@ -2215,6 +3352,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.remainder_near, 1, '2')
 
     def test_rotate(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.rotate(Decimal(1), Decimal(2))
         self.assertEqual(c.rotate(1, 2), d)
@@ -2224,12 +3364,18 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.rotate, 1, '2')
 
     def test_sqrt(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.sqrt(Decimal(10))
         self.assertEqual(c.sqrt(10), d)
         self.assertRaises(TypeError, c.sqrt, '10')
 
     def test_same_quantum(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.same_quantum(Decimal(1), Decimal(2))
         self.assertEqual(c.same_quantum(1, 2), d)
@@ -2239,6 +3385,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.same_quantum, 1, '2')
 
     def test_scaleb(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.scaleb(Decimal(1), Decimal(2))
         self.assertEqual(c.scaleb(1, 2), d)
@@ -2248,6 +3397,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.scaleb, 1, '2')
 
     def test_shift(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.shift(Decimal(1), Decimal(2))
         self.assertEqual(c.shift(1, 2), d)
@@ -2257,6 +3409,9 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.shift, 1, '2')
 
     def test_subtract(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.subtract(Decimal(1), Decimal(2))
         self.assertEqual(c.subtract(1, 2), d)
@@ -2266,35 +3421,56 @@ class ContextAPItests(unittest.TestCase):
         self.assertRaises(TypeError, c.subtract, 1, '2')
 
     def test_to_eng_string(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.to_eng_string(Decimal(10))
         self.assertEqual(c.to_eng_string(10), d)
         self.assertRaises(TypeError, c.to_eng_string, '10')
 
     def test_to_sci_string(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.to_sci_string(Decimal(10))
         self.assertEqual(c.to_sci_string(10), d)
         self.assertRaises(TypeError, c.to_sci_string, '10')
 
     def test_to_integral_exact(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.to_integral_exact(Decimal(10))
         self.assertEqual(c.to_integral_exact(10), d)
         self.assertRaises(TypeError, c.to_integral_exact, '10')
 
     def test_to_integral_value(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+
         c = Context()
         d = c.to_integral_value(Decimal(10))
         self.assertEqual(c.to_integral_value(10), d)
         self.assertRaises(TypeError, c.to_integral_value, '10')
+        self.assertRaises(TypeError, c.to_integral_value, 10, 'x')
+
+class CContextAPItests(ContextAPItests):
+    decimal = C
+class PyContextAPItests(ContextAPItests):
+    decimal = P
 
-class WithStatementTest(unittest.TestCase):
+class ContextWithStatement(unittest.TestCase):
     # Can't do these as docstrings until Python 2.6
     # as doctest can't handle __future__ statements
 
     def test_localcontext(self):
         # Use a copy of the current context in the block
+        getcontext = self.decimal.getcontext
+        localcontext = self.decimal.localcontext
+
         orig_ctx = getcontext()
         with localcontext() as enter_ctx:
             set_ctx = getcontext()
@@ -2305,6 +3481,11 @@ class WithStatementTest(unittest.TestCase):
 
     def test_localcontextarg(self):
         # Use a copy of the supplied context in the block
+        Context = self.decimal.Context
+        getcontext = self.decimal.getcontext
+        localcontext = self.decimal.localcontext
+
+        localcontext = self.decimal.localcontext
         orig_ctx = getcontext()
         new_ctx = Context(prec=42)
         with localcontext(new_ctx) as enter_ctx:
@@ -2315,17 +3496,130 @@ class WithStatementTest(unittest.TestCase):
         self.assertIsNot(new_ctx, set_ctx, 'did not copy the context')
         self.assertIs(set_ctx, enter_ctx, '__enter__ returned wrong context')
 
+    def test_nested_with_statements(self):
+        # Use a copy of the supplied context in the block
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+        getcontext = self.decimal.getcontext
+        localcontext = self.decimal.localcontext
+        Clamped = self.decimal.Clamped
+        Overflow = self.decimal.Overflow
+
+        orig_ctx = getcontext()
+        orig_ctx.clear_flags()
+        new_ctx = Context(Emax=384)
+        with localcontext() as c1:
+            self.assertEqual(c1.flags, orig_ctx.flags)
+            self.assertEqual(c1.traps, orig_ctx.traps)
+            c1.traps[Clamped] = True
+            c1.Emin = -383
+            self.assertNotEqual(orig_ctx.Emin, -383)
+            self.assertRaises(Clamped, c1.create_decimal, '0e-999')
+            self.assertTrue(c1.flags[Clamped])
+            with localcontext(new_ctx) as c2:
+                self.assertEqual(c2.flags, new_ctx.flags)
+                self.assertEqual(c2.traps, new_ctx.traps)
+                self.assertRaises(Overflow, c2.power, Decimal('3.4e200'), 2)
+                self.assertFalse(c2.flags[Clamped])
+                self.assertTrue(c2.flags[Overflow])
+                del c2
+            self.assertFalse(c1.flags[Overflow])
+            del c1
+        self.assertNotEqual(orig_ctx.Emin, -383)
+        self.assertFalse(orig_ctx.flags[Clamped])
+        self.assertFalse(orig_ctx.flags[Overflow])
+        self.assertFalse(new_ctx.flags[Clamped])
+        self.assertFalse(new_ctx.flags[Overflow])
+
+    def test_with_statements_gc1(self):
+        localcontext = self.decimal.localcontext
+
+        with localcontext() as c1:
+            del c1
+            with localcontext() as c2:
+                del c2
+                with localcontext() as c3:
+                    del c3
+                    with localcontext() as c4:
+                        del c4
+
+    def test_with_statements_gc2(self):
+        localcontext = self.decimal.localcontext
+
+        with localcontext() as c1:
+            with localcontext(c1) as c2:
+                del c1
+                with localcontext(c2) as c3:
+                    del c2
+                    with localcontext(c3) as c4:
+                        del c3
+                        del c4
+
+    def test_with_statements_gc3(self):
+        Context = self.decimal.Context
+        localcontext = self.decimal.localcontext
+        getcontext = self.decimal.getcontext
+        setcontext = self.decimal.setcontext
+
+        with localcontext() as c1:
+            del c1
+            n1 = Context(prec=1)
+            setcontext(n1)
+            with localcontext(n1) as c2:
+                del n1
+                self.assertEqual(c2.prec, 1)
+                del c2
+                n2 = Context(prec=2)
+                setcontext(n2)
+                del n2
+                self.assertEqual(getcontext().prec, 2)
+                n3 = Context(prec=3)
+                setcontext(n3)
+                self.assertEqual(getcontext().prec, 3)
+                with localcontext(n3) as c3:
+                    del n3
+                    self.assertEqual(c3.prec, 3)
+                    del c3
+                    n4 = Context(prec=4)
+                    setcontext(n4)
+                    del n4
+                    self.assertEqual(getcontext().prec, 4)
+                    with localcontext() as c4:
+                        self.assertEqual(c4.prec, 4)
+                        del c4
+
+class CContextWithStatement(ContextWithStatement):
+    decimal = C
+class PyContextWithStatement(ContextWithStatement):
+    decimal = P
+
 class ContextFlags(unittest.TestCase):
+
     def test_flags_irrelevant(self):
         # check that the result (numeric result + flags raised) of an
         # arithmetic operation doesn't depend on the current flags
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+        Inexact = self.decimal.Inexact
+        Rounded = self.decimal.Rounded
+        Underflow = self.decimal.Underflow
+        Clamped = self.decimal.Clamped
+        Subnormal = self.decimal.Subnormal
+
+        def raise_error(context, flag):
+            if self.decimal == C:
+                context.flags[flag] = True
+                if context.traps[flag]:
+                    raise flag
+            else:
+                context._raise_error(flag)
 
-        context = Context(prec=9, Emin = -999999999, Emax = 999999999,
-                    rounding=ROUND_HALF_EVEN, traps=[], flags=[])
+        context = Context(prec=9, Emin = -425000000, Emax = 425000000,
+                          rounding=ROUND_HALF_EVEN, traps=[], flags=[])
 
         # operations that raise various flags, in the form (function, arglist)
         operations = [
-            (context._apply, [Decimal("100E-1000000009")]),
+            (context._apply, [Decimal("100E-425000010")]),
             (context.sqrt, [Decimal(2)]),
             (context.add, [Decimal("1.23456789"), Decimal("9.87654321")]),
             (context.multiply, [Decimal("1.23456789"), Decimal("9.87654321")]),
@@ -2346,7 +3640,7 @@ class ContextFlags(unittest.TestCase):
                 # set flags, before calling operation
                 context.clear_flags()
                 for flag in extra_flags:
-                    context._raise_error(flag)
+                    raise_error(context, flag)
                 new_ans = fn(*args)
 
                 # flags that we expect to be set after the operation
@@ -2367,6 +3661,1742 @@ class ContextFlags(unittest.TestCase):
                                   "operation raises different flags depending on flags set: " +
                                   "expected %s, got %s" % (expected_flags, new_flags))
 
+    def test_flag_comparisons(self):
+        Context = self.decimal.Context
+        Inexact = self.decimal.Inexact
+        Rounded = self.decimal.Rounded
+
+        c = Context()
+
+        # Valid SignalDict
+        self.assertNotEqual(c.flags, c.traps)
+        self.assertNotEqual(c.traps, c.flags)
+
+        c.flags = c.traps
+        self.assertEqual(c.flags, c.traps)
+        self.assertEqual(c.traps, c.flags)
+
+        c.flags[Rounded] = True
+        c.traps = c.flags
+        self.assertEqual(c.flags, c.traps)
+        self.assertEqual(c.traps, c.flags)
+
+        d = {}
+        d.update(c.flags)
+        self.assertEqual(d, c.flags)
+        self.assertEqual(c.flags, d)
+
+        d[Inexact] = True
+        self.assertNotEqual(d, c.flags)
+        self.assertNotEqual(c.flags, d)
+
+        # Invalid SignalDict
+        d = {Inexact:False}
+        self.assertNotEqual(d, c.flags)
+        self.assertNotEqual(c.flags, d)
+
+        d = ["xyz"]
+        self.assertNotEqual(d, c.flags)
+        self.assertNotEqual(c.flags, d)
+
+    @requires_IEEE_754
+    def test_float_operation(self):
+        Decimal = self.decimal.Decimal
+        FloatOperation = self.decimal.FloatOperation
+        localcontext = self.decimal.localcontext
+
+        with localcontext() as c:
+            ##### trap is off by default
+            self.assertFalse(c.traps[FloatOperation])
+
+            # implicit conversion sets the flag
+            c.clear_flags()
+            self.assertEqual(Decimal(7.5), 7.5)
+            self.assertTrue(c.flags[FloatOperation])
+
+            c.clear_flags()
+            self.assertEqual(c.create_decimal(7.5), 7.5)
+            self.assertTrue(c.flags[FloatOperation])
+
+            # explicit conversion does not set the flag
+            c.clear_flags()
+            x = Decimal.from_float(7.5)
+            self.assertFalse(c.flags[FloatOperation])
+            # comparison sets the flag
+            self.assertEqual(x, 7.5)
+            self.assertTrue(c.flags[FloatOperation])
+
+            c.clear_flags()
+            x = c.create_decimal_from_float(7.5)
+            self.assertFalse(c.flags[FloatOperation])
+            self.assertEqual(x, 7.5)
+            self.assertTrue(c.flags[FloatOperation])
+
+            ##### set the trap
+            c.traps[FloatOperation] = True
+
+            # implicit conversion raises
+            c.clear_flags()
+            self.assertRaises(FloatOperation, Decimal, 7.5)
+            self.assertTrue(c.flags[FloatOperation])
+
+            c.clear_flags()
+            self.assertRaises(FloatOperation, c.create_decimal, 7.5)
+            self.assertTrue(c.flags[FloatOperation])
+
+            # explicit conversion is silent
+            c.clear_flags()
+            x = Decimal.from_float(7.5)
+            self.assertFalse(c.flags[FloatOperation])
+
+            c.clear_flags()
+            x = c.create_decimal_from_float(7.5)
+            self.assertFalse(c.flags[FloatOperation])
+
+    def test_float_comparison(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+        FloatOperation = self.decimal.FloatOperation
+        localcontext = self.decimal.localcontext
+
+        def assert_attr(a, b, attr, context, signal=None):
+            context.clear_flags()
+            f = getattr(a, attr)
+            if signal == FloatOperation:
+                self.assertRaises(signal, f, b)
+            else:
+                self.assertIs(f(b), True)
+            self.assertTrue(context.flags[FloatOperation])
+
+        small_d = Decimal('0.25')
+        big_d = Decimal('3.0')
+        small_f = 0.25
+        big_f = 3.0
+
+        zero_d = Decimal('0.0')
+        neg_zero_d = Decimal('-0.0')
+        zero_f = 0.0
+        neg_zero_f = -0.0
+
+        inf_d = Decimal('Infinity')
+        neg_inf_d = Decimal('-Infinity')
+        inf_f = float('inf')
+        neg_inf_f = float('-inf')
+
+        def doit(c, signal=None):
+            # Order
+            for attr in '__lt__', '__le__':
+                assert_attr(small_d, big_f, attr, c, signal)
+
+            for attr in '__gt__', '__ge__':
+                assert_attr(big_d, small_f, attr, c, signal)
+
+            # Equality
+            assert_attr(small_d, small_f, '__eq__', c, None)
+
+            assert_attr(neg_zero_d, neg_zero_f, '__eq__', c, None)
+            assert_attr(neg_zero_d, zero_f, '__eq__', c, None)
+
+            assert_attr(zero_d, neg_zero_f, '__eq__', c, None)
+            assert_attr(zero_d, zero_f, '__eq__', c, None)
+
+            assert_attr(neg_inf_d, neg_inf_f, '__eq__', c, None)
+            assert_attr(inf_d, inf_f, '__eq__', c, None)
+
+            # Inequality
+            assert_attr(small_d, big_f, '__ne__', c, None)
+
+            assert_attr(Decimal('0.1'), 0.1, '__ne__', c, None)
+
+            assert_attr(neg_inf_d, inf_f, '__ne__', c, None)
+            assert_attr(inf_d, neg_inf_f, '__ne__', c, None)
+
+            assert_attr(Decimal('NaN'), float('nan'), '__ne__', c, None)
+
+        def test_containers(c, signal=None):
+            c.clear_flags()
+            s = set([100.0, Decimal('100.0')])
+            self.assertEqual(len(s), 1)
+            self.assertTrue(c.flags[FloatOperation])
+
+            c.clear_flags()
+            if signal:
+                self.assertRaises(signal, sorted, [1.0, Decimal('10.0')])
+            else:
+                s = sorted([10.0, Decimal('10.0')])
+            self.assertTrue(c.flags[FloatOperation])
+
+            c.clear_flags()
+            b = 10.0 in [Decimal('10.0'), 1.0]
+            self.assertTrue(c.flags[FloatOperation])
+
+            c.clear_flags()
+            b = 10.0 in {Decimal('10.0'):'a', 1.0:'b'}
+            self.assertTrue(c.flags[FloatOperation])
+
+        nc = Context()
+        with localcontext(nc) as c:
+            self.assertFalse(c.traps[FloatOperation])
+            doit(c, signal=None)
+            test_containers(c, signal=None)
+
+            c.traps[FloatOperation] = True
+            doit(c, signal=FloatOperation)
+            test_containers(c, signal=FloatOperation)
+
+    def test_float_operation_default(self):
+        Decimal = self.decimal.Decimal
+        Context = self.decimal.Context
+        Inexact = self.decimal.Inexact
+        FloatOperation= self.decimal.FloatOperation
+
+        context = Context()
+        self.assertFalse(context.flags[FloatOperation])
+        self.assertFalse(context.traps[FloatOperation])
+
+        context.clear_traps()
+        context.traps[Inexact] = True
+        context.traps[FloatOperation] = True
+        self.assertTrue(context.traps[FloatOperation])
+        self.assertTrue(context.traps[Inexact])
+
+class CContextFlags(ContextFlags):
+    decimal = C
+class PyContextFlags(ContextFlags):
+    decimal = P
+
+class SpecialContexts(unittest.TestCase):
+    """Test the context templates."""
+
+    def test_context_templates(self):
+        BasicContext = self.decimal.BasicContext
+        ExtendedContext = self.decimal.ExtendedContext
+        getcontext = self.decimal.getcontext
+        setcontext = self.decimal.setcontext
+        InvalidOperation = self.decimal.InvalidOperation
+        DivisionByZero = self.decimal.DivisionByZero
+        Overflow = self.decimal.Overflow
+        Underflow = self.decimal.Underflow
+        Clamped = self.decimal.Clamped
+
+        assert_signals(self, BasicContext, 'traps',
+            [InvalidOperation, DivisionByZero, Overflow, Underflow, Clamped]
+        )
+
+        savecontext = getcontext().copy()
+        basic_context_prec = BasicContext.prec
+        extended_context_prec = ExtendedContext.prec
+
+        ex = None
+        try:
+            BasicContext.prec = ExtendedContext.prec = 441
+            for template in BasicContext, ExtendedContext:
+                setcontext(template)
+                c = getcontext()
+                self.assertIsNot(c, template)
+                self.assertEqual(c.prec, 441)
+        except Exception as e:
+            ex = e.__class__
+        finally:
+            BasicContext.prec = basic_context_prec
+            ExtendedContext.prec = extended_context_prec
+            setcontext(savecontext)
+            if ex:
+                raise ex
+
+    def test_default_context(self):
+        DefaultContext = self.decimal.DefaultContext
+        BasicContext = self.decimal.BasicContext
+        ExtendedContext = self.decimal.ExtendedContext
+        getcontext = self.decimal.getcontext
+        setcontext = self.decimal.setcontext
+        InvalidOperation = self.decimal.InvalidOperation
+        DivisionByZero = self.decimal.DivisionByZero
+        Overflow = self.decimal.Overflow
+
+        self.assertEqual(BasicContext.prec, 9)
+        self.assertEqual(ExtendedContext.prec, 9)
+
+        assert_signals(self, DefaultContext, 'traps',
+            [InvalidOperation, DivisionByZero, Overflow]
+        )
+
+        savecontext = getcontext().copy()
+        default_context_prec = DefaultContext.prec
+
+        ex = None
+        try:
+            c = getcontext()
+            saveprec = c.prec
+
+            DefaultContext.prec = 961
+            c = getcontext()
+            self.assertEqual(c.prec, saveprec)
+
+            setcontext(DefaultContext)
+            c = getcontext()
+            self.assertIsNot(c, DefaultContext)
+            self.assertEqual(c.prec, 961)
+        except Exception as e:
+            ex = e.__class__
+        finally:
+            DefaultContext.prec = default_context_prec
+            setcontext(savecontext)
+            if ex:
+                raise ex
+
+class CSpecialContexts(SpecialContexts):
+    decimal = C
+class PySpecialContexts(SpecialContexts):
+    decimal = P
+
+class ContextInputValidation(unittest.TestCase):
+
+    def test_invalid_context(self):
+        Context = self.decimal.Context
+        DefaultContext = self.decimal.DefaultContext
+
+        c = DefaultContext.copy()
+
+        # prec, Emax
+        for attr in ['prec', 'Emax']:
+            setattr(c, attr, 999999)
+            self.assertEqual(getattr(c, attr), 999999)
+            self.assertRaises(ValueError, setattr, c, attr, -1)
+            self.assertRaises(TypeError, setattr, c, attr, 'xyz')
+
+        # Emin
+        setattr(c, 'Emin', -999999)
+        self.assertEqual(getattr(c, 'Emin'), -999999)
+        self.assertRaises(ValueError, setattr, c, 'Emin', 1)
+        self.assertRaises(TypeError, setattr, c, 'Emin', (1,2,3))
+
+        self.assertRaises(TypeError, setattr, c, 'rounding', -1)
+        self.assertRaises(TypeError, setattr, c, 'rounding', 9)
+        self.assertRaises(TypeError, setattr, c, 'rounding', 1.0)
+        self.assertRaises(TypeError, setattr, c, 'rounding', 'xyz')
+
+        # capitals, clamp
+        for attr in ['capitals', 'clamp']:
+            self.assertRaises(ValueError, setattr, c, attr, -1)
+            self.assertRaises(ValueError, setattr, c, attr, 2)
+            self.assertRaises(TypeError, setattr, c, attr, [1,2,3])
+
+        # Invalid attribute
+        self.assertRaises(AttributeError, setattr, c, 'emax', 100)
+
+        # Invalid signal dict
+        self.assertRaises(TypeError, setattr, c, 'flags', [])
+        self.assertRaises(KeyError, setattr, c, 'flags', {})
+        self.assertRaises(KeyError, setattr, c, 'traps',
+                          {'InvalidOperation':0})
+
+        # Attributes cannot be deleted
+        for attr in ['prec', 'Emax', 'Emin', 'rounding', 'capitals', 'clamp',
+                     'flags', 'traps']:
+            self.assertRaises(AttributeError, c.__delattr__, attr)
+
+        # Invalid attributes
+        self.assertRaises(TypeError, getattr, c, 9)
+        self.assertRaises(TypeError, setattr, c, 9)
+
+        # Invalid values in constructor
+        self.assertRaises(TypeError, Context, rounding=999999)
+        self.assertRaises(TypeError, Context, rounding='xyz')
+        self.assertRaises(ValueError, Context, clamp=2)
+        self.assertRaises(ValueError, Context, capitals=-1)
+        self.assertRaises(KeyError, Context, flags=["P"])
+        self.assertRaises(KeyError, Context, traps=["Q"])
+
+        # Type error in conversion
+        self.assertRaises(TypeError, Context, flags=(0,1))
+        self.assertRaises(TypeError, Context, traps=(1,0))
+
+class CContextInputValidation(ContextInputValidation):
+    decimal = C
+class PyContextInputValidation(ContextInputValidation):
+    decimal = P
+
+class ContextSubclassing(unittest.TestCase):
+
+    def test_context_subclassing(self):
+        decimal = self.decimal
+        Decimal = decimal.Decimal
+        Context = decimal.Context
+        Clamped = decimal.Clamped
+        DivisionByZero = decimal.DivisionByZero
+        Inexact = decimal.Inexact
+        Overflow = decimal.Overflow
+        Rounded = decimal.Rounded
+        Subnormal = decimal.Subnormal
+        Underflow = decimal.Underflow
+        InvalidOperation = decimal.InvalidOperation
+
+        class MyContext(Context):
+            def __init__(self, prec=None, rounding=None, Emin=None, Emax=None,
+                               capitals=None, clamp=None, flags=None,
+                               traps=None):
+                Context.__init__(self)
+                if prec is not None:
+                    self.prec = prec
+                if rounding is not None:
+                    self.rounding = rounding
+                if Emin is not None:
+                    self.Emin = Emin
+                if Emax is not None:
+                    self.Emax = Emax
+                if capitals is not None:
+                    self.capitals = capitals
+                if clamp is not None:
+                    self.clamp = clamp
+                if flags is not None:
+                    if isinstance(flags, list):
+                        flags = {v:(v in flags) for v in OrderedSignals[decimal] + flags}
+                    self.flags = flags
+                if traps is not None:
+                    if isinstance(traps, list):
+                        traps = {v:(v in traps) for v in OrderedSignals[decimal] + traps}
+                    self.traps = traps
+
+        c = Context()
+        d = MyContext()
+        for attr in ('prec', 'rounding', 'Emin', 'Emax', 'capitals', 'clamp',
+                     'flags', 'traps'):
+            self.assertEqual(getattr(c, attr), getattr(d, attr))
+
+        # prec
+        self.assertRaises(ValueError, MyContext, **{'prec':-1})
+        c = MyContext(prec=1)
+        self.assertEqual(c.prec, 1)
+        self.assertRaises(InvalidOperation, c.quantize, Decimal('9e2'), 0)
+
+        # rounding
+        self.assertRaises(TypeError, MyContext, **{'rounding':'XYZ'})
+        c = MyContext(rounding=ROUND_DOWN, prec=1)
+        self.assertEqual(c.rounding, ROUND_DOWN)
+        self.assertEqual(c.plus(Decimal('9.9')), 9)
+
+        # Emin
+        self.assertRaises(ValueError, MyContext, **{'Emin':5})
+        c = MyContext(Emin=-1, prec=1)
+        self.assertEqual(c.Emin, -1)
+        x = c.add(Decimal('1e-99'), Decimal('2.234e-2000'))
+        self.assertEqual(x, Decimal('0.0'))
+        for signal in (Inexact, Underflow, Subnormal, Rounded, Clamped):
+            self.assertTrue(c.flags[signal])
+
+        # Emax
+        self.assertRaises(ValueError, MyContext, **{'Emax':-1})
+        c = MyContext(Emax=1, prec=1)
+        self.assertEqual(c.Emax, 1)
+        self.assertRaises(Overflow, c.add, Decimal('1e99'), Decimal('2.234e2000'))
+        if self.decimal == C:
+            for signal in (Inexact, Overflow, Rounded):
+                self.assertTrue(c.flags[signal])
+
+        # capitals
+        self.assertRaises(ValueError, MyContext, **{'capitals':-1})
+        c = MyContext(capitals=0)
+        self.assertEqual(c.capitals, 0)
+        x = c.create_decimal('1E222')
+        self.assertEqual(c.to_sci_string(x), '1e+222')
+
+        # clamp
+        self.assertRaises(ValueError, MyContext, **{'clamp':2})
+        c = MyContext(clamp=1, Emax=99)
+        self.assertEqual(c.clamp, 1)
+        x = c.plus(Decimal('1e99'))
+        self.assertEqual(str(x), '1.000000000000000000000000000E+99')
+
+        # flags
+        self.assertRaises(TypeError, MyContext, **{'flags':'XYZ'})
+        c = MyContext(flags=[Rounded, DivisionByZero])
+        for signal in (Rounded, DivisionByZero):
+            self.assertTrue(c.flags[signal])
+        c.clear_flags()
+        for signal in OrderedSignals[decimal]:
+            self.assertFalse(c.flags[signal])
+
+        # traps
+        self.assertRaises(TypeError, MyContext, **{'traps':'XYZ'})
+        c = MyContext(traps=[Rounded, DivisionByZero])
+        for signal in (Rounded, DivisionByZero):
+            self.assertTrue(c.traps[signal])
+        c.clear_traps()
+        for signal in OrderedSignals[decimal]:
+            self.assertFalse(c.traps[signal])
+
+class CContextSubclassing(ContextSubclassing):
+    decimal = C
+class PyContextSubclassing(ContextSubclassing):
+    decimal = P
+
+@skip_if_extra_functionality
+class CheckAttributes(unittest.TestCase):
+
+    def test_module_attributes(self):
+
+        # Architecture dependent context limits
+        self.assertEqual(C.MAX_PREC, P.MAX_PREC)
+        self.assertEqual(C.MAX_EMAX, P.MAX_EMAX)
+        self.assertEqual(C.MIN_EMIN, P.MIN_EMIN)
+        self.assertEqual(C.MIN_ETINY, P.MIN_ETINY)
+
+        self.assertTrue(C.HAVE_THREADS is True or C.HAVE_THREADS is False)
+        self.assertTrue(P.HAVE_THREADS is True or P.HAVE_THREADS is False)
+
+        self.assertEqual(C.__version__, P.__version__)
+
+        x = dir(C)
+        y = [s for s in dir(P) if '__' in s or not s.startswith('_')]
+        self.assertEqual(set(x) - set(y), set())
+
+    def test_context_attributes(self):
+
+        x = [s for s in dir(C.Context()) if '__' in s or not s.startswith('_')]
+        y = [s for s in dir(P.Context()) if '__' in s or not s.startswith('_')]
+        self.assertEqual(set(x) - set(y), set())
+
+    def test_decimal_attributes(self):
+
+        x = [s for s in dir(C.Decimal(9)) if '__' in s or not s.startswith('_')]
+        y = [s for s in dir(C.Decimal(9)) if '__' in s or not s.startswith('_')]
+        self.assertEqual(set(x) - set(y), set())
+
+class Coverage(unittest.TestCase):
+
+    def test_adjusted(self):
+        Decimal = self.decimal.Decimal
+
+        self.assertEqual(Decimal('1234e9999').adjusted(), 10002)
+        # XXX raise?
+        self.assertEqual(Decimal('nan').adjusted(), 0)
+        self.assertEqual(Decimal('inf').adjusted(), 0)
+
+    def test_canonical(self):
+        Decimal = self.decimal.Decimal
+        getcontext = self.decimal.getcontext
+
+        x = Decimal(9).canonical()
+        self.assertEqual(x, 9)
+
+        c = getcontext()
+        x = c.canonical(Decimal(9))
+        self.assertEqual(x, 9)
+
+    def test_context_repr(self):
+        c = self.decimal.DefaultContext.copy()
+
+        c.prec = 425000000
+        c.Emax = 425000000
+        c.Emin = -425000000
+        c.rounding = ROUND_HALF_DOWN
+        c.capitals = 0
+        c.clamp = 1
+        for sig in OrderedSignals[self.decimal]:
+            c.flags[sig] = False
+            c.traps[sig] = False
+
+        s = c.__repr__()
+        t = "Context(prec=425000000, rounding=ROUND_HALF_DOWN, " \
+            "Emin=-425000000, Emax=425000000, capitals=0, clamp=1, " \
+            "flags=[], traps=[])"
+        self.assertEqual(s, t)
+
+    def test_implicit_context(self):
+        Decimal = self.decimal.Decimal
+        localcontext = self.decimal.localcontext
+
+        with localcontext() as c:
+            c.prec = 1
+            c.Emax = 1
+            c.Emin = -1
+
+            # abs
+            self.assertEqual(abs(Decimal("-10")), 10)
+            # add
+            self.assertEqual(Decimal("7") + 1, 8)
+            # divide
+            self.assertEqual(Decimal("10") / 5, 2)
+            # divide_int
+            self.assertEqual(Decimal("10") // 7, 1)
+            # fma
+            self.assertEqual(Decimal("1.2").fma(Decimal("0.01"), 1), 1)
+            self.assertIs(Decimal("NaN").fma(7, 1).is_nan(), True)
+            # three arg power
+            self.assertEqual(pow(Decimal(10), 2, 7), 2)
+            # exp
+            self.assertEqual(Decimal("1.01").exp(), 3)
+            # is_normal
+            self.assertIs(Decimal("0.01").is_normal(), False)
+            # is_subnormal
+            self.assertIs(Decimal("0.01").is_subnormal(), True)
+            # ln
+            self.assertEqual(Decimal("20").ln(), 3)
+            # log10
+            self.assertEqual(Decimal("20").log10(), 1)
+            # logb
+            self.assertEqual(Decimal("580").logb(), 2)
+            # logical_invert
+            self.assertEqual(Decimal("10").logical_invert(), 1)
+            # minus
+            self.assertEqual(-Decimal("-10"), 10)
+            # multiply
+            self.assertEqual(Decimal("2") * 4, 8)
+            # next_minus
+            self.assertEqual(Decimal("10").next_minus(), 9)
+            # next_plus
+            self.assertEqual(Decimal("10").next_plus(), Decimal('2E+1'))
+            # normalize
+            self.assertEqual(Decimal("-10").normalize(), Decimal('-1E+1'))
+            # number_class
+            self.assertEqual(Decimal("10").number_class(), '+Normal')
+            # plus
+            self.assertEqual(+Decimal("-1"), -1)
+            # remainder
+            self.assertEqual(Decimal("10") % 7, 3)
+            # subtract
+            self.assertEqual(Decimal("10") - 7, 3)
+            # to_integral_exact
+            self.assertEqual(Decimal("1.12345").to_integral_exact(), 1)
+
+            # Boolean functions
+            self.assertTrue(Decimal("1").is_canonical())
+            self.assertTrue(Decimal("1").is_finite())
+            self.assertTrue(Decimal("1").is_finite())
+            self.assertTrue(Decimal("snan").is_snan())
+            self.assertTrue(Decimal("-1").is_signed())
+            self.assertTrue(Decimal("0").is_zero())
+            self.assertTrue(Decimal("0").is_zero())
+
+        # Copy
+        with localcontext() as c:
+            c.prec = 10000
+            x = 1228 ** 1523
+            y = -Decimal(x)
+
+            z = y.copy_abs()
+            self.assertEqual(z, x)
+
+            z = y.copy_negate()
+            self.assertEqual(z, x)
+
+            z = y.copy_sign(Decimal(1))
+            self.assertEqual(z, x)
+
+    def test_divmod(self):
+        Decimal = self.decimal.Decimal
+        localcontext = self.decimal.localcontext
+        InvalidOperation = self.decimal.InvalidOperation
+        DivisionByZero = self.decimal.DivisionByZero
+
+        with localcontext() as c:
+            q, r = divmod(Decimal("10912837129"), 1001)
+            self.assertEqual(q, Decimal('10901935'))
+            self.assertEqual(r, Decimal('194'))
+
+            q, r = divmod(Decimal("NaN"), 7)
+            self.assertTrue(q.is_nan() and r.is_nan())
+
+            c.traps[InvalidOperation] = False
+            q, r = divmod(Decimal("NaN"), 7)
+            self.assertTrue(q.is_nan() and r.is_nan())
+
+            c.traps[InvalidOperation] = False
+            c.clear_flags()
+            q, r = divmod(Decimal("inf"), Decimal("inf"))
+            self.assertTrue(q.is_nan() and r.is_nan())
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            q, r = divmod(Decimal("inf"), 101)
+            self.assertTrue(q.is_infinite() and r.is_nan())
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            q, r = divmod(Decimal(0), 0)
+            self.assertTrue(q.is_nan() and r.is_nan())
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.traps[DivisionByZero] = False
+            c.clear_flags()
+            q, r = divmod(Decimal(11), 0)
+            self.assertTrue(q.is_infinite() and r.is_nan())
+            self.assertTrue(c.flags[InvalidOperation] and
+                            c.flags[DivisionByZero])
+
+    def test_power(self):
+        Decimal = self.decimal.Decimal
+        localcontext = self.decimal.localcontext
+        Overflow = self.decimal.Overflow
+        Rounded = self.decimal.Rounded
+
+        with localcontext() as c:
+            c.prec = 3
+            c.clear_flags()
+            self.assertEqual(Decimal("1.0") ** 100, Decimal('1.00'))
+            self.assertTrue(c.flags[Rounded])
+
+            c.prec = 1
+            c.Emax = 1
+            c.Emin = -1
+            c.clear_flags()
+            c.traps[Overflow] = False
+            self.assertEqual(Decimal(10000) ** Decimal("0.5"), Decimal('inf'))
+            self.assertTrue(c.flags[Overflow])
+
+    def test_quantize(self):
+        Decimal = self.decimal.Decimal
+        localcontext = self.decimal.localcontext
+        InvalidOperation = self.decimal.InvalidOperation
+
+        with localcontext() as c:
+            c.prec = 1
+            c.Emax = 1
+            c.Emin = -1
+            c.traps[InvalidOperation] = False
+            x = Decimal(99).quantize(Decimal("1e1"))
+            self.assertTrue(x.is_nan())
+
+    def test_radix(self):
+        Decimal = self.decimal.Decimal
+        getcontext = self.decimal.getcontext
+
+        c = getcontext()
+        self.assertEqual(Decimal("1").radix(), 10)
+        self.assertEqual(c.radix(), 10)
+
+    def test_rop(self):
+        Decimal = self.decimal.Decimal
+
+        for attr in ('__radd__', '__rsub__', '__rmul__', '__rtruediv__',
+                     '__rdivmod__', '__rmod__', '__rfloordiv__', '__rpow__'):
+            self.assertIs(getattr(Decimal("1"), attr)("xyz"), NotImplemented)
+
+    def test_round(self):
+        # Python3 behavior: round() returns Decimal
+        Decimal = self.decimal.Decimal
+        getcontext = self.decimal.getcontext
+
+        c = getcontext()
+        c.prec = 28
+
+        self.assertEqual(str(Decimal("9.99").__round__()), "10")
+        self.assertEqual(str(Decimal("9.99e-5").__round__()), "0")
+        self.assertEqual(str(Decimal("1.23456789").__round__(5)), "1.23457")
+        self.assertEqual(str(Decimal("1.2345").__round__(10)), "1.2345000000")
+        self.assertEqual(str(Decimal("1.2345").__round__(-10)), "0E+10")
+
+        self.assertRaises(TypeError, Decimal("1.23").__round__, "5")
+        self.assertRaises(TypeError, Decimal("1.23").__round__, 5, 8)
+
+    def test_create_decimal(self):
+        c = self.decimal.Context()
+        self.assertRaises(ValueError, c.create_decimal, ["%"])
+
+    def test_int(self):
+        Decimal = self.decimal.Decimal
+        localcontext = self.decimal.localcontext
+
+        with localcontext() as c:
+            c.prec = 9999
+            x = Decimal(1221**1271) / 10**3923
+            self.assertEqual(int(x), 1)
+            self.assertEqual(x.to_integral(), 2)
+
+    def test_copy(self):
+        Context = self.decimal.Context
+
+        c = Context()
+        c.prec = 10000
+        x = -(1172 ** 1712)
+
+        y = c.copy_abs(x)
+        self.assertEqual(y, -x)
+
+        y = c.copy_negate(x)
+        self.assertEqual(y, -x)
+
+        y = c.copy_sign(x, 1)
+        self.assertEqual(y, -x)
+
+class CCoverage(Coverage):
+    decimal = C
+class PyCoverage(Coverage):
+    decimal = P
+
+class PyFunctionality(unittest.TestCase):
+    """Extra functionality in decimal.py"""
+
+    def test_py_quantize_watchexp(self):
+        # watchexp functionality
+        Decimal = P.Decimal
+        localcontext = P.localcontext
+
+        with localcontext() as c:
+            c.prec = 1
+            c.Emax = 1
+            c.Emin = -1
+            x = Decimal(99999).quantize(Decimal("1e3"), watchexp=False)
+            self.assertEqual(x, Decimal('1.00E+5'))
+
+    def test_py_alternate_formatting(self):
+        # triples giving a format, a Decimal, and the expected result
+        Decimal = P.Decimal
+        localcontext = P.localcontext
+
+        test_values = [
+            # Issue 7094: Alternate formatting (specified by #)
+            ('.0e', '1.0', '1e+0'),
+            ('#.0e', '1.0', '1.e+0'),
+            ('.0f', '1.0', '1'),
+            ('#.0f', '1.0', '1.'),
+            ('g', '1.1', '1.1'),
+            ('#g', '1.1', '1.1'),
+            ('.0g', '1', '1'),
+            ('#.0g', '1', '1.'),
+            ('.0%', '1.0', '100%'),
+            ('#.0%', '1.0', '100.%'),
+            ]
+        for fmt, d, result in test_values:
+            self.assertEqual(format(Decimal(d), fmt), result)
+
+class PyWhitebox(unittest.TestCase):
+    """White box testing for decimal.py"""
+
+    def test_py_exact_power(self):
+        # Rarely exercised lines in _power_exact.
+        Decimal = P.Decimal
+        localcontext = P.localcontext
+
+        with localcontext() as c:
+            c.prec = 8
+            x = Decimal(2**16) ** Decimal("-0.5")
+            self.assertEqual(x, Decimal('0.00390625'))
+
+            x = Decimal(2**16) ** Decimal("-0.6")
+            self.assertEqual(x, Decimal('0.0012885819'))
+
+            x = Decimal("256e7") ** Decimal("-0.5")
+
+            x = Decimal(152587890625) ** Decimal('-0.0625')
+            self.assertEqual(x, Decimal("0.2"))
+
+            x = Decimal("152587890625e7") ** Decimal('-0.0625')
+
+            x = Decimal(5**2659) ** Decimal('-0.0625')
+
+            c.prec = 1
+            x = Decimal("152587890625") ** Decimal('-0.5')
+            c.prec = 201
+            x = Decimal(2**578) ** Decimal("-0.5")
+
+    def test_py_immutability_operations(self):
+        # Do operations and check that it didn't change change internal objects.
+        Decimal = P.Decimal
+        DefaultContext = P.DefaultContext
+        setcontext = P.setcontext
+
+        c = DefaultContext.copy()
+        c.traps = dict((s, 0) for s in OrderedSignals[P])
+        setcontext(c)
+
+        d1 = Decimal('-25e55')
+        b1 = Decimal('-25e55')
+        d2 = Decimal('33e+33')
+        b2 = Decimal('33e+33')
+
+        def checkSameDec(operation, useOther=False):
+            if useOther:
+                eval("d1." + operation + "(d2)")
+                self.assertEqual(d1._sign, b1._sign)
+                self.assertEqual(d1._int, b1._int)
+                self.assertEqual(d1._exp, b1._exp)
+                self.assertEqual(d2._sign, b2._sign)
+                self.assertEqual(d2._int, b2._int)
+                self.assertEqual(d2._exp, b2._exp)
+            else:
+                eval("d1." + operation + "()")
+                self.assertEqual(d1._sign, b1._sign)
+                self.assertEqual(d1._int, b1._int)
+                self.assertEqual(d1._exp, b1._exp)
+            return
+
+        Decimal(d1)
+        self.assertEqual(d1._sign, b1._sign)
+        self.assertEqual(d1._int, b1._int)
+        self.assertEqual(d1._exp, b1._exp)
+
+        checkSameDec("__abs__")
+        checkSameDec("__add__", True)
+        checkSameDec("__divmod__", True)
+        checkSameDec("__eq__", True)
+        checkSameDec("__ne__", True)
+        checkSameDec("__le__", True)
+        checkSameDec("__lt__", True)
+        checkSameDec("__ge__", True)
+        checkSameDec("__gt__", True)
+        checkSameDec("__float__")
+        checkSameDec("__floordiv__", True)
+        checkSameDec("__hash__")
+        checkSameDec("__int__")
+        checkSameDec("__trunc__")
+        checkSameDec("__mod__", True)
+        checkSameDec("__mul__", True)
+        checkSameDec("__neg__")
+        checkSameDec("__bool__")
+        checkSameDec("__pos__")
+        checkSameDec("__pow__", True)
+        checkSameDec("__radd__", True)
+        checkSameDec("__rdivmod__", True)
+        checkSameDec("__repr__")
+        checkSameDec("__rfloordiv__", True)
+        checkSameDec("__rmod__", True)
+        checkSameDec("__rmul__", True)
+        checkSameDec("__rpow__", True)
+        checkSameDec("__rsub__", True)
+        checkSameDec("__str__")
+        checkSameDec("__sub__", True)
+        checkSameDec("__truediv__", True)
+        checkSameDec("adjusted")
+        checkSameDec("as_tuple")
+        checkSameDec("compare", True)
+        checkSameDec("max", True)
+        checkSameDec("min", True)
+        checkSameDec("normalize")
+        checkSameDec("quantize", True)
+        checkSameDec("remainder_near", True)
+        checkSameDec("same_quantum", True)
+        checkSameDec("sqrt")
+        checkSameDec("to_eng_string")
+        checkSameDec("to_integral")
+
+    def test_py_decimal_id(self):
+        Decimal = P.Decimal
+
+        d = Decimal(45)
+        e = Decimal(d)
+        self.assertEqual(str(e), '45')
+        self.assertNotEqual(id(d), id(e))
+
+    def test_py_rescale(self):
+        # Coverage
+        Decimal = P.Decimal
+        localcontext = P.localcontext
+
+        with localcontext() as c:
+            x = Decimal("NaN")._rescale(3, ROUND_UP)
+            self.assertTrue(x.is_nan())
+
+    def test_py__round(self):
+        # Coverage
+        Decimal = P.Decimal
+
+        self.assertRaises(ValueError, Decimal("3.1234")._round, 0, ROUND_UP)
+
+class CFunctionality(unittest.TestCase):
+    """Extra functionality in _decimal"""
+
+    @requires_extra_functionality
+    def test_c_ieee_context(self):
+        # issue 8786: Add support for IEEE 754 contexts to decimal module.
+        IEEEContext = C.IEEEContext
+        DECIMAL32 = C.DECIMAL32
+        DECIMAL64 = C.DECIMAL64
+        DECIMAL128 = C.DECIMAL128
+
+        def assert_rest(self, context):
+            self.assertEqual(context.clamp, 1)
+            assert_signals(self, context, 'traps', [])
+            assert_signals(self, context, 'flags', [])
+
+        c = IEEEContext(DECIMAL32)
+        self.assertEqual(c.prec, 7)
+        self.assertEqual(c.Emax, 96)
+        self.assertEqual(c.Emin, -95)
+        assert_rest(self, c)
+
+        c = IEEEContext(DECIMAL64)
+        self.assertEqual(c.prec, 16)
+        self.assertEqual(c.Emax, 384)
+        self.assertEqual(c.Emin, -383)
+        assert_rest(self, c)
+
+        c = IEEEContext(DECIMAL128)
+        self.assertEqual(c.prec, 34)
+        self.assertEqual(c.Emax, 6144)
+        self.assertEqual(c.Emin, -6143)
+        assert_rest(self, c)
+
+        # Invalid values
+        self.assertRaises(OverflowError, IEEEContext, 2**63)
+        self.assertRaises(ValueError, IEEEContext, -1)
+        self.assertRaises(ValueError, IEEEContext, 1024)
+
+    @requires_extra_functionality
+    def test_c_context(self):
+        Context = C.Context
+
+        c = Context(flags=C.DecClamped, traps=C.DecRounded)
+        self.assertEqual(c._flags, C.DecClamped)
+        self.assertEqual(c._traps, C.DecRounded)
+
+    @requires_extra_functionality
+    def test_constants(self):
+        # Condition flags
+        cond = (
+            C.DecClamped, C.DecConversionSyntax, C.DecDivisionByZero,
+            C.DecDivisionImpossible, C.DecDivisionUndefined,
+            C.DecFpuError, C.DecInexact, C.DecInvalidContext,
+            C.DecInvalidOperation, C.DecMallocError,
+            C.DecFloatOperation, C.DecOverflow, C.DecRounded,
+            C.DecSubnormal, C.DecUnderflow
+        )
+
+        # IEEEContext
+        self.assertEqual(C.DECIMAL32, 32)
+        self.assertEqual(C.DECIMAL64, 64)
+        self.assertEqual(C.DECIMAL128, 128)
+        self.assertEqual(C.IEEE_CONTEXT_MAX_BITS, 512)
+
+        # Conditions
+        for i, v in enumerate(cond):
+            self.assertEqual(v, 1<<i)
+
+        self.assertEqual(C.DecIEEEInvalidOperation,
+                         C.DecConversionSyntax|
+                         C.DecDivisionImpossible|
+                         C.DecDivisionUndefined|
+                         C.DecFpuError|
+                         C.DecInvalidContext|
+                         C.DecInvalidOperation|
+                         C.DecMallocError)
+
+        self.assertEqual(C.DecErrors,
+                         C.DecIEEEInvalidOperation|
+                         C.DecDivisionByZero)
+
+        self.assertEqual(C.DecTraps,
+                         C.DecErrors|C.DecOverflow|C.DecUnderflow)
+
+class CWhitebox(unittest.TestCase):
+    """Whitebox testing for _decimal"""
+
+    def test_bignum(self):
+        # Not exactly whitebox, but too slow with pydecimal.
+
+        Decimal = C.Decimal
+        localcontext = C.localcontext
+
+        b1 = 10**35
+        b2 = 10**36
+        with localcontext() as c:
+            c.prec = 1000000
+            for i in range(5):
+                a = random.randrange(b1, b2)
+                b = random.randrange(1000, 1200)
+                x = a ** b
+                y = Decimal(a) ** Decimal(b)
+                self.assertEqual(x, y)
+
+    def test_invalid_construction(self):
+        self.assertRaises(TypeError, C.Decimal, 9, "xyz")
+
+    def test_c_input_restriction(self):
+        # Too large for _decimal to be converted exactly
+        Decimal = C.Decimal
+        InvalidOperation = C.InvalidOperation
+        Context = C.Context
+        localcontext = C.localcontext
+
+        with localcontext(Context()):
+            self.assertRaises(InvalidOperation, Decimal,
+                              "1e9999999999999999999")
+
+    def test_c_context_repr(self):
+        # This test is _decimal-only because flags are not printed
+        # in the same order.
+        DefaultContext = C.DefaultContext
+        FloatOperation = C.FloatOperation
+
+        c = DefaultContext.copy()
+
+        c.prec = 425000000
+        c.Emax = 425000000
+        c.Emin = -425000000
+        c.rounding = ROUND_HALF_DOWN
+        c.capitals = 0
+        c.clamp = 1
+        for sig in OrderedSignals[C]:
+            c.flags[sig] = True
+            c.traps[sig] = True
+        c.flags[FloatOperation] = True
+        c.traps[FloatOperation] = True
+
+        s = c.__repr__()
+        t = "Context(prec=425000000, rounding=ROUND_HALF_DOWN, " \
+            "Emin=-425000000, Emax=425000000, capitals=0, clamp=1, " \
+            "flags=[Clamped, InvalidOperation, DivisionByZero, Inexact, " \
+                   "FloatOperation, Overflow, Rounded, Subnormal, Underflow], " \
+            "traps=[Clamped, InvalidOperation, DivisionByZero, Inexact, " \
+                   "FloatOperation, Overflow, Rounded, Subnormal, Underflow])"
+        self.assertEqual(s, t)
+
+    def test_c_context_errors(self):
+        Context = C.Context
+        InvalidOperation = C.InvalidOperation
+        Overflow = C.Overflow
+        FloatOperation = C.FloatOperation
+        localcontext = C.localcontext
+        getcontext = C.getcontext
+        setcontext = C.setcontext
+        HAVE_CONFIG_64 = (C.MAX_PREC > 425000000)
+
+        c = Context()
+
+        # SignalDict: input validation
+        self.assertRaises(KeyError, c.flags.__setitem__, 801, 0)
+        self.assertRaises(KeyError, c.traps.__setitem__, 801, 0)
+        self.assertRaises(ValueError, c.flags.__delitem__, Overflow)
+        self.assertRaises(ValueError, c.traps.__delitem__, InvalidOperation)
+        self.assertRaises(TypeError, setattr, c, 'flags', ['x'])
+        self.assertRaises(TypeError, setattr, c,'traps', ['y'])
+        self.assertRaises(KeyError, setattr, c, 'flags', {0:1})
+        self.assertRaises(KeyError, setattr, c, 'traps', {0:1})
+
+        # Test assignment from a signal dict with the correct length but
+        # one invalid key.
+        d = c.flags.copy()
+        del d[FloatOperation]
+        d["XYZ"] = 91283719
+        self.assertRaises(KeyError, setattr, c, 'flags', d)
+        self.assertRaises(KeyError, setattr, c, 'traps', d)
+
+        # Input corner cases
+        int_max = 2**63-1 if HAVE_CONFIG_64 else 2**31-1
+        gt_max_emax = 10**18 if HAVE_CONFIG_64 else 10**9
+
+        # prec, Emax, Emin
+        for attr in ['prec', 'Emax']:
+            self.assertRaises(ValueError, setattr, c, attr, gt_max_emax)
+        self.assertRaises(ValueError, setattr, c, 'Emin', -gt_max_emax)
+
+        # prec, Emax, Emin in context constructor
+        self.assertRaises(ValueError, Context, prec=gt_max_emax)
+        self.assertRaises(ValueError, Context, Emax=gt_max_emax)
+        self.assertRaises(ValueError, Context, Emin=-gt_max_emax)
+
+        # Overflow in conversion
+        self.assertRaises(OverflowError, Context, prec=int_max+1)
+        self.assertRaises(OverflowError, Context, Emax=int_max+1)
+        self.assertRaises(OverflowError, Context, Emin=-int_max-2)
+        self.assertRaises(OverflowError, Context, clamp=int_max+1)
+        self.assertRaises(OverflowError, Context, capitals=int_max+1)
+
+        # OverflowError, general ValueError
+        for attr in ('prec', 'Emin', 'Emax', 'capitals', 'clamp'):
+            self.assertRaises(OverflowError, setattr, c, attr, int_max+1)
+            self.assertRaises(OverflowError, setattr, c, attr, -int_max-2)
+            if sys.platform != 'win32':
+                self.assertRaises(ValueError, setattr, c, attr, int_max)
+                self.assertRaises(ValueError, setattr, c, attr, -int_max-1)
+
+        # OverflowError: _unsafe_setprec, _unsafe_setemin, _unsafe_setemax
+        if C.MAX_PREC == 425000000:
+            self.assertRaises(OverflowError, getattr(c, '_unsafe_setprec'),
+                              int_max+1)
+            self.assertRaises(OverflowError, getattr(c, '_unsafe_setemax'),
+                              int_max+1)
+            self.assertRaises(OverflowError, getattr(c, '_unsafe_setemin'),
+                              -int_max-2)
+
+        # ValueError: _unsafe_setprec, _unsafe_setemin, _unsafe_setemax
+        if C.MAX_PREC == 425000000:
+            self.assertRaises(ValueError, getattr(c, '_unsafe_setprec'), 0)
+            self.assertRaises(ValueError, getattr(c, '_unsafe_setprec'),
+                              1070000001)
+            self.assertRaises(ValueError, getattr(c, '_unsafe_setemax'), -1)
+            self.assertRaises(ValueError, getattr(c, '_unsafe_setemax'),
+                              1070000001)
+            self.assertRaises(ValueError, getattr(c, '_unsafe_setemin'),
+                              -1070000001)
+            self.assertRaises(ValueError, getattr(c, '_unsafe_setemin'), 1)
+
+        # capitals, clamp
+        for attr in ['capitals', 'clamp']:
+            self.assertRaises(ValueError, setattr, c, attr, -1)
+            self.assertRaises(ValueError, setattr, c, attr, 2)
+            self.assertRaises(TypeError, setattr, c, attr, [1,2,3])
+            if HAVE_CONFIG_64:
+                self.assertRaises(ValueError, setattr, c, attr, 2**32)
+                self.assertRaises(ValueError, setattr, c, attr, 2**32+1)
+
+        # Invalid local context
+        self.assertRaises(TypeError, exec, 'with localcontext("xyz"): pass',
+                          locals())
+        self.assertRaises(TypeError, exec,
+                          'with localcontext(context=getcontext()): pass',
+                          locals())
+
+        # setcontext
+        saved_context = getcontext()
+        self.assertRaises(TypeError, setcontext, "xyz")
+        setcontext(saved_context)
+
+    def test_rounding_strings_interned(self):
+
+        self.assertIs(C.ROUND_UP, P.ROUND_UP)
+        self.assertIs(C.ROUND_DOWN, P.ROUND_DOWN)
+        self.assertIs(C.ROUND_CEILING, P.ROUND_CEILING)
+        self.assertIs(C.ROUND_FLOOR, P.ROUND_FLOOR)
+        self.assertIs(C.ROUND_HALF_UP, P.ROUND_HALF_UP)
+        self.assertIs(C.ROUND_HALF_DOWN, P.ROUND_HALF_DOWN)
+        self.assertIs(C.ROUND_HALF_EVEN, P.ROUND_HALF_EVEN)
+        self.assertIs(C.ROUND_05UP, P.ROUND_05UP)
+
+    @requires_extra_functionality
+    def test_c_context_errors_extra(self):
+        Context = C.Context
+        InvalidOperation = C.InvalidOperation
+        Overflow = C.Overflow
+        localcontext = C.localcontext
+        getcontext = C.getcontext
+        setcontext = C.setcontext
+        HAVE_CONFIG_64 = (C.MAX_PREC > 425000000)
+
+        c = Context()
+
+        # Input corner cases
+        int_max = 2**63-1 if HAVE_CONFIG_64 else 2**31-1
+
+        # OverflowError, general ValueError
+        self.assertRaises(OverflowError, setattr, c, '_allcr', int_max+1)
+        self.assertRaises(OverflowError, setattr, c, '_allcr', -int_max-2)
+        if sys.platform != 'win32':
+            self.assertRaises(ValueError, setattr, c, '_allcr', int_max)
+            self.assertRaises(ValueError, setattr, c, '_allcr', -int_max-1)
+
+        # OverflowError, general TypeError
+        for attr in ('_flags', '_traps'):
+            self.assertRaises(OverflowError, setattr, c, attr, int_max+1)
+            self.assertRaises(OverflowError, setattr, c, attr, -int_max-2)
+            if sys.platform != 'win32':
+                self.assertRaises(TypeError, setattr, c, attr, int_max)
+                self.assertRaises(TypeError, setattr, c, attr, -int_max-1)
+
+        # _allcr
+        self.assertRaises(ValueError, setattr, c, '_allcr', -1)
+        self.assertRaises(ValueError, setattr, c, '_allcr', 2)
+        self.assertRaises(TypeError, setattr, c, '_allcr', [1,2,3])
+        if HAVE_CONFIG_64:
+            self.assertRaises(ValueError, setattr, c, '_allcr', 2**32)
+            self.assertRaises(ValueError, setattr, c, '_allcr', 2**32+1)
+
+        # _flags, _traps
+        for attr in ['_flags', '_traps']:
+            self.assertRaises(TypeError, setattr, c, attr, 999999)
+            self.assertRaises(TypeError, setattr, c, attr, 'x')
+
+    def test_c_valid_context(self):
+        # These tests are for code coverage in _decimal.
+        DefaultContext = C.DefaultContext
+        Clamped = C.Clamped
+        Underflow = C.Underflow
+        Inexact = C.Inexact
+        Rounded = C.Rounded
+        Subnormal = C.Subnormal
+
+        c = DefaultContext.copy()
+
+        # Exercise all getters and setters
+        c.prec = 34
+        c.rounding = ROUND_HALF_UP
+        c.Emax = 3000
+        c.Emin = -3000
+        c.capitals = 1
+        c.clamp = 0
+
+        self.assertEqual(c.prec, 34)
+        self.assertEqual(c.rounding, ROUND_HALF_UP)
+        self.assertEqual(c.Emin, -3000)
+        self.assertEqual(c.Emax, 3000)
+        self.assertEqual(c.capitals, 1)
+        self.assertEqual(c.clamp, 0)
+
+        self.assertEqual(c.Etiny(), -3033)
+        self.assertEqual(c.Etop(), 2967)
+
+        # Exercise all unsafe setters
+        if C.MAX_PREC == 425000000:
+            c._unsafe_setprec(999999999)
+            c._unsafe_setemax(999999999)
+            c._unsafe_setemin(-999999999)
+            self.assertEqual(c.prec, 999999999)
+            self.assertEqual(c.Emax, 999999999)
+            self.assertEqual(c.Emin, -999999999)
+
+    @requires_extra_functionality
+    def test_c_valid_context_extra(self):
+        DefaultContext = C.DefaultContext
+
+        c = DefaultContext.copy()
+        self.assertEqual(c._allcr, 1)
+        c._allcr = 0
+        self.assertEqual(c._allcr, 0)
+
+    def test_c_round(self):
+        # Restricted input.
+        Decimal = C.Decimal
+        InvalidOperation = C.InvalidOperation
+        localcontext = C.localcontext
+        MAX_EMAX = C.MAX_EMAX
+        MIN_ETINY = C.MIN_ETINY
+        int_max = 2**63-1 if C.MAX_PREC > 425000000 else 2**31-1
+
+        with localcontext() as c:
+            c.traps[InvalidOperation] = True
+            self.assertRaises(InvalidOperation, Decimal("1.23").__round__,
+                              -int_max-1)
+            self.assertRaises(InvalidOperation, Decimal("1.23").__round__,
+                              int_max)
+            self.assertRaises(InvalidOperation, Decimal("1").__round__,
+                              int(MAX_EMAX+1))
+            self.assertRaises(C.InvalidOperation, Decimal("1").__round__,
+                              -int(MIN_ETINY-1))
+            self.assertRaises(OverflowError, Decimal("1.23").__round__,
+                              -int_max-2)
+            self.assertRaises(OverflowError, Decimal("1.23").__round__,
+                              int_max+1)
+
+    def test_c_format(self):
+        # Restricted input
+        Decimal = C.Decimal
+        HAVE_CONFIG_64 = (C.MAX_PREC > 425000000)
+
+        self.assertRaises(TypeError, Decimal(1).__format__, "=10.10", [], 9)
+        self.assertRaises(TypeError, Decimal(1).__format__, "=10.10", 9)
+        self.assertRaises(TypeError, Decimal(1).__format__, [])
+
+        self.assertRaises(ValueError, Decimal(1).__format__, "<>=10.10")
+        maxsize = 2**63-1 if HAVE_CONFIG_64 else 2**31-1
+        self.assertRaises(ValueError, Decimal("1.23456789").__format__,
+                          "=%d.1" % maxsize)
+
+    def test_c_integral(self):
+        Decimal = C.Decimal
+        Inexact = C.Inexact
+        localcontext = C.localcontext
+
+        x = Decimal(10)
+        self.assertEqual(x.to_integral(), 10)
+        self.assertRaises(TypeError, x.to_integral, '10')
+        self.assertRaises(TypeError, x.to_integral, 10, 'x')
+        self.assertRaises(TypeError, x.to_integral, 10)
+
+        self.assertEqual(x.to_integral_value(), 10)
+        self.assertRaises(TypeError, x.to_integral_value, '10')
+        self.assertRaises(TypeError, x.to_integral_value, 10, 'x')
+        self.assertRaises(TypeError, x.to_integral_value, 10)
+
+        self.assertEqual(x.to_integral_exact(), 10)
+        self.assertRaises(TypeError, x.to_integral_exact, '10')
+        self.assertRaises(TypeError, x.to_integral_exact, 10, 'x')
+        self.assertRaises(TypeError, x.to_integral_exact, 10)
+
+        with localcontext() as c:
+            x = Decimal("99999999999999999999999999.9").to_integral_value(ROUND_UP)
+            self.assertEqual(x, Decimal('100000000000000000000000000'))
+
+            x = Decimal("99999999999999999999999999.9").to_integral_exact(ROUND_UP)
+            self.assertEqual(x, Decimal('100000000000000000000000000'))
+
+            c.traps[Inexact] = True
+            self.assertRaises(Inexact, Decimal("999.9").to_integral_exact, ROUND_UP)
+
+    def test_c_funcs(self):
+        # Invalid arguments
+        Decimal = C.Decimal
+        InvalidOperation = C.InvalidOperation
+        DivisionByZero = C.DivisionByZero
+        getcontext = C.getcontext
+        localcontext = C.localcontext
+
+        self.assertEqual(Decimal('9.99e10').to_eng_string(), '99.9E+9')
+
+        self.assertRaises(TypeError, pow, Decimal(1), 2, "3")
+        self.assertRaises(TypeError, Decimal(9).number_class, "x", "y")
+        self.assertRaises(TypeError, Decimal(9).same_quantum, 3, "x", "y")
+
+        self.assertRaises(
+            TypeError,
+            Decimal("1.23456789").quantize, Decimal('1e-100000'), []
+        )
+        self.assertRaises(
+            TypeError,
+            Decimal("1.23456789").quantize, Decimal('1e-100000'), getcontext()
+        )
+        self.assertRaises(
+            TypeError,
+            Decimal("1.23456789").quantize, Decimal('1e-100000'), 10
+        )
+        self.assertRaises(
+            TypeError,
+            Decimal("1.23456789").quantize, Decimal('1e-100000'), ROUND_UP, 1000
+        )
+
+        with localcontext() as c:
+            c.clear_traps()
+
+            # Invalid arguments
+            self.assertRaises(TypeError, c.copy_sign, Decimal(1), "x", "y")
+            self.assertRaises(TypeError, c.canonical, 200)
+            self.assertRaises(TypeError, c.is_canonical, 200)
+            self.assertRaises(TypeError, c.divmod, 9, 8, "x", "y")
+            self.assertRaises(TypeError, c.same_quantum, 9, 3, "x", "y")
+
+            self.assertEqual(str(c.canonical(Decimal(200))), '200')
+            self.assertEqual(c.radix(), 10)
+
+            c.traps[DivisionByZero] = True
+            self.assertRaises(DivisionByZero, Decimal(9).__divmod__, 0)
+            self.assertRaises(DivisionByZero, c.divmod, 9, 0)
+            self.assertTrue(c.flags[InvalidOperation])
+
+            c.clear_flags()
+            c.traps[InvalidOperation] = True
+            self.assertRaises(InvalidOperation, Decimal(9).__divmod__, 0)
+            self.assertRaises(InvalidOperation, c.divmod, 9, 0)
+            self.assertTrue(c.flags[DivisionByZero])
+
+            c.traps[InvalidOperation] = True
+            c.prec = 2
+            self.assertRaises(InvalidOperation, pow, Decimal(1000), 1, 501)
+
+    def test_va_args_exceptions(self):
+        Decimal = C.Decimal
+        Context = C.Context
+
+        x = Decimal("10001111111")
+
+        for attr in ['exp', 'is_normal', 'is_subnormal', 'ln', 'log10',
+                     'logb', 'logical_invert', 'next_minus', 'next_plus',
+                     'normalize', 'number_class', 'sqrt', 'to_eng_string']:
+            func = getattr(x, attr)
+            self.assertRaises(TypeError, func, context="x")
+            self.assertRaises(TypeError, func, "x", context=None)
+
+        for attr in ['compare', 'compare_signal', 'logical_and',
+                     'logical_or', 'max', 'max_mag', 'min', 'min_mag',
+                     'remainder_near', 'rotate', 'scaleb', 'shift']:
+            func = getattr(x, attr)
+            self.assertRaises(TypeError, func, context="x")
+            self.assertRaises(TypeError, func, "x", context=None)
+
+        self.assertRaises(TypeError, x.to_integral, rounding=None, context=[])
+        self.assertRaises(TypeError, x.to_integral, rounding={}, context=[])
+        self.assertRaises(TypeError, x.to_integral, [], [])
+
+        self.assertRaises(TypeError, x.to_integral_value, rounding=None, context=[])
+        self.assertRaises(TypeError, x.to_integral_value, rounding={}, context=[])
+        self.assertRaises(TypeError, x.to_integral_value, [], [])
+
+        self.assertRaises(TypeError, x.to_integral_exact, rounding=None, context=[])
+        self.assertRaises(TypeError, x.to_integral_exact, rounding={}, context=[])
+        self.assertRaises(TypeError, x.to_integral_exact, [], [])
+
+        self.assertRaises(TypeError, x.fma, 1, 2, context="x")
+        self.assertRaises(TypeError, x.fma, 1, 2, "x", context=None)
+
+        self.assertRaises(TypeError, x.quantize, 1, [], context=None)
+        self.assertRaises(TypeError, x.quantize, 1, [], rounding=None)
+        self.assertRaises(TypeError, x.quantize, 1, [], [])
+
+        c = Context()
+        self.assertRaises(TypeError, c.power, 1, 2, mod="x")
+        self.assertRaises(TypeError, c.power, 1, "x", mod=None)
+        self.assertRaises(TypeError, c.power, "x", 2, mod=None)
+
+    @requires_extra_functionality
+    def test_c_context_templates(self):
+        self.assertEqual(
+            C.BasicContext._traps,
+            C.DecIEEEInvalidOperation|C.DecDivisionByZero|C.DecOverflow|
+            C.DecUnderflow|C.DecClamped
+        )
+        self.assertEqual(
+            C.DefaultContext._traps,
+            C.DecIEEEInvalidOperation|C.DecDivisionByZero|C.DecOverflow
+        )
+
+    @requires_extra_functionality
+    def test_c_signal_dict(self):
+
+        # SignalDict coverage
+        Context = C.Context
+        DefaultContext = C.DefaultContext
+
+        InvalidOperation = C.InvalidOperation
+        DivisionByZero = C.DivisionByZero
+        Overflow = C.Overflow
+        Subnormal = C.Subnormal
+        Underflow = C.Underflow
+        Rounded = C.Rounded
+        Inexact = C.Inexact
+        Clamped = C.Clamped
+
+        DecClamped = C.DecClamped
+        DecInvalidOperation = C.DecInvalidOperation
+        DecIEEEInvalidOperation = C.DecIEEEInvalidOperation
+
+        def assertIsExclusivelySet(signal, signal_dict):
+            for sig in signal_dict:
+                if sig == signal:
+                    self.assertTrue(signal_dict[sig])
+                else:
+                    self.assertFalse(signal_dict[sig])
+
+        c = DefaultContext.copy()
+
+        # Signal dict methods
+        self.assertTrue(Overflow in c.traps)
+        c.clear_traps()
+        for k in c.traps.keys():
+            c.traps[k] = True
+        for v in c.traps.values():
+            self.assertTrue(v)
+        c.clear_traps()
+        for k, v in c.traps.items():
+            self.assertFalse(v)
+
+        self.assertFalse(c.flags.get(Overflow))
+        self.assertIs(c.flags.get("x"), None)
+        self.assertEqual(c.flags.get("x", "y"), "y")
+        self.assertRaises(TypeError, c.flags.get, "x", "y", "z")
+
+        self.assertEqual(len(c.flags), len(c.traps))
+        s = sys.getsizeof(c.flags)
+        s = sys.getsizeof(c.traps)
+        s = c.flags.__repr__()
+
+        # Set flags/traps.
+        c.clear_flags()
+        c._flags = DecClamped
+        self.assertTrue(c.flags[Clamped])
+
+        c.clear_traps()
+        c._traps = DecInvalidOperation
+        self.assertTrue(c.traps[InvalidOperation])
+
+        # Set flags/traps from dictionary.
+        c.clear_flags()
+        d = c.flags.copy()
+        d[DivisionByZero] = True
+        c.flags = d
+        assertIsExclusivelySet(DivisionByZero, c.flags)
+
+        c.clear_traps()
+        d = c.traps.copy()
+        d[Underflow] = True
+        c.traps = d
+        assertIsExclusivelySet(Underflow, c.traps)
+
+        # Random constructors
+        IntSignals = {
+          Clamped: C.DecClamped,
+          Rounded: C.DecRounded,
+          Inexact: C.DecInexact,
+          Subnormal: C.DecSubnormal,
+          Underflow: C.DecUnderflow,
+          Overflow: C.DecOverflow,
+          DivisionByZero: C.DecDivisionByZero,
+          InvalidOperation: C.DecIEEEInvalidOperation
+        }
+        IntCond = [
+          C.DecDivisionImpossible, C.DecDivisionUndefined, C.DecFpuError,
+          C.DecInvalidContext, C.DecInvalidOperation, C.DecMallocError,
+          C.DecConversionSyntax,
+        ]
+
+        lim = len(OrderedSignals[C])
+        for r in range(lim):
+            for t in range(lim):
+                for round in RoundingModes:
+                    flags = random.sample(OrderedSignals[C], r)
+                    traps = random.sample(OrderedSignals[C], t)
+                    prec = random.randrange(1, 10000)
+                    emin = random.randrange(-10000, 0)
+                    emax = random.randrange(0, 10000)
+                    clamp = random.randrange(0, 2)
+                    caps = random.randrange(0, 2)
+                    cr = random.randrange(0, 2)
+                    c = Context(prec=prec, rounding=round, Emin=emin, Emax=emax,
+                                capitals=caps, clamp=clamp, flags=list(flags),
+                                traps=list(traps))
+
+                    self.assertEqual(c.prec, prec)
+                    self.assertEqual(c.rounding, round)
+                    self.assertEqual(c.Emin, emin)
+                    self.assertEqual(c.Emax, emax)
+                    self.assertEqual(c.capitals, caps)
+                    self.assertEqual(c.clamp, clamp)
+
+                    f = 0
+                    for x in flags:
+                        f |= IntSignals[x]
+                    self.assertEqual(c._flags, f)
+
+                    f = 0
+                    for x in traps:
+                        f |= IntSignals[x]
+                    self.assertEqual(c._traps, f)
+
+        for cond in IntCond:
+            c._flags = cond
+            self.assertTrue(c._flags&DecIEEEInvalidOperation)
+            assertIsExclusivelySet(InvalidOperation, c.flags)
+
+        for cond in IntCond:
+            c._traps = cond
+            self.assertTrue(c._traps&DecIEEEInvalidOperation)
+            assertIsExclusivelySet(InvalidOperation, c.traps)
+
+    def test_invalid_override(self):
+        Decimal = C.Decimal
+
+        try:
+            from locale import CHAR_MAX
+        except ImportError:
+            return
+
+        def make_grouping(lst):
+            return ''.join([chr(x) for x in lst])
+
+        def get_fmt(x, override=None, fmt='n'):
+            return Decimal(x).__format__(fmt, override)
+
+        invalid_grouping = {
+            'decimal_point' : ',',
+            'grouping' : make_grouping([255, 255, 0]),
+            'thousands_sep' : ','
+        }
+        invalid_dot = {
+            'decimal_point' : 'xxxxx',
+            'grouping' : make_grouping([3, 3, 0]),
+            'thousands_sep' : ','
+        }
+        invalid_sep = {
+            'decimal_point' : '.',
+            'grouping' : make_grouping([3, 3, 0]),
+            'thousands_sep' : 'yyyyy'
+        }
+
+        if CHAR_MAX == 127: # negative grouping in override
+            self.assertRaises(ValueError, get_fmt, 12345,
+                              invalid_grouping, 'g')
+
+        self.assertRaises(ValueError, get_fmt, 12345, invalid_dot, 'g')
+        self.assertRaises(ValueError, get_fmt, 12345, invalid_sep, 'g')
+
+    def test_exact_conversion(self):
+        Decimal = C.Decimal
+        localcontext = C.localcontext
+        InvalidOperation = C.InvalidOperation
+
+        with localcontext() as c:
+
+            c.traps[InvalidOperation] = True
+
+            # Clamped
+            x = "0e%d" % sys.maxsize
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+            x = "0e%d" % (-sys.maxsize-1)
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+            # Overflow
+            x = "1e%d" % sys.maxsize
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+            # Underflow
+            x = "1e%d" % (-sys.maxsize-1)
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+    def test_from_tuple(self):
+        Decimal = C.Decimal
+        localcontext = C.localcontext
+        InvalidOperation = C.InvalidOperation
+        Overflow = C.Overflow
+        Underflow = C.Underflow
+
+        with localcontext() as c:
+
+            c.traps[InvalidOperation] = True
+            c.traps[Overflow] = True
+            c.traps[Underflow] = True
+
+            # SSIZE_MAX
+            x = (1, (), sys.maxsize)
+            self.assertEqual(str(c.create_decimal(x)), '-0E+999999')
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+            x = (1, (0, 1, 2), sys.maxsize)
+            self.assertRaises(Overflow, c.create_decimal, x)
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+            # SSIZE_MIN
+            x = (1, (), -sys.maxsize-1)
+            self.assertEqual(str(c.create_decimal(x)), '-0E-1000026')
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+            x = (1, (0, 1, 2), -sys.maxsize-1)
+            self.assertRaises(Underflow, c.create_decimal, x)
+            self.assertRaises(InvalidOperation, Decimal, x)
+
+            # OverflowError
+            x = (1, (), sys.maxsize+1)
+            self.assertRaises(OverflowError, c.create_decimal, x)
+            self.assertRaises(OverflowError, Decimal, x)
+
+            x = (1, (), -sys.maxsize-2)
+            self.assertRaises(OverflowError, c.create_decimal, x)
+            self.assertRaises(OverflowError, Decimal, x)
+
+            # Specials
+            x = (1, (), "N")
+            self.assertEqual(str(Decimal(x)), '-sNaN')
+            x = (1, (0,), "N")
+            self.assertEqual(str(Decimal(x)), '-sNaN')
+            x = (1, (0, 1), "N")
+            self.assertEqual(str(Decimal(x)), '-sNaN1')
+
+
+all_tests = [
+  CExplicitConstructionTest, PyExplicitConstructionTest,
+  CImplicitConstructionTest, PyImplicitConstructionTest,
+  CFormatTest,               PyFormatTest,
+  CArithmeticOperatorsTest,  PyArithmeticOperatorsTest,
+  CThreadingTest,            PyThreadingTest,
+  CUsabilityTest,            PyUsabilityTest,
+  CPythonAPItests,           PyPythonAPItests,
+  CContextAPItests,          PyContextAPItests,
+  CContextWithStatement,     PyContextWithStatement,
+  CContextFlags,             PyContextFlags,
+  CSpecialContexts,          PySpecialContexts,
+  CContextInputValidation,   PyContextInputValidation,
+  CContextSubclassing,       PyContextSubclassing,
+  CCoverage,                 PyCoverage,
+  CFunctionality,            PyFunctionality,
+  CWhitebox,                 PyWhitebox,
+  CIBMTestCases,             PyIBMTestCases,
+]
+
+# Delete C tests if _decimal.so is not present.
+if not C:
+    all_tests = all_tests[1::2]
+else:
+    all_tests.insert(0, CheckAttributes)
+
+
 def test_main(arith=False, verbose=None, todo_tests=None, debug=None):
     """ Execute the tests.
 
@@ -2374,27 +5404,16 @@ def test_main(arith=False, verbose=None, todo_tests=None, debug=None):
     is enabled in regrtest.py
     """
 
-    init()
+    init(C)
+    init(P)
     global TEST_ALL, DEBUG
     TEST_ALL = arith or is_resource_enabled('decimal')
     DEBUG = debug
 
     if todo_tests is None:
-        test_classes = [
-            DecimalExplicitConstructionTest,
-            DecimalImplicitConstructionTest,
-            DecimalArithmeticOperatorsTest,
-            DecimalFormatTest,
-            DecimalUseOfContextTest,
-            DecimalUsabilityTest,
-            DecimalPythonAPItests,
-            ContextAPItests,
-            DecimalTest,
-            WithStatementTest,
-            ContextFlags
-        ]
+        test_classes = all_tests
     else:
-        test_classes = [DecimalTest]
+        test_classes = [CIBMTestCases, PyIBMTestCases]
 
     # Dynamically build custom test definition for each file in the test
     # directory and add the definitions to the DecimalTest class.  This
@@ -2406,17 +5425,32 @@ def test_main(arith=False, verbose=None, todo_tests=None, debug=None):
         if todo_tests is not None and head not in todo_tests:
             continue
         tester = lambda self, f=filename: self.eval_file(directory + f)
-        setattr(DecimalTest, 'test_' + head, tester)
+        setattr(CIBMTestCases, 'test_' + head, tester)
+        setattr(PyIBMTestCases, 'test_' + head, tester)
         del filename, head, tail, tester
 
 
     try:
         run_unittest(*test_classes)
         if todo_tests is None:
-            import decimal as DecimalModule
-            run_doctest(DecimalModule, verbose)
+            from doctest import IGNORE_EXCEPTION_DETAIL
+            savedecimal = sys.modules['decimal']
+            if C:
+                sys.modules['decimal'] = C
+                run_doctest(C, verbose, optionflags=IGNORE_EXCEPTION_DETAIL)
+            sys.modules['decimal'] = P
+            run_doctest(P, verbose)
+            sys.modules['decimal'] = savedecimal
     finally:
-        setcontext(ORIGINAL_CONTEXT)
+        if C: C.setcontext(ORIGINAL_CONTEXT[C])
+        P.setcontext(ORIGINAL_CONTEXT[P])
+        if not C:
+            warnings.warn('C tests skipped: no module named _decimal.',
+                          UserWarning)
+        if not orig_sys_decimal is sys.modules['decimal']:
+            raise TestFailed("Internal error: unbalanced number of changes to "
+                             "sys.modules['decimal'].")
+
 
 if __name__ == '__main__':
     import optparse
diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
index f0afe1dcb7..a8487d2d95 100644
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -472,6 +472,19 @@ class TestBasic(unittest.TestCase):
 ##            self.assertNotEqual(id(d), id(e))
 ##            self.assertEqual(id(e), id(e[-1]))
 
+    def test_iterator_pickle(self):
+        data = deque(range(200))
+        it = itorg = iter(data)
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(type(itorg), type(it))
+        self.assertEqual(list(it), list(data))
+
+        it = pickle.loads(d)
+        next(it)
+        d = pickle.dumps(it)
+        self.assertEqual(list(it), list(data)[1:])
+
     def test_deepcopy(self):
         mut = [10]
         d = deque([mut])
@@ -524,7 +537,7 @@ class TestBasic(unittest.TestCase):
     @support.cpython_only
     def test_sizeof(self):
         BLOCKLEN = 62
-        basesize = support.calcobjsize('2P4PlP')
+        basesize = support.calcobjsize('2P4nlP')
         blocksize = struct.calcsize('2P%dP' % BLOCKLEN)
         self.assertEqual(object.__sizeof__(deque()), basesize)
         check = self.check_sizeof
diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
index cdaf7d2aca..b5a10edc12 100644
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -654,19 +654,19 @@ class ClassPropertiesAndMethods(unittest.TestCase):
         class A(metaclass=AMeta):
             pass
         self.assertEqual(['AMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
 
         class B(metaclass=BMeta):
             pass
         # BMeta.__new__ calls AMeta.__new__ with super:
         self.assertEqual(['BMeta', 'AMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
 
         class C(A, B):
             pass
         # The most derived metaclass is BMeta:
         self.assertEqual(['BMeta', 'AMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         # BMeta.__prepare__ should've been called:
         self.assertIn('BMeta_was_here', C.__dict__)
 
@@ -674,20 +674,20 @@ class ClassPropertiesAndMethods(unittest.TestCase):
         class C2(B, A):
             pass
         self.assertEqual(['BMeta', 'AMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertIn('BMeta_was_here', C2.__dict__)
 
         # Check correct metaclass calculation when a metaclass is declared:
         class D(C, metaclass=type):
             pass
         self.assertEqual(['BMeta', 'AMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertIn('BMeta_was_here', D.__dict__)
 
         class E(C, metaclass=AMeta):
             pass
         self.assertEqual(['BMeta', 'AMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertIn('BMeta_was_here', E.__dict__)
 
         # Special case: the given metaclass isn't a class,
@@ -729,33 +729,33 @@ class ClassPropertiesAndMethods(unittest.TestCase):
             pass
         self.assertIs(ANotMeta, type(A))
         self.assertEqual(['ANotMeta'], prepare_calls)
-        prepare_calls[:] = []
+        prepare_calls.clear()
         self.assertEqual(['ANotMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
 
         class B(metaclass=BNotMeta):
             pass
         self.assertIs(BNotMeta, type(B))
         self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls)
-        prepare_calls[:] = []
+        prepare_calls.clear()
         self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
 
         class C(A, B):
             pass
         self.assertIs(BNotMeta, type(C))
         self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls)
-        prepare_calls[:] = []
+        prepare_calls.clear()
 
         class C2(B, A):
             pass
         self.assertIs(BNotMeta, type(C2))
         self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls)
-        prepare_calls[:] = []
+        prepare_calls.clear()
 
         # This is a TypeError, because of a metaclass conflict:
         # BNotMeta is neither a subclass, nor a superclass of type
@@ -767,25 +767,25 @@ class ClassPropertiesAndMethods(unittest.TestCase):
             pass
         self.assertIs(BNotMeta, type(E))
         self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls)
-        prepare_calls[:] = []
+        prepare_calls.clear()
 
         class F(object(), C):
             pass
         self.assertIs(BNotMeta, type(F))
         self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls)
-        prepare_calls[:] = []
+        prepare_calls.clear()
 
         class F2(C, object()):
             pass
         self.assertIs(BNotMeta, type(F2))
         self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls)
-        new_calls[:] = []
+        new_calls.clear()
         self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls)
-        prepare_calls[:] = []
+        prepare_calls.clear()
 
         # TypeError: BNotMeta is neither a
         # subclass, nor a superclass of int
@@ -1444,6 +1444,14 @@ order (MRO) for bases """
         else:
             self.fail("classmethod shouldn't accept keyword args")
 
+        cm = classmethod(f)
+        self.assertEqual(cm.__dict__, {})
+        cm.x = 42
+        self.assertEqual(cm.x, 42)
+        self.assertEqual(cm.__dict__, {"x" : 42})
+        del cm.x
+        self.assertFalse(hasattr(cm, "x"))
+
     @support.impl_detail("the module 'xxsubtype' is internal")
     def test_classmethods_in_c(self):
         # Testing C-based class methods...
@@ -1491,6 +1499,13 @@ order (MRO) for bases """
         self.assertEqual(d.goo(1), (1,))
         self.assertEqual(d.foo(1), (d, 1))
         self.assertEqual(D.foo(d, 1), (d, 1))
+        sm = staticmethod(None)
+        self.assertEqual(sm.__dict__, {})
+        sm.x = 42
+        self.assertEqual(sm.x, 42)
+        self.assertEqual(sm.__dict__, {"x" : 42})
+        del sm.x
+        self.assertFalse(hasattr(sm, "x"))
 
     @support.impl_detail("the module 'xxsubtype' is internal")
     def test_staticmethods_in_c(self):
@@ -1771,6 +1786,7 @@ order (MRO) for bases """
             ("__format__", format, format_impl, set(), {}),
             ("__floor__", math.floor, zero, set(), {}),
             ("__trunc__", math.trunc, zero, set(), {}),
+            ("__trunc__", int, zero, set(), {}),
             ("__ceil__", math.ceil, zero, set(), {}),
             ("__dir__", dir, empty_seq, set(), {}),
             ]
@@ -1816,12 +1832,7 @@ order (MRO) for bases """
             for attr, obj in env.items():
                 setattr(X, attr, obj)
             setattr(X, name, ErrDescr())
-            try:
-                runner(X())
-            except MyException:
-                pass
-            else:
-                self.fail("{0!r} didn't raise".format(name))
+            self.assertRaises(MyException, runner, X())
 
     def test_specials(self):
         # Testing special operators...
@@ -2258,9 +2269,6 @@ order (MRO) for bases """
         # Two essentially featureless objects, just inheriting stuff from
         # object.
         self.assertEqual(dir(NotImplemented), dir(Ellipsis))
-        if support.check_impl_detail():
-            # None differs in PyPy: it has a __nonzero__
-            self.assertEqual(dir(None), dir(Ellipsis))
 
         # Nasty test case for proxied objects
         class Wrapper(object):
@@ -4456,6 +4464,62 @@ order (MRO) for bases """
         x.y = 42
         self.assertEqual(x["y"], 42)
 
+    def test_slot_shadows_class_variable(self):
+        with self.assertRaises(ValueError) as cm:
+            class X:
+                __slots__ = ["foo"]
+                foo = None
+        m = str(cm.exception)
+        self.assertEqual("'foo' in __slots__ conflicts with class variable", m)
+
+    def test_set_doc(self):
+        class X:
+            "elephant"
+        X.__doc__ = "banana"
+        self.assertEqual(X.__doc__, "banana")
+        with self.assertRaises(TypeError) as cm:
+            type(list).__dict__["__doc__"].__set__(list, "blah")
+        self.assertIn("can't set list.__doc__", str(cm.exception))
+        with self.assertRaises(TypeError) as cm:
+            type(X).__dict__["__doc__"].__delete__(X)
+        self.assertIn("can't delete X.__doc__", str(cm.exception))
+        self.assertEqual(X.__doc__, "banana")
+
+    def test_qualname(self):
+        descriptors = [str.lower, complex.real, float.real, int.__add__]
+        types = ['method', 'member', 'getset', 'wrapper']
+
+        # make sure we have an example of each type of descriptor
+        for d, n in zip(descriptors, types):
+            self.assertEqual(type(d).__name__, n + '_descriptor')
+
+        for d in descriptors:
+            qualname = d.__objclass__.__qualname__ + '.' + d.__name__
+            self.assertEqual(d.__qualname__, qualname)
+
+        self.assertEqual(str.lower.__qualname__, 'str.lower')
+        self.assertEqual(complex.real.__qualname__, 'complex.real')
+        self.assertEqual(float.real.__qualname__, 'float.real')
+        self.assertEqual(int.__add__.__qualname__, 'int.__add__')
+
+        class X:
+            pass
+        with self.assertRaises(TypeError):
+            del X.__qualname__
+
+        self.assertRaises(TypeError, type.__dict__['__qualname__'].__set__,
+                          str, 'Oink')
+
+    def test_qualname_dict(self):
+        ns = {'__qualname__': 'some.name'}
+        tp = type('Foo', (), ns)
+        self.assertEqual(tp.__qualname__, 'some.name')
+        self.assertNotIn('__qualname__', tp.__dict__)
+        self.assertEqual(ns, {'__qualname__': 'some.name'})
+
+        ns = {'__qualname__': 1}
+        self.assertRaises(TypeError, type, 'Foo', (), ns)
+
     def test_cycle_through_dict(self):
         # See bug #1469629
         class X(dict):
@@ -4471,6 +4535,27 @@ order (MRO) for bases """
         for o in gc.get_objects():
             self.assertIsNot(type(o), X)
 
+    def test_object_new_and_init_with_parameters(self):
+        # See issue #1683368
+        class OverrideNeither:
+            pass
+        self.assertRaises(TypeError, OverrideNeither, 1)
+        self.assertRaises(TypeError, OverrideNeither, kw=1)
+        class OverrideNew:
+            def __new__(cls, foo, kw=0, *args, **kwds):
+                return object.__new__(cls, *args, **kwds)
+        class OverrideInit:
+            def __init__(self, foo, kw=0, *args, **kwargs):
+                return object.__init__(self, *args, **kwargs)
+        class OverrideBoth(OverrideNew, OverrideInit):
+            pass
+        for case in OverrideNew, OverrideInit, OverrideBoth:
+            case(1)
+            case(1, kw=2)
+            self.assertRaises(TypeError, case, 1, 2, 3)
+            self.assertRaises(TypeError, case, 1, 2, foo=3)
+
+
 class DictProxyTests(unittest.TestCase):
     def setUp(self):
         class C(object):
@@ -4478,6 +4563,8 @@ class DictProxyTests(unittest.TestCase):
                 pass
         self.C = C
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                        'trace function introduces __local__')
     def test_iter_keys(self):
         # Testing dict-proxy keys...
         it = self.C.__dict__.keys()
@@ -4485,8 +4572,10 @@ class DictProxyTests(unittest.TestCase):
         keys = list(it)
         keys.sort()
         self.assertEqual(keys, ['__dict__', '__doc__', '__module__',
-            '__weakref__', 'meth'])
+                                '__weakref__', 'meth'])
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                        'trace function introduces __local__')
     def test_iter_values(self):
         # Testing dict-proxy values...
         it = self.C.__dict__.values()
@@ -4494,6 +4583,8 @@ class DictProxyTests(unittest.TestCase):
         values = list(it)
         self.assertEqual(len(values), 5)
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                        'trace function introduces __local__')
     def test_iter_items(self):
         # Testing dict-proxy iteritems...
         it = self.C.__dict__.items()
@@ -4501,7 +4592,7 @@ class DictProxyTests(unittest.TestCase):
         keys = [item[0] for item in it]
         keys.sort()
         self.assertEqual(keys, ['__dict__', '__doc__', '__module__',
-            '__weakref__', 'meth'])
+                                '__weakref__', 'meth'])
 
     def test_dict_type_with_metaclass(self):
         # Testing type of __dict__ when metaclass set...
@@ -4515,19 +4606,14 @@ class DictProxyTests(unittest.TestCase):
         self.assertEqual(type(C.__dict__), type(B.__dict__))
 
     def test_repr(self):
-        # Testing dict_proxy.__repr__
-        def sorted_dict_repr(repr_):
-            # Given the repr of a dict, sort the keys
-            assert repr_.startswith('{')
-            assert repr_.endswith('}')
-            kvs = repr_[1:-1].split(', ')
-            return '{' + ', '.join(sorted(kvs)) + '}'
-        dict_ = {k: v for k, v in self.C.__dict__.items()}
-        repr_ = repr(self.C.__dict__)
-        self.assertTrue(repr_.startswith('dict_proxy('))
-        self.assertTrue(repr_.endswith(')'))
-        self.assertEqual(sorted_dict_repr(repr_[len('dict_proxy('):-len(')')]),
-                         sorted_dict_repr('{!r}'.format(dict_)))
+        # Testing mappingproxy.__repr__.
+        # We can't blindly compare with the repr of another dict as ordering
+        # of keys and values is arbitrary and may differ.
+        r = repr(self.C.__dict__)
+        self.assertTrue(r.startswith('mappingproxy('), r)
+        self.assertTrue(r.endswith(')'), r)
+        for k, v in self.C.__dict__.items():
+            self.assertIn('{!r}: {!r}'.format(k, v), r)
 
 
 class PTypesLongInitTest(unittest.TestCase):
@@ -4552,10 +4638,38 @@ class PTypesLongInitTest(unittest.TestCase):
         type.mro(tuple)
 
 
+class MiscTests(unittest.TestCase):
+    def test_type_lookup_mro_reference(self):
+        # Issue #14199: _PyType_Lookup() has to keep a strong reference to
+        # the type MRO because it may be modified during the lookup, if
+        # __bases__ is set during the lookup for example.
+        class MyKey(object):
+            def __hash__(self):
+                return hash('mykey')
+
+            def __eq__(self, other):
+                X.__bases__ = (Base2,)
+
+        class Base(object):
+            mykey = 'from Base'
+            mykey2 = 'from Base'
+
+        class Base2(object):
+            mykey = 'from Base2'
+            mykey2 = 'from Base2'
+
+        X = type('X', (Base,), {MyKey(): 5})
+        # mykey is read from Base
+        self.assertEqual(X.mykey, 'from Base')
+        # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__
+        self.assertEqual(X.mykey2, 'from Base2')
+
+
 def test_main():
     # Run all local test cases, with PTypesLongInitTest first.
     support.run_unittest(PTypesLongInitTest, OperatorsTest,
-                              ClassPropertiesAndMethods, DictProxyTests)
+                         ClassPropertiesAndMethods, DictProxyTests,
+                         MiscTests)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py
index 2db3d33226..f495e1816b 100644
--- a/Lib/test/test_descrtut.py
+++ b/Lib/test/test_descrtut.py
@@ -170,6 +170,7 @@ You can get the information from the list type:
      '__contains__',
      '__delattr__',
      '__delitem__',
+     '__dir__',
      '__doc__',
      '__eq__',
      '__format__',
@@ -199,6 +200,8 @@ You can get the information from the list type:
      '__str__',
      '__subclasshook__',
      'append',
+     'clear',
+     'copy',
      'count',
      'extend',
      'index',
diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py
new file mode 100644
index 0000000000..bef4e1846b
--- /dev/null
+++ b/Lib/test/test_devpoll.py
@@ -0,0 +1,94 @@
+# Test case for the select.devpoll() function
+
+# Initial tests are copied as is from "test_poll.py"
+
+import os, select, random, unittest, sys
+from test.support import TESTFN, run_unittest
+
+try:
+    select.devpoll
+except AttributeError:
+    raise unittest.SkipTest("select.devpoll not defined -- skipping test_devpoll")
+
+
+def find_ready_matching(ready, flag):
+    match = []
+    for fd, mode in ready:
+        if mode & flag:
+            match.append(fd)
+    return match
+
+class DevPollTests(unittest.TestCase):
+
+    def test_devpoll1(self):
+        # Basic functional test of poll object
+        # Create a bunch of pipe and test that poll works with them.
+
+        p = select.devpoll()
+
+        NUM_PIPES = 12
+        MSG = b" This is a test."
+        MSG_LEN = len(MSG)
+        readers = []
+        writers = []
+        r2w = {}
+        w2r = {}
+
+        for i in range(NUM_PIPES):
+            rd, wr = os.pipe()
+            p.register(rd)
+            p.modify(rd, select.POLLIN)
+            p.register(wr, select.POLLOUT)
+            readers.append(rd)
+            writers.append(wr)
+            r2w[rd] = wr
+            w2r[wr] = rd
+
+        bufs = []
+
+        while writers:
+            ready = p.poll()
+            ready_writers = find_ready_matching(ready, select.POLLOUT)
+            if not ready_writers:
+                self.fail("no pipes ready for writing")
+            wr = random.choice(ready_writers)
+            os.write(wr, MSG)
+
+            ready = p.poll()
+            ready_readers = find_ready_matching(ready, select.POLLIN)
+            if not ready_readers:
+                self.fail("no pipes ready for reading")
+            self.assertEqual([w2r[wr]], ready_readers)
+            rd = ready_readers[0]
+            buf = os.read(rd, MSG_LEN)
+            self.assertEqual(len(buf), MSG_LEN)
+            bufs.append(buf)
+            os.close(r2w[rd]) ; os.close(rd)
+            p.unregister(r2w[rd])
+            p.unregister(rd)
+            writers.remove(r2w[rd])
+
+        self.assertEqual(bufs, [MSG] * NUM_PIPES)
+
+    def test_timeout_overflow(self):
+        pollster = select.devpoll()
+        w, r = os.pipe()
+        pollster.register(w)
+
+        pollster.poll(-1)
+        self.assertRaises(OverflowError, pollster.poll, -2)
+        self.assertRaises(OverflowError, pollster.poll, -1 << 31)
+        self.assertRaises(OverflowError, pollster.poll, -1 << 64)
+
+        pollster.poll(0)
+        pollster.poll(1)
+        pollster.poll(1 << 30)
+        self.assertRaises(OverflowError, pollster.poll, 1 << 31)
+        self.assertRaises(OverflowError, pollster.poll, 1 << 63)
+        self.assertRaises(OverflowError, pollster.poll, 1 << 64)
+
+def test_main():
+    run_unittest(DevPollTests)
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py
index 9666f9a31c..d1f3192370 100644
--- a/Lib/test/test_dict.py
+++ b/Lib/test/test_dict.py
@@ -2,7 +2,9 @@ import unittest
 from test import support
 
 import collections, random, string
+import collections.abc
 import gc, weakref
+import pickle
 
 
 class DictTest(unittest.TestCase):
@@ -327,6 +329,27 @@ class DictTest(unittest.TestCase):
         self.assertEqual(hashed2.hash_count, 1)
         self.assertEqual(hashed1.eq_count + hashed2.eq_count, 1)
 
+    def test_setitem_atomic_at_resize(self):
+        class Hashed(object):
+            def __init__(self):
+                self.hash_count = 0
+                self.eq_count = 0
+            def __hash__(self):
+                self.hash_count += 1
+                return 42
+            def __eq__(self, other):
+                self.eq_count += 1
+                return id(self) == id(other)
+        hashed1 = Hashed()
+        # 5 items
+        y = {hashed1: 5, 0: 0, 1: 1, 2: 2, 3: 3}
+        hashed2 = Hashed()
+        # 6th item forces a resize
+        y[hashed2] = []
+        self.assertEqual(hashed1.hash_count, 1)
+        self.assertEqual(hashed2.hash_count, 1)
+        self.assertEqual(hashed1.eq_count + hashed2.eq_count, 1)
+
     def test_popitem(self):
         # dict.popitem()
         for copymode in -1, +1:
@@ -387,7 +410,7 @@ class DictTest(unittest.TestCase):
         x.fail = True
         self.assertRaises(Exc, d.pop, x)
 
-    def test_mutatingiteration(self):
+    def test_mutating_iteration(self):
         # changing dict size during iteration
         d = {}
         d[1] = 1
@@ -395,6 +418,32 @@ class DictTest(unittest.TestCase):
             for i in d:
                 d[i+1] = 1
 
+    def test_mutating_lookup(self):
+        # changing dict during a lookup (issue #14417)
+        class NastyKey:
+            mutate_dict = None
+
+            def __init__(self, value):
+                self.value = value
+
+            def __hash__(self):
+                # hash collision!
+                return 1
+
+            def __eq__(self, other):
+                if NastyKey.mutate_dict:
+                    mydict, key = NastyKey.mutate_dict
+                    NastyKey.mutate_dict = None
+                    del mydict[key]
+                return self.value == other.value
+
+        key1 = NastyKey(1)
+        key2 = NastyKey(2)
+        d = {key1: 1}
+        NastyKey.mutate_dict = (d, key1)
+        d[key2] = 2
+        self.assertEqual(d, {key2: 2})
+
     def test_repr(self):
         d = {}
         self.assertEqual(repr(d), '{}')
@@ -784,6 +833,75 @@ class DictTest(unittest.TestCase):
             pass
         self._tracked(MyDict())
 
+    def test_iterator_pickling(self):
+        data = {1:"a", 2:"b", 3:"c"}
+        it = iter(data)
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(sorted(it), sorted(data))
+
+        it = pickle.loads(d)
+        try:
+            drop = next(it)
+        except StopIteration:
+            return
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        del data[drop]
+        self.assertEqual(sorted(it), sorted(data))
+
+    def test_itemiterator_pickling(self):
+        data = {1:"a", 2:"b", 3:"c"}
+        # dictviews aren't picklable, only their iterators
+        itorg = iter(data.items())
+        d = pickle.dumps(itorg)
+        it = pickle.loads(d)
+        # note that the type of type of the unpickled iterator
+        # is not necessarily the same as the original.  It is
+        # merely an object supporting the iterator protocol, yielding
+        # the same objects as the original one.
+        # self.assertEqual(type(itorg), type(it))
+        self.assertTrue(isinstance(it, collections.abc.Iterator))
+        self.assertEqual(dict(it), data)
+
+        it = pickle.loads(d)
+        drop = next(it)
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        del data[drop[0]]
+        self.assertEqual(dict(it), data)
+
+    def test_valuesiterator_pickling(self):
+        data = {1:"a", 2:"b", 3:"c"}
+        # data.values() isn't picklable, only its iterator
+        it = iter(data.values())
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(sorted(list(it)), sorted(list(data.values())))
+
+        it = pickle.loads(d)
+        drop = next(it)
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        values = list(it) + [drop]
+        self.assertEqual(sorted(values), sorted(list(data.values())))
+
+    def test_instance_dict_getattr_str_subclass(self):
+        class Foo:
+            def __init__(self, msg):
+                self.msg = msg
+        f = Foo('123')
+        class _str(str):
+            pass
+        self.assertEqual(f.msg, getattr(f, _str('msg')))
+        self.assertEqual(f.msg, f.__dict__[_str('msg')])
+
+    def test_object_set_item_single_instance_non_str_key(self):
+        class Foo: pass
+        f = Foo()
+        f.__dict__[1] = 1
+        f.a = 'a'
+        self.assertEqual(f.__dict__, {1:1, 'a':'a'})
 
 from test import mapping_tests
 
diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py
index 5a19319c11..3c8b95cf32 100644
--- a/Lib/test/test_dictcomps.py
+++ b/Lib/test/test_dictcomps.py
@@ -84,8 +84,5 @@ class DictComprehensionTest(unittest.TestCase):
                     "exec")
 
 
-def test_main():
-    support.run_unittest(__name__)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index 5c59eaa01c..1bcd693f63 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -1,11 +1,35 @@
 # Minimal tests for dis module
 
 from test.support import run_unittest, captured_stdout
+import difflib
 import unittest
 import sys
 import dis
 import io
 
+class _C:
+    def __init__(self, x):
+        self.x = x == 1
+
+dis_c_instance_method = """\
+ %-4d         0 LOAD_FAST                1 (x)
+              3 LOAD_CONST               1 (1)
+              6 COMPARE_OP               2 (==)
+              9 LOAD_FAST                0 (self)
+             12 STORE_ATTR               0 (x)
+             15 LOAD_CONST               0 (None)
+             18 RETURN_VALUE
+""" % (_C.__init__.__code__.co_firstlineno + 1,)
+
+dis_c_instance_method_bytes = """\
+          0 LOAD_FAST           1 (1)
+          3 LOAD_CONST          1 (1)
+          6 COMPARE_OP          2 (==)
+          9 LOAD_FAST           0 (0)
+         12 STORE_ATTR          0 (0)
+         15 LOAD_CONST          0 (0)
+         18 RETURN_VALUE
+"""
 
 def _f(a):
     print(a)
@@ -14,7 +38,7 @@ def _f(a):
 dis_f = """\
  %-4d         0 LOAD_GLOBAL              0 (print)
               3 LOAD_FAST                0 (a)
-              6 CALL_FUNCTION            1
+              6 CALL_FUNCTION            1 (1 positional, 0 keyword pair)
               9 POP_TOP
 
  %-4d        10 LOAD_CONST               1 (1)
@@ -23,6 +47,16 @@ dis_f = """\
        _f.__code__.co_firstlineno + 2)
 
 
+dis_f_co_code = """\
+          0 LOAD_GLOBAL         0 (0)
+          3 LOAD_FAST           0 (0)
+          6 CALL_FUNCTION       1 (1 positional, 0 keyword pair)
+          9 POP_TOP
+         10 LOAD_CONST          1 (1)
+         13 RETURN_VALUE
+"""
+
+
 def bug708901():
     for res in range(1,
                      10):
@@ -34,7 +68,7 @@ dis_bug708901 = """\
               6 LOAD_CONST               1 (1)
 
  %-4d         9 LOAD_CONST               2 (10)
-             12 CALL_FUNCTION            2
+             12 CALL_FUNCTION            2 (2 positional, 0 keyword pair)
              15 GET_ITER
         >>   16 FOR_ITER                 6 (to 25)
              19 STORE_FAST               0 (res)
@@ -138,18 +172,27 @@ dis_compound_stmt_str = """\
 """
 
 class DisTests(unittest.TestCase):
-    def do_disassembly_test(self, func, expected):
+
+    def get_disassembly(self, func, lasti=-1, wrapper=True):
         s = io.StringIO()
         save_stdout = sys.stdout
         sys.stdout = s
-        dis.dis(func)
-        sys.stdout = save_stdout
-        got = s.getvalue()
+        try:
+            if wrapper:
+                dis.dis(func)
+            else:
+                dis.disassemble(func, lasti)
+        finally:
+            sys.stdout = save_stdout
         # Trim trailing blanks (if any).
-        lines = got.split('\n')
-        lines = [line.rstrip() for line in lines]
-        expected = expected.split("\n")
-        import difflib
+        return [line.rstrip() for line in s.getvalue().splitlines()]
+
+    def get_disassemble_as_string(self, func, lasti=-1):
+        return '\n'.join(self.get_disassembly(func, lasti, False))
+
+    def do_disassembly_test(self, func, expected):
+        lines = self.get_disassembly(func)
+        expected = expected.splitlines()
         if expected != lines:
             self.fail(
                 "events did not match expectation:\n" +
@@ -157,7 +200,7 @@ class DisTests(unittest.TestCase):
                                         lines)))
 
     def test_opmap(self):
-        self.assertEqual(dis.opmap["STOP_CODE"], 0)
+        self.assertEqual(dis.opmap["NOP"], 9)
         self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst)
         self.assertIn(dis.opmap["STORE_NAME"], dis.hasname)
 
@@ -211,6 +254,44 @@ class DisTests(unittest.TestCase):
         self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str)
         self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str)
 
+    def test_disassemble_bytes(self):
+        self.do_disassembly_test(_f.__code__.co_code, dis_f_co_code)
+
+    def test_disassemble_method(self):
+        self.do_disassembly_test(_C(1).__init__, dis_c_instance_method)
+
+    def test_disassemble_method_bytes(self):
+        method_bytecode = _C(1).__init__.__code__.co_code
+        self.do_disassembly_test(method_bytecode, dis_c_instance_method_bytes)
+
+    def test_dis_none(self):
+        try:
+            del sys.last_traceback
+        except AttributeError:
+            pass
+        self.assertRaises(RuntimeError, dis.dis, None)
+
+    def test_dis_object(self):
+        self.assertRaises(TypeError, dis.dis, object())
+
+    def test_dis_traceback(self):
+        try:
+            del sys.last_traceback
+        except AttributeError:
+            pass
+
+        try:
+            1/0
+        except Exception as e:
+            tb = e.__traceback__
+            sys.last_traceback = tb
+
+        tb_dis = self.get_disassemble_as_string(tb.tb_frame.f_code, tb.tb_lasti)
+        self.do_disassembly_test(None, tb_dis)
+
+    def test_dis_object(self):
+        self.assertRaises(TypeError, dis.dis, object())
+
 code_info_code_info = """\
 Name:              code_info
 Filename:          (.*)
@@ -258,6 +339,7 @@ Flags:             OPTIMIZED, NEWLOCALS, VARARGS, VARKEYWORDS, GENERATOR
 Constants:
    0: None
    1: <code object f at (.*), file "(.*)", line (.*)>
+   2: 'tricky.<locals>.f'
 Variable names:
    0: x
    1: y
@@ -364,6 +446,13 @@ class CodeInfoTests(unittest.TestCase):
                 dis.show_code(x)
             self.assertRegex(output.getvalue(), expected+"\n")
 
+    def test_code_info_object(self):
+        self.assertRaises(TypeError, dis.code_info, object())
+
+    def test_pretty_flags_no_flags(self):
+        self.assertEqual(dis.pretty_flags(0), '0x0')
+
+
 def test_main():
     run_unittest(DisTests, CodeInfoTests)
 
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index a6c17cc432..8f8c7c7e82 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -5,6 +5,7 @@ Test script for doctest.
 from test import support
 import doctest
 import os
+import sys
 
 
 # NOTE: There are some additional tests relating to interaction with
@@ -432,7 +433,7 @@ We'll simulate a __file__ attr that ends in pyc:
     >>> tests = finder.find(sample_func)
 
     >>> print(tests)  # doctest: +ELLIPSIS
-    [<DocTest sample_func from ...:17 (1 example)>]
+    [<DocTest sample_func from ...:18 (1 example)>]
 
 The exact name depends on how test_doctest was invoked, so allow for
 leading path components.
@@ -1745,226 +1746,227 @@ Run the debugger on the docstring, and then restore sys.stdin.
 
 """
 
-def test_pdb_set_trace():
-    """Using pdb.set_trace from a doctest.
-
-    You can use pdb.set_trace from a doctest.  To do so, you must
-    retrieve the set_trace function from the pdb module at the time
-    you use it.  The doctest module changes sys.stdout so that it can
-    capture program output.  It also temporarily replaces pdb.set_trace
-    with a version that restores stdout.  This is necessary for you to
-    see debugger output.
-
-      >>> doc = '''
-      ... >>> x = 42
-      ... >>> raise Exception('clé')
-      ... Traceback (most recent call last):
-      ... Exception: clé
-      ... >>> import pdb; pdb.set_trace()
-      ... '''
-      >>> parser = doctest.DocTestParser()
-      >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0)
-      >>> runner = doctest.DocTestRunner(verbose=False)
-
-    To demonstrate this, we'll create a fake standard input that
-    captures our debugger input:
-
-      >>> import tempfile
-      >>> real_stdin = sys.stdin
-      >>> sys.stdin = _FakeInput([
-      ...    'print(x)',  # print data defined by the example
-      ...    'continue', # stop debugging
-      ...    ''])
-
-      >>> try: runner.run(test)
-      ... finally: sys.stdin = real_stdin
-      --Return--
-      > <doctest foo-bar@baz[2]>(1)<module>()->None
-      -> import pdb; pdb.set_trace()
-      (Pdb) print(x)
-      42
-      (Pdb) continue
-      TestResults(failed=0, attempted=3)
-
-      You can also put pdb.set_trace in a function called from a test:
-
-      >>> def calls_set_trace():
-      ...    y=2
-      ...    import pdb; pdb.set_trace()
-
-      >>> doc = '''
-      ... >>> x=1
-      ... >>> calls_set_trace()
-      ... '''
-      >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0)
-      >>> real_stdin = sys.stdin
-      >>> sys.stdin = _FakeInput([
-      ...    'print(y)',  # print data defined in the function
-      ...    'up',       # out of function
-      ...    'print(x)',  # print data defined by the example
-      ...    'continue', # stop debugging
-      ...    ''])
-
-      >>> try:
-      ...     runner.run(test)
-      ... finally:
-      ...     sys.stdin = real_stdin
-      --Return--
-      > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None
-      -> import pdb; pdb.set_trace()
-      (Pdb) print(y)
-      2
-      (Pdb) up
-      > <doctest foo-bar@baz[1]>(1)<module>()
-      -> calls_set_trace()
-      (Pdb) print(x)
-      1
-      (Pdb) continue
-      TestResults(failed=0, attempted=2)
-
-    During interactive debugging, source code is shown, even for
-    doctest examples:
-
-      >>> doc = '''
-      ... >>> def f(x):
-      ... ...     g(x*2)
-      ... >>> def g(x):
-      ... ...     print(x+3)
-      ... ...     import pdb; pdb.set_trace()
-      ... >>> f(3)
-      ... '''
-      >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0)
-      >>> real_stdin = sys.stdin
-      >>> sys.stdin = _FakeInput([
-      ...    'list',     # list source from example 2
-      ...    'next',     # return from g()
-      ...    'list',     # list source from example 1
-      ...    'next',     # return from f()
-      ...    'list',     # list source from example 3
-      ...    'continue', # stop debugging
-      ...    ''])
-      >>> try: runner.run(test)
-      ... finally: sys.stdin = real_stdin
-      ... # doctest: +NORMALIZE_WHITESPACE
-      --Return--
-      > <doctest foo-bar@baz[1]>(3)g()->None
-      -> import pdb; pdb.set_trace()
-      (Pdb) list
-        1     def g(x):
-        2         print(x+3)
-        3  ->     import pdb; pdb.set_trace()
-      [EOF]
-      (Pdb) next
-      --Return--
-      > <doctest foo-bar@baz[0]>(2)f()->None
-      -> g(x*2)
-      (Pdb) list
-        1     def f(x):
-        2  ->     g(x*2)
-      [EOF]
-      (Pdb) next
-      --Return--
-      > <doctest foo-bar@baz[2]>(1)<module>()->None
-      -> f(3)
-      (Pdb) list
-        1  -> f(3)
-      [EOF]
-      (Pdb) continue
-      **********************************************************************
-      File "foo-bar@baz.py", line 7, in foo-bar@baz
-      Failed example:
-          f(3)
-      Expected nothing
-      Got:
-          9
-      TestResults(failed=1, attempted=3)
-      """
-
-def test_pdb_set_trace_nested():
-    """This illustrates more-demanding use of set_trace with nested functions.
-
-    >>> class C(object):
-    ...     def calls_set_trace(self):
-    ...         y = 1
-    ...         import pdb; pdb.set_trace()
-    ...         self.f1()
-    ...         y = 2
-    ...     def f1(self):
-    ...         x = 1
-    ...         self.f2()
-    ...         x = 2
-    ...     def f2(self):
-    ...         z = 1
-    ...         z = 2
-
-    >>> calls_set_trace = C().calls_set_trace
-
-    >>> doc = '''
-    ... >>> a = 1
-    ... >>> calls_set_trace()
-    ... '''
-    >>> parser = doctest.DocTestParser()
-    >>> runner = doctest.DocTestRunner(verbose=False)
-    >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0)
-    >>> real_stdin = sys.stdin
-    >>> sys.stdin = _FakeInput([
-    ...    'print(y)',  # print data defined in the function
-    ...    'step', 'step', 'step', 'step', 'step', 'step', 'print(z)',
-    ...    'up', 'print(x)',
-    ...    'up', 'print(y)',
-    ...    'up', 'print(foo)',
-    ...    'continue', # stop debugging
-    ...    ''])
-
-    >>> try:
-    ...     runner.run(test)
-    ... finally:
-    ...     sys.stdin = real_stdin
-    ... # doctest: +REPORT_NDIFF
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace()
-    -> self.f1()
-    (Pdb) print(y)
-    1
-    (Pdb) step
-    --Call--
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(7)f1()
-    -> def f1(self):
-    (Pdb) step
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(8)f1()
-    -> x = 1
-    (Pdb) step
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1()
-    -> self.f2()
-    (Pdb) step
-    --Call--
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(11)f2()
-    -> def f2(self):
-    (Pdb) step
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(12)f2()
-    -> z = 1
-    (Pdb) step
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(13)f2()
-    -> z = 2
-    (Pdb) print(z)
-    1
-    (Pdb) up
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1()
-    -> self.f2()
-    (Pdb) print(x)
-    1
-    (Pdb) up
-    > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace()
-    -> self.f1()
-    (Pdb) print(y)
-    1
-    (Pdb) up
-    > <doctest foo-bar@baz[1]>(1)<module>()
-    -> calls_set_trace()
-    (Pdb) print(foo)
-    *** NameError: name 'foo' is not defined
-    (Pdb) continue
-    TestResults(failed=0, attempted=2)
-"""
+if not hasattr(sys, 'gettrace') or not sys.gettrace():
+    def test_pdb_set_trace():
+        """Using pdb.set_trace from a doctest.
+
+        You can use pdb.set_trace from a doctest.  To do so, you must
+        retrieve the set_trace function from the pdb module at the time
+        you use it.  The doctest module changes sys.stdout so that it can
+        capture program output.  It also temporarily replaces pdb.set_trace
+        with a version that restores stdout.  This is necessary for you to
+        see debugger output.
+
+          >>> doc = '''
+          ... >>> x = 42
+          ... >>> raise Exception('clé')
+          ... Traceback (most recent call last):
+          ... Exception: clé
+          ... >>> import pdb; pdb.set_trace()
+          ... '''
+          >>> parser = doctest.DocTestParser()
+          >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0)
+          >>> runner = doctest.DocTestRunner(verbose=False)
+
+        To demonstrate this, we'll create a fake standard input that
+        captures our debugger input:
+
+          >>> import tempfile
+          >>> real_stdin = sys.stdin
+          >>> sys.stdin = _FakeInput([
+          ...    'print(x)',  # print data defined by the example
+          ...    'continue', # stop debugging
+          ...    ''])
+
+          >>> try: runner.run(test)
+          ... finally: sys.stdin = real_stdin
+          --Return--
+          > <doctest foo-bar@baz[2]>(1)<module>()->None
+          -> import pdb; pdb.set_trace()
+          (Pdb) print(x)
+          42
+          (Pdb) continue
+          TestResults(failed=0, attempted=3)
+
+          You can also put pdb.set_trace in a function called from a test:
+
+          >>> def calls_set_trace():
+          ...    y=2
+          ...    import pdb; pdb.set_trace()
+
+          >>> doc = '''
+          ... >>> x=1
+          ... >>> calls_set_trace()
+          ... '''
+          >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0)
+          >>> real_stdin = sys.stdin
+          >>> sys.stdin = _FakeInput([
+          ...    'print(y)',  # print data defined in the function
+          ...    'up',       # out of function
+          ...    'print(x)',  # print data defined by the example
+          ...    'continue', # stop debugging
+          ...    ''])
+
+          >>> try:
+          ...     runner.run(test)
+          ... finally:
+          ...     sys.stdin = real_stdin
+          --Return--
+          > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None
+          -> import pdb; pdb.set_trace()
+          (Pdb) print(y)
+          2
+          (Pdb) up
+          > <doctest foo-bar@baz[1]>(1)<module>()
+          -> calls_set_trace()
+          (Pdb) print(x)
+          1
+          (Pdb) continue
+          TestResults(failed=0, attempted=2)
+
+        During interactive debugging, source code is shown, even for
+        doctest examples:
+
+          >>> doc = '''
+          ... >>> def f(x):
+          ... ...     g(x*2)
+          ... >>> def g(x):
+          ... ...     print(x+3)
+          ... ...     import pdb; pdb.set_trace()
+          ... >>> f(3)
+          ... '''
+          >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0)
+          >>> real_stdin = sys.stdin
+          >>> sys.stdin = _FakeInput([
+          ...    'list',     # list source from example 2
+          ...    'next',     # return from g()
+          ...    'list',     # list source from example 1
+          ...    'next',     # return from f()
+          ...    'list',     # list source from example 3
+          ...    'continue', # stop debugging
+          ...    ''])
+          >>> try: runner.run(test)
+          ... finally: sys.stdin = real_stdin
+          ... # doctest: +NORMALIZE_WHITESPACE
+          --Return--
+          > <doctest foo-bar@baz[1]>(3)g()->None
+          -> import pdb; pdb.set_trace()
+          (Pdb) list
+            1     def g(x):
+            2         print(x+3)
+            3  ->     import pdb; pdb.set_trace()
+          [EOF]
+          (Pdb) next
+          --Return--
+          > <doctest foo-bar@baz[0]>(2)f()->None
+          -> g(x*2)
+          (Pdb) list
+            1     def f(x):
+            2  ->     g(x*2)
+          [EOF]
+          (Pdb) next
+          --Return--
+          > <doctest foo-bar@baz[2]>(1)<module>()->None
+          -> f(3)
+          (Pdb) list
+            1  -> f(3)
+          [EOF]
+          (Pdb) continue
+          **********************************************************************
+          File "foo-bar@baz.py", line 7, in foo-bar@baz
+          Failed example:
+              f(3)
+          Expected nothing
+          Got:
+              9
+          TestResults(failed=1, attempted=3)
+          """
+
+    def test_pdb_set_trace_nested():
+        """This illustrates more-demanding use of set_trace with nested functions.
+
+        >>> class C(object):
+        ...     def calls_set_trace(self):
+        ...         y = 1
+        ...         import pdb; pdb.set_trace()
+        ...         self.f1()
+        ...         y = 2
+        ...     def f1(self):
+        ...         x = 1
+        ...         self.f2()
+        ...         x = 2
+        ...     def f2(self):
+        ...         z = 1
+        ...         z = 2
+
+        >>> calls_set_trace = C().calls_set_trace
+
+        >>> doc = '''
+        ... >>> a = 1
+        ... >>> calls_set_trace()
+        ... '''
+        >>> parser = doctest.DocTestParser()
+        >>> runner = doctest.DocTestRunner(verbose=False)
+        >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0)
+        >>> real_stdin = sys.stdin
+        >>> sys.stdin = _FakeInput([
+        ...    'print(y)',  # print data defined in the function
+        ...    'step', 'step', 'step', 'step', 'step', 'step', 'print(z)',
+        ...    'up', 'print(x)',
+        ...    'up', 'print(y)',
+        ...    'up', 'print(foo)',
+        ...    'continue', # stop debugging
+        ...    ''])
+
+        >>> try:
+        ...     runner.run(test)
+        ... finally:
+        ...     sys.stdin = real_stdin
+        ... # doctest: +REPORT_NDIFF
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace()
+        -> self.f1()
+        (Pdb) print(y)
+        1
+        (Pdb) step
+        --Call--
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(7)f1()
+        -> def f1(self):
+        (Pdb) step
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(8)f1()
+        -> x = 1
+        (Pdb) step
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1()
+        -> self.f2()
+        (Pdb) step
+        --Call--
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(11)f2()
+        -> def f2(self):
+        (Pdb) step
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(12)f2()
+        -> z = 1
+        (Pdb) step
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(13)f2()
+        -> z = 2
+        (Pdb) print(z)
+        1
+        (Pdb) up
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1()
+        -> self.f2()
+        (Pdb) print(x)
+        1
+        (Pdb) up
+        > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace()
+        -> self.f1()
+        (Pdb) print(y)
+        1
+        (Pdb) up
+        > <doctest foo-bar@baz[1]>(1)<module>()
+        -> calls_set_trace()
+        (Pdb) print(foo)
+        *** NameError: name 'foo' is not defined
+        (Pdb) continue
+        TestResults(failed=0, attempted=2)
+    """
 
 def test_DocTestSuite():
     """DocTestSuite creates a unittest test suite from a doctest.
@@ -2566,7 +2568,7 @@ import sys, re, io
 
 def test_coverage(coverdir):
     trace = support.import_module('trace')
-    tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix,],
+    tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,],
                          trace=0, count=1)
     tracer.run('test_main()')
     r = tracer.results()
diff --git a/Lib/test/test_dummy_thread.py b/Lib/test/test_dummy_thread.py
index c61078d620..2fafe1df85 100644
--- a/Lib/test/test_dummy_thread.py
+++ b/Lib/test/test_dummy_thread.py
@@ -35,8 +35,8 @@ class LockTests(unittest.TestCase):
                         "Lock object did not release properly.")
 
     def test_improper_release(self):
-        #Make sure release of an unlocked thread raises _thread.error
-        self.assertRaises(_thread.error, self.lock.release)
+        #Make sure release of an unlocked thread raises RuntimeError
+        self.assertRaises(RuntimeError, self.lock.release)
 
     def test_cond_acquire_success(self):
         #Make sure the conditional acquiring of the lock works.
diff --git a/Lib/test/test_email.py b/Lib/test/test_email.py
deleted file mode 100644
index 5eebba5590..0000000000
--- a/Lib/test/test_email.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright (C) 2001-2007 Python Software Foundation
-# email package unit tests
-
-# The specific tests now live in Lib/email/test
-from email.test.test_email import suite
-from email.test.test_email_codecs import suite as codecs_suite
-from test import support
-
-def test_main():
-    support.run_unittest(suite())
-    support.run_unittest(codecs_suite())
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py
new file mode 100644
index 0000000000..f206ace10c
--- /dev/null
+++ b/Lib/test/test_email/__init__.py
@@ -0,0 +1,150 @@
+import os
+import sys
+import unittest
+import test.support
+import email
+from email.message import Message
+from email._policybase import compat32
+from test.test_email import __file__ as landmark
+
+# Run all tests in package for '-m unittest test.test_email'
+def load_tests(loader, standard_tests, pattern):
+    this_dir = os.path.dirname(__file__)
+    if pattern is None:
+        pattern = "test*"
+    package_tests = loader.discover(start_dir=this_dir, pattern=pattern)
+    standard_tests.addTests(package_tests)
+    return standard_tests
+
+
+# used by regrtest and __main__.
+def test_main():
+    here = os.path.dirname(__file__)
+    # Unittest mucks with the path, so we have to save and restore
+    # it to keep regrtest happy.
+    savepath = sys.path[:]
+    test.support._run_suite(unittest.defaultTestLoader.discover(here))
+    sys.path[:] = savepath
+
+
+# helper code used by a number of test modules.
+
+def openfile(filename, *args, **kws):
+    path = os.path.join(os.path.dirname(landmark), 'data', filename)
+    return open(path, *args, **kws)
+
+
+# Base test class
+class TestEmailBase(unittest.TestCase):
+
+    maxDiff = None
+    # Currently the default policy is compat32.  By setting that as the default
+    # here we make minimal changes in the test_email tests compared to their
+    # pre-3.3 state.
+    policy = compat32
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self.addTypeEqualityFunc(bytes, self.assertBytesEqual)
+
+    # Backward compatibility to minimize test_email test changes.
+    ndiffAssertEqual = unittest.TestCase.assertEqual
+
+    def _msgobj(self, filename):
+        with openfile(filename) as fp:
+            return email.message_from_file(fp, policy=self.policy)
+
+    def _str_msg(self, string, message=Message, policy=None):
+        if policy is None:
+            policy = self.policy
+        return email.message_from_string(string, message, policy=policy)
+
+    def _bytes_repr(self, b):
+        return [repr(x) for x in b.splitlines(keepends=True)]
+
+    def assertBytesEqual(self, first, second, msg):
+        """Our byte strings are really encoded strings; improve diff output"""
+        self.assertEqual(self._bytes_repr(first), self._bytes_repr(second))
+
+    def assertDefectsEqual(self, actual, expected):
+        self.assertEqual(len(actual), len(expected), actual)
+        for i in range(len(actual)):
+            self.assertIsInstance(actual[i], expected[i],
+                                    'item {}'.format(i))
+
+
+def parameterize(cls):
+    """A test method parameterization class decorator.
+
+    Parameters are specified as the value of a class attribute that ends with
+    the string '_params'.  Call the portion before '_params' the prefix.  Then
+    a method to be parameterized must have the same prefix, the string
+    '_as_', and an arbitrary suffix.
+
+    The value of the _params attribute may be either a dictionary or a list.
+    The values in the dictionary and the elements of the list may either be
+    single values, or a list.  If single values, they are turned into single
+    element tuples.  However derived, the resulting sequence is passed via
+    *args to the parameterized test function.
+
+    In a _params dictioanry, the keys become part of the name of the generated
+    tests.  In a _params list, the values in the list are converted into a
+    string by joining the string values of the elements of the tuple by '_' and
+    converting any blanks into '_'s, and this become part of the name.
+    The  full name of a generated test is a 'test_' prefix, the portion of the
+    test function name after the  '_as_' separator, plus an '_', plus the name
+    derived as explained above.
+
+    For example, if we have:
+
+        count_params = range(2)
+
+        def count_as_foo_arg(self, foo):
+            self.assertEqual(foo+1, myfunc(foo))
+
+    we will get parameterized test methods named:
+        test_foo_arg_0
+        test_foo_arg_1
+        test_foo_arg_2
+
+    Or we could have:
+
+        example_params = {'foo': ('bar', 1), 'bing': ('bang', 2)}
+
+        def example_as_myfunc_input(self, name, count):
+            self.assertEqual(name+str(count), myfunc(name, count))
+
+    and get:
+        test_myfunc_input_foo
+        test_myfunc_input_bing
+
+    Note: if and only if the generated test name is a valid identifier can it
+    be used to select the test individually from the unittest command line.
+
+    """
+    paramdicts = {}
+    for name, attr in cls.__dict__.items():
+        if name.endswith('_params'):
+            if not hasattr(attr, 'keys'):
+                d = {}
+                for x in attr:
+                    if not hasattr(x, '__iter__'):
+                        x = (x,)
+                    n = '_'.join(str(v) for v in x).replace(' ', '_')
+                    d[n] = x
+                attr = d
+            paramdicts[name[:-7] + '_as_'] = attr
+    testfuncs = {}
+    for name, attr in cls.__dict__.items():
+        for paramsname, paramsdict in paramdicts.items():
+            if name.startswith(paramsname):
+                testnameroot = 'test_' + name[len(paramsname):]
+                for paramname, params in paramsdict.items():
+                    test = (lambda self, name=name, params=params:
+                                    getattr(self, name)(*params))
+                    testname = testnameroot + '_' + paramname
+                    test.__name__ = testname
+                    testfuncs[testname] = test
+    for key, value in testfuncs.items():
+        setattr(cls, key, value)
+    return cls
diff --git a/Lib/test/test_email/__main__.py b/Lib/test/test_email/__main__.py
new file mode 100644
index 0000000000..98af9ecbad
--- /dev/null
+++ b/Lib/test/test_email/__main__.py
@@ -0,0 +1,3 @@
+from test.test_email import test_main
+
+test_main()
diff --git a/Lib/email/test/data/PyBanner048.gif b/Lib/test/test_email/data/PyBanner048.gif
index 1a5c87f647..1a5c87f647 100644
--- a/Lib/email/test/data/PyBanner048.gif
+++ b/Lib/test/test_email/data/PyBanner048.gif
diff --git a/Lib/email/test/data/audiotest.au b/Lib/test/test_email/data/audiotest.au
index f76b0501b8..f76b0501b8 100644
--- a/Lib/email/test/data/audiotest.au
+++ b/Lib/test/test_email/data/audiotest.au
diff --git a/Lib/email/test/data/msg_01.txt b/Lib/test/test_email/data/msg_01.txt
index 7e33bcf96a..7e33bcf96a 100644
--- a/Lib/email/test/data/msg_01.txt
+++ b/Lib/test/test_email/data/msg_01.txt
diff --git a/Lib/email/test/data/msg_02.txt b/Lib/test/test_email/data/msg_02.txt
index 43f248038a..43f248038a 100644
--- a/Lib/email/test/data/msg_02.txt
+++ b/Lib/test/test_email/data/msg_02.txt
diff --git a/Lib/email/test/data/msg_03.txt b/Lib/test/test_email/data/msg_03.txt
index c748ebf117..c748ebf117 100644
--- a/Lib/email/test/data/msg_03.txt
+++ b/Lib/test/test_email/data/msg_03.txt
diff --git a/Lib/email/test/data/msg_04.txt b/Lib/test/test_email/data/msg_04.txt
index 1f633c4496..1f633c4496 100644
--- a/Lib/email/test/data/msg_04.txt
+++ b/Lib/test/test_email/data/msg_04.txt
diff --git a/Lib/email/test/data/msg_05.txt b/Lib/test/test_email/data/msg_05.txt
index 87d5e9cbf8..87d5e9cbf8 100644
--- a/Lib/email/test/data/msg_05.txt
+++ b/Lib/test/test_email/data/msg_05.txt
diff --git a/Lib/email/test/data/msg_06.txt b/Lib/test/test_email/data/msg_06.txt
index 69f3a47ff4..69f3a47ff4 100644
--- a/Lib/email/test/data/msg_06.txt
+++ b/Lib/test/test_email/data/msg_06.txt
diff --git a/Lib/email/test/data/msg_07.txt b/Lib/test/test_email/data/msg_07.txt
index 721f3a0d31..721f3a0d31 100644
--- a/Lib/email/test/data/msg_07.txt
+++ b/Lib/test/test_email/data/msg_07.txt
diff --git a/Lib/email/test/data/msg_08.txt b/Lib/test/test_email/data/msg_08.txt
index b5630836c5..b5630836c5 100644
--- a/Lib/email/test/data/msg_08.txt
+++ b/Lib/test/test_email/data/msg_08.txt
diff --git a/Lib/email/test/data/msg_09.txt b/Lib/test/test_email/data/msg_09.txt
index 575c4c205a..575c4c205a 100644
--- a/Lib/email/test/data/msg_09.txt
+++ b/Lib/test/test_email/data/msg_09.txt
diff --git a/Lib/email/test/data/msg_10.txt b/Lib/test/test_email/data/msg_10.txt
index 07903960f9..07903960f9 100644
--- a/Lib/email/test/data/msg_10.txt
+++ b/Lib/test/test_email/data/msg_10.txt
diff --git a/Lib/email/test/data/msg_11.txt b/Lib/test/test_email/data/msg_11.txt
index 8f7f1991cb..8f7f1991cb 100644
--- a/Lib/email/test/data/msg_11.txt
+++ b/Lib/test/test_email/data/msg_11.txt
diff --git a/Lib/email/test/data/msg_12.txt b/Lib/test/test_email/data/msg_12.txt
index 4bec8d9444..4bec8d9444 100644
--- a/Lib/email/test/data/msg_12.txt
+++ b/Lib/test/test_email/data/msg_12.txt
diff --git a/Lib/email/test/data/msg_12a.txt b/Lib/test/test_email/data/msg_12a.txt
index e94224ecfe..e94224ecfe 100644
--- a/Lib/email/test/data/msg_12a.txt
+++ b/Lib/test/test_email/data/msg_12a.txt
diff --git a/Lib/email/test/data/msg_13.txt b/Lib/test/test_email/data/msg_13.txt
index 8e6d52d5be..8e6d52d5be 100644
--- a/Lib/email/test/data/msg_13.txt
+++ b/Lib/test/test_email/data/msg_13.txt
diff --git a/Lib/email/test/data/msg_14.txt b/Lib/test/test_email/data/msg_14.txt
index 5d98d2fd14..5d98d2fd14 100644
--- a/Lib/email/test/data/msg_14.txt
+++ b/Lib/test/test_email/data/msg_14.txt
diff --git a/Lib/email/test/data/msg_15.txt b/Lib/test/test_email/data/msg_15.txt
index 0025624e75..0025624e75 100644
--- a/Lib/email/test/data/msg_15.txt
+++ b/Lib/test/test_email/data/msg_15.txt
diff --git a/Lib/email/test/data/msg_16.txt b/Lib/test/test_email/data/msg_16.txt
index 56167e9f5b..56167e9f5b 100644
--- a/Lib/email/test/data/msg_16.txt
+++ b/Lib/test/test_email/data/msg_16.txt
diff --git a/Lib/email/test/data/msg_17.txt b/Lib/test/test_email/data/msg_17.txt
index 8d86e4180d..8d86e4180d 100644
--- a/Lib/email/test/data/msg_17.txt
+++ b/Lib/test/test_email/data/msg_17.txt
diff --git a/Lib/email/test/data/msg_18.txt b/Lib/test/test_email/data/msg_18.txt
index f9f4904d36..f9f4904d36 100644
--- a/Lib/email/test/data/msg_18.txt
+++ b/Lib/test/test_email/data/msg_18.txt
diff --git a/Lib/email/test/data/msg_19.txt b/Lib/test/test_email/data/msg_19.txt
index 49bf7fccdd..49bf7fccdd 100644
--- a/Lib/email/test/data/msg_19.txt
+++ b/Lib/test/test_email/data/msg_19.txt
diff --git a/Lib/email/test/data/msg_20.txt b/Lib/test/test_email/data/msg_20.txt
index 1a6a88783e..1a6a88783e 100644
--- a/Lib/email/test/data/msg_20.txt
+++ b/Lib/test/test_email/data/msg_20.txt
diff --git a/Lib/email/test/data/msg_21.txt b/Lib/test/test_email/data/msg_21.txt
index 23590b255d..23590b255d 100644
--- a/Lib/email/test/data/msg_21.txt
+++ b/Lib/test/test_email/data/msg_21.txt
diff --git a/Lib/email/test/data/msg_22.txt b/Lib/test/test_email/data/msg_22.txt
index af9de5fa27..af9de5fa27 100644
--- a/Lib/email/test/data/msg_22.txt
+++ b/Lib/test/test_email/data/msg_22.txt
diff --git a/Lib/email/test/data/msg_23.txt b/Lib/test/test_email/data/msg_23.txt
index bb2e8ec36b..bb2e8ec36b 100644
--- a/Lib/email/test/data/msg_23.txt
+++ b/Lib/test/test_email/data/msg_23.txt
diff --git a/Lib/email/test/data/msg_24.txt b/Lib/test/test_email/data/msg_24.txt
index 4e52339e86..4e52339e86 100644
--- a/Lib/email/test/data/msg_24.txt
+++ b/Lib/test/test_email/data/msg_24.txt
diff --git a/Lib/email/test/data/msg_25.txt b/Lib/test/test_email/data/msg_25.txt
index 9e35275fe0..9e35275fe0 100644
--- a/Lib/email/test/data/msg_25.txt
+++ b/Lib/test/test_email/data/msg_25.txt
diff --git a/Lib/email/test/data/msg_26.txt b/Lib/test/test_email/data/msg_26.txt
index 58efaa9c9a..58efaa9c9a 100644
--- a/Lib/email/test/data/msg_26.txt
+++ b/Lib/test/test_email/data/msg_26.txt
diff --git a/Lib/email/test/data/msg_27.txt b/Lib/test/test_email/data/msg_27.txt
index d0191769d7..d0191769d7 100644
--- a/Lib/email/test/data/msg_27.txt
+++ b/Lib/test/test_email/data/msg_27.txt
diff --git a/Lib/email/test/data/msg_28.txt b/Lib/test/test_email/data/msg_28.txt
index 1e4824cabb..1e4824cabb 100644
--- a/Lib/email/test/data/msg_28.txt
+++ b/Lib/test/test_email/data/msg_28.txt
diff --git a/Lib/email/test/data/msg_29.txt b/Lib/test/test_email/data/msg_29.txt
index 1fab561617..1fab561617 100644
--- a/Lib/email/test/data/msg_29.txt
+++ b/Lib/test/test_email/data/msg_29.txt
diff --git a/Lib/email/test/data/msg_30.txt b/Lib/test/test_email/data/msg_30.txt
index 4334bb6e60..4334bb6e60 100644
--- a/Lib/email/test/data/msg_30.txt
+++ b/Lib/test/test_email/data/msg_30.txt
diff --git a/Lib/email/test/data/msg_31.txt b/Lib/test/test_email/data/msg_31.txt
index 1e58e56cf5..1e58e56cf5 100644
--- a/Lib/email/test/data/msg_31.txt
+++ b/Lib/test/test_email/data/msg_31.txt
diff --git a/Lib/email/test/data/msg_32.txt b/Lib/test/test_email/data/msg_32.txt
index 07ec5af9a3..07ec5af9a3 100644
--- a/Lib/email/test/data/msg_32.txt
+++ b/Lib/test/test_email/data/msg_32.txt
diff --git a/Lib/email/test/data/msg_33.txt b/Lib/test/test_email/data/msg_33.txt
index 042787a4fd..042787a4fd 100644
--- a/Lib/email/test/data/msg_33.txt
+++ b/Lib/test/test_email/data/msg_33.txt
diff --git a/Lib/email/test/data/msg_34.txt b/Lib/test/test_email/data/msg_34.txt
index 055dfea531..055dfea531 100644
--- a/Lib/email/test/data/msg_34.txt
+++ b/Lib/test/test_email/data/msg_34.txt
diff --git a/Lib/email/test/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt
index be7d5a2f7b..be7d5a2f7b 100644
--- a/Lib/email/test/data/msg_35.txt
+++ b/Lib/test/test_email/data/msg_35.txt
diff --git a/Lib/email/test/data/msg_36.txt b/Lib/test/test_email/data/msg_36.txt
index 5632c3062c..5632c3062c 100644
--- a/Lib/email/test/data/msg_36.txt
+++ b/Lib/test/test_email/data/msg_36.txt
diff --git a/Lib/email/test/data/msg_37.txt b/Lib/test/test_email/data/msg_37.txt
index 038d34a1a4..038d34a1a4 100644
--- a/Lib/email/test/data/msg_37.txt
+++ b/Lib/test/test_email/data/msg_37.txt
diff --git a/Lib/email/test/data/msg_38.txt b/Lib/test/test_email/data/msg_38.txt
index 006df81cb5..006df81cb5 100644
--- a/Lib/email/test/data/msg_38.txt
+++ b/Lib/test/test_email/data/msg_38.txt
diff --git a/Lib/email/test/data/msg_39.txt b/Lib/test/test_email/data/msg_39.txt
index 124b269192..124b269192 100644
--- a/Lib/email/test/data/msg_39.txt
+++ b/Lib/test/test_email/data/msg_39.txt
diff --git a/Lib/email/test/data/msg_40.txt b/Lib/test/test_email/data/msg_40.txt
index 1435fa1e1a..1435fa1e1a 100644
--- a/Lib/email/test/data/msg_40.txt
+++ b/Lib/test/test_email/data/msg_40.txt
diff --git a/Lib/email/test/data/msg_41.txt b/Lib/test/test_email/data/msg_41.txt
index 76cdd1cb7f..76cdd1cb7f 100644
--- a/Lib/email/test/data/msg_41.txt
+++ b/Lib/test/test_email/data/msg_41.txt
diff --git a/Lib/email/test/data/msg_42.txt b/Lib/test/test_email/data/msg_42.txt
index a75f8f4a02..a75f8f4a02 100644
--- a/Lib/email/test/data/msg_42.txt
+++ b/Lib/test/test_email/data/msg_42.txt
diff --git a/Lib/email/test/data/msg_43.txt b/Lib/test/test_email/data/msg_43.txt
index 797d12c568..797d12c568 100644
--- a/Lib/email/test/data/msg_43.txt
+++ b/Lib/test/test_email/data/msg_43.txt
diff --git a/Lib/email/test/data/msg_44.txt b/Lib/test/test_email/data/msg_44.txt
index 15a225287b..15a225287b 100644
--- a/Lib/email/test/data/msg_44.txt
+++ b/Lib/test/test_email/data/msg_44.txt
diff --git a/Lib/email/test/data/msg_45.txt b/Lib/test/test_email/data/msg_45.txt
index 58fde956e7..58fde956e7 100644
--- a/Lib/email/test/data/msg_45.txt
+++ b/Lib/test/test_email/data/msg_45.txt
diff --git a/Lib/email/test/data/msg_46.txt b/Lib/test/test_email/data/msg_46.txt
index 1e22c4f600..1e22c4f600 100644
--- a/Lib/email/test/data/msg_46.txt
+++ b/Lib/test/test_email/data/msg_46.txt
diff --git a/Lib/test/test_email/test__encoded_words.py b/Lib/test/test_email/test__encoded_words.py
new file mode 100644
index 0000000000..14395fed40
--- /dev/null
+++ b/Lib/test/test_email/test__encoded_words.py
@@ -0,0 +1,187 @@
+import unittest
+from email import _encoded_words as _ew
+from email import errors
+from test.test_email import TestEmailBase
+
+
+class TestDecodeQ(TestEmailBase):
+
+    def _test(self, source, ex_result, ex_defects=[]):
+        result, defects = _ew.decode_q(source)
+        self.assertEqual(result, ex_result)
+        self.assertDefectsEqual(defects, ex_defects)
+
+    def test_no_encoded(self):
+        self._test(b'foobar', b'foobar')
+
+    def test_spaces(self):
+        self._test(b'foo=20bar=20', b'foo bar ')
+        self._test(b'foo_bar_', b'foo bar ')
+
+    def test_run_of_encoded(self):
+        self._test(b'foo=20=20=21=2Cbar', b'foo  !,bar')
+
+
+class TestDecodeB(TestEmailBase):
+
+    def _test(self, source, ex_result, ex_defects=[]):
+        result, defects = _ew.decode_b(source)
+        self.assertEqual(result, ex_result)
+        self.assertDefectsEqual(defects, ex_defects)
+
+    def test_simple(self):
+        self._test(b'Zm9v', b'foo')
+
+    def test_missing_padding(self):
+        self._test(b'dmk', b'vi', [errors.InvalidBase64PaddingDefect])
+
+    def test_invalid_character(self):
+        self._test(b'dm\x01k===', b'vi', [errors.InvalidBase64CharactersDefect])
+
+    def test_invalid_character_and_bad_padding(self):
+        self._test(b'dm\x01k', b'vi', [errors.InvalidBase64CharactersDefect,
+                                       errors.InvalidBase64PaddingDefect])
+
+
+class TestDecode(TestEmailBase):
+
+    def test_wrong_format_input_raises(self):
+        with self.assertRaises(ValueError):
+            _ew.decode('=?badone?=')
+        with self.assertRaises(ValueError):
+            _ew.decode('=?')
+        with self.assertRaises(ValueError):
+            _ew.decode('')
+
+    def _test(self, source, result, charset='us-ascii', lang='', defects=[]):
+        res, char, l, d = _ew.decode(source)
+        self.assertEqual(res, result)
+        self.assertEqual(char, charset)
+        self.assertEqual(l, lang)
+        self.assertDefectsEqual(d, defects)
+
+    def test_simple_q(self):
+        self._test('=?us-ascii?q?foo?=', 'foo')
+
+    def test_simple_b(self):
+        self._test('=?us-ascii?b?dmk=?=', 'vi')
+
+    def test_q_case_ignored(self):
+        self._test('=?us-ascii?Q?foo?=', 'foo')
+
+    def test_b_case_ignored(self):
+        self._test('=?us-ascii?B?dmk=?=', 'vi')
+
+    def test_non_trivial_q(self):
+        self._test('=?latin-1?q?=20F=fcr=20Elise=20?=', ' Für Elise ', 'latin-1')
+
+    def test_q_escpaed_bytes_preserved(self):
+        self._test(b'=?us-ascii?q?=20\xACfoo?='.decode('us-ascii',
+                                                       'surrogateescape'),
+                   ' \uDCACfoo',
+                   defects = [errors.UndecodableBytesDefect])
+
+    def test_b_undecodable_bytes_ignored_with_defect(self):
+        self._test(b'=?us-ascii?b?dm\xACk?='.decode('us-ascii',
+                                                   'surrogateescape'),
+                   'vi',
+                   defects = [
+                    errors.InvalidBase64CharactersDefect,
+                    errors.InvalidBase64PaddingDefect])
+
+    def test_b_invalid_bytes_ignored_with_defect(self):
+        self._test('=?us-ascii?b?dm\x01k===?=',
+                   'vi',
+                   defects = [errors.InvalidBase64CharactersDefect])
+
+    def test_b_invalid_bytes_incorrect_padding(self):
+        self._test('=?us-ascii?b?dm\x01k?=',
+                   'vi',
+                   defects = [
+                    errors.InvalidBase64CharactersDefect,
+                    errors.InvalidBase64PaddingDefect])
+
+    def test_b_padding_defect(self):
+        self._test('=?us-ascii?b?dmk?=',
+                   'vi',
+                    defects = [errors.InvalidBase64PaddingDefect])
+
+    def test_nonnull_lang(self):
+        self._test('=?us-ascii*jive?q?test?=', 'test', lang='jive')
+
+    def test_unknown_8bit_charset(self):
+        self._test('=?unknown-8bit?q?foo=ACbar?=',
+                   b'foo\xacbar'.decode('ascii', 'surrogateescape'),
+                   charset = 'unknown-8bit',
+                   defects = [])
+
+    def test_unknown_charset(self):
+        self._test('=?foobar?q?foo=ACbar?=',
+                   b'foo\xacbar'.decode('ascii', 'surrogateescape'),
+                   charset = 'foobar',
+                   # XXX Should this be a new Defect instead?
+                   defects = [errors.CharsetError])
+
+
+class TestEncodeQ(TestEmailBase):
+
+    def _test(self, src, expected):
+        self.assertEqual(_ew.encode_q(src), expected)
+
+    def test_all_safe(self):
+        self._test(b'foobar', 'foobar')
+
+    def test_spaces(self):
+        self._test(b'foo bar ', 'foo_bar_')
+
+    def test_run_of_encodables(self):
+        self._test(b'foo  ,,bar', 'foo__=2C=2Cbar')
+
+
+class TestEncodeB(TestEmailBase):
+
+    def test_simple(self):
+        self.assertEqual(_ew.encode_b(b'foo'), 'Zm9v')
+
+    def test_padding(self):
+        self.assertEqual(_ew.encode_b(b'vi'), 'dmk=')
+
+
+class TestEncode(TestEmailBase):
+
+    def test_q(self):
+        self.assertEqual(_ew.encode('foo', 'utf-8', 'q'), '=?utf-8?q?foo?=')
+
+    def test_b(self):
+        self.assertEqual(_ew.encode('foo', 'utf-8', 'b'), '=?utf-8?b?Zm9v?=')
+
+    def test_auto_q(self):
+        self.assertEqual(_ew.encode('foo', 'utf-8'), '=?utf-8?q?foo?=')
+
+    def test_auto_q_if_short_mostly_safe(self):
+        self.assertEqual(_ew.encode('vi.', 'utf-8'), '=?utf-8?q?vi=2E?=')
+
+    def test_auto_b_if_enough_unsafe(self):
+        self.assertEqual(_ew.encode('.....', 'utf-8'), '=?utf-8?b?Li4uLi4=?=')
+
+    def test_auto_b_if_long_unsafe(self):
+        self.assertEqual(_ew.encode('vi.vi.vi.vi.vi.', 'utf-8'),
+                         '=?utf-8?b?dmkudmkudmkudmkudmku?=')
+
+    def test_auto_q_if_long_mostly_safe(self):
+        self.assertEqual(_ew.encode('vi vi vi.vi ', 'utf-8'),
+                         '=?utf-8?q?vi_vi_vi=2Evi_?=')
+
+    def test_utf8_default(self):
+        self.assertEqual(_ew.encode('foo'), '=?utf-8?q?foo?=')
+
+    def test_lang(self):
+        self.assertEqual(_ew.encode('foo', lang='jive'), '=?utf-8*jive?q?foo?=')
+
+    def test_unknown_8bit(self):
+        self.assertEqual(_ew.encode('foo\uDCACbar', charset='unknown-8bit'),
+                         '=?unknown-8bit?q?foo=ACbar?=')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
new file mode 100644
index 0000000000..6101e191c0
--- /dev/null
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -0,0 +1,2552 @@
+import string
+import unittest
+from email import _header_value_parser as parser
+from email import errors
+from email import policy
+from test.test_email import TestEmailBase, parameterize
+
+class TestTokens(TestEmailBase):
+
+    # EWWhiteSpaceTerminal
+
+    def test_EWWhiteSpaceTerminal(self):
+        x = parser.EWWhiteSpaceTerminal(' \t', 'fws')
+        self.assertEqual(x, ' \t')
+        self.assertEqual(str(x), '')
+        self.assertEqual(x.value, '')
+        self.assertEqual(x.encoded, ' \t')
+
+    # UnstructuredTokenList
+
+    def test_undecodable_bytes_error_preserved(self):
+        badstr = b"le pouf c\xaflebre".decode('ascii', 'surrogateescape')
+        unst = parser.get_unstructured(badstr)
+        self.assertDefectsEqual(unst.all_defects, [errors.UndecodableBytesDefect])
+        parts = list(unst.parts)
+        self.assertDefectsEqual(parts[0].all_defects, [])
+        self.assertDefectsEqual(parts[1].all_defects, [])
+        self.assertDefectsEqual(parts[2].all_defects, [errors.UndecodableBytesDefect])
+
+
+class TestParserMixin:
+
+    def _assert_results(self, tl, rest, string, value, defects, remainder,
+                        comments=None):
+        self.assertEqual(str(tl), string)
+        self.assertEqual(tl.value, value)
+        self.assertDefectsEqual(tl.all_defects, defects)
+        self.assertEqual(rest, remainder)
+        if comments is not None:
+            self.assertEqual(tl.comments, comments)
+
+    def _test_get_x(self, method, source, string, value, defects,
+                          remainder, comments=None):
+        tl, rest = method(source)
+        self._assert_results(tl, rest, string, value, defects, remainder,
+                             comments=None)
+        return tl
+
+    def _test_parse_x(self, method, input, string, value, defects,
+                             comments=None):
+        tl = method(input)
+        self._assert_results(tl, '', string, value, defects, '', comments)
+        return tl
+
+
+class TestParser(TestParserMixin, TestEmailBase):
+
+    # _wsp_splitter
+
+    rfc_printable_ascii = bytes(range(33, 127)).decode('ascii')
+    rfc_atext_chars = (string.ascii_letters + string.digits +
+                        "!#$%&\'*+-/=?^_`{}|~")
+    rfc_dtext_chars = rfc_printable_ascii.translate(str.maketrans('','',r'\[]'))
+
+    def test__wsp_splitter_one_word(self):
+        self.assertEqual(parser._wsp_splitter('foo', 1), ['foo'])
+
+    def test__wsp_splitter_two_words(self):
+        self.assertEqual(parser._wsp_splitter('foo def', 1),
+                                               ['foo', ' ', 'def'])
+
+    def test__wsp_splitter_ws_runs(self):
+        self.assertEqual(parser._wsp_splitter('foo \t def jik', 1),
+                                              ['foo', ' \t ', 'def jik'])
+
+
+    # get_fws
+
+    def test_get_fws_only(self):
+        fws = self._test_get_x(parser.get_fws, ' \t  ', ' \t  ', ' ', [], '')
+        self.assertEqual(fws.token_type, 'fws')
+
+    def test_get_fws_space(self):
+        self._test_get_x(parser.get_fws, ' foo', ' ', ' ', [], 'foo')
+
+    def test_get_fws_ws_run(self):
+        self._test_get_x(parser.get_fws, ' \t foo ', ' \t ', ' ', [], 'foo ')
+
+    # get_encoded_word
+
+    def test_get_encoded_word_missing_start_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_encoded_word('abc')
+
+    def test_get_encoded_word_missing_end_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_encoded_word('=?abc')
+
+    def test_get_encoded_word_missing_middle_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_encoded_word('=?abc?=')
+
+    def test_get_encoded_word_valid_ew(self):
+        self._test_get_x(parser.get_encoded_word,
+                         '=?us-ascii?q?this_is_a_test?=  bird',
+                         'this is a test',
+                         'this is a test',
+                         [],
+                         '  bird')
+
+    def test_get_encoded_word_internal_spaces(self):
+        self._test_get_x(parser.get_encoded_word,
+                         '=?us-ascii?q?this is a test?=  bird',
+                         'this is a test',
+                         'this is a test',
+                         [errors.InvalidHeaderDefect],
+                         '  bird')
+
+    def test_get_encoded_word_gets_first(self):
+        self._test_get_x(parser.get_encoded_word,
+                         '=?us-ascii?q?first?=  =?utf-8?q?second?=',
+                         'first',
+                         'first',
+                         [],
+                         '  =?utf-8?q?second?=')
+
+    def test_get_encoded_word_gets_first_even_if_no_space(self):
+        self._test_get_x(parser.get_encoded_word,
+                         '=?us-ascii?q?first?==?utf-8?q?second?=',
+                         'first',
+                         'first',
+                         [],
+                         '=?utf-8?q?second?=')
+
+    def test_get_encoded_word_sets_extra_attributes(self):
+        ew = self._test_get_x(parser.get_encoded_word,
+                         '=?us-ascii*jive?q?first_second?=',
+                         'first second',
+                         'first second',
+                         [],
+                         '')
+        self.assertEqual(ew.encoded, '=?us-ascii*jive?q?first_second?=')
+        self.assertEqual(ew.charset, 'us-ascii')
+        self.assertEqual(ew.lang, 'jive')
+
+    def test_get_encoded_word_lang_default_is_blank(self):
+        ew = self._test_get_x(parser.get_encoded_word,
+                         '=?us-ascii?q?first_second?=',
+                         'first second',
+                         'first second',
+                         [],
+                         '')
+        self.assertEqual(ew.encoded, '=?us-ascii?q?first_second?=')
+        self.assertEqual(ew.charset, 'us-ascii')
+        self.assertEqual(ew.lang, '')
+
+    def test_get_encoded_word_non_printable_defect(self):
+        self._test_get_x(parser.get_encoded_word,
+                         '=?us-ascii?q?first\x02second?=',
+                         'first\x02second',
+                         'first\x02second',
+                         [errors.NonPrintableDefect],
+                         '')
+
+    def test_get_encoded_word_leading_internal_space(self):
+        self._test_get_x(parser.get_encoded_word,
+                        '=?us-ascii?q?=20foo?=',
+                        ' foo',
+                        ' foo',
+                        [],
+                        '')
+
+    # get_unstructured
+
+    def _get_unst(self, value):
+        token = parser.get_unstructured(value)
+        return token, ''
+
+    def test_get_unstructured_null(self):
+        self._test_get_x(self._get_unst, '', '', '', [], '')
+
+    def test_get_unstructured_one_word(self):
+        self._test_get_x(self._get_unst, 'foo', 'foo', 'foo', [], '')
+
+    def test_get_unstructured_normal_phrase(self):
+        self._test_get_x(self._get_unst, 'foo bar bird',
+                                         'foo bar bird',
+                                         'foo bar bird',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_normal_phrase_with_whitespace(self):
+        self._test_get_x(self._get_unst, 'foo \t bar      bird',
+                                         'foo \t bar      bird',
+                                         'foo bar bird',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_leading_whitespace(self):
+        self._test_get_x(self._get_unst, '  foo bar',
+                                         '  foo bar',
+                                         ' foo bar',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_trailing_whitespace(self):
+        self._test_get_x(self._get_unst, 'foo bar  ',
+                                         'foo bar  ',
+                                         'foo bar ',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_leading_and_trailing_whitespace(self):
+        self._test_get_x(self._get_unst, '  foo bar  ',
+                                         '  foo bar  ',
+                                         ' foo bar ',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_one_valid_ew_no_ws(self):
+        self._test_get_x(self._get_unst, '=?us-ascii?q?bar?=',
+                                         'bar',
+                                         'bar',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_one_ew_trailing_ws(self):
+        self._test_get_x(self._get_unst, '=?us-ascii?q?bar?=  ',
+                                         'bar  ',
+                                         'bar ',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_one_valid_ew_trailing_text(self):
+        self._test_get_x(self._get_unst, '=?us-ascii?q?bar?= bird',
+                                         'bar bird',
+                                         'bar bird',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_phrase_with_ew_in_middle_of_text(self):
+        self._test_get_x(self._get_unst, 'foo =?us-ascii?q?bar?= bird',
+                                         'foo bar bird',
+                                         'foo bar bird',
+                                         [],
+                                         '')
+
+    def test_get_unstructured_phrase_with_two_ew(self):
+        self._test_get_x(self._get_unst,
+            'foo =?us-ascii?q?bar?= =?us-ascii?q?bird?=',
+            'foo barbird',
+            'foo barbird',
+            [],
+            '')
+
+    def test_get_unstructured_phrase_with_two_ew_trailing_ws(self):
+        self._test_get_x(self._get_unst,
+            'foo =?us-ascii?q?bar?= =?us-ascii?q?bird?=   ',
+            'foo barbird   ',
+            'foo barbird ',
+            [],
+            '')
+
+    def test_get_unstructured_phrase_with_ew_with_leading_ws(self):
+        self._test_get_x(self._get_unst,
+            '  =?us-ascii?q?bar?=',
+            '  bar',
+            ' bar',
+            [],
+            '')
+
+    def test_get_unstructured_phrase_with_two_ew_extra_ws(self):
+        self._test_get_x(self._get_unst,
+            'foo =?us-ascii?q?bar?= \t  =?us-ascii?q?bird?=',
+            'foo barbird',
+            'foo barbird',
+            [],
+            '')
+
+    def test_get_unstructured_two_ew_extra_ws_trailing_text(self):
+        self._test_get_x(self._get_unst,
+            '=?us-ascii?q?test?=   =?us-ascii?q?foo?=  val',
+            'testfoo  val',
+            'testfoo val',
+            [],
+            '')
+
+    def test_get_unstructured_ew_with_internal_ws(self):
+        self._test_get_x(self._get_unst,
+            '=?iso-8859-1?q?hello=20world?=',
+            'hello world',
+            'hello world',
+            [],
+            '')
+
+    def test_get_unstructured_ew_with_internal_leading_ws(self):
+        self._test_get_x(self._get_unst,
+            '   =?us-ascii?q?=20test?=   =?us-ascii?q?=20foo?=  val',
+            '    test foo  val',
+            '  test foo val',
+            [],
+            '')
+
+    def test_get_unstructured_invaild_ew(self):
+        self._test_get_x(self._get_unst,
+            '=?test val',
+            '=?test val',
+            '=?test val',
+            [],
+            '')
+
+    def test_get_unstructured_undecodable_bytes(self):
+        self._test_get_x(self._get_unst,
+            b'test \xACfoo  val'.decode('ascii', 'surrogateescape'),
+            'test \uDCACfoo  val',
+            'test \uDCACfoo val',
+            [errors.UndecodableBytesDefect],
+            '')
+
+    def test_get_unstructured_undecodable_bytes_in_EW(self):
+        self._test_get_x(self._get_unst,
+            (b'=?us-ascii?q?=20test?=   =?us-ascii?q?=20\xACfoo?='
+                b'  val').decode('ascii', 'surrogateescape'),
+            ' test \uDCACfoo  val',
+            ' test \uDCACfoo val',
+            [errors.UndecodableBytesDefect]*2,
+            '')
+
+    def test_get_unstructured_missing_base64_padding(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?b?dmk?=',
+            'vi',
+            'vi',
+            [errors.InvalidBase64PaddingDefect],
+            '')
+
+    def test_get_unstructured_invalid_base64_character(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?b?dm\x01k===?=',
+            'vi',
+            'vi',
+            [errors.InvalidBase64CharactersDefect],
+            '')
+
+    def test_get_unstructured_invalid_base64_character_and_bad_padding(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?b?dm\x01k?=',
+            'vi',
+            'vi',
+            [errors.InvalidBase64CharactersDefect,
+             errors.InvalidBase64PaddingDefect],
+            '')
+
+    def test_get_unstructured_no_whitespace_between_ews(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?q?foo?==?utf-8?q?bar?=',
+            'foobar',
+            'foobar',
+            [errors.InvalidHeaderDefect],
+            '')
+
+    # get_qp_ctext
+
+    def test_get_qp_ctext_only(self):
+        ptext = self._test_get_x(parser.get_qp_ctext,
+                                'foobar', 'foobar', ' ', [], '')
+        self.assertEqual(ptext.token_type, 'ptext')
+
+    def test_get_qp_ctext_all_printables(self):
+        with_qp = self.rfc_printable_ascii.replace('\\', '\\\\')
+        with_qp = with_qp.  replace('(', r'\(')
+        with_qp = with_qp.replace(')', r'\)')
+        ptext = self._test_get_x(parser.get_qp_ctext,
+                                 with_qp, self.rfc_printable_ascii, ' ', [], '')
+
+    def test_get_qp_ctext_two_words_gets_first(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo de', 'foo', ' ', [], ' de')
+
+    def test_get_qp_ctext_following_wsp_preserved(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo \t\tde', 'foo', ' ', [], ' \t\tde')
+
+    def test_get_qp_ctext_up_to_close_paren_only(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo)', 'foo', ' ', [], ')')
+
+    def test_get_qp_ctext_wsp_before_close_paren_preserved(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo  )', 'foo', ' ', [], '  )')
+
+    def test_get_qp_ctext_close_paren_mid_word(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo)bar', 'foo', ' ', [], ')bar')
+
+    def test_get_qp_ctext_up_to_open_paren_only(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo(', 'foo', ' ', [], '(')
+
+    def test_get_qp_ctext_wsp_before_open_paren_preserved(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo  (', 'foo', ' ', [], '  (')
+
+    def test_get_qp_ctext_open_paren_mid_word(self):
+        self._test_get_x(parser.get_qp_ctext,
+                        'foo(bar', 'foo', ' ', [], '(bar')
+
+    def test_get_qp_ctext_non_printables(self):
+        ptext = self._test_get_x(parser.get_qp_ctext,
+                                'foo\x00bar)', 'foo\x00bar', ' ',
+                                [errors.NonPrintableDefect], ')')
+        self.assertEqual(ptext.defects[0].non_printables[0], '\x00')
+
+    # get_qcontent
+
+    def test_get_qcontent_only(self):
+        ptext = self._test_get_x(parser.get_qcontent,
+                                'foobar', 'foobar', 'foobar', [], '')
+        self.assertEqual(ptext.token_type, 'ptext')
+
+    def test_get_qcontent_all_printables(self):
+        with_qp = self.rfc_printable_ascii.replace('\\', '\\\\')
+        with_qp = with_qp.  replace('"', r'\"')
+        ptext = self._test_get_x(parser.get_qcontent, with_qp,
+                                 self.rfc_printable_ascii,
+                                 self.rfc_printable_ascii, [], '')
+
+    def test_get_qcontent_two_words_gets_first(self):
+        self._test_get_x(parser.get_qcontent,
+                        'foo de', 'foo', 'foo', [], ' de')
+
+    def test_get_qcontent_following_wsp_preserved(self):
+        self._test_get_x(parser.get_qcontent,
+                        'foo \t\tde', 'foo', 'foo', [], ' \t\tde')
+
+    def test_get_qcontent_up_to_dquote_only(self):
+        self._test_get_x(parser.get_qcontent,
+                        'foo"', 'foo', 'foo', [], '"')
+
+    def test_get_qcontent_wsp_before_close_paren_preserved(self):
+        self._test_get_x(parser.get_qcontent,
+                        'foo  "', 'foo', 'foo', [], '  "')
+
+    def test_get_qcontent_close_paren_mid_word(self):
+        self._test_get_x(parser.get_qcontent,
+                        'foo"bar', 'foo', 'foo', [], '"bar')
+
+    def test_get_qcontent_non_printables(self):
+        ptext = self._test_get_x(parser.get_qcontent,
+                                'foo\x00fg"', 'foo\x00fg', 'foo\x00fg',
+                                [errors.NonPrintableDefect], '"')
+        self.assertEqual(ptext.defects[0].non_printables[0], '\x00')
+
+    # get_atext
+
+    def test_get_atext_only(self):
+        atext = self._test_get_x(parser.get_atext,
+                                'foobar', 'foobar', 'foobar', [], '')
+        self.assertEqual(atext.token_type, 'atext')
+
+    def test_get_atext_all_atext(self):
+        atext = self._test_get_x(parser.get_atext, self.rfc_atext_chars,
+                                 self.rfc_atext_chars,
+                                 self.rfc_atext_chars, [], '')
+
+    def test_get_atext_two_words_gets_first(self):
+        self._test_get_x(parser.get_atext,
+                        'foo bar', 'foo', 'foo', [], ' bar')
+
+    def test_get_atext_following_wsp_preserved(self):
+        self._test_get_x(parser.get_atext,
+                        'foo \t\tbar', 'foo', 'foo', [], ' \t\tbar')
+
+    def test_get_atext_up_to_special(self):
+        self._test_get_x(parser.get_atext,
+                        'foo@bar', 'foo', 'foo', [], '@bar')
+
+    def test_get_atext_non_printables(self):
+        atext = self._test_get_x(parser.get_atext,
+                                'foo\x00bar(', 'foo\x00bar', 'foo\x00bar',
+                                [errors.NonPrintableDefect], '(')
+        self.assertEqual(atext.defects[0].non_printables[0], '\x00')
+
+    # get_bare_quoted_string
+
+    def test_get_bare_quoted_string_only(self):
+        bqs = self._test_get_x(parser.get_bare_quoted_string,
+                               '"foo"', '"foo"', 'foo', [], '')
+        self.assertEqual(bqs.token_type, 'bare-quoted-string')
+
+    def test_get_bare_quoted_string_must_start_with_dquote(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_bare_quoted_string('foo"')
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_bare_quoted_string('  "foo"')
+
+    def test_get_bare_quoted_string_following_wsp_preserved(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+             '"foo"\t bar', '"foo"', 'foo', [], '\t bar')
+
+    def test_get_bare_quoted_string_multiple_words(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+             '"foo bar moo"', '"foo bar moo"', 'foo bar moo', [], '')
+
+    def test_get_bare_quoted_string_multiple_words_wsp_preserved(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+             '" foo  moo\t"', '" foo  moo\t"', ' foo  moo\t', [], '')
+
+    def test_get_bare_quoted_string_end_dquote_mid_word(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+             '"foo"bar', '"foo"', 'foo', [], 'bar')
+
+    def test_get_bare_quoted_string_quoted_dquote(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+             r'"foo\"in"a', r'"foo\"in"', 'foo"in', [], 'a')
+
+    def test_get_bare_quoted_string_non_printables(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+             '"a\x01a"', '"a\x01a"', 'a\x01a',
+             [errors.NonPrintableDefect], '')
+
+    def test_get_bare_quoted_string_no_end_dquote(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+             '"foo', '"foo"', 'foo',
+             [errors.InvalidHeaderDefect], '')
+        self._test_get_x(parser.get_bare_quoted_string,
+             '"foo ', '"foo "', 'foo ',
+             [errors.InvalidHeaderDefect], '')
+
+    def test_get_bare_quoted_string_empty_quotes(self):
+        self._test_get_x(parser.get_bare_quoted_string,
+            '""', '""', '', [], '')
+
+    # get_comment
+
+    def test_get_comment_only(self):
+        comment = self._test_get_x(parser.get_comment,
+            '(comment)', '(comment)', ' ', [], '', ['comment'])
+        self.assertEqual(comment.token_type, 'comment')
+
+    def test_get_comment_must_start_with_paren(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_comment('foo"')
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_comment('  (foo"')
+
+    def test_get_comment_following_wsp_preserved(self):
+        self._test_get_x(parser.get_comment,
+            '(comment)  \t', '(comment)', ' ', [], '  \t', ['comment'])
+
+    def test_get_comment_multiple_words(self):
+        self._test_get_x(parser.get_comment,
+            '(foo bar)  \t', '(foo bar)', ' ', [], '  \t', ['foo bar'])
+
+    def test_get_comment_multiple_words_wsp_preserved(self):
+        self._test_get_x(parser.get_comment,
+            '( foo  bar\t )  \t', '( foo  bar\t )', ' ', [], '  \t',
+                [' foo  bar\t '])
+
+    def test_get_comment_end_paren_mid_word(self):
+        self._test_get_x(parser.get_comment,
+            '(foo)bar', '(foo)', ' ', [], 'bar', ['foo'])
+
+    def test_get_comment_quoted_parens(self):
+        self._test_get_x(parser.get_comment,
+            '(foo\) \(\)bar)', '(foo\) \(\)bar)', ' ', [], '', ['foo) ()bar'])
+
+    def test_get_comment_non_printable(self):
+        self._test_get_x(parser.get_comment,
+            '(foo\x7Fbar)', '(foo\x7Fbar)', ' ',
+            [errors.NonPrintableDefect], '', ['foo\x7Fbar'])
+
+    def test_get_comment_no_end_paren(self):
+        self._test_get_x(parser.get_comment,
+            '(foo bar', '(foo bar)', ' ',
+            [errors.InvalidHeaderDefect], '', ['foo bar'])
+        self._test_get_x(parser.get_comment,
+            '(foo bar  ', '(foo bar  )', ' ',
+            [errors.InvalidHeaderDefect], '', ['foo bar  '])
+
+    def test_get_comment_nested_comment(self):
+        comment = self._test_get_x(parser.get_comment,
+            '(foo(bar))', '(foo(bar))', ' ', [], '', ['foo(bar)'])
+        self.assertEqual(comment[1].content, 'bar')
+
+    def test_get_comment_nested_comment_wsp(self):
+        comment = self._test_get_x(parser.get_comment,
+            '(foo ( bar ) )', '(foo ( bar ) )', ' ', [], '', ['foo ( bar ) '])
+        self.assertEqual(comment[2].content, ' bar ')
+
+    def test_get_comment_empty_comment(self):
+        self._test_get_x(parser.get_comment,
+            '()', '()', ' ', [], '', [''])
+
+    def test_get_comment_multiple_nesting(self):
+        comment = self._test_get_x(parser.get_comment,
+            '(((((foo)))))', '(((((foo)))))', ' ', [], '', ['((((foo))))'])
+        for i in range(4, 0, -1):
+            self.assertEqual(comment[0].content, '('*(i-1)+'foo'+')'*(i-1))
+            comment = comment[0]
+        self.assertEqual(comment.content, 'foo')
+
+    def test_get_comment_missing_end_of_nesting(self):
+        self._test_get_x(parser.get_comment,
+            '(((((foo)))', '(((((foo)))))', ' ',
+            [errors.InvalidHeaderDefect]*2, '', ['((((foo))))'])
+
+    def test_get_comment_qs_in_nested_comment(self):
+        comment = self._test_get_x(parser.get_comment,
+            '(foo (b\)))', '(foo (b\)))', ' ', [], '', ['foo (b\))'])
+        self.assertEqual(comment[2].content, 'b)')
+
+    # get_cfws
+
+    def test_get_cfws_only_ws(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            '  \t \t', '  \t \t', ' ', [], '', [])
+        self.assertEqual(cfws.token_type, 'cfws')
+
+    def test_get_cfws_only_comment(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            '(foo)', '(foo)', ' ', [], '', ['foo'])
+        self.assertEqual(cfws[0].content, 'foo')
+
+    def test_get_cfws_only_mixed(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            ' (foo )  ( bar) ', ' (foo )  ( bar) ', ' ', [], '',
+                ['foo ', ' bar'])
+        self.assertEqual(cfws[1].content, 'foo ')
+        self.assertEqual(cfws[3].content, ' bar')
+
+    def test_get_cfws_ends_at_non_leader(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            '(foo) bar', '(foo) ', ' ', [], 'bar', ['foo'])
+        self.assertEqual(cfws[0].content, 'foo')
+
+    def test_get_cfws_ends_at_non_printable(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            '(foo) \x07', '(foo) ', ' ', [], '\x07', ['foo'])
+        self.assertEqual(cfws[0].content, 'foo')
+
+    def test_get_cfws_non_printable_in_comment(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            '(foo \x07) "test"', '(foo \x07) ', ' ',
+            [errors.NonPrintableDefect], '"test"', ['foo \x07'])
+        self.assertEqual(cfws[0].content, 'foo \x07')
+
+    def test_get_cfws_header_ends_in_comment(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            '  (foo ', '  (foo )', ' ',
+            [errors.InvalidHeaderDefect], '', ['foo '])
+        self.assertEqual(cfws[1].content, 'foo ')
+
+    def test_get_cfws_multiple_nested_comments(self):
+        cfws = self._test_get_x(parser.get_cfws,
+            '(foo (bar)) ((a)(a))', '(foo (bar)) ((a)(a))', ' ', [],
+                '', ['foo (bar)', '(a)(a)'])
+        self.assertEqual(cfws[0].comments, ['foo (bar)'])
+        self.assertEqual(cfws[2].comments, ['(a)(a)'])
+
+    # get_quoted_string
+
+    def test_get_quoted_string_only(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            '"bob"', '"bob"', 'bob', [], '')
+        self.assertEqual(qs.token_type, 'quoted-string')
+        self.assertEqual(qs.quoted_value, '"bob"')
+        self.assertEqual(qs.content, 'bob')
+
+    def test_get_quoted_string_with_wsp(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            '\t "bob"  ', '\t "bob"  ', ' bob ', [], '')
+        self.assertEqual(qs.quoted_value, ' "bob" ')
+        self.assertEqual(qs.content, 'bob')
+
+    def test_get_quoted_string_with_comments_and_wsp(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            ' (foo) "bob"(bar)', ' (foo) "bob"(bar)', ' bob ', [], '')
+        self.assertEqual(qs[0][1].content, 'foo')
+        self.assertEqual(qs[2][0].content, 'bar')
+        self.assertEqual(qs.content, 'bob')
+        self.assertEqual(qs.quoted_value, ' "bob" ')
+
+    def test_get_quoted_string_with_multiple_comments(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            ' (foo) (bar) "bob"(bird)', ' (foo) (bar) "bob"(bird)', ' bob ',
+                [], '')
+        self.assertEqual(qs[0].comments, ['foo', 'bar'])
+        self.assertEqual(qs[2].comments, ['bird'])
+        self.assertEqual(qs.content, 'bob')
+        self.assertEqual(qs.quoted_value, ' "bob" ')
+
+    def test_get_quoted_string_non_printable_in_comment(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            ' (\x0A) "bob"', ' (\x0A) "bob"', ' bob',
+                [errors.NonPrintableDefect], '')
+        self.assertEqual(qs[0].comments, ['\x0A'])
+        self.assertEqual(qs.content, 'bob')
+        self.assertEqual(qs.quoted_value, ' "bob"')
+
+    def test_get_quoted_string_non_printable_in_qcontent(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            ' (a) "a\x0B"', ' (a) "a\x0B"', ' a\x0B',
+                [errors.NonPrintableDefect], '')
+        self.assertEqual(qs[0].comments, ['a'])
+        self.assertEqual(qs.content, 'a\x0B')
+        self.assertEqual(qs.quoted_value, ' "a\x0B"')
+
+    def test_get_quoted_string_internal_ws(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            ' (a) "foo  bar "', ' (a) "foo  bar "', ' foo  bar ',
+                [], '')
+        self.assertEqual(qs[0].comments, ['a'])
+        self.assertEqual(qs.content, 'foo  bar ')
+        self.assertEqual(qs.quoted_value, ' "foo  bar "')
+
+    def test_get_quoted_string_header_ends_in_comment(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            ' (a) "bob" (a', ' (a) "bob" (a)', ' bob ',
+                [errors.InvalidHeaderDefect], '')
+        self.assertEqual(qs[0].comments, ['a'])
+        self.assertEqual(qs[2].comments, ['a'])
+        self.assertEqual(qs.content, 'bob')
+        self.assertEqual(qs.quoted_value, ' "bob" ')
+
+    def test_get_quoted_string_header_ends_in_qcontent(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            ' (a) "bob', ' (a) "bob"', ' bob',
+                [errors.InvalidHeaderDefect], '')
+        self.assertEqual(qs[0].comments, ['a'])
+        self.assertEqual(qs.content, 'bob')
+        self.assertEqual(qs.quoted_value, ' "bob"')
+
+    def test_get_quoted_string_no_quoted_string(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_quoted_string(' (ab) xyz')
+
+    def test_get_quoted_string_qs_ends_at_noncfws(self):
+        qs = self._test_get_x(parser.get_quoted_string,
+            '\t "bob" fee', '\t "bob" ', ' bob ', [], 'fee')
+        self.assertEqual(qs.content, 'bob')
+        self.assertEqual(qs.quoted_value, ' "bob" ')
+
+    # get_atom
+
+    def test_get_atom_only(self):
+        atom = self._test_get_x(parser.get_atom,
+            'bob', 'bob', 'bob', [], '')
+        self.assertEqual(atom.token_type, 'atom')
+
+    def test_get_atom_with_wsp(self):
+        self._test_get_x(parser.get_atom,
+            '\t bob  ', '\t bob  ', ' bob ', [], '')
+
+    def test_get_atom_with_comments_and_wsp(self):
+        atom = self._test_get_x(parser.get_atom,
+            ' (foo) bob(bar)', ' (foo) bob(bar)', ' bob ', [], '')
+        self.assertEqual(atom[0][1].content, 'foo')
+        self.assertEqual(atom[2][0].content, 'bar')
+
+    def test_get_atom_with_multiple_comments(self):
+        atom = self._test_get_x(parser.get_atom,
+            ' (foo) (bar) bob(bird)', ' (foo) (bar) bob(bird)', ' bob ',
+                [], '')
+        self.assertEqual(atom[0].comments, ['foo', 'bar'])
+        self.assertEqual(atom[2].comments, ['bird'])
+
+    def test_get_atom_non_printable_in_comment(self):
+        atom = self._test_get_x(parser.get_atom,
+            ' (\x0A) bob', ' (\x0A) bob', ' bob',
+                [errors.NonPrintableDefect], '')
+        self.assertEqual(atom[0].comments, ['\x0A'])
+
+    def test_get_atom_non_printable_in_atext(self):
+        atom = self._test_get_x(parser.get_atom,
+            ' (a) a\x0B', ' (a) a\x0B', ' a\x0B',
+                [errors.NonPrintableDefect], '')
+        self.assertEqual(atom[0].comments, ['a'])
+
+    def test_get_atom_header_ends_in_comment(self):
+        atom = self._test_get_x(parser.get_atom,
+            ' (a) bob (a', ' (a) bob (a)', ' bob ',
+                [errors.InvalidHeaderDefect], '')
+        self.assertEqual(atom[0].comments, ['a'])
+        self.assertEqual(atom[2].comments, ['a'])
+
+    def test_get_atom_no_atom(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_atom(' (ab) ')
+
+    def test_get_atom_no_atom_before_special(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_atom(' (ab) @')
+
+    def test_get_atom_atom_ends_at_special(self):
+        atom = self._test_get_x(parser.get_atom,
+            ' (foo) bob(bar)  @bang', ' (foo) bob(bar)  ', ' bob ', [], '@bang')
+        self.assertEqual(atom[0].comments, ['foo'])
+        self.assertEqual(atom[2].comments, ['bar'])
+
+    def test_get_atom_atom_ends_at_noncfws(self):
+        atom = self._test_get_x(parser.get_atom,
+            'bob  fred', 'bob  ', 'bob ', [], 'fred')
+
+    # get_dot_atom_text
+
+    def test_get_dot_atom_text(self):
+        dot_atom_text = self._test_get_x(parser.get_dot_atom_text,
+            'foo.bar.bang', 'foo.bar.bang', 'foo.bar.bang', [], '')
+        self.assertEqual(dot_atom_text.token_type, 'dot-atom-text')
+        self.assertEqual(len(dot_atom_text), 5)
+
+    def test_get_dot_atom_text_lone_atom_is_valid(self):
+        dot_atom_text = self._test_get_x(parser.get_dot_atom_text,
+            'foo', 'foo', 'foo', [], '')
+
+    def test_get_dot_atom_text_raises_on_leading_dot(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom_text('.foo.bar')
+
+    def test_get_dot_atom_text_raises_on_trailing_dot(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom_text('foo.bar.')
+
+    def test_get_dot_atom_text_raises_on_leading_non_atext(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom_text(' foo.bar')
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom_text('@foo.bar')
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom_text('"foo.bar"')
+
+    def test_get_dot_atom_text_trailing_text_preserved(self):
+        dot_atom_text = self._test_get_x(parser.get_dot_atom_text,
+            'foo@bar', 'foo', 'foo', [], '@bar')
+
+    def test_get_dot_atom_text_trailing_ws_preserved(self):
+        dot_atom_text = self._test_get_x(parser.get_dot_atom_text,
+            'foo .bar', 'foo', 'foo', [], ' .bar')
+
+    # get_dot_atom
+
+    def test_get_dot_atom_only(self):
+        dot_atom = self._test_get_x(parser.get_dot_atom,
+            'foo.bar.bing', 'foo.bar.bing', 'foo.bar.bing', [], '')
+        self.assertEqual(dot_atom.token_type, 'dot-atom')
+        self.assertEqual(len(dot_atom), 1)
+
+    def test_get_dot_atom_with_wsp(self):
+        self._test_get_x(parser.get_dot_atom,
+            '\t  foo.bar.bing  ', '\t  foo.bar.bing  ', ' foo.bar.bing ', [], '')
+
+    def test_get_dot_atom_with_comments_and_wsp(self):
+        self._test_get_x(parser.get_dot_atom,
+            ' (sing)  foo.bar.bing (here) ', ' (sing)  foo.bar.bing (here) ',
+                ' foo.bar.bing ', [], '')
+
+    def test_get_dot_atom_space_ends_dot_atom(self):
+        self._test_get_x(parser.get_dot_atom,
+            ' (sing)  foo.bar .bing (here) ', ' (sing)  foo.bar ',
+                ' foo.bar ', [], '.bing (here) ')
+
+    def test_get_dot_atom_no_atom_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom(' (foo) ')
+
+    def test_get_dot_atom_leading_dot_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom(' (foo) .bar')
+
+    def test_get_dot_atom_two_dots_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom('bar..bang')
+
+    def test_get_dot_atom_trailing_dot_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_dot_atom(' (foo) bar.bang. foo')
+
+    # get_word (if this were black box we'd repeat all the qs/atom tests)
+
+    def test_get_word_atom_yields_atom(self):
+        word = self._test_get_x(parser.get_word,
+            ' (foo) bar (bang) :ah', ' (foo) bar (bang) ', ' bar ', [], ':ah')
+        self.assertEqual(word.token_type, 'atom')
+        self.assertEqual(word[0].token_type, 'cfws')
+
+    def test_get_word_qs_yields_qs(self):
+        word = self._test_get_x(parser.get_word,
+            '"bar " (bang) ah', '"bar " (bang) ', 'bar  ', [], 'ah')
+        self.assertEqual(word.token_type, 'quoted-string')
+        self.assertEqual(word[0].token_type, 'bare-quoted-string')
+        self.assertEqual(word[0].value, 'bar ')
+        self.assertEqual(word.content, 'bar ')
+
+    def test_get_word_ends_at_dot(self):
+        self._test_get_x(parser.get_word,
+            'foo.', 'foo', 'foo', [], '.')
+
+    # get_phrase
+
+    def test_get_phrase_simple(self):
+        phrase = self._test_get_x(parser.get_phrase,
+            '"Fred A. Johnson" is his name, oh.',
+            '"Fred A. Johnson" is his name',
+            'Fred A. Johnson is his name',
+            [],
+            ', oh.')
+        self.assertEqual(phrase.token_type, 'phrase')
+
+    def test_get_phrase_complex(self):
+        phrase = self._test_get_x(parser.get_phrase,
+            ' (A) bird (in (my|your)) "hand  " is messy\t<>\t',
+            ' (A) bird (in (my|your)) "hand  " is messy\t',
+            ' bird hand   is messy ',
+            [],
+            '<>\t')
+        self.assertEqual(phrase[0][0].comments, ['A'])
+        self.assertEqual(phrase[0][2].comments, ['in (my|your)'])
+
+    def test_get_phrase_obsolete(self):
+        phrase = self._test_get_x(parser.get_phrase,
+            'Fred A.(weird).O Johnson',
+            'Fred A.(weird).O Johnson',
+            'Fred A. .O Johnson',
+            [errors.ObsoleteHeaderDefect]*3,
+            '')
+        self.assertEqual(len(phrase), 7)
+        self.assertEqual(phrase[3].comments, ['weird'])
+
+    def test_get_phrase_pharse_must_start_with_word(self):
+        phrase = self._test_get_x(parser.get_phrase,
+            '(even weirder).name',
+            '(even weirder).name',
+            ' .name',
+            [errors.InvalidHeaderDefect] + [errors.ObsoleteHeaderDefect]*2,
+            '')
+        self.assertEqual(len(phrase), 3)
+        self.assertEqual(phrase[0].comments, ['even weirder'])
+
+    def test_get_phrase_ending_with_obsolete(self):
+        phrase = self._test_get_x(parser.get_phrase,
+            'simple phrase.(with trailing comment):boo',
+            'simple phrase.(with trailing comment)',
+            'simple phrase. ',
+            [errors.ObsoleteHeaderDefect]*2,
+            ':boo')
+        self.assertEqual(len(phrase), 4)
+        self.assertEqual(phrase[3].comments, ['with trailing comment'])
+
+    def get_phrase_cfws_only_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_phrase(' (foo) ')
+
+    # get_local_part
+
+    def test_get_local_part_simple(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            'dinsdale@python.org', 'dinsdale', 'dinsdale', [], '@python.org')
+        self.assertEqual(local_part.token_type, 'local-part')
+        self.assertEqual(local_part.local_part, 'dinsdale')
+
+    def test_get_local_part_with_dot(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            'Fred.A.Johnson@python.org',
+            'Fred.A.Johnson',
+            'Fred.A.Johnson',
+            [],
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred.A.Johnson')
+
+    def test_get_local_part_with_whitespace(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' Fred.A.Johnson  @python.org',
+            ' Fred.A.Johnson  ',
+            ' Fred.A.Johnson ',
+            [],
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred.A.Johnson')
+
+    def test_get_local_part_with_cfws(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' (foo) Fred.A.Johnson (bar (bird))  @python.org',
+            ' (foo) Fred.A.Johnson (bar (bird))  ',
+            ' Fred.A.Johnson ',
+            [],
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred.A.Johnson')
+        self.assertEqual(local_part[0][0].comments, ['foo'])
+        self.assertEqual(local_part[0][2].comments, ['bar (bird)'])
+
+    def test_get_local_part_simple_quoted(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            '"dinsdale"@python.org', '"dinsdale"', '"dinsdale"', [], '@python.org')
+        self.assertEqual(local_part.token_type, 'local-part')
+        self.assertEqual(local_part.local_part, 'dinsdale')
+
+    def test_get_local_part_with_quoted_dot(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            '"Fred.A.Johnson"@python.org',
+            '"Fred.A.Johnson"',
+            '"Fred.A.Johnson"',
+            [],
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred.A.Johnson')
+
+    def test_get_local_part_quoted_with_whitespace(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' "Fred A. Johnson"  @python.org',
+            ' "Fred A. Johnson"  ',
+            ' "Fred A. Johnson" ',
+            [],
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred A. Johnson')
+
+    def test_get_local_part_quoted_with_cfws(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' (foo) " Fred A. Johnson " (bar (bird))  @python.org',
+            ' (foo) " Fred A. Johnson " (bar (bird))  ',
+            ' " Fred A. Johnson " ',
+            [],
+            '@python.org')
+        self.assertEqual(local_part.local_part, ' Fred A. Johnson ')
+        self.assertEqual(local_part[0][0].comments, ['foo'])
+        self.assertEqual(local_part[0][2].comments, ['bar (bird)'])
+
+
+    def test_get_local_part_simple_obsolete(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            'Fred. A.Johnson@python.org',
+            'Fred. A.Johnson',
+            'Fred. A.Johnson',
+            [errors.ObsoleteHeaderDefect],
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred.A.Johnson')
+
+    def test_get_local_part_complex_obsolete_1(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' (foo )Fred (bar).(bird) A.(sheep)Johnson."and  dogs "@python.org',
+            ' (foo )Fred (bar).(bird) A.(sheep)Johnson."and  dogs "',
+            ' Fred . A. Johnson.and  dogs ',
+            [errors.ObsoleteHeaderDefect],
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred.A.Johnson.and  dogs ')
+
+    def test_get_local_part_complex_obsolete_invalid(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' (foo )Fred (bar).(bird) A.(sheep)Johnson "and  dogs"@python.org',
+            ' (foo )Fred (bar).(bird) A.(sheep)Johnson "and  dogs"',
+            ' Fred . A. Johnson and  dogs',
+            [errors.InvalidHeaderDefect]*2,
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'Fred.A.Johnson and  dogs')
+
+    def test_get_local_part_no_part_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_local_part(' (foo) ')
+
+    def test_get_local_part_special_instead_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_local_part(' (foo) @python.org')
+
+    def test_get_local_part_trailing_dot(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' borris.@python.org',
+            ' borris.',
+            ' borris.',
+            [errors.InvalidHeaderDefect]*2,
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'borris.')
+
+    def test_get_local_part_trailing_dot_with_ws(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' borris. @python.org',
+            ' borris. ',
+            ' borris. ',
+            [errors.InvalidHeaderDefect]*2,
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'borris.')
+
+    def test_get_local_part_leading_dot(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            '.borris@python.org',
+            '.borris',
+            '.borris',
+            [errors.InvalidHeaderDefect]*2,
+            '@python.org')
+        self.assertEqual(local_part.local_part, '.borris')
+
+    def test_get_local_part_leading_dot_after_ws(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' .borris@python.org',
+            ' .borris',
+            ' .borris',
+            [errors.InvalidHeaderDefect]*2,
+            '@python.org')
+        self.assertEqual(local_part.local_part, '.borris')
+
+    def test_get_local_part_double_dot_raises(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            ' borris.(foo).natasha@python.org',
+            ' borris.(foo).natasha',
+            ' borris. .natasha',
+            [errors.InvalidHeaderDefect]*2,
+            '@python.org')
+        self.assertEqual(local_part.local_part, 'borris..natasha')
+
+    def test_get_local_part_quoted_strings_in_atom_list(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            '""example" example"@example.com',
+            '""example" example"',
+            'example example',
+            [errors.InvalidHeaderDefect]*3,
+            '@example.com')
+        self.assertEqual(local_part.local_part, 'example example')
+
+    def test_get_local_part_valid_and_invalid_qp_in_atom_list(self):
+        local_part = self._test_get_x(parser.get_local_part,
+            r'"\\"example\\" example"@example.com',
+            r'"\\"example\\" example"',
+            r'\example\\ example',
+            [errors.InvalidHeaderDefect]*5,
+            '@example.com')
+        self.assertEqual(local_part.local_part, r'\example\\ example')
+
+    def test_get_local_part_unicode_defect(self):
+        # Currently this only happens when parsing unicode, not when parsing
+        # stuff that was originally binary.
+        local_part = self._test_get_x(parser.get_local_part,
+            'exámple@example.com',
+            'exámple',
+            'exámple',
+            [errors.NonASCIILocalPartDefect],
+            '@example.com')
+        self.assertEqual(local_part.local_part, 'exámple')
+
+    # get_dtext
+
+    def test_get_dtext_only(self):
+        dtext = self._test_get_x(parser.get_dtext,
+                                'foobar', 'foobar', 'foobar', [], '')
+        self.assertEqual(dtext.token_type, 'ptext')
+
+    def test_get_dtext_all_dtext(self):
+        dtext = self._test_get_x(parser.get_dtext, self.rfc_dtext_chars,
+                                 self.rfc_dtext_chars,
+                                 self.rfc_dtext_chars, [], '')
+
+    def test_get_dtext_two_words_gets_first(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo bar', 'foo', 'foo', [], ' bar')
+
+    def test_get_dtext_following_wsp_preserved(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo \t\tbar', 'foo', 'foo', [], ' \t\tbar')
+
+    def test_get_dtext_non_printables(self):
+        dtext = self._test_get_x(parser.get_dtext,
+                                'foo\x00bar]', 'foo\x00bar', 'foo\x00bar',
+                                [errors.NonPrintableDefect], ']')
+        self.assertEqual(dtext.defects[0].non_printables[0], '\x00')
+
+    def test_get_dtext_with_qp(self):
+        ptext = self._test_get_x(parser.get_dtext,
+                                 r'foo\]\[\\bar\b\e\l\l',
+                                 r'foo][\barbell',
+                                 r'foo][\barbell',
+                                 [errors.ObsoleteHeaderDefect],
+                                 '')
+
+    def test_get_dtext_up_to_close_bracket_only(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo]', 'foo', 'foo', [], ']')
+
+    def test_get_dtext_wsp_before_close_bracket_preserved(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo  ]', 'foo', 'foo', [], '  ]')
+
+    def test_get_dtext_close_bracket_mid_word(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo]bar', 'foo', 'foo', [], ']bar')
+
+    def test_get_dtext_up_to_open_bracket_only(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo[', 'foo', 'foo', [], '[')
+
+    def test_get_dtext_wsp_before_open_bracket_preserved(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo  [', 'foo', 'foo', [], '  [')
+
+    def test_get_dtext_open_bracket_mid_word(self):
+        self._test_get_x(parser.get_dtext,
+                        'foo[bar', 'foo', 'foo', [], '[bar')
+
+    # get_domain_literal
+
+    def test_get_domain_literal_only(self):
+        domain_literal = domain_literal = self._test_get_x(parser.get_domain_literal,
+                                '[127.0.0.1]',
+                                '[127.0.0.1]',
+                                '[127.0.0.1]',
+                                [],
+                                '')
+        self.assertEqual(domain_literal.token_type, 'domain-literal')
+        self.assertEqual(domain_literal.domain, '[127.0.0.1]')
+        self.assertEqual(domain_literal.ip, '127.0.0.1')
+
+    def test_get_domain_literal_with_internal_ws(self):
+        domain_literal = self._test_get_x(parser.get_domain_literal,
+                                '[  127.0.0.1\t ]',
+                                '[  127.0.0.1\t ]',
+                                '[ 127.0.0.1 ]',
+                                [],
+                                '')
+        self.assertEqual(domain_literal.domain, '[127.0.0.1]')
+        self.assertEqual(domain_literal.ip, '127.0.0.1')
+
+    def test_get_domain_literal_with_surrounding_cfws(self):
+        domain_literal = self._test_get_x(parser.get_domain_literal,
+                                '(foo)[  127.0.0.1] (bar)',
+                                '(foo)[  127.0.0.1] (bar)',
+                                ' [ 127.0.0.1] ',
+                                [],
+                                '')
+        self.assertEqual(domain_literal.domain, '[127.0.0.1]')
+        self.assertEqual(domain_literal.ip, '127.0.0.1')
+
+    def test_get_domain_literal_no_start_char_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_domain_literal('(foo) ')
+
+    def test_get_domain_literal_no_start_char_before_special_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_domain_literal('(foo) @')
+
+    def test_get_domain_literal_bad_dtext_char_before_special_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_domain_literal('(foo) [abc[@')
+
+    # get_domain
+
+    def test_get_domain_regular_domain_only(self):
+        domain = self._test_get_x(parser.get_domain,
+                                  'example.com',
+                                  'example.com',
+                                  'example.com',
+                                  [],
+                                  '')
+        self.assertEqual(domain.token_type, 'domain')
+        self.assertEqual(domain.domain, 'example.com')
+
+    def test_get_domain_domain_literal_only(self):
+        domain = self._test_get_x(parser.get_domain,
+                                  '[127.0.0.1]',
+                                  '[127.0.0.1]',
+                                  '[127.0.0.1]',
+                                  [],
+                                  '')
+        self.assertEqual(domain.token_type, 'domain')
+        self.assertEqual(domain.domain, '[127.0.0.1]')
+
+    def test_get_domain_with_cfws(self):
+        domain = self._test_get_x(parser.get_domain,
+                                  '(foo) example.com(bar)\t',
+                                  '(foo) example.com(bar)\t',
+                                  ' example.com ',
+                                  [],
+                                  '')
+        self.assertEqual(domain.domain, 'example.com')
+
+    def test_get_domain_domain_literal_with_cfws(self):
+        domain = self._test_get_x(parser.get_domain,
+                                  '(foo)[127.0.0.1]\t(bar)',
+                                  '(foo)[127.0.0.1]\t(bar)',
+                                  ' [127.0.0.1] ',
+                                  [],
+                                  '')
+        self.assertEqual(domain.domain, '[127.0.0.1]')
+
+    def test_get_domain_domain_with_cfws_ends_at_special(self):
+        domain = self._test_get_x(parser.get_domain,
+                                  '(foo)example.com\t(bar), next',
+                                  '(foo)example.com\t(bar)',
+                                  ' example.com ',
+                                  [],
+                                  ', next')
+        self.assertEqual(domain.domain, 'example.com')
+
+    def test_get_domain_domain_literal_with_cfws_ends_at_special(self):
+        domain = self._test_get_x(parser.get_domain,
+                                  '(foo)[127.0.0.1]\t(bar), next',
+                                  '(foo)[127.0.0.1]\t(bar)',
+                                  ' [127.0.0.1] ',
+                                  [],
+                                  ', next')
+        self.assertEqual(domain.domain, '[127.0.0.1]')
+
+    def test_get_domain_obsolete(self):
+        domain = self._test_get_x(parser.get_domain,
+                                  '(foo) example . (bird)com(bar)\t',
+                                  '(foo) example . (bird)com(bar)\t',
+                                  ' example . com ',
+                                  [errors.ObsoleteHeaderDefect],
+                                  '')
+        self.assertEqual(domain.domain, 'example.com')
+
+    def test_get_domain_no_non_cfws_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_domain("  (foo)\t")
+
+    def test_get_domain_no_atom_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_domain("  (foo)\t, broken")
+
+
+    # get_addr_spec
+
+    def test_get_addr_spec_normal(self):
+        addr_spec = self._test_get_x(parser.get_addr_spec,
+                                    'dinsdale@example.com',
+                                    'dinsdale@example.com',
+                                    'dinsdale@example.com',
+                                    [],
+                                    '')
+        self.assertEqual(addr_spec.token_type, 'addr-spec')
+        self.assertEqual(addr_spec.local_part, 'dinsdale')
+        self.assertEqual(addr_spec.domain, 'example.com')
+        self.assertEqual(addr_spec.addr_spec, 'dinsdale@example.com')
+
+    def test_get_addr_spec_with_doamin_literal(self):
+        addr_spec = self._test_get_x(parser.get_addr_spec,
+                                    'dinsdale@[127.0.0.1]',
+                                    'dinsdale@[127.0.0.1]',
+                                    'dinsdale@[127.0.0.1]',
+                                    [],
+                                    '')
+        self.assertEqual(addr_spec.local_part, 'dinsdale')
+        self.assertEqual(addr_spec.domain, '[127.0.0.1]')
+        self.assertEqual(addr_spec.addr_spec, 'dinsdale@[127.0.0.1]')
+
+    def test_get_addr_spec_with_cfws(self):
+        addr_spec = self._test_get_x(parser.get_addr_spec,
+                '(foo) dinsdale(bar)@ (bird) example.com (bog)',
+                '(foo) dinsdale(bar)@ (bird) example.com (bog)',
+                ' dinsdale@example.com ',
+                [],
+                '')
+        self.assertEqual(addr_spec.local_part, 'dinsdale')
+        self.assertEqual(addr_spec.domain, 'example.com')
+        self.assertEqual(addr_spec.addr_spec, 'dinsdale@example.com')
+
+    def test_get_addr_spec_with_qouoted_string_and_cfws(self):
+        addr_spec = self._test_get_x(parser.get_addr_spec,
+                '(foo) "roy a bug"(bar)@ (bird) example.com (bog)',
+                '(foo) "roy a bug"(bar)@ (bird) example.com (bog)',
+                ' "roy a bug"@example.com ',
+                [],
+                '')
+        self.assertEqual(addr_spec.local_part, 'roy a bug')
+        self.assertEqual(addr_spec.domain, 'example.com')
+        self.assertEqual(addr_spec.addr_spec, '"roy a bug"@example.com')
+
+    def test_get_addr_spec_ends_at_special(self):
+        addr_spec = self._test_get_x(parser.get_addr_spec,
+                '(foo) "roy a bug"(bar)@ (bird) example.com (bog) , next',
+                '(foo) "roy a bug"(bar)@ (bird) example.com (bog) ',
+                ' "roy a bug"@example.com ',
+                [],
+                ', next')
+        self.assertEqual(addr_spec.local_part, 'roy a bug')
+        self.assertEqual(addr_spec.domain, 'example.com')
+        self.assertEqual(addr_spec.addr_spec, '"roy a bug"@example.com')
+
+    def test_get_addr_spec_quoted_strings_in_atom_list(self):
+        addr_spec = self._test_get_x(parser.get_addr_spec,
+            '""example" example"@example.com',
+            '""example" example"@example.com',
+            'example example@example.com',
+            [errors.InvalidHeaderDefect]*3,
+            '')
+        self.assertEqual(addr_spec.local_part, 'example example')
+        self.assertEqual(addr_spec.domain, 'example.com')
+        self.assertEqual(addr_spec.addr_spec, '"example example"@example.com')
+
+    def test_get_addr_spec_dot_atom(self):
+        addr_spec = self._test_get_x(parser.get_addr_spec,
+            'star.a.star@example.com',
+            'star.a.star@example.com',
+            'star.a.star@example.com',
+            [],
+            '')
+        self.assertEqual(addr_spec.local_part, 'star.a.star')
+        self.assertEqual(addr_spec.domain, 'example.com')
+        self.assertEqual(addr_spec.addr_spec, 'star.a.star@example.com')
+
+    # get_obs_route
+
+    def test_get_obs_route_simple(self):
+        obs_route = self._test_get_x(parser.get_obs_route,
+            '@example.com, @two.example.com:',
+            '@example.com, @two.example.com:',
+            '@example.com, @two.example.com:',
+            [],
+            '')
+        self.assertEqual(obs_route.token_type, 'obs-route')
+        self.assertEqual(obs_route.domains, ['example.com', 'two.example.com'])
+
+    def test_get_obs_route_complex(self):
+        obs_route = self._test_get_x(parser.get_obs_route,
+            '(foo),, (blue)@example.com (bar),@two.(foo) example.com (bird):',
+            '(foo),, (blue)@example.com (bar),@two.(foo) example.com (bird):',
+            ' ,, @example.com ,@two. example.com :',
+            [errors.ObsoleteHeaderDefect],  # This is the obs-domain
+            '')
+        self.assertEqual(obs_route.token_type, 'obs-route')
+        self.assertEqual(obs_route.domains, ['example.com', 'two.example.com'])
+
+    def test_get_obs_route_no_route_before_end_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_obs_route('(foo) @example.com,')
+
+    def test_get_obs_route_no_route_before_special_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_obs_route('(foo) [abc],')
+
+    def test_get_obs_route_no_route_before_special_raises2(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_obs_route('(foo) @example.com [abc],')
+
+    # get_angle_addr
+
+    def test_get_angle_addr_simple(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '<dinsdale@example.com>',
+            '<dinsdale@example.com>',
+            '<dinsdale@example.com>',
+            [],
+            '')
+        self.assertEqual(angle_addr.token_type, 'angle-addr')
+        self.assertEqual(angle_addr.local_part, 'dinsdale')
+        self.assertEqual(angle_addr.domain, 'example.com')
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_angle_addr_empty(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '<>',
+            '<>',
+            '<>',
+            [errors.InvalidHeaderDefect],
+            '')
+        self.assertEqual(angle_addr.token_type, 'angle-addr')
+        self.assertIsNone(angle_addr.local_part)
+        self.assertIsNone(angle_addr.domain)
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, '<>')
+
+    def test_get_angle_addr_with_cfws(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            ' (foo) <dinsdale@example.com>(bar)',
+            ' (foo) <dinsdale@example.com>(bar)',
+            ' <dinsdale@example.com> ',
+            [],
+            '')
+        self.assertEqual(angle_addr.token_type, 'angle-addr')
+        self.assertEqual(angle_addr.local_part, 'dinsdale')
+        self.assertEqual(angle_addr.domain, 'example.com')
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_angle_addr_qs_and_domain_literal(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '<"Fred Perfect"@[127.0.0.1]>',
+            '<"Fred Perfect"@[127.0.0.1]>',
+            '<"Fred Perfect"@[127.0.0.1]>',
+            [],
+            '')
+        self.assertEqual(angle_addr.local_part, 'Fred Perfect')
+        self.assertEqual(angle_addr.domain, '[127.0.0.1]')
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, '"Fred Perfect"@[127.0.0.1]')
+
+    def test_get_angle_addr_internal_cfws(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '<(foo) dinsdale@example.com(bar)>',
+            '<(foo) dinsdale@example.com(bar)>',
+            '< dinsdale@example.com >',
+            [],
+            '')
+        self.assertEqual(angle_addr.local_part, 'dinsdale')
+        self.assertEqual(angle_addr.domain, 'example.com')
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_angle_addr_obs_route(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '(foo)<@example.com, (bird) @two.example.com: dinsdale@example.com> (bar) ',
+            '(foo)<@example.com, (bird) @two.example.com: dinsdale@example.com> (bar) ',
+            ' <@example.com, @two.example.com: dinsdale@example.com> ',
+            [errors.ObsoleteHeaderDefect],
+            '')
+        self.assertEqual(angle_addr.local_part, 'dinsdale')
+        self.assertEqual(angle_addr.domain, 'example.com')
+        self.assertEqual(angle_addr.route, ['example.com', 'two.example.com'])
+        self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_angle_addr_missing_closing_angle(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '<dinsdale@example.com',
+            '<dinsdale@example.com>',
+            '<dinsdale@example.com>',
+            [errors.InvalidHeaderDefect],
+            '')
+        self.assertEqual(angle_addr.local_part, 'dinsdale')
+        self.assertEqual(angle_addr.domain, 'example.com')
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_angle_addr_missing_closing_angle_with_cfws(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '<dinsdale@example.com (foo)',
+            '<dinsdale@example.com (foo)>',
+            '<dinsdale@example.com >',
+            [errors.InvalidHeaderDefect],
+            '')
+        self.assertEqual(angle_addr.local_part, 'dinsdale')
+        self.assertEqual(angle_addr.domain, 'example.com')
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_angle_addr_ends_at_special(self):
+        angle_addr = self._test_get_x(parser.get_angle_addr,
+            '<dinsdale@example.com> (foo), next',
+            '<dinsdale@example.com> (foo)',
+            '<dinsdale@example.com> ',
+            [],
+            ', next')
+        self.assertEqual(angle_addr.local_part, 'dinsdale')
+        self.assertEqual(angle_addr.domain, 'example.com')
+        self.assertIsNone(angle_addr.route)
+        self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_angle_addr_no_angle_raise(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_angle_addr('(foo) ')
+
+    def test_get_angle_addr_no_angle_before_special_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_angle_addr('(foo) , next')
+
+    def test_get_angle_addr_no_angle_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_angle_addr('bar')
+
+    def test_get_angle_addr_special_after_angle_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_angle_addr('(foo) <, bar')
+
+    # get_display_name  This is phrase but with a different value.
+
+    def test_get_display_name_simple(self):
+        display_name = self._test_get_x(parser.get_display_name,
+            'Fred A Johnson',
+            'Fred A Johnson',
+            'Fred A Johnson',
+            [],
+            '')
+        self.assertEqual(display_name.token_type, 'display-name')
+        self.assertEqual(display_name.display_name, 'Fred A Johnson')
+
+    def test_get_display_name_complex1(self):
+        display_name = self._test_get_x(parser.get_display_name,
+            '"Fred A. Johnson" is his name, oh.',
+            '"Fred A. Johnson" is his name',
+            '"Fred A. Johnson is his name"',
+            [],
+            ', oh.')
+        self.assertEqual(display_name.token_type, 'display-name')
+        self.assertEqual(display_name.display_name, 'Fred A. Johnson is his name')
+
+    def test_get_display_name_complex2(self):
+        display_name = self._test_get_x(parser.get_display_name,
+            ' (A) bird (in (my|your)) "hand  " is messy\t<>\t',
+            ' (A) bird (in (my|your)) "hand  " is messy\t',
+            ' "bird hand   is messy" ',
+            [],
+            '<>\t')
+        self.assertEqual(display_name[0][0].comments, ['A'])
+        self.assertEqual(display_name[0][2].comments, ['in (my|your)'])
+        self.assertEqual(display_name.display_name, 'bird hand   is messy')
+
+    def test_get_display_name_obsolete(self):
+        display_name = self._test_get_x(parser.get_display_name,
+            'Fred A.(weird).O Johnson',
+            'Fred A.(weird).O Johnson',
+            '"Fred A. .O Johnson"',
+            [errors.ObsoleteHeaderDefect]*3,
+            '')
+        self.assertEqual(len(display_name), 7)
+        self.assertEqual(display_name[3].comments, ['weird'])
+        self.assertEqual(display_name.display_name, 'Fred A. .O Johnson')
+
+    def test_get_display_name_pharse_must_start_with_word(self):
+        display_name = self._test_get_x(parser.get_display_name,
+            '(even weirder).name',
+            '(even weirder).name',
+            ' ".name"',
+            [errors.InvalidHeaderDefect] + [errors.ObsoleteHeaderDefect]*2,
+            '')
+        self.assertEqual(len(display_name), 3)
+        self.assertEqual(display_name[0].comments, ['even weirder'])
+        self.assertEqual(display_name.display_name, '.name')
+
+    def test_get_display_name_ending_with_obsolete(self):
+        display_name = self._test_get_x(parser.get_display_name,
+            'simple phrase.(with trailing comment):boo',
+            'simple phrase.(with trailing comment)',
+            '"simple phrase." ',
+            [errors.ObsoleteHeaderDefect]*2,
+            ':boo')
+        self.assertEqual(len(display_name), 4)
+        self.assertEqual(display_name[3].comments, ['with trailing comment'])
+        self.assertEqual(display_name.display_name, 'simple phrase.')
+
+    # get_name_addr
+
+    def test_get_name_addr_angle_addr_only(self):
+        name_addr = self._test_get_x(parser.get_name_addr,
+            '<dinsdale@example.com>',
+            '<dinsdale@example.com>',
+            '<dinsdale@example.com>',
+            [],
+            '')
+        self.assertEqual(name_addr.token_type, 'name-addr')
+        self.assertIsNone(name_addr.display_name)
+        self.assertEqual(name_addr.local_part, 'dinsdale')
+        self.assertEqual(name_addr.domain, 'example.com')
+        self.assertIsNone(name_addr.route)
+        self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_name_addr_atom_name(self):
+        name_addr = self._test_get_x(parser.get_name_addr,
+            'Dinsdale <dinsdale@example.com>',
+            'Dinsdale <dinsdale@example.com>',
+            'Dinsdale <dinsdale@example.com>',
+            [],
+            '')
+        self.assertEqual(name_addr.token_type, 'name-addr')
+        self.assertEqual(name_addr.display_name, 'Dinsdale')
+        self.assertEqual(name_addr.local_part, 'dinsdale')
+        self.assertEqual(name_addr.domain, 'example.com')
+        self.assertIsNone(name_addr.route)
+        self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_name_addr_atom_name_with_cfws(self):
+        name_addr = self._test_get_x(parser.get_name_addr,
+            '(foo) Dinsdale (bar) <dinsdale@example.com> (bird)',
+            '(foo) Dinsdale (bar) <dinsdale@example.com> (bird)',
+            ' Dinsdale <dinsdale@example.com> ',
+            [],
+            '')
+        self.assertEqual(name_addr.display_name, 'Dinsdale')
+        self.assertEqual(name_addr.local_part, 'dinsdale')
+        self.assertEqual(name_addr.domain, 'example.com')
+        self.assertIsNone(name_addr.route)
+        self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_name_addr_name_with_cfws_and_dots(self):
+        name_addr = self._test_get_x(parser.get_name_addr,
+            '(foo) Roy.A.Bear (bar) <dinsdale@example.com> (bird)',
+            '(foo) Roy.A.Bear (bar) <dinsdale@example.com> (bird)',
+            ' "Roy.A.Bear" <dinsdale@example.com> ',
+            [errors.ObsoleteHeaderDefect]*2,
+            '')
+        self.assertEqual(name_addr.display_name, 'Roy.A.Bear')
+        self.assertEqual(name_addr.local_part, 'dinsdale')
+        self.assertEqual(name_addr.domain, 'example.com')
+        self.assertIsNone(name_addr.route)
+        self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_name_addr_qs_name(self):
+        name_addr = self._test_get_x(parser.get_name_addr,
+            '"Roy.A.Bear" <dinsdale@example.com>',
+            '"Roy.A.Bear" <dinsdale@example.com>',
+            '"Roy.A.Bear" <dinsdale@example.com>',
+            [],
+            '')
+        self.assertEqual(name_addr.display_name, 'Roy.A.Bear')
+        self.assertEqual(name_addr.local_part, 'dinsdale')
+        self.assertEqual(name_addr.domain, 'example.com')
+        self.assertIsNone(name_addr.route)
+        self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_name_addr_with_route(self):
+        name_addr = self._test_get_x(parser.get_name_addr,
+            '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>',
+            '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>',
+            '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>',
+            [errors.ObsoleteHeaderDefect],
+            '')
+        self.assertEqual(name_addr.display_name, 'Roy.A.Bear')
+        self.assertEqual(name_addr.local_part, 'dinsdale')
+        self.assertEqual(name_addr.domain, 'example.com')
+        self.assertEqual(name_addr.route, ['two.example.com'])
+        self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_name_addr_ends_at_special(self):
+        name_addr = self._test_get_x(parser.get_name_addr,
+            '"Roy.A.Bear" <dinsdale@example.com>, next',
+            '"Roy.A.Bear" <dinsdale@example.com>',
+            '"Roy.A.Bear" <dinsdale@example.com>',
+            [],
+            ', next')
+        self.assertEqual(name_addr.display_name, 'Roy.A.Bear')
+        self.assertEqual(name_addr.local_part, 'dinsdale')
+        self.assertEqual(name_addr.domain, 'example.com')
+        self.assertIsNone(name_addr.route)
+        self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com')
+
+    def test_get_name_addr_no_content_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_name_addr(' (foo) ')
+
+    def test_get_name_addr_no_content_before_special_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_name_addr(' (foo) ,')
+
+    def test_get_name_addr_no_angle_after_display_name_raises(self):
+        with self.assertRaises(errors.HeaderParseError):
+            parser.get_name_addr('foo bar')
+
+    # get_mailbox
+
+    def test_get_mailbox_addr_spec_only(self):
+        mailbox = self._test_get_x(parser.get_mailbox,
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            [],
+            '')
+        self.assertEqual(mailbox.token_type, 'mailbox')
+        self.assertIsNone(mailbox.display_name)
+        self.assertEqual(mailbox.local_part, 'dinsdale')
+        self.assertEqual(mailbox.domain, 'example.com')
+        self.assertIsNone(mailbox.route)
+        self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com')
+
+    def test_get_mailbox_angle_addr_only(self):
+        mailbox = self._test_get_x(parser.get_mailbox,
+            '<dinsdale@example.com>',
+            '<dinsdale@example.com>',
+            '<dinsdale@example.com>',
+            [],
+            '')
+        self.assertEqual(mailbox.token_type, 'mailbox')
+        self.assertIsNone(mailbox.display_name)
+        self.assertEqual(mailbox.local_part, 'dinsdale')
+        self.assertEqual(mailbox.domain, 'example.com')
+        self.assertIsNone(mailbox.route)
+        self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com')
+
+    def test_get_mailbox_name_addr(self):
+        mailbox = self._test_get_x(parser.get_mailbox,
+            '"Roy A. Bear" <dinsdale@example.com>',
+            '"Roy A. Bear" <dinsdale@example.com>',
+            '"Roy A. Bear" <dinsdale@example.com>',
+            [],
+            '')
+        self.assertEqual(mailbox.token_type, 'mailbox')
+        self.assertEqual(mailbox.display_name, 'Roy A. Bear')
+        self.assertEqual(mailbox.local_part, 'dinsdale')
+        self.assertEqual(mailbox.domain, 'example.com')
+        self.assertIsNone(mailbox.route)
+        self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com')
+
+    def test_get_mailbox_ends_at_special(self):
+        mailbox = self._test_get_x(parser.get_mailbox,
+            '"Roy A. Bear" <dinsdale@example.com>, rest',
+            '"Roy A. Bear" <dinsdale@example.com>',
+            '"Roy A. Bear" <dinsdale@example.com>',
+            [],
+            ', rest')
+        self.assertEqual(mailbox.token_type, 'mailbox')
+        self.assertEqual(mailbox.display_name, 'Roy A. Bear')
+        self.assertEqual(mailbox.local_part, 'dinsdale')
+        self.assertEqual(mailbox.domain, 'example.com')
+        self.assertIsNone(mailbox.route)
+        self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com')
+
+    def test_get_mailbox_quoted_strings_in_atom_list(self):
+        mailbox = self._test_get_x(parser.get_mailbox,
+            '""example" example"@example.com',
+            '""example" example"@example.com',
+            'example example@example.com',
+            [errors.InvalidHeaderDefect]*3,
+            '')
+        self.assertEqual(mailbox.local_part, 'example example')
+        self.assertEqual(mailbox.domain, 'example.com')
+        self.assertEqual(mailbox.addr_spec, '"example example"@example.com')
+
+    # get_mailbox_list
+
+    def test_get_mailbox_list_single_addr(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            [],
+            '')
+        self.assertEqual(mailbox_list.token_type, 'mailbox-list')
+        self.assertEqual(len(mailbox_list.mailboxes), 1)
+        mailbox = mailbox_list.mailboxes[0]
+        self.assertIsNone(mailbox.display_name)
+        self.assertEqual(mailbox.local_part, 'dinsdale')
+        self.assertEqual(mailbox.domain, 'example.com')
+        self.assertIsNone(mailbox.route)
+        self.assertEqual(mailbox.addr_spec, 'dinsdale@example.com')
+        self.assertEqual(mailbox_list.mailboxes,
+                         mailbox_list.all_mailboxes)
+
+    def test_get_mailbox_list_two_simple_addr(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            'dinsdale@example.com, dinsdale@test.example.com',
+            'dinsdale@example.com, dinsdale@test.example.com',
+            'dinsdale@example.com, dinsdale@test.example.com',
+            [],
+            '')
+        self.assertEqual(mailbox_list.token_type, 'mailbox-list')
+        self.assertEqual(len(mailbox_list.mailboxes), 2)
+        self.assertEqual(mailbox_list.mailboxes[0].addr_spec,
+                        'dinsdale@example.com')
+        self.assertEqual(mailbox_list.mailboxes[1].addr_spec,
+                        'dinsdale@test.example.com')
+        self.assertEqual(mailbox_list.mailboxes,
+                         mailbox_list.all_mailboxes)
+
+    def test_get_mailbox_list_two_name_addr(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            ('"Roy A. Bear" <dinsdale@example.com>,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            [],
+            '')
+        self.assertEqual(len(mailbox_list.mailboxes), 2)
+        self.assertEqual(mailbox_list.mailboxes[0].addr_spec,
+                        'dinsdale@example.com')
+        self.assertEqual(mailbox_list.mailboxes[0].display_name,
+                        'Roy A. Bear')
+        self.assertEqual(mailbox_list.mailboxes[1].addr_spec,
+                        'dinsdale@test.example.com')
+        self.assertEqual(mailbox_list.mailboxes[1].display_name,
+                        'Fred Flintstone')
+        self.assertEqual(mailbox_list.mailboxes,
+                         mailbox_list.all_mailboxes)
+
+    def test_get_mailbox_list_two_complex(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            ('(foo) "Roy A. Bear" <dinsdale@example.com>(bar),'
+                ' "Fred Flintstone" <dinsdale@test.(bird)example.com>'),
+            ('(foo) "Roy A. Bear" <dinsdale@example.com>(bar),'
+                ' "Fred Flintstone" <dinsdale@test.(bird)example.com>'),
+            (' "Roy A. Bear" <dinsdale@example.com> ,'
+                ' "Fred Flintstone" <dinsdale@test. example.com>'),
+            [errors.ObsoleteHeaderDefect],
+            '')
+        self.assertEqual(len(mailbox_list.mailboxes), 2)
+        self.assertEqual(mailbox_list.mailboxes[0].addr_spec,
+                        'dinsdale@example.com')
+        self.assertEqual(mailbox_list.mailboxes[0].display_name,
+                        'Roy A. Bear')
+        self.assertEqual(mailbox_list.mailboxes[1].addr_spec,
+                        'dinsdale@test.example.com')
+        self.assertEqual(mailbox_list.mailboxes[1].display_name,
+                        'Fred Flintstone')
+        self.assertEqual(mailbox_list.mailboxes,
+                         mailbox_list.all_mailboxes)
+
+    def test_get_mailbox_list_unparseable_mailbox_null(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            ('"Roy A. Bear"[] dinsdale@example.com,'
+                ' "Fred Flintstone" <dinsdale@test.(bird)example.com>'),
+            ('"Roy A. Bear"[] dinsdale@example.com,'
+                ' "Fred Flintstone" <dinsdale@test.(bird)example.com>'),
+            ('"Roy A. Bear"[] dinsdale@example.com,'
+                ' "Fred Flintstone" <dinsdale@test. example.com>'),
+            [errors.InvalidHeaderDefect,   # the 'extra' text after the local part
+             errors.InvalidHeaderDefect,   # the local part with no angle-addr
+             errors.ObsoleteHeaderDefect,  # period in extra text (example.com)
+             errors.ObsoleteHeaderDefect], # (bird) in valid address.
+            '')
+        self.assertEqual(len(mailbox_list.mailboxes), 1)
+        self.assertEqual(len(mailbox_list.all_mailboxes), 2)
+        self.assertEqual(mailbox_list.all_mailboxes[0].token_type,
+                        'invalid-mailbox')
+        self.assertIsNone(mailbox_list.all_mailboxes[0].display_name)
+        self.assertEqual(mailbox_list.all_mailboxes[0].local_part,
+                        'Roy A. Bear')
+        self.assertIsNone(mailbox_list.all_mailboxes[0].domain)
+        self.assertEqual(mailbox_list.all_mailboxes[0].addr_spec,
+                        '"Roy A. Bear"')
+        self.assertIs(mailbox_list.all_mailboxes[1],
+                        mailbox_list.mailboxes[0])
+        self.assertEqual(mailbox_list.mailboxes[0].addr_spec,
+                        'dinsdale@test.example.com')
+        self.assertEqual(mailbox_list.mailboxes[0].display_name,
+                        'Fred Flintstone')
+
+    def test_get_mailbox_list_junk_after_valid_address(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            ('"Roy A. Bear" <dinsdale@example.com>@@,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>@@,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>@@,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            [errors.InvalidHeaderDefect],
+            '')
+        self.assertEqual(len(mailbox_list.mailboxes), 1)
+        self.assertEqual(len(mailbox_list.all_mailboxes), 2)
+        self.assertEqual(mailbox_list.all_mailboxes[0].addr_spec,
+                        'dinsdale@example.com')
+        self.assertEqual(mailbox_list.all_mailboxes[0].display_name,
+                        'Roy A. Bear')
+        self.assertEqual(mailbox_list.all_mailboxes[0].token_type,
+                        'invalid-mailbox')
+        self.assertIs(mailbox_list.all_mailboxes[1],
+                        mailbox_list.mailboxes[0])
+        self.assertEqual(mailbox_list.mailboxes[0].addr_spec,
+                        'dinsdale@test.example.com')
+        self.assertEqual(mailbox_list.mailboxes[0].display_name,
+                        'Fred Flintstone')
+
+    def test_get_mailbox_list_empty_list_element(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            ('"Roy A. Bear" <dinsdale@example.com>, (bird),,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>, (bird),,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>, ,,'
+                ' "Fred Flintstone" <dinsdale@test.example.com>'),
+            [errors.ObsoleteHeaderDefect]*2,
+            '')
+        self.assertEqual(len(mailbox_list.mailboxes), 2)
+        self.assertEqual(mailbox_list.all_mailboxes,
+                         mailbox_list.mailboxes)
+        self.assertEqual(mailbox_list.all_mailboxes[0].addr_spec,
+                        'dinsdale@example.com')
+        self.assertEqual(mailbox_list.all_mailboxes[0].display_name,
+                        'Roy A. Bear')
+        self.assertEqual(mailbox_list.mailboxes[1].addr_spec,
+                        'dinsdale@test.example.com')
+        self.assertEqual(mailbox_list.mailboxes[1].display_name,
+                        'Fred Flintstone')
+
+    def test_get_mailbox_list_only_empty_elements(self):
+        mailbox_list = self._test_get_x(parser.get_mailbox_list,
+            '(foo),, (bar)',
+            '(foo),, (bar)',
+            ' ,, ',
+            [errors.ObsoleteHeaderDefect]*3,
+            '')
+        self.assertEqual(len(mailbox_list.mailboxes), 0)
+        self.assertEqual(mailbox_list.all_mailboxes,
+                         mailbox_list.mailboxes)
+
+    # get_group_list
+
+    def test_get_group_list_cfws_only(self):
+        group_list = self._test_get_x(parser.get_group_list,
+            '(hidden);',
+            '(hidden)',
+            ' ',
+            [],
+            ';')
+        self.assertEqual(group_list.token_type, 'group-list')
+        self.assertEqual(len(group_list.mailboxes), 0)
+        self.assertEqual(group_list.mailboxes,
+                         group_list.all_mailboxes)
+
+    def test_get_group_list_mailbox_list(self):
+        group_list = self._test_get_x(parser.get_group_list,
+            'dinsdale@example.org, "Fred A. Bear" <dinsdale@example.org>',
+            'dinsdale@example.org, "Fred A. Bear" <dinsdale@example.org>',
+            'dinsdale@example.org, "Fred A. Bear" <dinsdale@example.org>',
+            [],
+            '')
+        self.assertEqual(group_list.token_type, 'group-list')
+        self.assertEqual(len(group_list.mailboxes), 2)
+        self.assertEqual(group_list.mailboxes,
+                         group_list.all_mailboxes)
+        self.assertEqual(group_list.mailboxes[1].display_name,
+                         'Fred A. Bear')
+
+    def test_get_group_list_obs_group_list(self):
+        group_list = self._test_get_x(parser.get_group_list,
+            ', (foo),,(bar)',
+            ', (foo),,(bar)',
+            ', ,, ',
+            [errors.ObsoleteHeaderDefect],
+            '')
+        self.assertEqual(group_list.token_type, 'group-list')
+        self.assertEqual(len(group_list.mailboxes), 0)
+        self.assertEqual(group_list.mailboxes,
+                         group_list.all_mailboxes)
+
+    def test_get_group_list_comment_only_invalid(self):
+        group_list = self._test_get_x(parser.get_group_list,
+            '(bar)',
+            '(bar)',
+            ' ',
+            [errors.InvalidHeaderDefect],
+            '')
+        self.assertEqual(group_list.token_type, 'group-list')
+        self.assertEqual(len(group_list.mailboxes), 0)
+        self.assertEqual(group_list.mailboxes,
+                         group_list.all_mailboxes)
+
+    # get_group
+
+    def test_get_group_empty(self):
+        group = self._test_get_x(parser.get_group,
+            'Monty Python:;',
+            'Monty Python:;',
+            'Monty Python:;',
+            [],
+            '')
+        self.assertEqual(group.token_type, 'group')
+        self.assertEqual(group.display_name, 'Monty Python')
+        self.assertEqual(len(group.mailboxes), 0)
+        self.assertEqual(group.mailboxes,
+                         group.all_mailboxes)
+
+    def test_get_group_null_addr_spec(self):
+        group = self._test_get_x(parser.get_group,
+            'foo: <>;',
+            'foo: <>;',
+            'foo: <>;',
+            [errors.InvalidHeaderDefect],
+            '')
+        self.assertEqual(group.display_name, 'foo')
+        self.assertEqual(len(group.mailboxes), 0)
+        self.assertEqual(len(group.all_mailboxes), 1)
+        self.assertEqual(group.all_mailboxes[0].value, '<>')
+
+    def test_get_group_cfws_only(self):
+        group = self._test_get_x(parser.get_group,
+            'Monty Python: (hidden);',
+            'Monty Python: (hidden);',
+            'Monty Python: ;',
+            [],
+            '')
+        self.assertEqual(group.token_type, 'group')
+        self.assertEqual(group.display_name, 'Monty Python')
+        self.assertEqual(len(group.mailboxes), 0)
+        self.assertEqual(group.mailboxes,
+                         group.all_mailboxes)
+
+    def test_get_group_single_mailbox(self):
+        group = self._test_get_x(parser.get_group,
+            'Monty Python: "Fred A. Bear" <dinsdale@example.com>;',
+            'Monty Python: "Fred A. Bear" <dinsdale@example.com>;',
+            'Monty Python: "Fred A. Bear" <dinsdale@example.com>;',
+            [],
+            '')
+        self.assertEqual(group.token_type, 'group')
+        self.assertEqual(group.display_name, 'Monty Python')
+        self.assertEqual(len(group.mailboxes), 1)
+        self.assertEqual(group.mailboxes,
+                         group.all_mailboxes)
+        self.assertEqual(group.mailboxes[0].addr_spec,
+                         'dinsdale@example.com')
+
+    def test_get_group_mixed_list(self):
+        group = self._test_get_x(parser.get_group,
+            ('Monty Python: "Fred A. Bear" <dinsdale@example.com>,'
+                '(foo) Roger <ping@exampele.com>, x@test.example.com;'),
+            ('Monty Python: "Fred A. Bear" <dinsdale@example.com>,'
+                '(foo) Roger <ping@exampele.com>, x@test.example.com;'),
+            ('Monty Python: "Fred A. Bear" <dinsdale@example.com>,'
+                ' Roger <ping@exampele.com>, x@test.example.com;'),
+            [],
+            '')
+        self.assertEqual(group.token_type, 'group')
+        self.assertEqual(group.display_name, 'Monty Python')
+        self.assertEqual(len(group.mailboxes), 3)
+        self.assertEqual(group.mailboxes,
+                         group.all_mailboxes)
+        self.assertEqual(group.mailboxes[0].display_name,
+                         'Fred A. Bear')
+        self.assertEqual(group.mailboxes[1].display_name,
+                         'Roger')
+        self.assertEqual(group.mailboxes[2].local_part, 'x')
+
+    def test_get_group_one_invalid(self):
+        group = self._test_get_x(parser.get_group,
+            ('Monty Python: "Fred A. Bear" <dinsdale@example.com>,'
+                '(foo) Roger ping@exampele.com, x@test.example.com;'),
+            ('Monty Python: "Fred A. Bear" <dinsdale@example.com>,'
+                '(foo) Roger ping@exampele.com, x@test.example.com;'),
+            ('Monty Python: "Fred A. Bear" <dinsdale@example.com>,'
+                ' Roger ping@exampele.com, x@test.example.com;'),
+            [errors.InvalidHeaderDefect,   # non-angle addr makes local part invalid
+             errors.InvalidHeaderDefect],   # and its not obs-local either: no dots.
+            '')
+        self.assertEqual(group.token_type, 'group')
+        self.assertEqual(group.display_name, 'Monty Python')
+        self.assertEqual(len(group.mailboxes), 2)
+        self.assertEqual(len(group.all_mailboxes), 3)
+        self.assertEqual(group.mailboxes[0].display_name,
+                         'Fred A. Bear')
+        self.assertEqual(group.mailboxes[1].local_part, 'x')
+        self.assertIsNone(group.all_mailboxes[1].display_name)
+
+    # get_address
+
+    def test_get_address_simple(self):
+        address = self._test_get_x(parser.get_address,
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            [],
+            '')
+        self.assertEqual(address.token_type, 'address')
+        self.assertEqual(len(address.mailboxes), 1)
+        self.assertEqual(address.mailboxes,
+                         address.all_mailboxes)
+        self.assertEqual(address.mailboxes[0].domain,
+                         'example.com')
+        self.assertEqual(address[0].token_type,
+                         'mailbox')
+
+    def test_get_address_complex(self):
+        address = self._test_get_x(parser.get_address,
+            '(foo) "Fred A. Bear" <(bird)dinsdale@example.com>',
+            '(foo) "Fred A. Bear" <(bird)dinsdale@example.com>',
+            ' "Fred A. Bear" < dinsdale@example.com>',
+            [],
+            '')
+        self.assertEqual(address.token_type, 'address')
+        self.assertEqual(len(address.mailboxes), 1)
+        self.assertEqual(address.mailboxes,
+                         address.all_mailboxes)
+        self.assertEqual(address.mailboxes[0].display_name,
+                         'Fred A. Bear')
+        self.assertEqual(address[0].token_type,
+                         'mailbox')
+
+    def test_get_address_empty_group(self):
+        address = self._test_get_x(parser.get_address,
+            'Monty Python:;',
+            'Monty Python:;',
+            'Monty Python:;',
+            [],
+            '')
+        self.assertEqual(address.token_type, 'address')
+        self.assertEqual(len(address.mailboxes), 0)
+        self.assertEqual(address.mailboxes,
+                         address.all_mailboxes)
+        self.assertEqual(address[0].token_type,
+                         'group')
+        self.assertEqual(address[0].display_name,
+                         'Monty Python')
+
+    def test_get_address_group(self):
+        address = self._test_get_x(parser.get_address,
+            'Monty Python: x@example.com, y@example.com;',
+            'Monty Python: x@example.com, y@example.com;',
+            'Monty Python: x@example.com, y@example.com;',
+            [],
+            '')
+        self.assertEqual(address.token_type, 'address')
+        self.assertEqual(len(address.mailboxes), 2)
+        self.assertEqual(address.mailboxes,
+                         address.all_mailboxes)
+        self.assertEqual(address[0].token_type,
+                         'group')
+        self.assertEqual(address[0].display_name,
+                         'Monty Python')
+        self.assertEqual(address.mailboxes[0].local_part, 'x')
+
+    def test_get_address_quoted_local_part(self):
+        address = self._test_get_x(parser.get_address,
+            '"foo bar"@example.com',
+            '"foo bar"@example.com',
+            '"foo bar"@example.com',
+            [],
+            '')
+        self.assertEqual(address.token_type, 'address')
+        self.assertEqual(len(address.mailboxes), 1)
+        self.assertEqual(address.mailboxes,
+                         address.all_mailboxes)
+        self.assertEqual(address.mailboxes[0].domain,
+                         'example.com')
+        self.assertEqual(address.mailboxes[0].local_part,
+                         'foo bar')
+        self.assertEqual(address[0].token_type, 'mailbox')
+
+    def test_get_address_ends_at_special(self):
+        address = self._test_get_x(parser.get_address,
+            'dinsdale@example.com, next',
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            [],
+            ', next')
+        self.assertEqual(address.token_type, 'address')
+        self.assertEqual(len(address.mailboxes), 1)
+        self.assertEqual(address.mailboxes,
+                         address.all_mailboxes)
+        self.assertEqual(address.mailboxes[0].domain,
+                         'example.com')
+        self.assertEqual(address[0].token_type, 'mailbox')
+
+    def test_get_address_invalid_mailbox_invalid(self):
+        address = self._test_get_x(parser.get_address,
+            'ping example.com, next',
+            'ping example.com',
+            'ping example.com',
+            [errors.InvalidHeaderDefect,    # addr-spec with no domain
+             errors.InvalidHeaderDefect,    # invalid local-part
+             errors.InvalidHeaderDefect,    # missing .s in local-part
+            ],
+            ', next')
+        self.assertEqual(address.token_type, 'address')
+        self.assertEqual(len(address.mailboxes), 0)
+        self.assertEqual(len(address.all_mailboxes), 1)
+        self.assertIsNone(address.all_mailboxes[0].domain)
+        self.assertEqual(address.all_mailboxes[0].local_part, 'ping example.com')
+        self.assertEqual(address[0].token_type, 'invalid-mailbox')
+
+    def test_get_address_quoted_strings_in_atom_list(self):
+        address = self._test_get_x(parser.get_address,
+            '""example" example"@example.com',
+            '""example" example"@example.com',
+            'example example@example.com',
+            [errors.InvalidHeaderDefect]*3,
+            '')
+        self.assertEqual(address.all_mailboxes[0].local_part, 'example example')
+        self.assertEqual(address.all_mailboxes[0].domain, 'example.com')
+        self.assertEqual(address.all_mailboxes[0].addr_spec, '"example example"@example.com')
+
+
+    # get_address_list
+
+    def test_get_address_list_mailboxes_simple(self):
+        address_list = self._test_get_x(parser.get_address_list,
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            'dinsdale@example.com',
+            [],
+            '')
+        self.assertEqual(address_list.token_type, 'address-list')
+        self.assertEqual(len(address_list.mailboxes), 1)
+        self.assertEqual(address_list.mailboxes,
+                         address_list.all_mailboxes)
+        self.assertEqual([str(x) for x in address_list.mailboxes],
+                         [str(x) for x in address_list.addresses])
+        self.assertEqual(address_list.mailboxes[0].domain, 'example.com')
+        self.assertEqual(address_list[0].token_type, 'address')
+        self.assertIsNone(address_list[0].display_name)
+
+    def test_get_address_list_mailboxes_two_simple(self):
+        address_list = self._test_get_x(parser.get_address_list,
+            'foo@example.com, "Fred A. Bar" <bar@example.com>',
+            'foo@example.com, "Fred A. Bar" <bar@example.com>',
+            'foo@example.com, "Fred A. Bar" <bar@example.com>',
+            [],
+            '')
+        self.assertEqual(address_list.token_type, 'address-list')
+        self.assertEqual(len(address_list.mailboxes), 2)
+        self.assertEqual(address_list.mailboxes,
+                         address_list.all_mailboxes)
+        self.assertEqual([str(x) for x in address_list.mailboxes],
+                         [str(x) for x in address_list.addresses])
+        self.assertEqual(address_list.mailboxes[0].local_part, 'foo')
+        self.assertEqual(address_list.mailboxes[1].display_name, "Fred A. Bar")
+
+    def test_get_address_list_mailboxes_complex(self):
+        address_list = self._test_get_x(parser.get_address_list,
+            ('"Roy A. Bear" <dinsdale@example.com>, '
+                '(ping) Foo <x@example.com>,'
+                'Nobody Is. Special <y@(bird)example.(bad)com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>, '
+                '(ping) Foo <x@example.com>,'
+                'Nobody Is. Special <y@(bird)example.(bad)com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>, '
+                'Foo <x@example.com>,'
+                '"Nobody Is. Special" <y@example. com>'),
+            [errors.ObsoleteHeaderDefect, # period in Is.
+            errors.ObsoleteHeaderDefect], # cfws in domain
+            '')
+        self.assertEqual(address_list.token_type, 'address-list')
+        self.assertEqual(len(address_list.mailboxes), 3)
+        self.assertEqual(address_list.mailboxes,
+                         address_list.all_mailboxes)
+        self.assertEqual([str(x) for x in address_list.mailboxes],
+                         [str(x) for x in address_list.addresses])
+        self.assertEqual(address_list.mailboxes[0].domain, 'example.com')
+        self.assertEqual(address_list.mailboxes[0].token_type, 'mailbox')
+        self.assertEqual(address_list.addresses[0].token_type, 'address')
+        self.assertEqual(address_list.mailboxes[1].local_part, 'x')
+        self.assertEqual(address_list.mailboxes[2].display_name,
+                         'Nobody Is. Special')
+
+    def test_get_address_list_mailboxes_invalid_addresses(self):
+        address_list = self._test_get_x(parser.get_address_list,
+            ('"Roy A. Bear" <dinsdale@example.com>, '
+                '(ping) Foo x@example.com[],'
+                'Nobody Is. Special <(bird)example.(bad)com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>, '
+                '(ping) Foo x@example.com[],'
+                'Nobody Is. Special <(bird)example.(bad)com>'),
+            ('"Roy A. Bear" <dinsdale@example.com>, '
+                'Foo x@example.com[],'
+                '"Nobody Is. Special" < example. com>'),
+             [errors.InvalidHeaderDefect,   # invalid address in list
+              errors.InvalidHeaderDefect,   # 'Foo x' local part invalid.
+              errors.InvalidHeaderDefect,   # Missing . in 'Foo x' local part
+              errors.ObsoleteHeaderDefect,  # period in 'Is.' disp-name phrase
+              errors.InvalidHeaderDefect,   # no domain part in addr-spec
+              errors.ObsoleteHeaderDefect], # addr-spec has comment in it
+            '')
+        self.assertEqual(address_list.token_type, 'address-list')
+        self.assertEqual(len(address_list.mailboxes), 1)
+        self.assertEqual(len(address_list.all_mailboxes), 3)
+        self.assertEqual([str(x) for x in address_list.all_mailboxes],
+                         [str(x) for x in address_list.addresses])
+        self.assertEqual(address_list.mailboxes[0].domain, 'example.com')
+        self.assertEqual(address_list.mailboxes[0].token_type, 'mailbox')
+        self.assertEqual(address_list.addresses[0].token_type, 'address')
+        self.assertEqual(address_list.addresses[1].token_type, 'address')
+        self.assertEqual(len(address_list.addresses[0].mailboxes), 1)
+        self.assertEqual(len(address_list.addresses[1].mailboxes), 0)
+        self.assertEqual(len(address_list.addresses[1].mailboxes), 0)
+        self.assertEqual(
+            address_list.addresses[1].all_mailboxes[0].local_part, 'Foo x')
+        self.assertEqual(
+            address_list.addresses[2].all_mailboxes[0].display_name,
+                "Nobody Is. Special")
+
+    def test_get_address_list_group_empty(self):
+        address_list = self._test_get_x(parser.get_address_list,
+            'Monty Python: ;',
+            'Monty Python: ;',
+            'Monty Python: ;',
+            [],
+            '')
+        self.assertEqual(address_list.token_type, 'address-list')
+        self.assertEqual(len(address_list.mailboxes), 0)
+        self.assertEqual(address_list.mailboxes,
+                         address_list.all_mailboxes)
+        self.assertEqual(len(address_list.addresses), 1)
+        self.assertEqual(address_list.addresses[0].token_type, 'address')
+        self.assertEqual(address_list.addresses[0].display_name, 'Monty Python')
+        self.assertEqual(len(address_list.addresses[0].mailboxes), 0)
+
+    def test_get_address_list_group_simple(self):
+        address_list = self._test_get_x(parser.get_address_list,
+            'Monty Python: dinsdale@example.com;',
+            'Monty Python: dinsdale@example.com;',
+            'Monty Python: dinsdale@example.com;',
+            [],
+            '')
+        self.assertEqual(address_list.token_type, 'address-list')
+        self.assertEqual(len(address_list.mailboxes), 1)
+        self.assertEqual(address_list.mailboxes,
+                         address_list.all_mailboxes)
+        self.assertEqual(address_list.mailboxes[0].domain, 'example.com')
+        self.assertEqual(address_list.addresses[0].display_name,
+                         'Monty Python')
+        self.assertEqual(address_list.addresses[0].mailboxes[0].domain,
+                         'example.com')
+
+    def test_get_address_list_group_and_mailboxes(self):
+        address_list = self._test_get_x(parser.get_address_list,
+            ('Monty Python: dinsdale@example.com, "Fred" <flint@example.com>;, '
+                'Abe <x@example.com>, Bee <y@example.com>'),
+            ('Monty Python: dinsdale@example.com, "Fred" <flint@example.com>;, '
+                'Abe <x@example.com>, Bee <y@example.com>'),
+            ('Monty Python: dinsdale@example.com, "Fred" <flint@example.com>;, '
+                'Abe <x@example.com>, Bee <y@example.com>'),
+            [],
+            '')
+        self.assertEqual(address_list.token_type, 'address-list')
+        self.assertEqual(len(address_list.mailboxes), 4)
+        self.assertEqual(address_list.mailboxes,
+                         address_list.all_mailboxes)
+        self.assertEqual(len(address_list.addresses), 3)
+        self.assertEqual(address_list.mailboxes[0].local_part, 'dinsdale')
+        self.assertEqual(address_list.addresses[0].display_name,
+                         'Monty Python')
+        self.assertEqual(address_list.addresses[0].mailboxes[0].domain,
+                         'example.com')
+        self.assertEqual(address_list.addresses[0].mailboxes[1].local_part,
+                         'flint')
+        self.assertEqual(address_list.addresses[1].mailboxes[0].local_part,
+                         'x')
+        self.assertEqual(address_list.addresses[2].mailboxes[0].local_part,
+                         'y')
+        self.assertEqual(str(address_list.addresses[1]),
+                         str(address_list.mailboxes[2]))
+
+
+@parameterize
+class Test_parse_mime_version(TestParserMixin, TestEmailBase):
+
+    def mime_version_as_value(self,
+                              value,
+                              tl_str,
+                              tl_value,
+                              major,
+                              minor,
+                              defects):
+        mime_version = self._test_parse_x(parser.parse_mime_version,
+            value, tl_str, tl_value, defects)
+        self.assertEqual(mime_version.major, major)
+        self.assertEqual(mime_version.minor, minor)
+
+    mime_version_params = {
+
+        'rfc_2045_1': (
+            '1.0',
+            '1.0',
+            '1.0',
+            1,
+            0,
+            []),
+
+        'RFC_2045_2': (
+            '1.0 (produced by MetaSend Vx.x)',
+            '1.0 (produced by MetaSend Vx.x)',
+            '1.0 ',
+            1,
+            0,
+            []),
+
+        'RFC_2045_3': (
+            '(produced by MetaSend Vx.x) 1.0',
+            '(produced by MetaSend Vx.x) 1.0',
+            ' 1.0',
+            1,
+            0,
+            []),
+
+        'RFC_2045_4': (
+            '1.(produced by MetaSend Vx.x)0',
+            '1.(produced by MetaSend Vx.x)0',
+            '1. 0',
+            1,
+            0,
+            []),
+
+        'empty': (
+            '',
+            '',
+            '',
+            None,
+            None,
+            [errors.HeaderMissingRequiredValue]),
+
+        }
+
+
+
+class TestFolding(TestEmailBase):
+
+    policy = policy.default
+
+    def _test(self, tl, folded, policy=policy):
+        self.assertEqual(tl.fold(policy=policy), folded, tl.ppstr())
+
+    def test_simple_unstructured_no_folds(self):
+        self._test(parser.get_unstructured("This is a test"),
+                   "This is a test\n")
+
+    def test_simple_unstructured_folded(self):
+        self._test(parser.get_unstructured("This is also a test, but this "
+                        "time there are enough words (and even some "
+                        "symbols) to make it wrap; at least in theory."),
+                   "This is also a test, but this time there are enough "
+                        "words (and even some\n"
+                   " symbols) to make it wrap; at least in theory.\n")
+
+    def test_unstructured_with_unicode_no_folds(self):
+        self._test(parser.get_unstructured("hübsch kleiner beißt"),
+                   "=?utf-8?q?h=C3=BCbsch_kleiner_bei=C3=9Ft?=\n")
+
+    def test_one_ew_on_each_of_two_wrapped_lines(self):
+        self._test(parser.get_unstructured("Mein kleiner Kaktus ist sehr "
+                                           "hübsch.  Es hat viele Stacheln "
+                                           "und oft beißt mich."),
+                   "Mein kleiner Kaktus ist sehr =?utf-8?q?h=C3=BCbsch=2E?=  "
+                        "Es hat viele Stacheln\n"
+                   " und oft =?utf-8?q?bei=C3=9Ft?= mich.\n")
+
+    def test_ews_combined_before_wrap(self):
+        self._test(parser.get_unstructured("Mein Kaktus ist hübsch.  "
+                                           "Es beißt mich.  "
+                                           "And that's all I'm sayin."),
+                   "Mein Kaktus ist =?utf-8?q?h=C3=BCbsch=2E__Es_bei=C3=9Ft?= "
+                        "mich.  And that's\n"
+                   " all I'm sayin.\n")
+
+    # XXX Need test of an encoded word so long that it needs to be wrapped
+
+    def test_simple_address(self):
+        self._test(parser.get_address_list("abc <xyz@example.com>")[0],
+                   "abc <xyz@example.com>\n")
+
+    def test_address_list_folding_at_commas(self):
+        self._test(parser.get_address_list('abc <xyz@example.com>, '
+                                            '"Fred Blunt" <sharp@example.com>, '
+                                            '"J.P.Cool" <hot@example.com>, '
+                                            '"K<>y" <key@example.com>, '
+                                            'Firesale <cheap@example.com>, '
+                                            '<end@example.com>')[0],
+                    'abc <xyz@example.com>, "Fred Blunt" <sharp@example.com>,\n'
+                    ' "J.P.Cool" <hot@example.com>, "K<>y" <key@example.com>,\n'
+                    ' Firesale <cheap@example.com>, <end@example.com>\n')
+
+    def test_address_list_with_unicode_names(self):
+        self._test(parser.get_address_list(
+            'Hübsch Kaktus <beautiful@example.com>, '
+                'beißt beißt <biter@example.com>')[0],
+            '=?utf-8?q?H=C3=BCbsch?= Kaktus <beautiful@example.com>,\n'
+                ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= <biter@example.com>\n')
+
+    def test_address_list_with_unicode_names_in_quotes(self):
+        self._test(parser.get_address_list(
+            '"Hübsch Kaktus" <beautiful@example.com>, '
+                '"beißt" beißt <biter@example.com>')[0],
+            '=?utf-8?q?H=C3=BCbsch?= Kaktus <beautiful@example.com>,\n'
+                ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= <biter@example.com>\n')
+
+    # XXX Need tests with comments on various sides of a unicode token,
+    # and with unicode tokens in the comments.  Spaces inside the quotes
+    # currently don't do the right thing.
+
+    def test_initial_whitespace_splitting(self):
+        body = parser.get_unstructured('   ' + 'x'*77)
+        header = parser.Header([
+            parser.HeaderLabel([parser.ValueTerminal('test:', 'atext')]),
+            parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), body])
+        self._test(header, 'test:   \n ' + 'x'*77 + '\n')
+
+    def test_whitespace_splitting(self):
+        self._test(parser.get_unstructured('xxx   ' + 'y'*77),
+                   'xxx  \n ' + 'y'*77 + '\n')
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/email/test/test_email_codecs.py b/Lib/test/test_email/test_asian_codecs.py
index ca85f5731e..089269f544 100644
--- a/Lib/email/test/test_email_codecs.py
+++ b/Lib/test/test_email/test_asian_codecs.py
@@ -5,7 +5,7 @@
 import unittest
 from test.support import run_unittest
 
-from email.test.test_email import TestEmailBase
+from test.test_email.test_email import TestEmailBase
 from email.charset import Charset
 from email.header import Header, decode_header
 from email.message import Message
@@ -41,7 +41,7 @@ class TestEmailAsianCodecs(TestEmailBase):
 Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?=
  =?iso-8859-1?q?Gr=FC=DF_Gott!?=""")
         eq(decode_header(h.encode()),
-           [(b'Hello World!', None),
+           [(b'Hello World! ', None),
             (b'\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'),
             (b'Gr\xfc\xdf Gott!', gcode)])
         subject_bytes = (b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5'
@@ -78,16 +78,5 @@ Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?=
 
 
 
-def suite():
-    suite = unittest.TestSuite()
-    suite.addTest(unittest.makeSuite(TestEmailAsianCodecs))
-    return suite
-
-
-def test_main():
-    run_unittest(TestEmailAsianCodecs)
-
-
-
 if __name__ == '__main__':
-    unittest.main(defaultTest='suite')
+    unittest.main()
diff --git a/Lib/test/test_email/test_defect_handling.py b/Lib/test/test_email/test_defect_handling.py
new file mode 100644
index 0000000000..305432df37
--- /dev/null
+++ b/Lib/test/test_email/test_defect_handling.py
@@ -0,0 +1,320 @@
+import textwrap
+import unittest
+import contextlib
+from email import policy
+from email import errors
+from test.test_email import TestEmailBase
+
+
+class TestDefectsBase:
+
+    policy = policy.default
+    raise_expected = False
+
+    @contextlib.contextmanager
+    def _raise_point(self, defect):
+        yield
+
+    def test_same_boundary_inner_outer(self):
+        source = textwrap.dedent("""\
+            Subject: XX
+            From: xx@xx.dk
+            To: XX
+            Mime-version: 1.0
+            Content-type: multipart/mixed;
+               boundary="MS_Mac_OE_3071477847_720252_MIME_Part"
+
+            --MS_Mac_OE_3071477847_720252_MIME_Part
+            Content-type: multipart/alternative;
+               boundary="MS_Mac_OE_3071477847_720252_MIME_Part"
+
+            --MS_Mac_OE_3071477847_720252_MIME_Part
+            Content-type: text/plain; charset="ISO-8859-1"
+            Content-transfer-encoding: quoted-printable
+
+            text
+
+            --MS_Mac_OE_3071477847_720252_MIME_Part
+            Content-type: text/html; charset="ISO-8859-1"
+            Content-transfer-encoding: quoted-printable
+
+            <HTML></HTML>
+
+            --MS_Mac_OE_3071477847_720252_MIME_Part--
+
+            --MS_Mac_OE_3071477847_720252_MIME_Part
+            Content-type: image/gif; name="xx.gif";
+            Content-disposition: attachment
+            Content-transfer-encoding: base64
+
+            Some removed base64 encoded chars.
+
+            --MS_Mac_OE_3071477847_720252_MIME_Part--
+
+            """)
+        # XXX better would be to actually detect the duplicate.
+        with self._raise_point(errors.StartBoundaryNotFoundDefect):
+            msg = self._str_msg(source)
+        if self.raise_expected: return
+        inner = msg.get_payload(0)
+        self.assertTrue(hasattr(inner, 'defects'))
+        self.assertEqual(len(self.get_defects(inner)), 1)
+        self.assertTrue(isinstance(self.get_defects(inner)[0],
+                                   errors.StartBoundaryNotFoundDefect))
+
+    def test_multipart_no_boundary(self):
+        source = textwrap.dedent("""\
+            Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800)
+            From: foobar
+            Subject: broken mail
+            MIME-Version: 1.0
+            Content-Type: multipart/report; report-type=delivery-status;
+
+            --JAB03225.986577786/zinfandel.lacita.com
+
+            One part
+
+            --JAB03225.986577786/zinfandel.lacita.com
+            Content-Type: message/delivery-status
+
+            Header: Another part
+
+            --JAB03225.986577786/zinfandel.lacita.com--
+            """)
+        with self._raise_point(errors.NoBoundaryInMultipartDefect):
+            msg = self._str_msg(source)
+        if self.raise_expected: return
+        self.assertTrue(isinstance(msg.get_payload(), str))
+        self.assertEqual(len(self.get_defects(msg)), 2)
+        self.assertTrue(isinstance(self.get_defects(msg)[0],
+                                   errors.NoBoundaryInMultipartDefect))
+        self.assertTrue(isinstance(self.get_defects(msg)[1],
+                                   errors.MultipartInvariantViolationDefect))
+
+    multipart_msg = textwrap.dedent("""\
+        Date: Wed, 14 Nov 2007 12:56:23 GMT
+        From: foo@bar.invalid
+        To: foo@bar.invalid
+        Subject: Content-Transfer-Encoding: base64 and multipart
+        MIME-Version: 1.0
+        Content-Type: multipart/mixed;
+            boundary="===============3344438784458119861=="{}
+
+        --===============3344438784458119861==
+        Content-Type: text/plain
+
+        Test message
+
+        --===============3344438784458119861==
+        Content-Type: application/octet-stream
+        Content-Transfer-Encoding: base64
+
+        YWJj
+
+        --===============3344438784458119861==--
+        """)
+
+    def test_multipart_invalid_cte(self):
+        with self._raise_point(
+                errors.InvalidMultipartContentTransferEncodingDefect):
+            msg = self._str_msg(
+                    self.multipart_msg.format(
+                        "\nContent-Transfer-Encoding: base64"))
+        if self.raise_expected: return
+        self.assertEqual(len(self.get_defects(msg)), 1)
+        self.assertIsInstance(self.get_defects(msg)[0],
+            errors.InvalidMultipartContentTransferEncodingDefect)
+
+    def test_multipart_no_cte_no_defect(self):
+        if self.raise_expected: return
+        msg = self._str_msg(self.multipart_msg.format(''))
+        self.assertEqual(len(self.get_defects(msg)), 0)
+
+    def test_multipart_valid_cte_no_defect(self):
+        if self.raise_expected: return
+        for cte in ('7bit', '8bit', 'BINary'):
+            msg = self._str_msg(
+                self.multipart_msg.format("\nContent-Transfer-Encoding: "+cte))
+            self.assertEqual(len(self.get_defects(msg)), 0, "cte="+cte)
+
+    def test_lying_multipart(self):
+        source = textwrap.dedent("""\
+            From: "Allison Dunlap" <xxx@example.com>
+            To: yyy@example.com
+            Subject: 64423
+            Date: Sun, 11 Jul 2004 16:09:27 -0300
+            MIME-Version: 1.0
+            Content-Type: multipart/alternative;
+
+            Blah blah blah
+            """)
+        with self._raise_point(errors.NoBoundaryInMultipartDefect):
+            msg = self._str_msg(source)
+        if self.raise_expected: return
+        self.assertTrue(hasattr(msg, 'defects'))
+        self.assertEqual(len(self.get_defects(msg)), 2)
+        self.assertTrue(isinstance(self.get_defects(msg)[0],
+                                   errors.NoBoundaryInMultipartDefect))
+        self.assertTrue(isinstance(self.get_defects(msg)[1],
+                                   errors.MultipartInvariantViolationDefect))
+
+    def test_missing_start_boundary(self):
+        source = textwrap.dedent("""\
+            Content-Type: multipart/mixed; boundary="AAA"
+            From: Mail Delivery Subsystem <xxx@example.com>
+            To: yyy@example.com
+
+            --AAA
+
+            Stuff
+
+            --AAA
+            Content-Type: message/rfc822
+
+            From: webmaster@python.org
+            To: zzz@example.com
+            Content-Type: multipart/mixed; boundary="BBB"
+
+            --BBB--
+
+            --AAA--
+
+            """)
+        # The message structure is:
+        #
+        # multipart/mixed
+        #    text/plain
+        #    message/rfc822
+        #        multipart/mixed [*]
+        #
+        # [*] This message is missing its start boundary
+        with self._raise_point(errors.StartBoundaryNotFoundDefect):
+            outer = self._str_msg(source)
+        if self.raise_expected: return
+        bad = outer.get_payload(1).get_payload(0)
+        self.assertEqual(len(self.get_defects(bad)), 1)
+        self.assertTrue(isinstance(self.get_defects(bad)[0],
+                                   errors.StartBoundaryNotFoundDefect))
+
+    def test_first_line_is_continuation_header(self):
+        with self._raise_point(errors.FirstHeaderLineIsContinuationDefect):
+            msg = self._str_msg(' Line 1\nSubject: test\n\nbody')
+        if self.raise_expected: return
+        self.assertEqual(msg.keys(), ['Subject'])
+        self.assertEqual(msg.get_payload(), 'body')
+        self.assertEqual(len(self.get_defects(msg)), 1)
+        self.assertDefectsEqual(self.get_defects(msg),
+                                 [errors.FirstHeaderLineIsContinuationDefect])
+        self.assertEqual(self.get_defects(msg)[0].line, ' Line 1\n')
+
+    def test_missing_header_body_separator(self):
+        # Our heuristic if we see a line that doesn't look like a header (no
+        # leading whitespace but no ':') is to assume that the blank line that
+        # separates the header from the body is missing, and to stop parsing
+        # headers and start parsing the body.
+        with self._raise_point(errors.MissingHeaderBodySeparatorDefect):
+            msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n')
+        if self.raise_expected: return
+        self.assertEqual(msg.keys(), ['Subject'])
+        self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n')
+        self.assertDefectsEqual(self.get_defects(msg),
+                                [errors.MissingHeaderBodySeparatorDefect])
+
+    def test_bad_padding_in_base64_payload(self):
+        source = textwrap.dedent("""\
+            Subject: test
+            MIME-Version: 1.0
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: base64
+
+            dmk
+            """)
+        msg = self._str_msg(source)
+        with self._raise_point(errors.InvalidBase64PaddingDefect):
+            payload = msg.get_payload(decode=True)
+        if self.raise_expected: return
+        self.assertEqual(payload, b'vi')
+        self.assertDefectsEqual(self.get_defects(msg),
+                                [errors.InvalidBase64PaddingDefect])
+
+    def test_invalid_chars_in_base64_payload(self):
+        source = textwrap.dedent("""\
+            Subject: test
+            MIME-Version: 1.0
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: base64
+
+            dm\x01k===
+            """)
+        msg = self._str_msg(source)
+        with self._raise_point(errors.InvalidBase64CharactersDefect):
+            payload = msg.get_payload(decode=True)
+        if self.raise_expected: return
+        self.assertEqual(payload, b'vi')
+        self.assertDefectsEqual(self.get_defects(msg),
+                                [errors.InvalidBase64CharactersDefect])
+
+    def test_missing_ending_boundary(self):
+        source = textwrap.dedent("""\
+            To: 1@harrydomain4.com
+            Subject: Fwd: 1
+            MIME-Version: 1.0
+            Content-Type: multipart/alternative;
+             boundary="------------000101020201080900040301"
+
+            --------------000101020201080900040301
+            Content-Type: text/plain; charset=ISO-8859-1
+            Content-Transfer-Encoding: 7bit
+
+            Alternative 1
+
+            --------------000101020201080900040301
+            Content-Type: text/html; charset=ISO-8859-1
+            Content-Transfer-Encoding: 7bit
+
+            Alternative 2
+
+            """)
+        with self._raise_point(errors.CloseBoundaryNotFoundDefect):
+            msg = self._str_msg(source)
+        if self.raise_expected: return
+        self.assertEqual(len(msg.get_payload()), 2)
+        self.assertEqual(msg.get_payload(1).get_payload(), 'Alternative 2\n')
+        self.assertDefectsEqual(self.get_defects(msg),
+                                [errors.CloseBoundaryNotFoundDefect])
+
+
+class TestDefectDetection(TestDefectsBase, TestEmailBase):
+
+    def get_defects(self, obj):
+        return obj.defects
+
+
+class TestDefectCapture(TestDefectsBase, TestEmailBase):
+
+    class CapturePolicy(policy.EmailPolicy):
+        captured = None
+        def register_defect(self, obj, defect):
+            self.captured.append(defect)
+
+    def setUp(self):
+        self.policy = self.CapturePolicy(captured=list())
+
+    def get_defects(self, obj):
+        return self.policy.captured
+
+
+class TestDefectRaising(TestDefectsBase, TestEmailBase):
+
+    policy = TestDefectsBase.policy
+    policy = policy.clone(raise_on_defect=True)
+    raise_expected = True
+
+    @contextlib.contextmanager
+    def _raise_point(self, defect):
+        with self.assertRaises(defect):
+            yield
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/email/test/test_email.py b/Lib/test/test_email/test_email.py
index a1ceb7f1b5..23f062fd80 100644
--- a/Lib/email/test/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -2,12 +2,9 @@
 # Contact: email-sig@python.org
 # email package unit tests
 
-import os
 import re
-import sys
 import time
 import base64
-import difflib
 import unittest
 import textwrap
 
@@ -15,6 +12,7 @@ from io import StringIO, BytesIO
 from itertools import chain
 
 import email
+import email.policy
 
 from email.charset import Charset
 from email.header import Header, decode_header, make_header
@@ -35,41 +33,14 @@ from email import iterators
 from email import base64mime
 from email import quoprimime
 
-from test.support import findfile, run_unittest, unlink
-from email.test import __file__ as landmark
-
+from test.support import unlink
+from test.test_email import openfile, TestEmailBase
 
 NL = '\n'
 EMPTYSTRING = ''
 SPACE = ' '
 
 
-
-def openfile(filename, *args, **kws):
-    path = os.path.join(os.path.dirname(landmark), 'data', filename)
-    return open(path, *args, **kws)
-
-
-
-# Base test class
-class TestEmailBase(unittest.TestCase):
-    def ndiffAssertEqual(self, first, second):
-        """Like assertEqual except use ndiff for readable output."""
-        if first != second:
-            sfirst = str(first)
-            ssecond = str(second)
-            rfirst = [repr(line) for line in sfirst.splitlines()]
-            rsecond = [repr(line) for line in ssecond.splitlines()]
-            diff = difflib.ndiff(rfirst, rsecond)
-            raise self.failureException(NL + NL.join(diff))
-
-    def _msgobj(self, filename):
-        with openfile(findfile(filename)) as fp:
-            return email.message_from_file(fp)
-
-    maxDiff = None
-
-
 # Test various aspects of the Message class's API
 class TestMessageAPI(TestEmailBase):
     def test_get_all(self):
@@ -194,7 +165,7 @@ class TestMessageAPI(TestEmailBase):
     def test_message_rfc822_only(self):
         # Issue 7970: message/rfc822 not in multipart parsed by
         # HeaderParser caused an exception when flattened.
-        with openfile(findfile('msg_46.txt')) as fp:
+        with openfile('msg_46.txt') as fp:
             msgdata = fp.read()
         parser = HeaderParser()
         msg = parser.parsestr(msgdata)
@@ -203,6 +174,17 @@ class TestMessageAPI(TestEmailBase):
         gen.flatten(msg, False)
         self.assertEqual(out.getvalue(), msgdata)
 
+    def test_byte_message_rfc822_only(self):
+        # Make sure new bytes header parser also passes this.
+        with openfile('msg_46.txt', 'rb') as fp:
+            msgdata = fp.read()
+        parser = email.parser.BytesHeaderParser()
+        msg = parser.parsebytes(msgdata)
+        out = BytesIO()
+        gen = email.generator.BytesGenerator(out)
+        gen.flatten(msg)
+        self.assertEqual(out.getvalue(), msgdata)
+
     def test_get_decoded_payload(self):
         eq = self.assertEqual
         msg = self._msgobj('msg_10.txt')
@@ -273,6 +255,7 @@ class TestMessageAPI(TestEmailBase):
         self.assertTrue(lines[0].startswith('From '))
         eq(text, NL.join(lines[1:]))
 
+    # test_headerregistry.TestContentTypeHeader.bad_params
     def test_bad_param(self):
         msg = email.message_from_string("Content-Type: blarg; baz; boo\n")
         self.assertEqual(msg.get_param('baz'), '')
@@ -306,6 +289,7 @@ class TestMessageAPI(TestEmailBase):
         eq(msg.get_params(header='x-header'),
            [('foo', ''), ('bar', 'one'), ('baz', 'two')])
 
+    # test_headerregistry.TestContentTypeHeader.spaces_around_param_equals
     def test_get_param_liberal(self):
         msg = Message()
         msg['Content-Type'] = 'Content-Type: Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"'
@@ -328,10 +312,12 @@ class TestMessageAPI(TestEmailBase):
         # msg.get_param("weird")
         # yet.
 
+    # test_headerregistry.TestContentTypeHeader.spaces_around_semis
     def test_get_param_funky_continuation_lines(self):
         msg = self._msgobj('msg_22.txt')
         self.assertEqual(msg.get_payload(1).get_param('name'), 'wibble.JPG')
 
+    # test_headerregistry.TestContentTypeHeader.semis_inside_quotes
     def test_get_param_with_semis_in_quotes(self):
         msg = email.message_from_string(
             'Content-Type: image/pjpeg; name="Jim&amp;&amp;Jill"\n')
@@ -339,6 +325,7 @@ class TestMessageAPI(TestEmailBase):
         self.assertEqual(msg.get_param('name', unquote=False),
                          '"Jim&amp;&amp;Jill"')
 
+    # test_headerregistry.TestContentTypeHeader.quotes_inside_rfc2231_value
     def test_get_param_with_quotes(self):
         msg = email.message_from_string(
             'Content-Type: foo; bar*0="baz\\"foobar"; bar*1="\\"baz"')
@@ -527,6 +514,7 @@ class TestMessageAPI(TestEmailBase):
         eq(msg.values(), ['One Hundred', 'Twenty', 'Three', 'Eleven'])
         self.assertRaises(KeyError, msg.replace_header, 'Fourth', 'Missing')
 
+    # test_defect_handling:test_invalid_chars_in_base64_payload
     def test_broken_base64_payload(self):
         x = 'AwDp0P7//y6LwKEAcPa/6Q=9'
         msg = Message()
@@ -534,7 +522,10 @@ class TestMessageAPI(TestEmailBase):
         msg['content-transfer-encoding'] = 'base64'
         msg.set_payload(x)
         self.assertEqual(msg.get_payload(decode=True),
-                         bytes(x, 'raw-unicode-escape'))
+                         (b'\x03\x00\xe9\xd0\xfe\xff\xff.\x8b\xc0'
+                          b'\xa1\x00p\xf6\xbf\xe9\x0f'))
+        self.assertIsInstance(msg.defects[0],
+                              errors.InvalidBase64CharactersDefect)
 
     def test_broken_unicode_payload(self):
         # This test improves coverage but is not a compliance test.
@@ -632,6 +623,18 @@ class TestMessageAPI(TestEmailBase):
             abc
             """))
 
+    def test_unicode_body_defaults_to_utf8_encoding(self):
+        # Issue 14291
+        m = MIMEText('É testabc\n')
+        self.assertEqual(str(m),textwrap.dedent("""\
+            Content-Type: text/plain; charset="utf-8"
+            MIME-Version: 1.0
+            Content-Transfer-Encoding: base64
+
+            w4kgdGVzdGFiYwo=
+            """))
+
+
 # Test the email.encoders module
 class TestEncoders(unittest.TestCase):
 
@@ -657,7 +660,7 @@ class TestEncoders(unittest.TestCase):
         eq(msg['content-transfer-encoding'], '7bit')
         # Similar, but with 8bit data
         msg = MIMEText('hello \xf8 world')
-        eq(msg['content-transfer-encoding'], '8bit')
+        eq(msg['content-transfer-encoding'], 'base64')
         # And now with a different charset
         msg = MIMEText('hello \xf8 world', _charset='iso-8859-1')
         eq(msg['content-transfer-encoding'], 'quoted-printable')
@@ -1264,6 +1267,7 @@ List: List-Unsubscribe:
              =?utf-8?q?_folding_white_space_works?=""")+'\n')
 
 
+
 # Test mangling of "From " lines in the body of a message
 class TestFromMangling(unittest.TestCase):
     def setUp(self):
@@ -1337,13 +1341,7 @@ Blah blah blah
 # Test the basic MIMEAudio class
 class TestMIMEAudio(unittest.TestCase):
     def setUp(self):
-        # Make sure we pick up the audiotest.au that lives in email/test/data.
-        # In Python, there's an audiotest.au living in Lib/test but that isn't
-        # included in some binary distros that don't include the test
-        # package.  The trailing empty string on the .join() is significant
-        # since findfile() will do a dirname().
-        datadir = os.path.join(os.path.dirname(landmark), 'data', '')
-        with open(findfile('audiotest.au', datadir), 'rb') as fp:
+        with openfile('audiotest.au', 'rb') as fp:
             self._audiodata = fp.read()
         self._au = MIMEAudio(self._audiodata)
 
@@ -1904,6 +1902,7 @@ YXNkZg==
 
 # Test some badly formatted messages
 class TestNonConformant(TestEmailBase):
+
     def test_parse_missing_minor_type(self):
         eq = self.assertEqual
         msg = self._msgobj('msg_14.txt')
@@ -1911,6 +1910,7 @@ class TestNonConformant(TestEmailBase):
         eq(msg.get_content_maintype(), 'text')
         eq(msg.get_content_subtype(), 'plain')
 
+    # test_defect_handling
     def test_same_boundary_inner_outer(self):
         unless = self.assertTrue
         msg = self._msgobj('msg_15.txt')
@@ -1921,15 +1921,62 @@ class TestNonConformant(TestEmailBase):
         unless(isinstance(inner.defects[0],
                           errors.StartBoundaryNotFoundDefect))
 
+    # test_defect_handling
     def test_multipart_no_boundary(self):
         unless = self.assertTrue
         msg = self._msgobj('msg_25.txt')
         unless(isinstance(msg.get_payload(), str))
         self.assertEqual(len(msg.defects), 2)
-        unless(isinstance(msg.defects[0], errors.NoBoundaryInMultipartDefect))
+        unless(isinstance(msg.defects[0],
+                          errors.NoBoundaryInMultipartDefect))
         unless(isinstance(msg.defects[1],
                           errors.MultipartInvariantViolationDefect))
 
+    multipart_msg = textwrap.dedent("""\
+        Date: Wed, 14 Nov 2007 12:56:23 GMT
+        From: foo@bar.invalid
+        To: foo@bar.invalid
+        Subject: Content-Transfer-Encoding: base64 and multipart
+        MIME-Version: 1.0
+        Content-Type: multipart/mixed;
+            boundary="===============3344438784458119861=="{}
+
+        --===============3344438784458119861==
+        Content-Type: text/plain
+
+        Test message
+
+        --===============3344438784458119861==
+        Content-Type: application/octet-stream
+        Content-Transfer-Encoding: base64
+
+        YWJj
+
+        --===============3344438784458119861==--
+        """)
+
+    # test_defect_handling
+    def test_multipart_invalid_cte(self):
+        msg = self._str_msg(
+            self.multipart_msg.format("\nContent-Transfer-Encoding: base64"))
+        self.assertEqual(len(msg.defects), 1)
+        self.assertIsInstance(msg.defects[0],
+            errors.InvalidMultipartContentTransferEncodingDefect)
+
+    # test_defect_handling
+    def test_multipart_no_cte_no_defect(self):
+        msg = self._str_msg(self.multipart_msg.format(''))
+        self.assertEqual(len(msg.defects), 0)
+
+    # test_defect_handling
+    def test_multipart_valid_cte_no_defect(self):
+        for cte in ('7bit', '8bit', 'BINary'):
+            msg = self._str_msg(
+                self.multipart_msg.format(
+                    "\nContent-Transfer-Encoding: {}".format(cte)))
+            self.assertEqual(len(msg.defects), 0)
+
+    # test_headerregistry.TestContentTyopeHeader invalid_1 and invalid_2.
     def test_invalid_content_type(self):
         eq = self.assertEqual
         neq = self.ndiffAssertEqual
@@ -1979,15 +2026,18 @@ Subject: here's something interesting
 counter to RFC 2822, there's no separating newline here
 """)
 
+    # test_defect_handling
     def test_lying_multipart(self):
         unless = self.assertTrue
         msg = self._msgobj('msg_41.txt')
         unless(hasattr(msg, 'defects'))
         self.assertEqual(len(msg.defects), 2)
-        unless(isinstance(msg.defects[0], errors.NoBoundaryInMultipartDefect))
+        unless(isinstance(msg.defects[0],
+                          errors.NoBoundaryInMultipartDefect))
         unless(isinstance(msg.defects[1],
                           errors.MultipartInvariantViolationDefect))
 
+    # test_defect_handling
     def test_missing_start_boundary(self):
         outer = self._msgobj('msg_42.txt')
         # The message structure is:
@@ -2003,17 +2053,29 @@ counter to RFC 2822, there's no separating newline here
         self.assertTrue(isinstance(bad.defects[0],
                                    errors.StartBoundaryNotFoundDefect))
 
+    # test_defect_handling
     def test_first_line_is_continuation_header(self):
         eq = self.assertEqual
-        m = ' Line 1\nLine 2\nLine 3'
+        m = ' Line 1\nSubject: test\n\nbody'
         msg = email.message_from_string(m)
-        eq(msg.keys(), [])
-        eq(msg.get_payload(), 'Line 2\nLine 3')
+        eq(msg.keys(), ['Subject'])
+        eq(msg.get_payload(), 'body')
         eq(len(msg.defects), 1)
-        self.assertTrue(isinstance(msg.defects[0],
-                                   errors.FirstHeaderLineIsContinuationDefect))
+        self.assertDefectsEqual(msg.defects,
+                                 [errors.FirstHeaderLineIsContinuationDefect])
         eq(msg.defects[0].line, ' Line 1\n')
 
+    # test_defect_handling
+    def test_missing_header_body_separator(self):
+        # Our heuristic if we see a line that doesn't look like a header (no
+        # leading whitespace but no ':') is to assume that the blank line that
+        # separates the header from the body is missing, and to stop parsing
+        # headers and start parsing the body.
+        msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n')
+        self.assertEqual(msg.keys(), ['Subject'])
+        self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n')
+        self.assertDefectsEqual(msg.defects,
+                                [errors.MissingHeaderBodySeparatorDefect])
 
 
 # Test RFC 2047 header encoding and decoding
@@ -2024,9 +2086,9 @@ class TestRFC2047(TestEmailBase):
  foo bar =?mac-iceland?q?r=8Aksm=9Arg=8Cs?="""
         dh = decode_header(s)
         eq(dh, [
-            (b'Re:', None),
+            (b'Re: ', None),
             (b'r\x8aksm\x9arg\x8cs', 'mac-iceland'),
-            (b'baz foo bar', None),
+            (b' baz foo bar ', None),
             (b'r\x8aksm\x9arg\x8cs', 'mac-iceland')])
         header = make_header(dh)
         eq(str(header),
@@ -2035,35 +2097,37 @@ class TestRFC2047(TestEmailBase):
 Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar =?mac-iceland?q?r=8Aksm?=
  =?mac-iceland?q?=9Arg=8Cs?=""")
 
-    def test_whitespace_eater_unicode(self):
+    def test_whitespace_keeper_unicode(self):
         eq = self.assertEqual
         s = '=?ISO-8859-1?Q?Andr=E9?= Pirard <pirard@dom.ain>'
         dh = decode_header(s)
         eq(dh, [(b'Andr\xe9', 'iso-8859-1'),
-                (b'Pirard <pirard@dom.ain>', None)])
+                (b' Pirard <pirard@dom.ain>', None)])
         header = str(make_header(dh))
         eq(header, 'Andr\xe9 Pirard <pirard@dom.ain>')
 
-    def test_whitespace_eater_unicode_2(self):
+    def test_whitespace_keeper_unicode_2(self):
         eq = self.assertEqual
         s = 'The =?iso-8859-1?b?cXVpY2sgYnJvd24gZm94?= jumped over the =?iso-8859-1?b?bGF6eSBkb2c=?='
         dh = decode_header(s)
-        eq(dh, [(b'The', None), (b'quick brown fox', 'iso-8859-1'),
-                (b'jumped over the', None), (b'lazy dog', 'iso-8859-1')])
+        eq(dh, [(b'The ', None), (b'quick brown fox', 'iso-8859-1'),
+                (b' jumped over the ', None), (b'lazy dog', 'iso-8859-1')])
         hu = str(make_header(dh))
         eq(hu, 'The quick brown fox jumped over the lazy dog')
 
     def test_rfc2047_missing_whitespace(self):
         s = 'Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord'
         dh = decode_header(s)
-        self.assertEqual(dh, [(s, None)])
+        self.assertEqual(dh, [(b'Sm', None), (b'\xf6', 'iso-8859-1'),
+                              (b'rg', None), (b'\xe5', 'iso-8859-1'),
+                              (b'sbord', None)])
 
     def test_rfc2047_with_whitespace(self):
         s = 'Sm =?ISO-8859-1?B?9g==?= rg =?ISO-8859-1?B?5Q==?= sbord'
         dh = decode_header(s)
-        self.assertEqual(dh, [(b'Sm', None), (b'\xf6', 'iso-8859-1'),
-                              (b'rg', None), (b'\xe5', 'iso-8859-1'),
-                              (b'sbord', None)])
+        self.assertEqual(dh, [(b'Sm ', None), (b'\xf6', 'iso-8859-1'),
+                              (b' rg ', None), (b'\xe5', 'iso-8859-1'),
+                              (b' sbord', None)])
 
     def test_rfc2047_B_bad_padding(self):
         s = '=?iso-8859-1?B?%s?='
@@ -2081,6 +2145,67 @@ Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar =?mac-iceland?q?r=8Aksm?=
         self.assertEqual(decode_header(s),
                         [(b'andr\xe9=zz', 'iso-8659-1')])
 
+    def test_rfc2047_rfc2047_1(self):
+        # 1st testcase at end of rfc2047
+        s = '(=?ISO-8859-1?Q?a?=)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'a', 'iso-8859-1'), (b')', None)])
+
+    def test_rfc2047_rfc2047_2(self):
+        # 2nd testcase at end of rfc2047
+        s = '(=?ISO-8859-1?Q?a?= b)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'a', 'iso-8859-1'), (b' b)', None)])
+
+    def test_rfc2047_rfc2047_3(self):
+        # 3rd testcase at end of rfc2047
+        s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)])
+
+    def test_rfc2047_rfc2047_4(self):
+        # 4th testcase at end of rfc2047
+        s = '(=?ISO-8859-1?Q?a?=  =?ISO-8859-1?Q?b?=)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)])
+
+    def test_rfc2047_rfc2047_5a(self):
+        # 5th testcase at end of rfc2047 newline is \r\n
+        s = '(=?ISO-8859-1?Q?a?=\r\n    =?ISO-8859-1?Q?b?=)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)])
+
+    def test_rfc2047_rfc2047_5b(self):
+        # 5th testcase at end of rfc2047 newline is \n
+        s = '(=?ISO-8859-1?Q?a?=\n    =?ISO-8859-1?Q?b?=)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)])
+
+    def test_rfc2047_rfc2047_6(self):
+        # 6th testcase at end of rfc2047
+        s = '(=?ISO-8859-1?Q?a_b?=)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'a b', 'iso-8859-1'), (b')', None)])
+
+    def test_rfc2047_rfc2047_7(self):
+        # 7th testcase at end of rfc2047
+        s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=)'
+        self.assertEqual(decode_header(s),
+            [(b'(', None), (b'a', 'iso-8859-1'), (b' b', 'iso-8859-2'),
+             (b')', None)])
+        self.assertEqual(make_header(decode_header(s)).encode(), s.lower())
+        self.assertEqual(str(make_header(decode_header(s))), '(a b)')
+
+    def test_multiline_header(self):
+        s = '=?windows-1252?q?=22M=FCller_T=22?=\r\n <T.Mueller@xxx.com>'
+        self.assertEqual(decode_header(s),
+            [(b'"M\xfcller T"', 'windows-1252'),
+             (b'<T.Mueller@xxx.com>', None)])
+        self.assertEqual(make_header(decode_header(s)).encode(),
+                         ''.join(s.splitlines()))
+        self.assertEqual(str(make_header(decode_header(s))),
+                         '"Müller T" <T.Mueller@xxx.com>')
+
 
 # Test the MIMEMessage class
 class TestMIMEMessage(TestEmailBase):
@@ -2606,6 +2731,13 @@ class TestMiscellaneous(TestEmailBase):
         for subpart in msg.walk():
             unless(isinstance(subpart, MyMessage))
 
+    def test_custom_message_does_not_require_arguments(self):
+        class MyMessage(Message):
+            def __init__(self):
+                super().__init__()
+        msg = self._str_msg("Subject: test\n\ntest", MyMessage)
+        self.assertTrue(isinstance(msg, MyMessage))
+
     def test__all__(self):
         module = __import__('email')
         self.assertEqual(sorted(module.__all__), [
@@ -2636,8 +2768,17 @@ class TestMiscellaneous(TestEmailBase):
             utils.formatdate(now, localtime=False, usegmt=True),
             time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(now)))
 
-    def test_parsedate_none(self):
-        self.assertEqual(utils.parsedate(''), None)
+    # parsedate and parsedate_tz will become deprecated interfaces someday
+    def test_parsedate_returns_None_for_invalid_strings(self):
+        self.assertIsNone(utils.parsedate(''))
+        self.assertIsNone(utils.parsedate_tz(''))
+        self.assertIsNone(utils.parsedate('0'))
+        self.assertIsNone(utils.parsedate_tz('0'))
+        self.assertIsNone(utils.parsedate('A Complete Waste of Time'))
+        self.assertIsNone(utils.parsedate_tz('A Complete Waste of Time'))
+        # Not a part of the spec but, but this has historically worked:
+        self.assertIsNone(utils.parsedate(None))
+        self.assertIsNone(utils.parsedate_tz(None))
 
     def test_parsedate_compact(self):
         # The FWS after the comma is optional
@@ -2664,6 +2805,13 @@ class TestMiscellaneous(TestEmailBase):
            (2002, 4, 3, 14, 58, 26, 0, 1, -1, -28800))
 
 
+    def test_parsedate_accepts_time_with_dots(self):
+        eq = self.assertEqual
+        eq(utils.parsedate_tz('5 Feb 2003 13.47.26 -0800'),
+           (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800))
+        eq(utils.parsedate_tz('5 Feb 2003 13.47 -0800'),
+           (2003, 2, 5, 13, 47, 0, 0, 1, -1, -28800))
+
     def test_parsedate_acceptable_to_time_functions(self):
         eq = self.assertEqual
         timetup = utils.parsedate('5 Feb 2003 13:47:26 -0800')
@@ -2706,7 +2854,10 @@ class TestMiscellaneous(TestEmailBase):
     def test_escape_dump(self):
         self.assertEqual(
             utils.formataddr(('A (Very) Silly Person', 'person@dom.ain')),
-            r'"A \(Very\) Silly Person" <person@dom.ain>')
+            r'"A (Very) Silly Person" <person@dom.ain>')
+        self.assertEqual(
+            utils.parseaddr(r'"A \(Very\) Silly Person" <person@dom.ain>'),
+            ('A (Very) Silly Person', 'person@dom.ain'))
         a = r'A \(Special\) Person'
         b = 'person@dom.ain'
         self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b))
@@ -2719,6 +2870,46 @@ class TestMiscellaneous(TestEmailBase):
         b = 'person@dom.ain'
         self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b))
 
+    def test_quotes_unicode_names(self):
+        # issue 1690608.  email.utils.formataddr() should be rfc2047 aware.
+        name = "H\u00e4ns W\u00fcrst"
+        addr = 'person@dom.ain'
+        utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= <person@dom.ain>"
+        latin1_quopri = "=?iso-8859-1?q?H=E4ns_W=FCrst?= <person@dom.ain>"
+        self.assertEqual(utils.formataddr((name, addr)), utf8_base64)
+        self.assertEqual(utils.formataddr((name, addr), 'iso-8859-1'),
+            latin1_quopri)
+
+    def test_accepts_any_charset_like_object(self):
+        # issue 1690608.  email.utils.formataddr() should be rfc2047 aware.
+        name = "H\u00e4ns W\u00fcrst"
+        addr = 'person@dom.ain'
+        utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= <person@dom.ain>"
+        foobar = "FOOBAR"
+        class CharsetMock:
+            def header_encode(self, string):
+                return foobar
+        mock = CharsetMock()
+        mock_expected = "%s <%s>" % (foobar, addr)
+        self.assertEqual(utils.formataddr((name, addr), mock), mock_expected)
+        self.assertEqual(utils.formataddr((name, addr), Charset('utf-8')),
+            utf8_base64)
+
+    def test_invalid_charset_like_object_raises_error(self):
+        # issue 1690608.  email.utils.formataddr() should be rfc2047 aware.
+        name = "H\u00e4ns W\u00fcrst"
+        addr = 'person@dom.ain'
+        # A object without a header_encode method:
+        bad_charset = object()
+        self.assertRaises(AttributeError, utils.formataddr, (name, addr),
+            bad_charset)
+
+    def test_unicode_address_raises_error(self):
+        # issue 1690608.  email.utils.formataddr() should be rfc2047 aware.
+        addr = 'pers\u00f6n@dom.in'
+        self.assertRaises(UnicodeError, utils.formataddr, (None, addr))
+        self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr))
+
     def test_name_with_dot(self):
         x = 'John X. Doe <jxd@example.com>'
         y = '"John X. Doe" <jxd@example.com>'
@@ -2764,6 +2955,15 @@ class TestMiscellaneous(TestEmailBase):
         self.assertEqual(('', 'merwok.wok.wok@xample.com'),
             utils.parseaddr('merwok. wok .  wok@xample.com'))
 
+    def test_formataddr_does_not_quote_parens_in_quoted_string(self):
+        addr = ("'foo@example.com' (foo@example.com)",
+                'foo@example.com')
+        addrstr = ('"\'foo@example.com\' '
+                            '(foo@example.com)" <foo@example.com>')
+        self.assertEqual(utils.parseaddr(addrstr), addr)
+        self.assertEqual(utils.formataddr(addr), addrstr)
+
+
     def test_multiline_from_comment(self):
         x = """\
 Foo
@@ -3033,6 +3233,7 @@ Do you like this message?
 
 
 class TestParsers(TestEmailBase):
+
     def test_header_parser(self):
         eq = self.assertEqual
         # Parse only the headers of a complex multipart MIME document
@@ -3044,6 +3245,18 @@ class TestParsers(TestEmailBase):
         self.assertFalse(msg.is_multipart())
         self.assertTrue(isinstance(msg.get_payload(), str))
 
+    def test_bytes_header_parser(self):
+        eq = self.assertEqual
+        # Parse only the headers of a complex multipart MIME document
+        with openfile('msg_02.txt', 'rb') as fp:
+            msg = email.parser.BytesHeaderParser().parse(fp)
+        eq(msg['from'], 'ppp-request@zzz.org')
+        eq(msg['to'], 'ppp@zzz.org')
+        eq(msg.get_content_type(), 'multipart/mixed')
+        self.assertFalse(msg.is_multipart())
+        self.assertTrue(isinstance(msg.get_payload(), str))
+        self.assertTrue(isinstance(msg.get_payload(decode=True), bytes))
+
     def test_whitespace_continuation(self):
         eq = self.assertEqual
         # This message contains a line after the Subject: header that has only
@@ -3266,15 +3479,19 @@ class Test8BitBytesHandling(unittest.TestCase):
         self.assertEqual(msg.get_payload(decode=True),
                         'pöstál\n'.encode('utf-8'))
 
+    # test_defect_handling:test_invalid_chars_in_base64_payload
     def test_8bit_in_base64_body(self):
-        # Sticking an 8bit byte in a base64 block makes it undecodable by
-        # normal means, so the block is returned undecoded, but as bytes.
+        # If we get 8bit bytes in a base64 body, we can just ignore them
+        # as being outside the base64 alphabet and decode anyway.  But
+        # we register a defect.
         m = self.bodytest_msg.format(charset='utf-8',
                                      cte='base64',
                                      bodyline='cMO2c3RhbAá=').encode('utf-8')
         msg = email.message_from_bytes(m)
         self.assertEqual(msg.get_payload(decode=True),
-                         'cMO2c3RhbAá=\n'.encode('utf-8'))
+                         'pöstal'.encode('utf-8'))
+        self.assertIsInstance(msg.defects[0],
+                              errors.InvalidBase64CharactersDefect)
 
     def test_8bit_in_uuencode_body(self):
         # Sticking an 8bit byte in a uuencode block makes it undecodable by
@@ -3355,6 +3572,7 @@ class Test8BitBytesHandling(unittest.TestCase):
         self.assertEqual(msg.get_content_maintype(), "text")
         self.assertEqual(msg.get_content_subtype(), "pl\uFFFDin")
 
+    # test_headerregistry.TestContentTypeHeader.non_ascii_in_params
     def test_get_params_with_8bit(self):
         msg = email.message_from_bytes(
             'X-Header: foo=\xa7ne; b\xa7r=two; baz=three\n'.encode('latin-1'))
@@ -3364,6 +3582,7 @@ class Test8BitBytesHandling(unittest.TestCase):
         # XXX: someday you might be able to get 'b\xa7r', for now you can't.
         self.assertEqual(msg.get_param('b\xa7r', header='x-header'), None)
 
+    # test_headerregistry.TestContentTypeHeader.non_ascii_in_rfc2231_value
     def test_get_rfc2231_params_with_8bit(self):
         msg = email.message_from_bytes(textwrap.dedent("""\
             Content-Type: text/plain; charset=us-ascii;
@@ -3609,12 +3828,7 @@ class BaseTestBytesGeneratorIdempotent:
         b = BytesIO()
         g = email.generator.BytesGenerator(b, maxheaderlen=0)
         g.flatten(msg, unixfrom=unixfrom, linesep=self.linesep)
-        self.assertByteStringsEqual(data, b.getvalue())
-
-    def assertByteStringsEqual(self, str1, str2):
-        # Not using self.blinesep here is intentional.  This way the output
-        # is more useful when the failure results in mixed line endings.
-        self.assertListEqual(str1.split(b'\n'), str2.split(b'\n'))
+        self.assertEqual(data, b.getvalue())
 
 
 class TestBytesGeneratorIdempotentNL(BaseTestBytesGeneratorIdempotent,
@@ -4380,11 +4594,11 @@ A very long line that must get split to something other than at the
         h = make_header(decode_header(s))
         eq(h.encode(), s)
 
-    def test_whitespace_eater(self):
+    def test_whitespace_keeper(self):
         eq = self.assertEqual
         s = 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztk=?= =?koi8-r?q?=CA?= zz.'
         parts = decode_header(s)
-        eq(parts, [(b'Subject:', None), (b'\xf0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 \xce\xc1 \xc6\xc9\xce\xc1\xcc\xd8\xce\xd9\xca', 'koi8-r'), (b'zz.', None)])
+        eq(parts, [(b'Subject: ', None), (b'\xf0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 \xce\xc1 \xc6\xc9\xce\xc1\xcc\xd8\xce\xd9\xca', 'koi8-r'), (b' zz.', None)])
         hdr = make_header(parts)
         eq(hdr.encode(),
            'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztnK?= zz.')
@@ -4414,6 +4628,9 @@ A very long line that must get split to something other than at the
 
 # Test RFC 2231 header parameters (en/de)coding
 class TestRFC2231(TestEmailBase):
+
+    # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_with_double_quotes
+    # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_inside_double_quotes
     def test_get_param(self):
         eq = self.assertEqual
         msg = self._msgobj('msg_29.txt')
@@ -4499,11 +4716,15 @@ Do you like this message?
 -Me
 """)
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_charset
+    # I changed the charset name, though, because the one in the file isn't
+    # a legal charset name.  Should add a test for an illegal charset.
     def test_rfc2231_get_content_charset(self):
         eq = self.assertEqual
         msg = self._msgobj('msg_32.txt')
         eq(msg.get_content_charset(), 'us-ascii')
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_no_double_quotes
     def test_rfc2231_parse_rfc_quoting(self):
         m = textwrap.dedent('''\
             Content-Disposition: inline;
@@ -4517,6 +4738,7 @@ Do you like this message?
                          'This is even more ***fun*** is it not.pdf')
         self.assertEqual(m, msg.as_string())
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_with_double_quotes
     def test_rfc2231_parse_extra_quoting(self):
         m = textwrap.dedent('''\
             Content-Disposition: inline;
@@ -4530,6 +4752,9 @@ Do you like this message?
                          'This is even more ***fun*** is it not.pdf')
         self.assertEqual(m, msg.as_string())
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_no_language_or_charset
+    # but new test uses *0* because otherwise lang/charset is not valid.
+    # test_headerregistry.TestContentTypeHeader.rfc2231_segmented_normal_values
     def test_rfc2231_no_language_or_charset(self):
         m = '''\
 Content-Transfer-Encoding: 8bit
@@ -4544,6 +4769,7 @@ Content-Type: text/html; NAME*0=file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOC
             param,
             'file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm')
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_no_charset
     def test_rfc2231_no_language_or_charset_in_filename(self):
         m = '''\
 Content-Disposition: inline;
@@ -4556,6 +4782,7 @@ Content-Disposition: inline;
         self.assertEqual(msg.get_filename(),
                          'This is even more ***fun*** is it not.pdf')
 
+    # Duplicate of previous test?
     def test_rfc2231_no_language_or_charset_in_filename_encoded(self):
         m = '''\
 Content-Disposition: inline;
@@ -4568,6 +4795,8 @@ Content-Disposition: inline;
         self.assertEqual(msg.get_filename(),
                          'This is even more ***fun*** is it not.pdf')
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_partly_encoded,
+    # but the test below is wrong (the first part should be decoded).
     def test_rfc2231_partly_encoded(self):
         m = '''\
 Content-Disposition: inline;
@@ -4619,6 +4848,7 @@ Content-Type: text/plain;
         self.assertEqual(msg.get_content_charset(),
                          'this is even more ***fun*** is it not.pdf')
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_unknown_charset_treated_as_ascii
     def test_rfc2231_bad_encoding_in_filename(self):
         m = '''\
 Content-Disposition: inline;
@@ -4685,6 +4915,7 @@ Content-Type: application/x-foo;
         eq(language, None)
         eq(s, "Frank's Document")
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_inside_double_quotes
     def test_rfc2231_single_tick_in_filename(self):
         m = """\
 Content-Type: application/x-foo; name*0=\"Frank's\"; name*1=\" Document\"
@@ -4695,6 +4926,7 @@ Content-Type: application/x-foo; name*0=\"Frank's\"; name*1=\" Document\"
         self.assertFalse(isinstance(param, tuple))
         self.assertEqual(param, "Frank's Document")
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_in_value_with_charset_and_lang
     def test_rfc2231_tick_attack_extended(self):
         eq = self.assertEqual
         m = """\
@@ -4708,6 +4940,7 @@ Content-Type: application/x-foo;
         eq(language, 'en-us')
         eq(s, "Frank's Document")
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_single_quote_in_non_encoded_value
     def test_rfc2231_tick_attack(self):
         m = """\
 Content-Type: application/x-foo;
@@ -4719,6 +4952,7 @@ Content-Type: application/x-foo;
         self.assertFalse(isinstance(param, tuple))
         self.assertEqual(param, "us-ascii'en-us'Frank's Document")
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_single_quotes_inside_quotes
     def test_rfc2231_no_extended_values(self):
         eq = self.assertEqual
         m = """\
@@ -4728,6 +4962,7 @@ Content-Type: application/x-foo; name=\"Frank's Document\"
         msg = email.message_from_string(m)
         eq(msg.get_param('name'), "Frank's Document")
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_encoded_then_unencoded_segments
     def test_rfc2231_encoded_then_unencoded_segments(self):
         eq = self.assertEqual
         m = """\
@@ -4743,6 +4978,8 @@ Content-Type: application/x-foo;
         eq(language, 'en-us')
         eq(s, 'My Document For You')
 
+    # test_headerregistry.TestContentTypeHeader.rfc2231_unencoded_then_encoded_segments
+    # test_headerregistry.TestContentTypeHeader.rfc2231_quoted_unencoded_then_encoded_segments
     def test_rfc2231_unencoded_then_encoded_segments(self):
         eq = self.assertEqual
         m = """\
@@ -4766,7 +5003,7 @@ Content-Type: application/x-foo;
 class TestSigned(TestEmailBase):
 
     def _msg_and_obj(self, filename):
-        with openfile(findfile(filename)) as fp:
+        with openfile(filename) as fp:
             original = fp.read()
             msg = email.message_from_string(original)
         return original, msg
@@ -4798,23 +5035,5 @@ class TestSigned(TestEmailBase):
 
 
 
-def _testclasses():
-    mod = sys.modules[__name__]
-    return [getattr(mod, name) for name in dir(mod) if name.startswith('Test')]
-
-
-def suite():
-    suite = unittest.TestSuite()
-    for testclass in _testclasses():
-        suite.addTest(unittest.makeSuite(testclass))
-    return suite
-
-
-def test_main():
-    for testclass in _testclasses():
-        run_unittest(testclass)
-
-
-
 if __name__ == '__main__':
-    unittest.main(defaultTest='suite')
+    unittest.main()
diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py
new file mode 100644
index 0000000000..8917408171
--- /dev/null
+++ b/Lib/test/test_email/test_generator.py
@@ -0,0 +1,199 @@
+import io
+import textwrap
+import unittest
+from email import message_from_string, message_from_bytes
+from email.generator import Generator, BytesGenerator
+from email import policy
+from test.test_email import TestEmailBase, parameterize
+
+
+@parameterize
+class TestGeneratorBase:
+
+    policy = policy.default
+
+    def msgmaker(self, msg, policy=None):
+        policy = self.policy if policy is None else policy
+        return self.msgfunc(msg, policy=policy)
+
+    refold_long_expected = {
+        0: textwrap.dedent("""\
+            To: whom_it_may_concern@example.com
+            From: nobody_you_want_to_know@example.com
+            Subject: We the willing led by the unknowing are doing the
+             impossible for the ungrateful. We have done so much for so long with so little
+             we are now qualified to do anything with nothing.
+
+            None
+            """),
+        # From is wrapped because wrapped it fits in 40.
+        40: textwrap.dedent("""\
+            To: whom_it_may_concern@example.com
+            From:
+             nobody_you_want_to_know@example.com
+            Subject: We the willing led by the
+             unknowing are doing the impossible for
+             the ungrateful. We have done so much
+             for so long with so little we are now
+             qualified to do anything with nothing.
+
+            None
+            """),
+        # Neither to nor from fit even if put on a new line,
+        # so we leave them sticking out on the first line.
+        20: textwrap.dedent("""\
+            To: whom_it_may_concern@example.com
+            From: nobody_you_want_to_know@example.com
+            Subject: We the
+             willing led by the
+             unknowing are doing
+             the impossible for
+             the ungrateful. We
+             have done so much
+             for so long with so
+             little we are now
+             qualified to do
+             anything with
+             nothing.
+
+            None
+            """),
+        }
+    refold_long_expected[100] = refold_long_expected[0]
+
+    refold_all_expected = refold_long_expected.copy()
+    refold_all_expected[0] = (
+            "To: whom_it_may_concern@example.com\n"
+            "From: nobody_you_want_to_know@example.com\n"
+            "Subject: We the willing led by the unknowing are doing the "
+              "impossible for the ungrateful. We have done so much for "
+              "so long with so little we are now qualified to do anything "
+              "with nothing.\n"
+              "\n"
+              "None\n")
+    refold_all_expected[100] = (
+            "To: whom_it_may_concern@example.com\n"
+            "From: nobody_you_want_to_know@example.com\n"
+            "Subject: We the willing led by the unknowing are doing the "
+                "impossible for the ungrateful. We have\n"
+              " done so much for so long with so little we are now qualified "
+                "to do anything with nothing.\n"
+              "\n"
+              "None\n")
+
+    length_params = [n for n in refold_long_expected]
+
+    def length_as_maxheaderlen_parameter(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
+        s = self.ioclass()
+        g = self.genclass(s, maxheaderlen=n, policy=self.policy)
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n]))
+
+    def length_as_max_line_length_policy(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
+        s = self.ioclass()
+        g = self.genclass(s, policy=self.policy.clone(max_line_length=n))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n]))
+
+    def length_as_maxheaderlen_parm_overrides_policy(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
+        s = self.ioclass()
+        g = self.genclass(s, maxheaderlen=n,
+                          policy=self.policy.clone(max_line_length=10))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[n]))
+
+    def length_as_max_line_length_with_refold_none_does_not_fold(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
+        s = self.ioclass()
+        g = self.genclass(s, policy=self.policy.clone(refold_source='none',
+                                                      max_line_length=n))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0]))
+
+    def length_as_max_line_length_with_refold_all_folds(self, n):
+        msg = self.msgmaker(self.typ(self.refold_long_expected[0]))
+        s = self.ioclass()
+        g = self.genclass(s, policy=self.policy.clone(refold_source='all',
+                                                      max_line_length=n))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(self.refold_all_expected[n]))
+
+    def test_crlf_control_via_policy(self):
+        source = "Subject: test\r\n\r\ntest body\r\n"
+        expected = source
+        msg = self.msgmaker(self.typ(source))
+        s = self.ioclass()
+        g = self.genclass(s, policy=policy.SMTP)
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), self.typ(expected))
+
+    def test_flatten_linesep_overrides_policy(self):
+        source = "Subject: test\n\ntest body\n"
+        expected = source
+        msg = self.msgmaker(self.typ(source))
+        s = self.ioclass()
+        g = self.genclass(s, policy=policy.SMTP)
+        g.flatten(msg, linesep='\n')
+        self.assertEqual(s.getvalue(), self.typ(expected))
+
+
+class TestGenerator(TestGeneratorBase, TestEmailBase):
+
+    msgfunc = staticmethod(message_from_string)
+    genclass = Generator
+    ioclass = io.StringIO
+    typ = str
+
+
+class TestBytesGenerator(TestGeneratorBase, TestEmailBase):
+
+    msgfunc = staticmethod(message_from_bytes)
+    genclass = BytesGenerator
+    ioclass = io.BytesIO
+    typ = lambda self, x: x.encode('ascii')
+
+    def test_cte_type_7bit_handles_unknown_8bit(self):
+        source = ("Subject: Maintenant je vous présente mon "
+                 "collègue\n\n").encode('utf-8')
+        expected = ('Subject: Maintenant je vous =?unknown-8bit?q?'
+                    'pr=C3=A9sente_mon_coll=C3=A8gue?=\n\n').encode('ascii')
+        msg = message_from_bytes(source)
+        s = io.BytesIO()
+        g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit'))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), expected)
+
+    def test_cte_type_7bit_transforms_8bit_cte(self):
+        source = textwrap.dedent("""\
+            From: foo@bar.com
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink
+            Mime-Version: 1.0
+            Content-Type: text/plain; charset="latin-1"
+            Content-Transfer-Encoding: 8bit
+
+            oh là là, know what I mean, know what I mean?
+            """).encode('latin1')
+        msg = message_from_bytes(source)
+        expected =  textwrap.dedent("""\
+            From: foo@bar.com
+            To: Dinsdale
+            Subject: Nudge nudge, wink, wink
+            Mime-Version: 1.0
+            Content-Type: text/plain; charset="iso-8859-1"
+            Content-Transfer-Encoding: quoted-printable
+
+            oh l=E0 l=E0, know what I mean, know what I mean?
+            """).encode('ascii')
+        s = io.BytesIO()
+        g = BytesGenerator(s, policy=self.policy.clone(cte_type='7bit',
+                                                       linesep='\n'))
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), expected)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py
new file mode 100644
index 0000000000..c0c81c1caa
--- /dev/null
+++ b/Lib/test/test_email/test_headerregistry.py
@@ -0,0 +1,1515 @@
+import datetime
+import textwrap
+import unittest
+from email import errors
+from email import policy
+from email.message import Message
+from test.test_email import TestEmailBase, parameterize
+from email import headerregistry
+from email.headerregistry import Address, Group
+
+
+DITTO = object()
+
+
+class TestHeaderRegistry(TestEmailBase):
+
+    def test_arbitrary_name_unstructured(self):
+        factory = headerregistry.HeaderRegistry()
+        h = factory('foobar', 'test')
+        self.assertIsInstance(h, headerregistry.BaseHeader)
+        self.assertIsInstance(h, headerregistry.UnstructuredHeader)
+
+    def test_name_case_ignored(self):
+        factory = headerregistry.HeaderRegistry()
+        # Whitebox check that test is valid
+        self.assertNotIn('Subject', factory.registry)
+        h = factory('Subject', 'test')
+        self.assertIsInstance(h, headerregistry.BaseHeader)
+        self.assertIsInstance(h, headerregistry.UniqueUnstructuredHeader)
+
+    class FooBase:
+        def __init__(self, *args, **kw):
+            pass
+
+    def test_override_default_base_class(self):
+        factory = headerregistry.HeaderRegistry(base_class=self.FooBase)
+        h = factory('foobar', 'test')
+        self.assertIsInstance(h, self.FooBase)
+        self.assertIsInstance(h, headerregistry.UnstructuredHeader)
+
+    class FooDefault:
+        parse = headerregistry.UnstructuredHeader.parse
+
+    def test_override_default_class(self):
+        factory = headerregistry.HeaderRegistry(default_class=self.FooDefault)
+        h = factory('foobar', 'test')
+        self.assertIsInstance(h, headerregistry.BaseHeader)
+        self.assertIsInstance(h, self.FooDefault)
+
+    def test_override_default_class_only_overrides_default(self):
+        factory = headerregistry.HeaderRegistry(default_class=self.FooDefault)
+        h = factory('subject', 'test')
+        self.assertIsInstance(h, headerregistry.BaseHeader)
+        self.assertIsInstance(h, headerregistry.UniqueUnstructuredHeader)
+
+    def test_dont_use_default_map(self):
+        factory = headerregistry.HeaderRegistry(use_default_map=False)
+        h = factory('subject', 'test')
+        self.assertIsInstance(h, headerregistry.BaseHeader)
+        self.assertIsInstance(h, headerregistry.UnstructuredHeader)
+
+    def test_map_to_type(self):
+        factory = headerregistry.HeaderRegistry()
+        h1 = factory('foobar', 'test')
+        factory.map_to_type('foobar', headerregistry.UniqueUnstructuredHeader)
+        h2 = factory('foobar', 'test')
+        self.assertIsInstance(h1, headerregistry.BaseHeader)
+        self.assertIsInstance(h1, headerregistry.UnstructuredHeader)
+        self.assertIsInstance(h2, headerregistry.BaseHeader)
+        self.assertIsInstance(h2, headerregistry.UniqueUnstructuredHeader)
+
+
+class TestHeaderBase(TestEmailBase):
+
+    factory = headerregistry.HeaderRegistry()
+
+    def make_header(self, name, value):
+        return self.factory(name, value)
+
+
+class TestBaseHeaderFeatures(TestHeaderBase):
+
+    def test_str(self):
+        h = self.make_header('subject', 'this is a test')
+        self.assertIsInstance(h, str)
+        self.assertEqual(h, 'this is a test')
+        self.assertEqual(str(h), 'this is a test')
+
+    def test_substr(self):
+        h = self.make_header('subject', 'this is a test')
+        self.assertEqual(h[5:7], 'is')
+
+    def test_has_name(self):
+        h = self.make_header('subject', 'this is a test')
+        self.assertEqual(h.name, 'subject')
+
+    def _test_attr_ro(self, attr):
+        h = self.make_header('subject', 'this is a test')
+        with self.assertRaises(AttributeError):
+            setattr(h, attr, 'foo')
+
+    def test_name_read_only(self):
+        self._test_attr_ro('name')
+
+    def test_defects_read_only(self):
+        self._test_attr_ro('defects')
+
+    def test_defects_is_tuple(self):
+        h = self.make_header('subject', 'this is a test')
+        self.assertEqual(len(h.defects), 0)
+        self.assertIsInstance(h.defects, tuple)
+        # Make sure it is still true when there are defects.
+        h = self.make_header('date', '')
+        self.assertEqual(len(h.defects), 1)
+        self.assertIsInstance(h.defects, tuple)
+
+    # XXX: FIXME
+    #def test_CR_in_value(self):
+    #    # XXX: this also re-raises the issue of embedded headers,
+    #    # need test and solution for that.
+    #    value = '\r'.join(['this is', ' a test'])
+    #    h = self.make_header('subject', value)
+    #    self.assertEqual(h, value)
+    #    self.assertDefectsEqual(h.defects, [errors.ObsoleteHeaderDefect])
+
+    def test_RFC2047_value_decoded(self):
+        value = '=?utf-8?q?this_is_a_test?='
+        h = self.make_header('subject', value)
+        self.assertEqual(h, 'this is a test')
+
+
+class TestDateHeader(TestHeaderBase):
+
+    datestring = 'Sun, 23 Sep 2001 20:10:55 -0700'
+    utcoffset = datetime.timedelta(hours=-7)
+    tz = datetime.timezone(utcoffset)
+    dt = datetime.datetime(2001, 9, 23, 20, 10, 55, tzinfo=tz)
+
+    def test_parse_date(self):
+        h = self.make_header('date', self.datestring)
+        self.assertEqual(h, self.datestring)
+        self.assertEqual(h.datetime, self.dt)
+        self.assertEqual(h.datetime.utcoffset(), self.utcoffset)
+        self.assertEqual(h.defects, ())
+
+    def test_set_from_datetime(self):
+        h = self.make_header('date', self.dt)
+        self.assertEqual(h, self.datestring)
+        self.assertEqual(h.datetime, self.dt)
+        self.assertEqual(h.defects, ())
+
+    def test_date_header_properties(self):
+        h = self.make_header('date', self.datestring)
+        self.assertIsInstance(h, headerregistry.UniqueDateHeader)
+        self.assertEqual(h.max_count, 1)
+        self.assertEqual(h.defects, ())
+
+    def test_resent_date_header_properties(self):
+        h = self.make_header('resent-date', self.datestring)
+        self.assertIsInstance(h, headerregistry.DateHeader)
+        self.assertEqual(h.max_count, None)
+        self.assertEqual(h.defects, ())
+
+    def test_no_value_is_defect(self):
+        h = self.make_header('date', '')
+        self.assertEqual(len(h.defects), 1)
+        self.assertIsInstance(h.defects[0], errors.HeaderMissingRequiredValue)
+
+    def test_datetime_read_only(self):
+        h = self.make_header('date', self.datestring)
+        with self.assertRaises(AttributeError):
+            h.datetime = 'foo'
+
+    def test_set_date_header_from_datetime(self):
+        m = Message(policy=policy.default)
+        m['Date'] = self.dt
+        self.assertEqual(m['Date'], self.datestring)
+        self.assertEqual(m['Date'].datetime, self.dt)
+
+
+@parameterize
+class TestContentTypeHeader(TestHeaderBase):
+
+    def content_type_as_value(self,
+                              source,
+                              content_type,
+                              maintype,
+                              subtype,
+                              *args):
+        l = len(args)
+        parmdict = args[0] if l>0 else {}
+        defects =  args[1] if l>1 else []
+        decoded =  args[2] if l>2 and args[2] is not DITTO else source
+        header = 'Content-Type:' + ' ' if source else ''
+        folded = args[3] if l>3 else header + source + '\n'
+        h = self.make_header('Content-Type', source)
+        self.assertEqual(h.content_type, content_type)
+        self.assertEqual(h.maintype, maintype)
+        self.assertEqual(h.subtype, subtype)
+        self.assertEqual(h.params, parmdict)
+        self.assertDefectsEqual(h.defects, defects)
+        self.assertEqual(h, decoded)
+        self.assertEqual(h.fold(policy=policy.default), folded)
+
+    content_type_params = {
+
+        # Examples from RFC 2045.
+
+        'RFC_2045_1': (
+            'text/plain; charset=us-ascii (Plain text)',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'us-ascii'},
+            [],
+            'text/plain; charset="us-ascii"'),
+
+        'RFC_2045_2': (
+            'text/plain; charset=us-ascii',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'us-ascii'},
+            [],
+            'text/plain; charset="us-ascii"'),
+
+        'RFC_2045_3': (
+            'text/plain; charset="us-ascii"',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'us-ascii'}),
+
+        # RFC 2045 5.2 says syntactically invalid values are to be treated as
+        # text/plain.
+
+        'no_subtype_in_content_type': (
+            'text/',
+            'text/plain',
+            'text',
+            'plain',
+            {},
+            [errors.InvalidHeaderDefect]),
+
+        'no_slash_in_content_type': (
+            'foo',
+            'text/plain',
+            'text',
+            'plain',
+            {},
+            [errors.InvalidHeaderDefect]),
+
+        'junk_text_in_content_type': (
+            '<crazy "stuff">',
+            'text/plain',
+            'text',
+            'plain',
+            {},
+            [errors.InvalidHeaderDefect]),
+
+        'too_many_slashes_in_content_type': (
+            'image/jpeg/foo',
+            'text/plain',
+            'text',
+            'plain',
+            {},
+            [errors.InvalidHeaderDefect]),
+
+        # But unknown names are OK.  We could make non-IANA names a defect, but
+        # by not doing so we make ourselves future proof.  The fact that they
+        # are unknown will be detectable by the fact that they don't appear in
+        # the mime_registry...and the application is free to extend that list
+        # to handle them even if the core library doesn't.
+
+        'unknown_content_type': (
+            'bad/names',
+            'bad/names',
+            'bad',
+            'names'),
+
+        # The content type is case insensitive, and CFWS is ignored.
+
+        'mixed_case_content_type': (
+            'ImAge/JPeg',
+            'image/jpeg',
+            'image',
+            'jpeg'),
+
+        'spaces_in_content_type': (
+            '  text  /  plain  ',
+            'text/plain',
+            'text',
+            'plain'),
+
+        'cfws_in_content_type': (
+            '(foo) text (bar)/(baz)plain(stuff)',
+            'text/plain',
+            'text',
+            'plain'),
+
+        # test some parameters (more tests could be added for parameters
+        # associated with other content types, but since parameter parsing is
+        # generic they would be redundant for the current implementation).
+
+        'charset_param': (
+            'text/plain; charset="utf-8"',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'utf-8'}),
+
+        'capitalized_charset': (
+            'text/plain; charset="US-ASCII"',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'US-ASCII'}),
+
+        'unknown_charset': (
+            'text/plain; charset="fOo"',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'fOo'}),
+
+        'capitalized_charset_param_name_and_comment': (
+            'text/plain; (interjection) Charset="utf-8"',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'utf-8'},
+            [],
+            # Should the parameter name be lowercased here?
+            'text/plain; Charset="utf-8"'),
+
+        # Since this is pretty much the ur-mimeheader, we'll put all the tests
+        # that exercise the parameter parsing and formatting here.
+        #
+        # XXX: question: is minimal quoting preferred?
+
+        'unquoted_param_value': (
+            'text/plain; title=foo',
+            'text/plain',
+            'text',
+            'plain',
+            {'title': 'foo'},
+            [],
+            'text/plain; title="foo"'),
+
+        'param_value_with_tspecials': (
+            'text/plain; title="(bar)foo blue"',
+            'text/plain',
+            'text',
+            'plain',
+            {'title': '(bar)foo blue'}),
+
+        'param_with_extra_quoted_whitespace': (
+            'text/plain; title="  a     loong  way \t home   "',
+            'text/plain',
+            'text',
+            'plain',
+            {'title': '  a     loong  way \t home   '}),
+
+        'bad_params': (
+            'blarg; baz; boo',
+            'text/plain',
+            'text',
+            'plain',
+            {'baz': '', 'boo': ''},
+            [errors.InvalidHeaderDefect]*3),
+
+        'spaces_around_param_equals': (
+            'Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"',
+            'multipart/mixed',
+            'multipart',
+            'mixed',
+            {'boundary': 'CPIMSSMTPC06p5f3tG'},
+            [],
+            'Multipart/mixed; boundary="CPIMSSMTPC06p5f3tG"'),
+
+        'spaces_around_semis': (
+            ('image/jpeg; name="wibble.JPG" ; x-mac-type="4A504547" ; '
+                'x-mac-creator="474B4F4E"'),
+            'image/jpeg',
+            'image',
+            'jpeg',
+            {'name': 'wibble.JPG',
+             'x-mac-type': '4A504547',
+             'x-mac-creator': '474B4F4E'},
+            [],
+            ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; '
+                'x-mac-creator="474B4F4E"'),
+            # XXX: it could be that we will eventually prefer to fold starting
+            # from the decoded value, in which case these spaces and similar
+            # spaces in other tests will be wrong.
+            ('Content-Type: image/jpeg; name="wibble.JPG" ; '
+                'x-mac-type="4A504547" ;\n'
+             ' x-mac-creator="474B4F4E"\n'),
+            ),
+
+        'semis_inside_quotes': (
+            'image/jpeg; name="Jim&amp;&amp;Jill"',
+            'image/jpeg',
+            'image',
+            'jpeg',
+            {'name': 'Jim&amp;&amp;Jill'}),
+
+        'single_quotes_inside_quotes': (
+            'image/jpeg; name="Jim \'Bob\' Jill"',
+            'image/jpeg',
+            'image',
+            'jpeg',
+            {'name': "Jim 'Bob' Jill"}),
+
+        'double_quotes_inside_quotes': (
+            r'image/jpeg; name="Jim \"Bob\" Jill"',
+            'image/jpeg',
+            'image',
+            'jpeg',
+            {'name': 'Jim "Bob" Jill'},
+            [],
+            r'image/jpeg; name="Jim \"Bob\" Jill"'),
+
+        # XXX: This test works except for the refolding of the header.  I'll
+        # deal with that bug when I deal with the other folding bugs.
+        #'non_ascii_in_params': (
+        #    ('foo\xa7/bar; b\xa7r=two; '
+        #        'baz=thr\xa7e'.encode('latin-1').decode('us-ascii',
+        #                                                'surrogateescape')),
+        #    'foo\uFFFD/bar',
+        #    'foo\uFFFD',
+        #    'bar',
+        #    {'b\uFFFDr': 'two', 'baz': 'thr\uFFFDe'},
+        #    [errors.UndecodableBytesDefect]*3,
+        #    'foo�/bar; b�r="two"; baz="thr�e"',
+        #    ),
+
+        # RFC 2231 parameter tests.
+
+        'rfc2231_segmented_normal_values': (
+            'image/jpeg; name*0="abc"; name*1=".html"',
+            'image/jpeg',
+            'image',
+            'jpeg',
+            {'name': "abc.html"},
+            [],
+            'image/jpeg; name="abc.html"'),
+
+        'quotes_inside_rfc2231_value': (
+            r'image/jpeg; bar*0="baz\"foobar"; bar*1="\"baz"',
+            'image/jpeg',
+            'image',
+            'jpeg',
+            {'bar': 'baz"foobar"baz'},
+            [],
+            r'image/jpeg; bar="baz\"foobar\"baz"'),
+
+        # XXX: This test works except for the refolding of the header.  I'll
+        # deal with that bug when I deal with the other folding bugs.
+        #'non_ascii_rfc2231_value': (
+        #    ('text/plain; charset=us-ascii; '
+        #     "title*=us-ascii'en'This%20is%20"
+        #     'not%20f\xa7n').encode('latin-1').decode('us-ascii',
+        #                                             'surrogateescape'),
+        #    'text/plain',
+        #    'text',
+        #    'plain',
+        #    {'charset': 'us-ascii', 'title': 'This is not f\uFFFDn'},
+        #     [errors.UndecodableBytesDefect],
+        #     'text/plain; charset="us-ascii"; title="This is not f�n"'),
+
+        'rfc2231_encoded_charset': (
+            'text/plain; charset*=ansi-x3.4-1968\'\'us-ascii',
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'us-ascii'},
+            [],
+            'text/plain; charset="us-ascii"'),
+
+        # This follows the RFC: no double quotes around encoded values.
+        'rfc2231_encoded_no_double_quotes': (
+            ("text/plain;"
+                "\tname*0*=''This%20is%20;"
+                "\tname*1*=%2A%2A%2Afun%2A%2A%2A%20;"
+                '\tname*2="is it not.pdf"'),
+            'text/plain',
+            'text',
+            'plain',
+            {'name': 'This is ***fun*** is it not.pdf'},
+            [],
+            'text/plain; name="This is ***fun*** is it not.pdf"',
+            ('Content-Type: text/plain;\tname*0*=\'\'This%20is%20;\n'
+             '\tname*1*=%2A%2A%2Afun%2A%2A%2A%20;\tname*2="is it not.pdf"\n'),
+            ),
+
+        # Make sure we also handle it if there are spurrious double qoutes.
+        'rfc2231_encoded_with_double_quotes': (
+            ("text/plain;"
+                '\tname*0*="us-ascii\'\'This%20is%20even%20more%20";'
+                '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";'
+                '\tname*2="is it not.pdf"'),
+            'text/plain',
+            'text',
+            'plain',
+            {'name': 'This is even more ***fun*** is it not.pdf'},
+            [errors.InvalidHeaderDefect]*2,
+            'text/plain; name="This is even more ***fun*** is it not.pdf"',
+            ('Content-Type: text/plain;\t'
+                'name*0*="us-ascii\'\'This%20is%20even%20more%20";\n'
+             '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it not.pdf"\n'),
+            ),
+
+        'rfc2231_single_quote_inside_double_quotes': (
+            ('text/plain; charset=us-ascii;'
+               '\ttitle*0*="us-ascii\'en\'This%20is%20really%20";'
+               '\ttitle*1*="%2A%2A%2Afun%2A%2A%2A%20";'
+               '\ttitle*2="isn\'t it!"'),
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'us-ascii', 'title': "This is really ***fun*** isn't it!"},
+            [errors.InvalidHeaderDefect]*2,
+            ('text/plain; charset="us-ascii"; '
+               'title="This is really ***fun*** isn\'t it!"'),
+            ('Content-Type: text/plain; charset=us-ascii;\n'
+             '\ttitle*0*="us-ascii\'en\'This%20is%20really%20";\n'
+             '\ttitle*1*="%2A%2A%2Afun%2A%2A%2A%20";\ttitle*2="isn\'t it!"\n'),
+            ),
+
+        'rfc2231_single_quote_in_value_with_charset_and_lang': (
+            ('application/x-foo;'
+                "\tname*0*=\"us-ascii'en-us'Frank's\"; name*1*=\" Document\""),
+            'application/x-foo',
+            'application',
+            'x-foo',
+            {'name': "Frank's Document"},
+            [errors.InvalidHeaderDefect]*2,
+            'application/x-foo; name="Frank\'s Document"',
+            ('Content-Type: application/x-foo;\t'
+                'name*0*="us-ascii\'en-us\'Frank\'s";\n'
+             ' name*1*=" Document"\n'),
+            ),
+
+        'rfc2231_single_quote_in_non_encoded_value': (
+            ('application/x-foo;'
+                "\tname*0=\"us-ascii'en-us'Frank's\"; name*1=\" Document\""),
+            'application/x-foo',
+            'application',
+            'x-foo',
+            {'name': "us-ascii'en-us'Frank's Document"},
+            [],
+            'application/x-foo; name="us-ascii\'en-us\'Frank\'s Document"',
+            ('Content-Type: application/x-foo;\t'
+                'name*0="us-ascii\'en-us\'Frank\'s";\n'
+             ' name*1=" Document"\n'),
+             ),
+
+        'rfc2231_no_language_or_charset': (
+            'text/plain; NAME*0*=english_is_the_default.html',
+            'text/plain',
+            'text',
+            'plain',
+            {'name': 'english_is_the_default.html'},
+            [errors.InvalidHeaderDefect],
+            'text/plain; NAME="english_is_the_default.html"'),
+
+        'rfc2231_encoded_no_charset': (
+            ("text/plain;"
+                '\tname*0*="\'\'This%20is%20even%20more%20";'
+                '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";'
+                '\tname*2="is it.pdf"'),
+            'text/plain',
+            'text',
+            'plain',
+            {'name': 'This is even more ***fun*** is it.pdf'},
+            [errors.InvalidHeaderDefect]*2,
+            'text/plain; name="This is even more ***fun*** is it.pdf"',
+            ('Content-Type: text/plain;\t'
+                'name*0*="\'\'This%20is%20even%20more%20";\n'
+             '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'),
+            ),
+
+        # XXX: see below...the first name line here should be *0 not *0*.
+        'rfc2231_partly_encoded': (
+            ("text/plain;"
+                '\tname*0*="\'\'This%20is%20even%20more%20";'
+                '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";'
+                '\tname*2="is it.pdf"'),
+            'text/plain',
+            'text',
+            'plain',
+            {'name': 'This is even more ***fun*** is it.pdf'},
+            [errors.InvalidHeaderDefect]*2,
+            'text/plain; name="This is even more ***fun*** is it.pdf"',
+            ('Content-Type: text/plain;\t'
+                'name*0*="\'\'This%20is%20even%20more%20";\n'
+             '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'),
+            ),
+
+        'rfc2231_partly_encoded_2': (
+            ("text/plain;"
+                '\tname*0*="\'\'This%20is%20even%20more%20";'
+                '\tname*1="%2A%2A%2Afun%2A%2A%2A%20";'
+                '\tname*2="is it.pdf"'),
+            'text/plain',
+            'text',
+            'plain',
+            {'name': 'This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf'},
+            [errors.InvalidHeaderDefect],
+            'text/plain; name="This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf"',
+            ('Content-Type: text/plain;\t'
+                'name*0*="\'\'This%20is%20even%20more%20";\n'
+             '\tname*1="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'),
+            ),
+
+        'rfc2231_unknown_charset_treated_as_ascii': (
+            "text/plain; name*0*=bogus'xx'ascii_is_the_default",
+            'text/plain',
+            'text',
+            'plain',
+            {'name': 'ascii_is_the_default'},
+            [],
+            'text/plain; name="ascii_is_the_default"'),
+
+        'rfc2231_bad_character_in_charset_parameter_value': (
+            "text/plain; charset*=ascii''utf-8%E2%80%9D",
+            'text/plain',
+            'text',
+            'plain',
+            {'charset': 'utf-8\uFFFD\uFFFD\uFFFD'},
+            [errors.UndecodableBytesDefect],
+            'text/plain; charset="utf-8\uFFFD\uFFFD\uFFFD"'),
+
+        'rfc2231_encoded_then_unencoded_segments': (
+            ('application/x-foo;'
+                '\tname*0*="us-ascii\'en-us\'My";'
+                '\tname*1=" Document";'
+                '\tname*2=" For You"'),
+            'application/x-foo',
+            'application',
+            'x-foo',
+            {'name': 'My Document For You'},
+            [errors.InvalidHeaderDefect],
+            'application/x-foo; name="My Document For You"',
+            ('Content-Type: application/x-foo;\t'
+                'name*0*="us-ascii\'en-us\'My";\n'
+             '\tname*1=" Document";\tname*2=" For You"\n'),
+            ),
+
+        # My reading of the RFC is that this is an invalid header.  The RFC
+        # says that if charset and language information is given, the first
+        # segment *must* be encoded.
+        'rfc2231_unencoded_then_encoded_segments': (
+            ('application/x-foo;'
+                '\tname*0=us-ascii\'en-us\'My;'
+                '\tname*1*=" Document";'
+                '\tname*2*=" For You"'),
+            'application/x-foo',
+            'application',
+            'x-foo',
+            {'name': 'My Document For You'},
+            [errors.InvalidHeaderDefect]*3,
+            'application/x-foo; name="My Document For You"',
+            ("Content-Type: application/x-foo;\tname*0=us-ascii'en-us'My;\t"
+                # XXX: the newline is in the wrong place, come back and fix
+                # this when the rest of tests pass.
+                'name*1*=" Document"\n;'
+             '\tname*2*=" For You"\n'),
+            ),
+
+        # XXX: I would say this one should default to ascii/en for the
+        # "encoded" segment, since the first segment is not encoded and is
+        # in double quotes, making the value a valid non-encoded string.  The
+        # old parser decodes this just like the previous case, which may be the
+        # better Postel rule, but could equally result in borking headers that
+        # intentially have quoted quotes in them.  We could get this 98% right
+        # if we treat it as a quoted string *unless* it matches the
+        # charset'lang'value pattern exactly *and* there is at least one
+        # encoded segment.  Implementing that algorithm will require some
+        # refactoring, so I haven't done it (yet).
+
+        'rfc2231_qouted_unencoded_then_encoded_segments': (
+            ('application/x-foo;'
+                '\tname*0="us-ascii\'en-us\'My";'
+                '\tname*1*=" Document";'
+                '\tname*2*=" For You"'),
+            'application/x-foo',
+            'application',
+            'x-foo',
+            {'name': "us-ascii'en-us'My Document For You"},
+            [errors.InvalidHeaderDefect]*2,
+            'application/x-foo; name="us-ascii\'en-us\'My Document For You"',
+            ('Content-Type: application/x-foo;\t'
+                'name*0="us-ascii\'en-us\'My";\n'
+             '\tname*1*=" Document";\tname*2*=" For You"\n'),
+            ),
+
+    }
+
+
+@parameterize
+class TestContentTransferEncoding(TestHeaderBase):
+
+    def cte_as_value(self,
+                     source,
+                     cte,
+                     *args):
+        l = len(args)
+        defects =  args[0] if l>0 else []
+        decoded =  args[1] if l>1 and args[1] is not DITTO else source
+        header = 'Content-Transfer-Encoding:' + ' ' if source else ''
+        folded = args[2] if l>2 else header + source + '\n'
+        h = self.make_header('Content-Transfer-Encoding', source)
+        self.assertEqual(h.cte, cte)
+        self.assertDefectsEqual(h.defects, defects)
+        self.assertEqual(h, decoded)
+        self.assertEqual(h.fold(policy=policy.default), folded)
+
+    cte_params = {
+
+        'RFC_2183_1': (
+            'base64',
+            'base64',),
+
+        'no_value': (
+            '',
+            '7bit',
+            [errors.HeaderMissingRequiredValue],
+            '',
+            'Content-Transfer-Encoding:\n',
+            ),
+
+        'junk_after_cte': (
+            '7bit and a bunch more',
+            '7bit',
+            [errors.InvalidHeaderDefect]),
+
+    }
+
+
+@parameterize
+class TestContentDisposition(TestHeaderBase):
+
+    def content_disp_as_value(self,
+                              source,
+                              content_disposition,
+                              *args):
+        l = len(args)
+        parmdict = args[0] if l>0 else {}
+        defects =  args[1] if l>1 else []
+        decoded =  args[2] if l>2 and args[2] is not DITTO else source
+        header = 'Content-Disposition:' + ' ' if source else ''
+        folded = args[3] if l>3 else header + source + '\n'
+        h = self.make_header('Content-Disposition', source)
+        self.assertEqual(h.content_disposition, content_disposition)
+        self.assertEqual(h.params, parmdict)
+        self.assertDefectsEqual(h.defects, defects)
+        self.assertEqual(h, decoded)
+        self.assertEqual(h.fold(policy=policy.default), folded)
+
+    content_disp_params = {
+
+        # Examples from RFC 2183.
+
+        'RFC_2183_1': (
+            'inline',
+            'inline',),
+
+        'RFC_2183_2': (
+            ('attachment; filename=genome.jpeg;'
+             '  modification-date="Wed, 12 Feb 1997 16:29:51 -0500";'),
+            'attachment',
+            {'filename': 'genome.jpeg',
+             'modification-date': 'Wed, 12 Feb 1997 16:29:51 -0500'},
+            [],
+            ('attachment; filename="genome.jpeg"; '
+                 'modification-date="Wed, 12 Feb 1997 16:29:51 -0500"'),
+            ('Content-Disposition: attachment; filename=genome.jpeg;\n'
+             '  modification-date="Wed, 12 Feb 1997 16:29:51 -0500";\n'),
+            ),
+
+        'no_value': (
+            '',
+            None,
+            {},
+            [errors.HeaderMissingRequiredValue],
+            '',
+            'Content-Disposition:\n'),
+
+        'invalid_value': (
+            'ab./k',
+            'ab.',
+            {},
+            [errors.InvalidHeaderDefect]),
+
+        'invalid_value_with_params': (
+            'ab./k; filename="foo"',
+            'ab.',
+            {'filename': 'foo'},
+            [errors.InvalidHeaderDefect]),
+
+    }
+
+
+@parameterize
+class TestMIMEVersionHeader(TestHeaderBase):
+
+    def version_string_as_MIME_Version(self,
+                                       source,
+                                       decoded,
+                                       version,
+                                       major,
+                                       minor,
+                                       defects):
+        h = self.make_header('MIME-Version', source)
+        self.assertEqual(h, decoded)
+        self.assertEqual(h.version, version)
+        self.assertEqual(h.major, major)
+        self.assertEqual(h.minor, minor)
+        self.assertDefectsEqual(h.defects, defects)
+        if source:
+            source = ' ' + source
+        self.assertEqual(h.fold(policy=policy.default),
+                        'MIME-Version:' + source + '\n')
+
+    version_string_params = {
+
+        # Examples from the RFC.
+
+        'RFC_2045_1': (
+            '1.0',
+            '1.0',
+            '1.0',
+            1,
+            0,
+            []),
+
+        'RFC_2045_2': (
+            '1.0 (produced by MetaSend Vx.x)',
+            '1.0 (produced by MetaSend Vx.x)',
+            '1.0',
+            1,
+            0,
+            []),
+
+        'RFC_2045_3': (
+            '(produced by MetaSend Vx.x) 1.0',
+            '(produced by MetaSend Vx.x) 1.0',
+            '1.0',
+            1,
+            0,
+            []),
+
+        'RFC_2045_4': (
+            '1.(produced by MetaSend Vx.x)0',
+            '1.(produced by MetaSend Vx.x)0',
+            '1.0',
+            1,
+            0,
+            []),
+
+        # Other valid values.
+
+        '1_1': (
+            '1.1',
+            '1.1',
+            '1.1',
+            1,
+            1,
+            []),
+
+        '2_1': (
+            '2.1',
+            '2.1',
+            '2.1',
+            2,
+            1,
+            []),
+
+        'whitespace': (
+            '1 .0',
+            '1 .0',
+            '1.0',
+            1,
+            0,
+            []),
+
+        'leading_trailing_whitespace_ignored': (
+            '  1.0  ',
+            '  1.0  ',
+            '1.0',
+            1,
+            0,
+            []),
+
+        # Recoverable invalid values.  We can recover here only because we
+        # already have a valid value by the time we encounter the garbage.
+        # Anywhere else, and we don't know where the garbage ends.
+
+        'non_comment_garbage_after': (
+            '1.0 <abc>',
+            '1.0 <abc>',
+            '1.0',
+            1,
+            0,
+            [errors.InvalidHeaderDefect]),
+
+        # Unrecoverable invalid values.  We *could* apply more heuristics to
+        # get someing out of the first two, but doing so is not worth the
+        # effort.
+
+        'non_comment_garbage_before': (
+            '<abc> 1.0',
+            '<abc> 1.0',
+            None,
+            None,
+            None,
+            [errors.InvalidHeaderDefect]),
+
+        'non_comment_garbage_inside': (
+            '1.<abc>0',
+            '1.<abc>0',
+            None,
+            None,
+            None,
+            [errors.InvalidHeaderDefect]),
+
+        'two_periods': (
+            '1..0',
+            '1..0',
+            None,
+            None,
+            None,
+            [errors.InvalidHeaderDefect]),
+
+        '2_x': (
+            '2.x',
+            '2.x',
+            None,  # This could be 2, but it seems safer to make it None.
+            None,
+            None,
+            [errors.InvalidHeaderDefect]),
+
+        'foo': (
+            'foo',
+            'foo',
+            None,
+            None,
+            None,
+            [errors.InvalidHeaderDefect]),
+
+        'missing': (
+            '',
+            '',
+            None,
+            None,
+            None,
+            [errors.HeaderMissingRequiredValue]),
+
+        }
+
+
+@parameterize
+class TestAddressHeader(TestHeaderBase):
+
+    example_params = {
+
+        'empty':
+            ('<>',
+             [errors.InvalidHeaderDefect],
+             '<>',
+             '',
+             '<>',
+             '',
+             '',
+             None),
+
+        'address_only':
+            ('zippy@pinhead.com',
+             [],
+             'zippy@pinhead.com',
+             '',
+             'zippy@pinhead.com',
+             'zippy',
+             'pinhead.com',
+             None),
+
+        'name_and_address':
+            ('Zaphrod Beblebrux <zippy@pinhead.com>',
+             [],
+             'Zaphrod Beblebrux <zippy@pinhead.com>',
+             'Zaphrod Beblebrux',
+             'zippy@pinhead.com',
+             'zippy',
+             'pinhead.com',
+             None),
+
+        'quoted_local_part':
+            ('Zaphrod Beblebrux <"foo bar"@pinhead.com>',
+             [],
+             'Zaphrod Beblebrux <"foo bar"@pinhead.com>',
+             'Zaphrod Beblebrux',
+             '"foo bar"@pinhead.com',
+             'foo bar',
+             'pinhead.com',
+             None),
+
+        'quoted_parens_in_name':
+            (r'"A \(Special\) Person" <person@dom.ain>',
+             [],
+             '"A (Special) Person" <person@dom.ain>',
+             'A (Special) Person',
+             'person@dom.ain',
+             'person',
+             'dom.ain',
+             None),
+
+        'quoted_backslashes_in_name':
+            (r'"Arthur \\Backslash\\ Foobar" <person@dom.ain>',
+             [],
+             r'"Arthur \\Backslash\\ Foobar" <person@dom.ain>',
+             r'Arthur \Backslash\ Foobar',
+             'person@dom.ain',
+             'person',
+             'dom.ain',
+             None),
+
+        'name_with_dot':
+            ('John X. Doe <jxd@example.com>',
+             [errors.ObsoleteHeaderDefect],
+             '"John X. Doe" <jxd@example.com>',
+             'John X. Doe',
+             'jxd@example.com',
+             'jxd',
+             'example.com',
+             None),
+
+        'quoted_strings_in_local_part':
+            ('""example" example"@example.com',
+             [errors.InvalidHeaderDefect]*3,
+             '"example example"@example.com',
+             '',
+             '"example example"@example.com',
+             'example example',
+             'example.com',
+             None),
+
+        'escaped_quoted_strings_in_local_part':
+            (r'"\"example\" example"@example.com',
+             [],
+             r'"\"example\" example"@example.com',
+             '',
+             r'"\"example\" example"@example.com',
+             r'"example" example',
+             'example.com',
+            None),
+
+        'escaped_escapes_in_local_part':
+            (r'"\\"example\\" example"@example.com',
+             [errors.InvalidHeaderDefect]*5,
+             r'"\\example\\\\ example"@example.com',
+             '',
+             r'"\\example\\\\ example"@example.com',
+             r'\example\\ example',
+             'example.com',
+            None),
+
+        'spaces_in_unquoted_local_part_collapsed':
+            ('merwok  wok  @example.com',
+             [errors.InvalidHeaderDefect]*2,
+             '"merwok wok"@example.com',
+             '',
+             '"merwok wok"@example.com',
+             'merwok wok',
+             'example.com',
+             None),
+
+        'spaces_around_dots_in_local_part_removed':
+            ('merwok. wok .  wok@example.com',
+             [errors.ObsoleteHeaderDefect],
+             'merwok.wok.wok@example.com',
+             '',
+             'merwok.wok.wok@example.com',
+             'merwok.wok.wok',
+             'example.com',
+             None),
+
+        }
+
+        # XXX: Need many more examples, and in particular some with names in
+        # trailing comments, which aren't currently handled.  comments in
+        # general are not handled yet.
+
+    def example_as_address(self, source, defects, decoded, display_name,
+                           addr_spec, username, domain, comment):
+        h = self.make_header('sender', source)
+        self.assertEqual(h, decoded)
+        self.assertDefectsEqual(h.defects, defects)
+        a = h.address
+        self.assertEqual(str(a), decoded)
+        self.assertEqual(len(h.groups), 1)
+        self.assertEqual([a], list(h.groups[0].addresses))
+        self.assertEqual([a], list(h.addresses))
+        self.assertEqual(a.display_name, display_name)
+        self.assertEqual(a.addr_spec, addr_spec)
+        self.assertEqual(a.username, username)
+        self.assertEqual(a.domain, domain)
+        # XXX: we have no comment support yet.
+        #self.assertEqual(a.comment, comment)
+
+    def example_as_group(self, source, defects, decoded, display_name,
+                         addr_spec, username, domain, comment):
+        source = 'foo: {};'.format(source)
+        gdecoded = 'foo: {};'.format(decoded) if decoded else 'foo:;'
+        h = self.make_header('to', source)
+        self.assertEqual(h, gdecoded)
+        self.assertDefectsEqual(h.defects, defects)
+        self.assertEqual(h.groups[0].addresses, h.addresses)
+        self.assertEqual(len(h.groups), 1)
+        self.assertEqual(len(h.addresses), 1)
+        a = h.addresses[0]
+        self.assertEqual(str(a), decoded)
+        self.assertEqual(a.display_name, display_name)
+        self.assertEqual(a.addr_spec, addr_spec)
+        self.assertEqual(a.username, username)
+        self.assertEqual(a.domain, domain)
+
+    def test_simple_address_list(self):
+        value = ('Fred <dinsdale@python.org>, foo@example.com, '
+                    '"Harry W. Hastings" <hasty@example.com>')
+        h = self.make_header('to', value)
+        self.assertEqual(h, value)
+        self.assertEqual(len(h.groups), 3)
+        self.assertEqual(len(h.addresses), 3)
+        for i in range(3):
+            self.assertEqual(h.groups[i].addresses[0], h.addresses[i])
+        self.assertEqual(str(h.addresses[0]), 'Fred <dinsdale@python.org>')
+        self.assertEqual(str(h.addresses[1]), 'foo@example.com')
+        self.assertEqual(str(h.addresses[2]),
+            '"Harry W. Hastings" <hasty@example.com>')
+        self.assertEqual(h.addresses[2].display_name,
+            'Harry W. Hastings')
+
+    def test_complex_address_list(self):
+        examples = list(self.example_params.values())
+        source = ('dummy list:;, another: (empty);,' +
+                 ', '.join([x[0] for x in examples[:4]]) + ', ' +
+                 r'"A \"list\"": ' +
+                    ', '.join([x[0] for x in examples[4:6]]) + ';,' +
+                 ', '.join([x[0] for x in examples[6:]])
+            )
+        # XXX: the fact that (empty) disappears here is a potential API design
+        # bug.  We don't currently have a way to preserve comments.
+        expected = ('dummy list:;, another:;, ' +
+                 ', '.join([x[2] for x in examples[:4]]) + ', ' +
+                 r'"A \"list\"": ' +
+                    ', '.join([x[2] for x in examples[4:6]]) + ';, ' +
+                 ', '.join([x[2] for x in examples[6:]])
+            )
+
+        h = self.make_header('to', source)
+        self.assertEqual(h.split(','), expected.split(','))
+        self.assertEqual(h, expected)
+        self.assertEqual(len(h.groups), 7 + len(examples) - 6)
+        self.assertEqual(h.groups[0].display_name, 'dummy list')
+        self.assertEqual(h.groups[1].display_name, 'another')
+        self.assertEqual(h.groups[6].display_name, 'A "list"')
+        self.assertEqual(len(h.addresses), len(examples))
+        for i in range(4):
+            self.assertIsNone(h.groups[i+2].display_name)
+            self.assertEqual(str(h.groups[i+2].addresses[0]), examples[i][2])
+        for i in range(7, 7 + len(examples) - 6):
+            self.assertIsNone(h.groups[i].display_name)
+            self.assertEqual(str(h.groups[i].addresses[0]), examples[i-1][2])
+        for i in range(len(examples)):
+            self.assertEqual(str(h.addresses[i]), examples[i][2])
+            self.assertEqual(h.addresses[i].addr_spec, examples[i][4])
+
+    def test_address_read_only(self):
+        h = self.make_header('sender', 'abc@xyz.com')
+        with self.assertRaises(AttributeError):
+            h.address = 'foo'
+
+    def test_addresses_read_only(self):
+        h = self.make_header('sender', 'abc@xyz.com')
+        with self.assertRaises(AttributeError):
+            h.addresses = 'foo'
+
+    def test_groups_read_only(self):
+        h = self.make_header('sender', 'abc@xyz.com')
+        with self.assertRaises(AttributeError):
+            h.groups = 'foo'
+
+    def test_addresses_types(self):
+        source = 'me <who@example.com>'
+        h = self.make_header('to', source)
+        self.assertIsInstance(h.addresses, tuple)
+        self.assertIsInstance(h.addresses[0], Address)
+
+    def test_groups_types(self):
+        source = 'me <who@example.com>'
+        h = self.make_header('to', source)
+        self.assertIsInstance(h.groups, tuple)
+        self.assertIsInstance(h.groups[0], Group)
+
+    def test_set_from_Address(self):
+        h = self.make_header('to', Address('me', 'foo', 'example.com'))
+        self.assertEqual(h, 'me <foo@example.com>')
+
+    def test_set_from_Address_list(self):
+        h = self.make_header('to', [Address('me', 'foo', 'example.com'),
+                                    Address('you', 'bar', 'example.com')])
+        self.assertEqual(h, 'me <foo@example.com>, you <bar@example.com>')
+
+    def test_set_from_Address_and_Group_list(self):
+        h = self.make_header('to', [Address('me', 'foo', 'example.com'),
+                                    Group('bing', [Address('fiz', 'z', 'b.com'),
+                                                   Address('zif', 'f', 'c.com')]),
+                                    Address('you', 'bar', 'example.com')])
+        self.assertEqual(h, 'me <foo@example.com>, bing: fiz <z@b.com>, '
+                            'zif <f@c.com>;, you <bar@example.com>')
+        self.assertEqual(h.fold(policy=policy.default.clone(max_line_length=40)),
+                        'to: me <foo@example.com>,\n'
+                        ' bing: fiz <z@b.com>, zif <f@c.com>;,\n'
+                        ' you <bar@example.com>\n')
+
+    def test_set_from_Group_list(self):
+        h = self.make_header('to', [Group('bing', [Address('fiz', 'z', 'b.com'),
+                                                   Address('zif', 'f', 'c.com')])])
+        self.assertEqual(h, 'bing: fiz <z@b.com>, zif <f@c.com>;')
+
+
+class TestAddressAndGroup(TestEmailBase):
+
+    def _test_attr_ro(self, obj, attr):
+        with self.assertRaises(AttributeError):
+            setattr(obj, attr, 'foo')
+
+    def test_address_display_name_ro(self):
+        self._test_attr_ro(Address('foo', 'bar', 'baz'), 'display_name')
+
+    def test_address_username_ro(self):
+        self._test_attr_ro(Address('foo', 'bar', 'baz'), 'username')
+
+    def test_address_domain_ro(self):
+        self._test_attr_ro(Address('foo', 'bar', 'baz'), 'domain')
+
+    def test_group_display_name_ro(self):
+        self._test_attr_ro(Group('foo'), 'display_name')
+
+    def test_group_addresses_ro(self):
+        self._test_attr_ro(Group('foo'), 'addresses')
+
+    def test_address_from_username_domain(self):
+        a = Address('foo', 'bar', 'baz')
+        self.assertEqual(a.display_name, 'foo')
+        self.assertEqual(a.username, 'bar')
+        self.assertEqual(a.domain, 'baz')
+        self.assertEqual(a.addr_spec, 'bar@baz')
+        self.assertEqual(str(a), 'foo <bar@baz>')
+
+    def test_address_from_addr_spec(self):
+        a = Address('foo', addr_spec='bar@baz')
+        self.assertEqual(a.display_name, 'foo')
+        self.assertEqual(a.username, 'bar')
+        self.assertEqual(a.domain, 'baz')
+        self.assertEqual(a.addr_spec, 'bar@baz')
+        self.assertEqual(str(a), 'foo <bar@baz>')
+
+    def test_address_with_no_display_name(self):
+        a = Address(addr_spec='bar@baz')
+        self.assertEqual(a.display_name, '')
+        self.assertEqual(a.username, 'bar')
+        self.assertEqual(a.domain, 'baz')
+        self.assertEqual(a.addr_spec, 'bar@baz')
+        self.assertEqual(str(a), 'bar@baz')
+
+    def test_null_address(self):
+        a = Address()
+        self.assertEqual(a.display_name, '')
+        self.assertEqual(a.username, '')
+        self.assertEqual(a.domain, '')
+        self.assertEqual(a.addr_spec, '<>')
+        self.assertEqual(str(a), '<>')
+
+    def test_domain_only(self):
+        # This isn't really a valid address.
+        a = Address(domain='buzz')
+        self.assertEqual(a.display_name, '')
+        self.assertEqual(a.username, '')
+        self.assertEqual(a.domain, 'buzz')
+        self.assertEqual(a.addr_spec, '@buzz')
+        self.assertEqual(str(a), '@buzz')
+
+    def test_username_only(self):
+        # This isn't really a valid address.
+        a = Address(username='buzz')
+        self.assertEqual(a.display_name, '')
+        self.assertEqual(a.username, 'buzz')
+        self.assertEqual(a.domain, '')
+        self.assertEqual(a.addr_spec, 'buzz')
+        self.assertEqual(str(a), 'buzz')
+
+    def test_display_name_only(self):
+        a = Address('buzz')
+        self.assertEqual(a.display_name, 'buzz')
+        self.assertEqual(a.username, '')
+        self.assertEqual(a.domain, '')
+        self.assertEqual(a.addr_spec, '<>')
+        self.assertEqual(str(a), 'buzz <>')
+
+    def test_quoting(self):
+        # Ideally we'd check every special individually, but I'm not up for
+        # writing that many tests.
+        a = Address('Sara J.', 'bad name', 'example.com')
+        self.assertEqual(a.display_name, 'Sara J.')
+        self.assertEqual(a.username, 'bad name')
+        self.assertEqual(a.domain, 'example.com')
+        self.assertEqual(a.addr_spec, '"bad name"@example.com')
+        self.assertEqual(str(a), '"Sara J." <"bad name"@example.com>')
+
+    def test_il8n(self):
+        a = Address('Éric', 'wok', 'exàmple.com')
+        self.assertEqual(a.display_name, 'Éric')
+        self.assertEqual(a.username, 'wok')
+        self.assertEqual(a.domain, 'exàmple.com')
+        self.assertEqual(a.addr_spec, 'wok@exàmple.com')
+        self.assertEqual(str(a), 'Éric <wok@exàmple.com>')
+
+    # XXX: there is an API design issue that needs to be solved here.
+    #def test_non_ascii_username_raises(self):
+    #    with self.assertRaises(ValueError):
+    #        Address('foo', 'wők', 'example.com')
+
+    def test_non_ascii_username_in_addr_spec_raises(self):
+        with self.assertRaises(ValueError):
+            Address('foo', addr_spec='wők@example.com')
+
+    def test_address_addr_spec_and_username_raises(self):
+        with self.assertRaises(TypeError):
+            Address('foo', username='bing', addr_spec='bar@baz')
+
+    def test_address_addr_spec_and_domain_raises(self):
+        with self.assertRaises(TypeError):
+            Address('foo', domain='bing', addr_spec='bar@baz')
+
+    def test_address_addr_spec_and_username_and_domain_raises(self):
+        with self.assertRaises(TypeError):
+            Address('foo', username='bong', domain='bing', addr_spec='bar@baz')
+
+    def test_space_in_addr_spec_username_raises(self):
+        with self.assertRaises(ValueError):
+            Address('foo', addr_spec="bad name@example.com")
+
+    def test_bad_addr_sepc_raises(self):
+        with self.assertRaises(ValueError):
+            Address('foo', addr_spec="name@ex[]ample.com")
+
+    def test_empty_group(self):
+        g = Group('foo')
+        self.assertEqual(g.display_name, 'foo')
+        self.assertEqual(g.addresses, tuple())
+        self.assertEqual(str(g), 'foo:;')
+
+    def test_empty_group_list(self):
+        g = Group('foo', addresses=[])
+        self.assertEqual(g.display_name, 'foo')
+        self.assertEqual(g.addresses, tuple())
+        self.assertEqual(str(g), 'foo:;')
+
+    def test_null_group(self):
+        g = Group()
+        self.assertIsNone(g.display_name)
+        self.assertEqual(g.addresses, tuple())
+        self.assertEqual(str(g), 'None:;')
+
+    def test_group_with_addresses(self):
+        addrs = [Address('b', 'b', 'c'), Address('a', 'b','c')]
+        g = Group('foo', addrs)
+        self.assertEqual(g.display_name, 'foo')
+        self.assertEqual(g.addresses, tuple(addrs))
+        self.assertEqual(str(g), 'foo: b <b@c>, a <b@c>;')
+
+    def test_group_with_addresses_no_display_name(self):
+        addrs = [Address('b', 'b', 'c'), Address('a', 'b','c')]
+        g = Group(addresses=addrs)
+        self.assertIsNone(g.display_name)
+        self.assertEqual(g.addresses, tuple(addrs))
+        self.assertEqual(str(g), 'None: b <b@c>, a <b@c>;')
+
+    def test_group_with_one_address_no_display_name(self):
+        addrs = [Address('b', 'b', 'c')]
+        g = Group(addresses=addrs)
+        self.assertIsNone(g.display_name)
+        self.assertEqual(g.addresses, tuple(addrs))
+        self.assertEqual(str(g), 'b <b@c>')
+
+    def test_display_name_quoting(self):
+        g = Group('foo.bar')
+        self.assertEqual(g.display_name, 'foo.bar')
+        self.assertEqual(g.addresses, tuple())
+        self.assertEqual(str(g), '"foo.bar":;')
+
+    def test_display_name_blanks_not_quoted(self):
+        g = Group('foo bar')
+        self.assertEqual(g.display_name, 'foo bar')
+        self.assertEqual(g.addresses, tuple())
+        self.assertEqual(str(g), 'foo bar:;')
+
+    def test_set_message_header_from_address(self):
+        a = Address('foo', 'bar', 'example.com')
+        m = Message(policy=policy.default)
+        m['To'] = a
+        self.assertEqual(m['to'], 'foo <bar@example.com>')
+        self.assertEqual(m['to'].addresses, (a,))
+
+    def test_set_message_header_from_group(self):
+        g = Group('foo bar')
+        m = Message(policy=policy.default)
+        m['To'] = g
+        self.assertEqual(m['to'], 'foo bar:;')
+        self.assertEqual(m['to'].addresses, g.addresses)
+
+
+class TestFolding(TestHeaderBase):
+
+    def test_short_unstructured(self):
+        h = self.make_header('subject', 'this is a test')
+        self.assertEqual(h.fold(policy=policy.default),
+                         'subject: this is a test\n')
+
+    def test_long_unstructured(self):
+        h = self.make_header('Subject', 'This is a long header '
+            'line that will need to be folded into two lines '
+            'and will demonstrate basic folding')
+        self.assertEqual(h.fold(policy=policy.default),
+                        'Subject: This is a long header line that will '
+                            'need to be folded into two lines\n'
+                        ' and will demonstrate basic folding\n')
+
+    def test_unstructured_short_max_line_length(self):
+        h = self.make_header('Subject', 'this is a short header '
+            'that will be folded anyway')
+        self.assertEqual(
+            h.fold(policy=policy.default.clone(max_line_length=20)),
+            textwrap.dedent("""\
+                Subject: this is a
+                 short header that
+                 will be folded
+                 anyway
+                """))
+
+    def test_fold_unstructured_single_word(self):
+        h = self.make_header('Subject', 'test')
+        self.assertEqual(h.fold(policy=policy.default), 'Subject: test\n')
+
+    def test_fold_unstructured_short(self):
+        h = self.make_header('Subject', 'test test test')
+        self.assertEqual(h.fold(policy=policy.default),
+                        'Subject: test test test\n')
+
+    def test_fold_unstructured_with_overlong_word(self):
+        h = self.make_header('Subject', 'thisisaverylonglineconsistingofa'
+            'singlewordthatwontfit')
+        self.assertEqual(
+            h.fold(policy=policy.default.clone(max_line_length=20)),
+            'Subject: thisisaverylonglineconsistingofasinglewordthatwontfit\n')
+
+    def test_fold_unstructured_with_two_overlong_words(self):
+        h = self.make_header('Subject', 'thisisaverylonglineconsistingofa'
+            'singlewordthatwontfit plusanotherverylongwordthatwontfit')
+        self.assertEqual(
+            h.fold(policy=policy.default.clone(max_line_length=20)),
+            'Subject: thisisaverylonglineconsistingofasinglewordthatwontfit\n'
+                ' plusanotherverylongwordthatwontfit\n')
+
+    def test_fold_unstructured_with_slightly_long_word(self):
+        h = self.make_header('Subject', 'thislongwordislessthanmaxlinelen')
+        self.assertEqual(
+            h.fold(policy=policy.default.clone(max_line_length=35)),
+            'Subject:\n thislongwordislessthanmaxlinelen\n')
+
+    def test_fold_unstructured_with_commas(self):
+        # The old wrapper would fold this at the commas.
+        h = self.make_header('Subject', "This header is intended to "
+            "demonstrate, in a fairly susinct way, that we now do "
+            "not give a , special treatment in unstructured headers.")
+        self.assertEqual(
+            h.fold(policy=policy.default.clone(max_line_length=60)),
+            textwrap.dedent("""\
+                Subject: This header is intended to demonstrate, in a fairly
+                 susinct way, that we now do not give a , special treatment
+                 in unstructured headers.
+                 """))
+
+    def test_fold_address_list(self):
+        h = self.make_header('To', '"Theodore H. Perfect" <yes@man.com>, '
+            '"My address is very long because my name is long" <foo@bar.com>, '
+            '"Only A. Friend" <no@yes.com>')
+        self.assertEqual(h.fold(policy=policy.default), textwrap.dedent("""\
+            To: "Theodore H. Perfect" <yes@man.com>,
+             "My address is very long because my name is long" <foo@bar.com>,
+             "Only A. Friend" <no@yes.com>
+             """))
+
+    def test_fold_date_header(self):
+        h = self.make_header('Date', 'Sat, 2 Feb 2002 17:00:06 -0800')
+        self.assertEqual(h.fold(policy=policy.default),
+                        'Date: Sat, 02 Feb 2002 17:00:06 -0800\n')
+
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_email/test_inversion.py b/Lib/test/test_email/test_inversion.py
new file mode 100644
index 0000000000..6f5c534676
--- /dev/null
+++ b/Lib/test/test_email/test_inversion.py
@@ -0,0 +1,45 @@
+"""Test the parser and generator are inverses.
+
+Note that this is only strictly true if we are parsing RFC valid messages and
+producing RFC valid messages.
+"""
+
+import io
+import unittest
+from email import policy, message_from_bytes
+from email.generator import BytesGenerator
+from test.test_email import TestEmailBase, parameterize
+
+# This is like textwrap.dedent for bytes, except that it uses \r\n for the line
+# separators on the rebuilt string.
+def dedent(bstr):
+    lines = bstr.splitlines()
+    if not lines[0].strip():
+        raise ValueError("First line must contain text")
+    stripamt = len(lines[0]) - len(lines[0].lstrip())
+    return b'\r\n'.join(
+        [x[stripamt:] if len(x)>=stripamt else b''
+            for x in lines])
+
+
+@parameterize
+class TestInversion(TestEmailBase, unittest.TestCase):
+
+    def msg_as_input(self, msg):
+        m = message_from_bytes(msg, policy=policy.SMTP)
+        b = io.BytesIO()
+        g = BytesGenerator(b)
+        g.flatten(m)
+        self.assertEqual(b.getvalue(), msg)
+
+    # XXX: spaces are not preserved correctly here yet in the general case.
+    msg_params = {
+        'header_with_one_space_body': (dedent(b"""\
+            From: abc@xyz.com
+            X-Status:\x20
+            Subject: test
+
+            foo
+            """),),
+
+            }
diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py
new file mode 100644
index 0000000000..8cc3f80e46
--- /dev/null
+++ b/Lib/test/test_email/test_message.py
@@ -0,0 +1,18 @@
+import unittest
+from email import policy
+from test.test_email import TestEmailBase
+
+
+class Test(TestEmailBase):
+
+    policy = policy.default
+
+    def test_error_on_setitem_if_max_count_exceeded(self):
+        m = self._str_msg("")
+        m['To'] = 'abc@xyz'
+        with self.assertRaises(ValueError):
+            m['To'] = 'xyz@abc'
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_email/test_parser.py b/Lib/test/test_email/test_parser.py
new file mode 100644
index 0000000000..3abd11a45c
--- /dev/null
+++ b/Lib/test/test_email/test_parser.py
@@ -0,0 +1,36 @@
+import io
+import email
+import unittest
+from email.message import Message
+from test.test_email import TestEmailBase
+
+
+class TestCustomMessage(TestEmailBase):
+
+    class MyMessage(Message):
+        def __init__(self, policy):
+            self.check_policy = policy
+            super().__init__()
+
+    MyPolicy = TestEmailBase.policy.clone(linesep='boo')
+
+    def test_custom_message_gets_policy_if_possible_from_string(self):
+        msg = email.message_from_string("Subject: bogus\n\nmsg\n",
+                                        self.MyMessage,
+                                        policy=self.MyPolicy)
+        self.assertTrue(isinstance(msg, self.MyMessage))
+        self.assertIs(msg.check_policy, self.MyPolicy)
+
+    def test_custom_message_gets_policy_if_possible_from_file(self):
+        source_file = io.StringIO("Subject: bogus\n\nmsg\n")
+        msg = email.message_from_file(source_file,
+                                      self.MyMessage,
+                                      policy=self.MyPolicy)
+        self.assertTrue(isinstance(msg, self.MyMessage))
+        self.assertIs(msg.check_policy, self.MyPolicy)
+
+    # XXX add tests for other functions that take Message arg.
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_email/test_pickleable.py b/Lib/test/test_email/test_pickleable.py
new file mode 100644
index 0000000000..daa8d25053
--- /dev/null
+++ b/Lib/test/test_email/test_pickleable.py
@@ -0,0 +1,74 @@
+import unittest
+import textwrap
+import copy
+import pickle
+import email
+import email.message
+from email import policy
+from email.headerregistry import HeaderRegistry
+from test.test_email import TestEmailBase, parameterize
+
+
+@parameterize
+class TestPickleCopyHeader(TestEmailBase):
+
+    header_factory = HeaderRegistry()
+
+    unstructured = header_factory('subject', 'this is a test')
+
+    header_params = {
+        'subject': ('subject', 'this is a test'),
+        'from':    ('from',    'frodo@mordor.net'),
+        'to':      ('to',      'a: k@b.com, y@z.com;, j@f.com'),
+        'date':    ('date',    'Tue, 29 May 2012 09:24:26 +1000'),
+        }
+
+    def header_as_deepcopy(self, name, value):
+        header = self.header_factory(name, value)
+        h = copy.deepcopy(header)
+        self.assertEqual(str(h), str(header))
+
+    def header_as_pickle(self, name, value):
+        header = self.header_factory(name, value)
+        p = pickle.dumps(header)
+        h = pickle.loads(p)
+        self.assertEqual(str(h), str(header))
+
+
+@parameterize
+class TestPickleCopyMessage(TestEmailBase):
+
+    # Message objects are a sequence, so we have to make them a one-tuple in
+    # msg_params so they get passed to the parameterized test method as a
+    # single argument instead of as a list of headers.
+    msg_params = {}
+
+    # Note: there will be no custom header objects in the parsed message.
+    msg_params['parsed'] = (email.message_from_string(textwrap.dedent("""\
+        Date: Tue, 29 May 2012 09:24:26 +1000
+        From: frodo@mordor.net
+        To: bilbo@underhill.org
+        Subject: help
+
+        I think I forgot the ring.
+        """), policy=policy.default),)
+
+    msg_params['created'] = (email.message.Message(policy=policy.default),)
+    msg_params['created'][0]['Date'] = 'Tue, 29 May 2012 09:24:26 +1000'
+    msg_params['created'][0]['From'] = 'frodo@mordor.net'
+    msg_params['created'][0]['To'] = 'bilbo@underhill.org'
+    msg_params['created'][0]['Subject'] = 'help'
+    msg_params['created'][0].set_payload('I think I forgot the ring.')
+
+    def msg_as_deepcopy(self, msg):
+        msg2 = copy.deepcopy(msg)
+        self.assertEqual(msg2.as_string(), msg.as_string())
+
+    def msg_as_pickle(self, msg):
+        p = pickle.dumps(msg)
+        msg2 = pickle.loads(p)
+        self.assertEqual(msg2.as_string(), msg.as_string())
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py
new file mode 100644
index 0000000000..983bd49a11
--- /dev/null
+++ b/Lib/test/test_email/test_policy.py
@@ -0,0 +1,322 @@
+import io
+import types
+import textwrap
+import unittest
+import email.policy
+import email.parser
+import email.generator
+from email import headerregistry
+
+def make_defaults(base_defaults, differences):
+    defaults = base_defaults.copy()
+    defaults.update(differences)
+    return defaults
+
+class PolicyAPITests(unittest.TestCase):
+
+    longMessage = True
+
+    # Base default values.
+    compat32_defaults = {
+        'max_line_length':          78,
+        'linesep':                  '\n',
+        'cte_type':                 '8bit',
+        'raise_on_defect':          False,
+        }
+    # These default values are the ones set on email.policy.default.
+    # If any of these defaults change, the docs must be updated.
+    policy_defaults = compat32_defaults.copy()
+    policy_defaults.update({
+        'raise_on_defect':          False,
+        'header_factory':           email.policy.EmailPolicy.header_factory,
+        'refold_source':            'long',
+        })
+
+    # For each policy under test, we give here what we expect the defaults to
+    # be for that policy.  The second argument to make defaults is the
+    # difference between the base defaults and that for the particular policy.
+    new_policy = email.policy.EmailPolicy()
+    policies = {
+        email.policy.compat32: make_defaults(compat32_defaults, {}),
+        email.policy.default: make_defaults(policy_defaults, {}),
+        email.policy.SMTP: make_defaults(policy_defaults,
+                                         {'linesep': '\r\n'}),
+        email.policy.HTTP: make_defaults(policy_defaults,
+                                         {'linesep': '\r\n',
+                                          'max_line_length': None}),
+        email.policy.strict: make_defaults(policy_defaults,
+                                           {'raise_on_defect': True}),
+        new_policy: make_defaults(policy_defaults, {}),
+        }
+    # Creating a new policy creates a new header factory.  There is a test
+    # later that proves this.
+    policies[new_policy]['header_factory'] = new_policy.header_factory
+
+    def test_defaults(self):
+        for policy, expected in self.policies.items():
+            for attr, value in expected.items():
+                self.assertEqual(getattr(policy, attr), value,
+                                ("change {} docs/docstrings if defaults have "
+                                "changed").format(policy))
+
+    def test_all_attributes_covered(self):
+        for policy, expected in self.policies.items():
+            for attr in dir(policy):
+                if (attr.startswith('_') or
+                        isinstance(getattr(email.policy.EmailPolicy, attr),
+                              types.FunctionType)):
+                    continue
+                else:
+                    self.assertIn(attr, expected,
+                                  "{} is not fully tested".format(attr))
+
+    def test_abc(self):
+        with self.assertRaises(TypeError) as cm:
+            email.policy.Policy()
+        msg = str(cm.exception)
+        abstract_methods = ('fold',
+                            'fold_binary',
+                            'header_fetch_parse',
+                            'header_source_parse',
+                            'header_store_parse')
+        for method in abstract_methods:
+            self.assertIn(method, msg)
+
+    def test_policy_is_immutable(self):
+        for policy, defaults in self.policies.items():
+            for attr in defaults:
+                with self.assertRaisesRegex(AttributeError, attr+".*read-only"):
+                    setattr(policy, attr, None)
+            with self.assertRaisesRegex(AttributeError, 'no attribute.*foo'):
+                policy.foo = None
+
+    def test_set_policy_attrs_when_cloned(self):
+        # None of the attributes has a default value of None, so we set them
+        # all to None in the clone call and check that it worked.
+        for policyclass, defaults in self.policies.items():
+            testattrdict = {attr: None for attr in defaults}
+            policy = policyclass.clone(**testattrdict)
+            for attr in defaults:
+                self.assertIsNone(getattr(policy, attr))
+
+    def test_reject_non_policy_keyword_when_called(self):
+        for policyclass in self.policies:
+            with self.assertRaises(TypeError):
+                policyclass(this_keyword_should_not_be_valid=None)
+            with self.assertRaises(TypeError):
+                policyclass(newtline=None)
+
+    def test_policy_addition(self):
+        expected = self.policy_defaults.copy()
+        p1 = email.policy.default.clone(max_line_length=100)
+        p2 = email.policy.default.clone(max_line_length=50)
+        added = p1 + p2
+        expected.update(max_line_length=50)
+        for attr, value in expected.items():
+            self.assertEqual(getattr(added, attr), value)
+        added = p2 + p1
+        expected.update(max_line_length=100)
+        for attr, value in expected.items():
+            self.assertEqual(getattr(added, attr), value)
+        added = added + email.policy.default
+        for attr, value in expected.items():
+            self.assertEqual(getattr(added, attr), value)
+
+    def test_register_defect(self):
+        class Dummy:
+            def __init__(self):
+                self.defects = []
+        obj = Dummy()
+        defect = object()
+        policy = email.policy.EmailPolicy()
+        policy.register_defect(obj, defect)
+        self.assertEqual(obj.defects, [defect])
+        defect2 = object()
+        policy.register_defect(obj, defect2)
+        self.assertEqual(obj.defects, [defect, defect2])
+
+    class MyObj:
+        def __init__(self):
+            self.defects = []
+
+    class MyDefect(Exception):
+        pass
+
+    def test_handle_defect_raises_on_strict(self):
+        foo = self.MyObj()
+        defect = self.MyDefect("the telly is broken")
+        with self.assertRaisesRegex(self.MyDefect, "the telly is broken"):
+            email.policy.strict.handle_defect(foo, defect)
+
+    def test_handle_defect_registers_defect(self):
+        foo = self.MyObj()
+        defect1 = self.MyDefect("one")
+        email.policy.default.handle_defect(foo, defect1)
+        self.assertEqual(foo.defects, [defect1])
+        defect2 = self.MyDefect("two")
+        email.policy.default.handle_defect(foo, defect2)
+        self.assertEqual(foo.defects, [defect1, defect2])
+
+    class MyPolicy(email.policy.EmailPolicy):
+        defects = None
+        def __init__(self, *args, **kw):
+            super().__init__(*args, defects=[], **kw)
+        def register_defect(self, obj, defect):
+            self.defects.append(defect)
+
+    def test_overridden_register_defect_still_raises(self):
+        foo = self.MyObj()
+        defect = self.MyDefect("the telly is broken")
+        with self.assertRaisesRegex(self.MyDefect, "the telly is broken"):
+            self.MyPolicy(raise_on_defect=True).handle_defect(foo, defect)
+
+    def test_overriden_register_defect_works(self):
+        foo = self.MyObj()
+        defect1 = self.MyDefect("one")
+        my_policy = self.MyPolicy()
+        my_policy.handle_defect(foo, defect1)
+        self.assertEqual(my_policy.defects, [defect1])
+        self.assertEqual(foo.defects, [])
+        defect2 = self.MyDefect("two")
+        my_policy.handle_defect(foo, defect2)
+        self.assertEqual(my_policy.defects, [defect1, defect2])
+        self.assertEqual(foo.defects, [])
+
+    def test_default_header_factory(self):
+        h = email.policy.default.header_factory('Test', 'test')
+        self.assertEqual(h.name, 'Test')
+        self.assertIsInstance(h, headerregistry.UnstructuredHeader)
+        self.assertIsInstance(h, headerregistry.BaseHeader)
+
+    class Foo:
+        parse = headerregistry.UnstructuredHeader.parse
+
+    def test_each_Policy_gets_unique_factory(self):
+        policy1 = email.policy.EmailPolicy()
+        policy2 = email.policy.EmailPolicy()
+        policy1.header_factory.map_to_type('foo', self.Foo)
+        h = policy1.header_factory('foo', 'test')
+        self.assertIsInstance(h, self.Foo)
+        self.assertNotIsInstance(h, headerregistry.UnstructuredHeader)
+        h = policy2.header_factory('foo', 'test')
+        self.assertNotIsInstance(h, self.Foo)
+        self.assertIsInstance(h, headerregistry.UnstructuredHeader)
+
+    def test_clone_copies_factory(self):
+        policy1 = email.policy.EmailPolicy()
+        policy2 = policy1.clone()
+        policy1.header_factory.map_to_type('foo', self.Foo)
+        h = policy1.header_factory('foo', 'test')
+        self.assertIsInstance(h, self.Foo)
+        h = policy2.header_factory('foo', 'test')
+        self.assertIsInstance(h, self.Foo)
+
+    def test_new_factory_overrides_default(self):
+        mypolicy = email.policy.EmailPolicy()
+        myfactory = mypolicy.header_factory
+        newpolicy = mypolicy + email.policy.strict
+        self.assertEqual(newpolicy.header_factory, myfactory)
+        newpolicy = email.policy.strict + mypolicy
+        self.assertEqual(newpolicy.header_factory, myfactory)
+
+    def test_adding_default_policies_preserves_default_factory(self):
+        newpolicy = email.policy.default + email.policy.strict
+        self.assertEqual(newpolicy.header_factory,
+                         email.policy.EmailPolicy.header_factory)
+        self.assertEqual(newpolicy.__dict__, {'raise_on_defect': True})
+
+    # XXX: Need subclassing tests.
+    # For adding subclassed objects, make sure the usual rules apply (subclass
+    # wins), but that the order still works (right overrides left).
+
+
+class TestPolicyPropagation(unittest.TestCase):
+
+    # The abstract methods are used by the parser but not by the wrapper
+    # functions that call it, so if the exception gets raised we know that the
+    # policy was actually propagated all the way to feedparser.
+    class MyPolicy(email.policy.Policy):
+        def badmethod(self, *args, **kw):
+            raise Exception("test")
+        fold = fold_binary = header_fetch_parser = badmethod
+        header_source_parse = header_store_parse = badmethod
+
+    def test_message_from_string(self):
+        with self.assertRaisesRegex(Exception, "^test$"):
+            email.message_from_string("Subject: test\n\n",
+                                      policy=self.MyPolicy)
+
+    def test_message_from_bytes(self):
+        with self.assertRaisesRegex(Exception, "^test$"):
+            email.message_from_bytes(b"Subject: test\n\n",
+                                     policy=self.MyPolicy)
+
+    def test_message_from_file(self):
+        f = io.StringIO('Subject: test\n\n')
+        with self.assertRaisesRegex(Exception, "^test$"):
+            email.message_from_file(f, policy=self.MyPolicy)
+
+    def test_message_from_binary_file(self):
+        f = io.BytesIO(b'Subject: test\n\n')
+        with self.assertRaisesRegex(Exception, "^test$"):
+            email.message_from_binary_file(f, policy=self.MyPolicy)
+
+    # These are redundant, but we need them for black-box completeness.
+
+    def test_parser(self):
+        p = email.parser.Parser(policy=self.MyPolicy)
+        with self.assertRaisesRegex(Exception, "^test$"):
+            p.parsestr('Subject: test\n\n')
+
+    def test_bytes_parser(self):
+        p = email.parser.BytesParser(policy=self.MyPolicy)
+        with self.assertRaisesRegex(Exception, "^test$"):
+            p.parsebytes(b'Subject: test\n\n')
+
+    # Now that we've established that all the parse methods get the
+    # policy in to feedparser, we can use message_from_string for
+    # the rest of the propagation tests.
+
+    def _make_msg(self, source='Subject: test\n\n', policy=None):
+        self.policy = email.policy.default.clone() if policy is None else policy
+        return email.message_from_string(source, policy=self.policy)
+
+    def test_parser_propagates_policy_to_message(self):
+        msg = self._make_msg()
+        self.assertIs(msg.policy, self.policy)
+
+    def test_parser_propagates_policy_to_sub_messages(self):
+        msg = self._make_msg(textwrap.dedent("""\
+            Subject: mime test
+            MIME-Version: 1.0
+            Content-Type: multipart/mixed, boundary="XXX"
+
+            --XXX
+            Content-Type: text/plain
+
+            test
+            --XXX
+            Content-Type: text/plain
+
+            test2
+            --XXX--
+            """))
+        for part in msg.walk():
+            self.assertIs(part.policy, self.policy)
+
+    def test_message_policy_propagates_to_generator(self):
+        msg = self._make_msg("Subject: test\nTo: foo\n\n",
+                             policy=email.policy.default.clone(linesep='X'))
+        s = io.StringIO()
+        g = email.generator.Generator(s)
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), "Subject: testXTo: fooXX")
+
+    def test_message_policy_used_by_as_string(self):
+        msg = self._make_msg("Subject: test\nTo: foo\n\n",
+                             policy=email.policy.default.clone(linesep='X'))
+        self.assertEqual(msg.as_string(), "Subject: testXTo: fooXX")
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py
new file mode 100644
index 0000000000..622ef3a9c9
--- /dev/null
+++ b/Lib/test/test_email/test_utils.py
@@ -0,0 +1,136 @@
+import datetime
+from email import utils
+import test.support
+import time
+import unittest
+import sys
+
+class DateTimeTests(unittest.TestCase):
+
+    datestring = 'Sun, 23 Sep 2001 20:10:55'
+    dateargs = (2001, 9, 23, 20, 10, 55)
+    offsetstring = ' -0700'
+    utcoffset = datetime.timedelta(hours=-7)
+    tz = datetime.timezone(utcoffset)
+    naive_dt = datetime.datetime(*dateargs)
+    aware_dt = datetime.datetime(*dateargs, tzinfo=tz)
+
+    def test_naive_datetime(self):
+        self.assertEqual(utils.format_datetime(self.naive_dt),
+                         self.datestring + ' -0000')
+
+    def test_aware_datetime(self):
+        self.assertEqual(utils.format_datetime(self.aware_dt),
+                         self.datestring + self.offsetstring)
+
+    def test_usegmt(self):
+        utc_dt = datetime.datetime(*self.dateargs,
+                                   tzinfo=datetime.timezone.utc)
+        self.assertEqual(utils.format_datetime(utc_dt, usegmt=True),
+                         self.datestring + ' GMT')
+
+    def test_usegmt_with_naive_datetime_raises(self):
+        with self.assertRaises(ValueError):
+            utils.format_datetime(self.naive_dt, usegmt=True)
+
+    def test_usegmt_with_non_utc_datetime_raises(self):
+        with self.assertRaises(ValueError):
+            utils.format_datetime(self.aware_dt, usegmt=True)
+
+    def test_parsedate_to_datetime(self):
+        self.assertEqual(
+            utils.parsedate_to_datetime(self.datestring + self.offsetstring),
+            self.aware_dt)
+
+    def test_parsedate_to_datetime_naive(self):
+        self.assertEqual(
+            utils.parsedate_to_datetime(self.datestring + ' -0000'),
+            self.naive_dt)
+
+
+class LocaltimeTests(unittest.TestCase):
+
+    def test_localtime_is_tz_aware_daylight_true(self):
+        test.support.patch(self, time, 'daylight', True)
+        t = utils.localtime()
+        self.assertIsNot(t.tzinfo, None)
+
+    def test_localtime_is_tz_aware_daylight_false(self):
+        test.support.patch(self, time, 'daylight', False)
+        t = utils.localtime()
+        self.assertIsNot(t.tzinfo, None)
+
+    def test_localtime_daylight_true_dst_false(self):
+        test.support.patch(self, time, 'daylight', True)
+        t0 = datetime.datetime(2012, 3, 12, 1, 1)
+        t1 = utils.localtime(t0, isdst=-1)
+        t2 = utils.localtime(t1)
+        self.assertEqual(t1, t2)
+
+    def test_localtime_daylight_false_dst_false(self):
+        test.support.patch(self, time, 'daylight', False)
+        t0 = datetime.datetime(2012, 3, 12, 1, 1)
+        t1 = utils.localtime(t0, isdst=-1)
+        t2 = utils.localtime(t1)
+        self.assertEqual(t1, t2)
+
+    def test_localtime_daylight_true_dst_true(self):
+        test.support.patch(self, time, 'daylight', True)
+        t0 = datetime.datetime(2012, 3, 12, 1, 1)
+        t1 = utils.localtime(t0, isdst=1)
+        t2 = utils.localtime(t1)
+        self.assertEqual(t1, t2)
+
+    def test_localtime_daylight_false_dst_true(self):
+        test.support.patch(self, time, 'daylight', False)
+        t0 = datetime.datetime(2012, 3, 12, 1, 1)
+        t1 = utils.localtime(t0, isdst=1)
+        t2 = utils.localtime(t1)
+        self.assertEqual(t1, t2)
+
+    @test.support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0')
+    def test_localtime_epoch_utc_daylight_true(self):
+        test.support.patch(self, time, 'daylight', True)
+        t0 = datetime.datetime(1990, 1, 1, tzinfo = datetime.timezone.utc)
+        t1 = utils.localtime(t0)
+        t2 = t0 - datetime.timedelta(hours=5)
+        t2 = t2.replace(tzinfo = datetime.timezone(datetime.timedelta(hours=-5)))
+        self.assertEqual(t1, t2)
+
+    @test.support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0')
+    def test_localtime_epoch_utc_daylight_false(self):
+        test.support.patch(self, time, 'daylight', False)
+        t0 = datetime.datetime(1990, 1, 1, tzinfo = datetime.timezone.utc)
+        t1 = utils.localtime(t0)
+        t2 = t0 - datetime.timedelta(hours=5)
+        t2 = t2.replace(tzinfo = datetime.timezone(datetime.timedelta(hours=-5)))
+        self.assertEqual(t1, t2)
+
+    def test_localtime_epoch_notz_daylight_true(self):
+        test.support.patch(self, time, 'daylight', True)
+        t0 = datetime.datetime(1990, 1, 1)
+        t1 = utils.localtime(t0)
+        t2 = utils.localtime(t0.replace(tzinfo=None))
+        self.assertEqual(t1, t2)
+
+    def test_localtime_epoch_notz_daylight_false(self):
+        test.support.patch(self, time, 'daylight', False)
+        t0 = datetime.datetime(1990, 1, 1)
+        t1 = utils.localtime(t0)
+        t2 = utils.localtime(t0.replace(tzinfo=None))
+        self.assertEqual(t1, t2)
+
+    # XXX: Need a more robust test for Olson's tzdata
+    @unittest.skipIf(sys.platform.startswith('win'),
+                     "Windows does not use Olson's TZ database")
+    @test.support.run_with_tz('Europe/Kiev')
+    def test_variable_tzname(self):
+        t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc)
+        t1 = utils.localtime(t0)
+        self.assertEqual(t1.tzname(), 'MSK')
+        t0 = datetime.datetime(1994, 1, 1, tzinfo=datetime.timezone.utc)
+        t1 = utils.localtime(t0)
+        self.assertEqual(t1.tzname(), 'EET')
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/email/test/test_email_torture.py b/Lib/test/test_email/torture_test.py
index 544b1bbb39..544b1bbb39 100644
--- a/Lib/email/test/test_email_torture.py
+++ b/Lib/test/test_email/torture_test.py
diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py
index 095820b45f..2e904cf878 100644
--- a/Lib/test/test_enumerate.py
+++ b/Lib/test/test_enumerate.py
@@ -1,5 +1,6 @@
 import unittest
 import sys
+import pickle
 
 from test import support
 
@@ -61,7 +62,25 @@ class N:
     def __iter__(self):
         return self
 
-class EnumerateTestCase(unittest.TestCase):
+class PickleTest:
+    # Helper to check picklability
+    def check_pickle(self, itorg, seq):
+        d = pickle.dumps(itorg)
+        it = pickle.loads(d)
+        self.assertEqual(type(itorg), type(it))
+        self.assertEqual(list(it), seq)
+
+        it = pickle.loads(d)
+        try:
+            next(it)
+        except StopIteration:
+            self.assertFalse(seq[1:])
+            return
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(list(it), seq[1:])
+
+class EnumerateTestCase(unittest.TestCase, PickleTest):
 
     enum = enumerate
     seq, res = 'abc', [(0,'a'), (1,'b'), (2,'c')]
@@ -73,6 +92,9 @@ class EnumerateTestCase(unittest.TestCase):
         self.assertEqual(list(self.enum(self.seq)), self.res)
         self.enum.__doc__
 
+    def test_pickle(self):
+        self.check_pickle(self.enum(self.seq), self.res)
+
     def test_getitemseqn(self):
         self.assertEqual(list(self.enum(G(self.seq))), self.res)
         e = self.enum(G(''))
@@ -126,7 +148,7 @@ class TestBig(EnumerateTestCase):
     seq = range(10,20000,2)
     res = list(zip(range(20000), seq))
 
-class TestReversed(unittest.TestCase):
+class TestReversed(unittest.TestCase, PickleTest):
 
     def test_simple(self):
         class A:
@@ -212,6 +234,10 @@ class TestReversed(unittest.TestCase):
         ngi = NoGetItem()
         self.assertRaises(TypeError, reversed, ngi)
 
+    def test_pickle(self):
+        for data in 'abc', range(5), tuple(enumerate('abc')), range(1,17,5):
+            self.check_pickle(reversed(data), list(data)[::-1])
+
 
 class EnumerateStartTestCase(EnumerateTestCase):
 
diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py
index 083fd7f79d..7f9547ff95 100644
--- a/Lib/test/test_epoll.py
+++ b/Lib/test/test_epoll.py
@@ -75,6 +75,9 @@ class TestEPoll(unittest.TestCase):
         ep.close()
         self.assertTrue(ep.closed)
         self.assertRaises(ValueError, ep.fileno)
+        if hasattr(select, "EPOLL_CLOEXEC"):
+            select.epoll(select.EPOLL_CLOEXEC).close()
+            self.assertRaises(OSError, select.epoll, flags=12356)
 
     def test_badcreate(self):
         self.assertRaises(TypeError, select.epoll, 1, 2, 3)
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 28dcb24ea0..1ad7f97b74 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -8,7 +8,8 @@ import weakref
 import errno
 
 from test.support import (TESTFN, captured_output, check_impl_detail,
-                          cpython_only, gc_collect, run_unittest, unlink)
+                          cpython_only, gc_collect, run_unittest, no_tracing,
+                          unlink)
 
 class NaiveException(Exception):
     def __init__(self, x):
@@ -55,8 +56,8 @@ class ExceptionTests(unittest.TestCase):
             fp.close()
             unlink(TESTFN)
 
-        self.raise_catch(IOError, "IOError")
-        self.assertRaises(IOError, open, 'this file does not exist', 'r')
+        self.raise_catch(OSError, "OSError")
+        self.assertRaises(OSError, open, 'this file does not exist', 'r')
 
         self.raise_catch(ImportError, "ImportError")
         self.assertRaises(ImportError, __import__, "undefined_module")
@@ -201,11 +202,35 @@ class ExceptionTests(unittest.TestCase):
         except NameError:
             pass
         else:
-            self.assertEqual(str(WindowsError(1001)), "1001")
-            self.assertEqual(str(WindowsError(1001, "message")),
-                             "[Error 1001] message")
-            self.assertEqual(WindowsError(1001, "message").errno, 22)
-            self.assertEqual(WindowsError(1001, "message").winerror, 1001)
+            self.assertIs(WindowsError, OSError)
+            self.assertEqual(str(OSError(1001)), "1001")
+            self.assertEqual(str(OSError(1001, "message")),
+                             "[Errno 1001] message")
+            # POSIX errno (9 aka EBADF) is untranslated
+            w = OSError(9, 'foo', 'bar')
+            self.assertEqual(w.errno, 9)
+            self.assertEqual(w.winerror, None)
+            self.assertEqual(str(w), "[Errno 9] foo: 'bar'")
+            # ERROR_PATH_NOT_FOUND (win error 3) becomes ENOENT (2)
+            w = OSError(0, 'foo', 'bar', 3)
+            self.assertEqual(w.errno, 2)
+            self.assertEqual(w.winerror, 3)
+            self.assertEqual(w.strerror, 'foo')
+            self.assertEqual(w.filename, 'bar')
+            self.assertEqual(str(w), "[WinError 3] foo: 'bar'")
+            # Unknown win error becomes EINVAL (22)
+            w = OSError(0, 'foo', None, 1001)
+            self.assertEqual(w.errno, 22)
+            self.assertEqual(w.winerror, 1001)
+            self.assertEqual(w.strerror, 'foo')
+            self.assertEqual(w.filename, None)
+            self.assertEqual(str(w), "[WinError 1001] foo")
+            # Non-numeric "errno"
+            w = OSError('bar', 'foo')
+            self.assertEqual(w.errno, 'bar')
+            self.assertEqual(w.winerror, None)
+            self.assertEqual(w.strerror, 'foo')
+            self.assertEqual(w.filename, None)
 
     def testAttributes(self):
         # test that exception attributes are happy
@@ -287,11 +312,12 @@ class ExceptionTests(unittest.TestCase):
                 {'args': ('foo',), 'x': 'foo'}),
         ]
         try:
+            # More tests are in test_WindowsError
             exceptionList.append(
                 (WindowsError, (1, 'strErrorStr', 'filenameStr'),
                     {'args' : (1, 'strErrorStr'),
-                     'strerror' : 'strErrorStr', 'winerror' : 1,
-                     'errno' : 22, 'filename' : 'filenameStr'})
+                     'strerror' : 'strErrorStr', 'winerror' : None,
+                     'errno' : 1, 'filename' : 'filenameStr'})
             )
         except NameError:
             pass
@@ -376,19 +402,37 @@ class ExceptionTests(unittest.TestCase):
 
     def testChainingAttrs(self):
         e = Exception()
-        self.assertEqual(e.__context__, None)
-        self.assertEqual(e.__cause__, None)
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
 
         e = TypeError()
-        self.assertEqual(e.__context__, None)
-        self.assertEqual(e.__cause__, None)
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
 
         class MyException(EnvironmentError):
             pass
 
         e = MyException()
-        self.assertEqual(e.__context__, None)
-        self.assertEqual(e.__cause__, None)
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
+
+    def testChainingDescriptors(self):
+        try:
+            raise Exception()
+        except Exception as exc:
+            e = exc
+
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
+        self.assertFalse(e.__suppress_context__)
+
+        e.__context__ = NameError()
+        e.__cause__ = None
+        self.assertIsInstance(e.__context__, NameError)
+        self.assertIsNone(e.__cause__)
+        self.assertTrue(e.__suppress_context__)
+        e.__suppress_context__ = False
+        self.assertFalse(e.__suppress_context__)
 
     def testKeywordArgs(self):
         # test that builtin exception don't take keyword args,
@@ -403,6 +447,7 @@ class ExceptionTests(unittest.TestCase):
         x = DerivedException(fancy_arg=42)
         self.assertEqual(x.fancy_arg, 42)
 
+    @no_tracing
     def testInfiniteRecursion(self):
         def f():
             return f()
@@ -742,6 +787,7 @@ class ExceptionTests(unittest.TestCase):
         u.start = 1000
         self.assertEqual(str(u), "can't translate characters in position 1000-4: 965230951443685724997")
 
+    @no_tracing
     def test_badisinstance(self):
         # Bug #2542: if issubclass(e, MyException) raises an exception,
         # it should be ignored
@@ -852,6 +898,7 @@ class ExceptionTests(unittest.TestCase):
             self.fail("MemoryError not raised")
         self.assertEqual(wr(), None)
 
+    @no_tracing
     def test_recursion_error_cleanup(self):
         # Same test as above, but with "recursion exceeded" errors
         class C:
@@ -878,8 +925,35 @@ class ExceptionTests(unittest.TestCase):
         self.assertEqual(cm.exception.errno, errno.ENOTDIR, cm.exception)
 
 
+class ImportErrorTests(unittest.TestCase):
+
+    def test_attributes(self):
+        # Setting 'name' and 'path' should not be a problem.
+        exc = ImportError('test')
+        self.assertIsNone(exc.name)
+        self.assertIsNone(exc.path)
+
+        exc = ImportError('test', name='somemodule')
+        self.assertEqual(exc.name, 'somemodule')
+        self.assertIsNone(exc.path)
+
+        exc = ImportError('test', path='somepath')
+        self.assertEqual(exc.path, 'somepath')
+        self.assertIsNone(exc.name)
+
+        exc = ImportError('test', path='somepath', name='somename')
+        self.assertEqual(exc.name, 'somename')
+        self.assertEqual(exc.path, 'somepath')
+
+    def test_non_str_argument(self):
+        # Issue #15778
+        arg = b'abc'
+        exc = ImportError(arg)
+        self.assertEqual(str(arg), str(exc))
+
+
 def test_main():
-    run_unittest(ExceptionTests)
+    run_unittest(ExceptionTests, ImportErrorTests)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py
index 1f7f63042e..6b6c12de2e 100644
--- a/Lib/test/test_extcall.py
+++ b/Lib/test/test_extcall.py
@@ -66,17 +66,17 @@ Verify clearing of SF bug #733667
     >>> g()
     Traceback (most recent call last):
       ...
-    TypeError: g() takes at least 1 argument (0 given)
+    TypeError: g() missing 1 required positional argument: 'x'
 
     >>> g(*())
     Traceback (most recent call last):
       ...
-    TypeError: g() takes at least 1 argument (0 given)
+    TypeError: g() missing 1 required positional argument: 'x'
 
     >>> g(*(), **{})
     Traceback (most recent call last):
       ...
-    TypeError: g() takes at least 1 argument (0 given)
+    TypeError: g() missing 1 required positional argument: 'x'
 
     >>> g(1)
     1 () {}
@@ -151,7 +151,7 @@ What about willful misconduct?
     >>> g(1, 2, 3, **{'x': 4, 'y': 5})
     Traceback (most recent call last):
       ...
-    TypeError: g() got multiple values for keyword argument 'x'
+    TypeError: g() got multiple values for argument 'x'
 
     >>> f(**{1:2})
     Traceback (most recent call last):
@@ -263,29 +263,80 @@ the function call setup. See <http://bugs.python.org/issue2016>.
     >>> f(**x)
     1 2
 
-A obscure message:
+Too many arguments:
 
-    >>> def f(a, b):
-    ...    pass
-    >>> f(b=1)
+    >>> def f(): pass
+    >>> f(1)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() takes 0 positional arguments but 1 was given
+    >>> def f(a): pass
+    >>> f(1, 2)
     Traceback (most recent call last):
       ...
-    TypeError: f() takes exactly 2 arguments (1 given)
+    TypeError: f() takes 1 positional argument but 2 were given
+    >>> def f(a, b=1): pass
+    >>> f(1, 2, 3)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() takes from 1 to 2 positional arguments but 3 were given
+    >>> def f(*, kw): pass
+    >>> f(1, kw=3)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given
+    >>> def f(*, kw, b): pass
+    >>> f(1, 2, 3, b=3, kw=3)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() takes 0 positional arguments but 3 positional arguments (and 2 keyword-only arguments) were given
+    >>> def f(a, b=2, *, kw): pass
+    >>> f(2, 3, 4, kw=4)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() takes from 1 to 2 positional arguments but 3 positional arguments (and 1 keyword-only argument) were given
 
-The number of arguments passed in includes keywords:
+Too few and missing arguments:
 
-    >>> def f(a):
-    ...    pass
-    >>> f(6, a=4, *(1, 2, 3))
+    >>> def f(a): pass
+    >>> f()
     Traceback (most recent call last):
       ...
-    TypeError: f() takes exactly 1 positional argument (5 given)
-    >>> def f(a, *, kw):
-    ...    pass
-    >>> f(6, 4, kw=4)
+    TypeError: f() missing 1 required positional argument: 'a'
+    >>> def f(a, b): pass
+    >>> f()
     Traceback (most recent call last):
       ...
-    TypeError: f() takes exactly 1 positional argument (3 given)
+    TypeError: f() missing 2 required positional arguments: 'a' and 'b'
+    >>> def f(a, b, c): pass
+    >>> f()
+    Traceback (most recent call last):
+      ...
+    TypeError: f() missing 3 required positional arguments: 'a', 'b', and 'c'
+    >>> def f(a, b, c, d, e): pass
+    >>> f()
+    Traceback (most recent call last):
+      ...
+    TypeError: f() missing 5 required positional arguments: 'a', 'b', 'c', 'd', and 'e'
+    >>> def f(a, b=4, c=5, d=5): pass
+    >>> f(c=12, b=9)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() missing 1 required positional argument: 'a'
+
+Same with keyword only args:
+
+    >>> def f(*, w): pass
+    >>> f()
+    Traceback (most recent call last):
+      ...
+    TypeError: f() missing 1 required keyword-only argument: 'w'
+    >>> def f(*, a, b, c, d, e): pass
+    >>> f()
+    Traceback (most recent call last):
+      ...
+    TypeError: f() missing 5 required keyword-only arguments: 'a', 'b', 'c', 'd', and 'e'
+
 """
 
 import sys
diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
new file mode 100644
index 0000000000..c171faf230
--- /dev/null
+++ b/Lib/test/test_faulthandler.py
@@ -0,0 +1,595 @@
+from contextlib import contextmanager
+import datetime
+import faulthandler
+import os
+import re
+import signal
+import subprocess
+import sys
+from test import support, script_helper
+from test.script_helper import assert_python_ok
+import tempfile
+import unittest
+
+try:
+    import threading
+    HAVE_THREADS = True
+except ImportError:
+    HAVE_THREADS = False
+
+TIMEOUT = 0.5
+
+try:
+    from resource import setrlimit, RLIMIT_CORE, error as resource_error
+except ImportError:
+    prepare_subprocess = None
+else:
+    def prepare_subprocess():
+        # don't create core file
+        try:
+            setrlimit(RLIMIT_CORE, (0, 0))
+        except (ValueError, resource_error):
+            pass
+
+def expected_traceback(lineno1, lineno2, header, min_count=1):
+    regex = header
+    regex += '  File "<string>", line %s in func\n' % lineno1
+    regex += '  File "<string>", line %s in <module>' % lineno2
+    if 1 < min_count:
+        return '^' + (regex + '\n') * (min_count - 1) + regex
+    else:
+        return '^' + regex + '$'
+
+@contextmanager
+def temporary_filename():
+    filename = tempfile.mktemp()
+    try:
+        yield filename
+    finally:
+        support.unlink(filename)
+
+class FaultHandlerTests(unittest.TestCase):
+    def get_output(self, code, filename=None):
+        """
+        Run the specified code in Python (in a new child process) and read the
+        output from the standard error or from a file (if filename is set).
+        Return the output lines as a list.
+
+        Strip the reference count from the standard error for Python debug
+        build, and replace "Current thread 0x00007f8d8fbd9700" by "Current
+        thread XXX".
+        """
+        options = {}
+        if prepare_subprocess:
+            options['preexec_fn'] = prepare_subprocess
+        process = script_helper.spawn_python('-c', code, **options)
+        stdout, stderr = process.communicate()
+        exitcode = process.wait()
+        output = support.strip_python_stderr(stdout)
+        output = output.decode('ascii', 'backslashreplace')
+        if filename:
+            self.assertEqual(output, '')
+            with open(filename, "rb") as fp:
+                output = fp.read()
+            output = output.decode('ascii', 'backslashreplace')
+        output = re.sub('Current thread 0x[0-9a-f]+',
+                        'Current thread XXX',
+                        output)
+        return output.splitlines(), exitcode
+
+    def check_fatal_error(self, code, line_number, name_regex,
+                          filename=None, all_threads=True, other_regex=None):
+        """
+        Check that the fault handler for fatal errors is enabled and check the
+        traceback from the child process output.
+
+        Raise an error if the output doesn't match the expected format.
+        """
+        if all_threads:
+            header = 'Current thread XXX'
+        else:
+            header = 'Traceback (most recent call first)'
+        regex = """
+^Fatal Python error: {name}
+
+{header}:
+  File "<string>", line {lineno} in <module>
+""".strip()
+        regex = regex.format(
+            lineno=line_number,
+            name=name_regex,
+            header=re.escape(header))
+        if other_regex:
+            regex += '|' + other_regex
+        with support.suppress_crash_popup():
+            output, exitcode = self.get_output(code, filename)
+        output = '\n'.join(output)
+        self.assertRegex(output, regex)
+        self.assertNotEqual(exitcode, 0)
+
+    def test_read_null(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._read_null()
+""".strip(),
+            3,
+            # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion
+            '(?:Segmentation fault|Bus error|Illegal instruction)')
+
+    def test_sigsegv(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._sigsegv()
+""".strip(),
+            3,
+            'Segmentation fault')
+
+    def test_sigabrt(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._sigabrt()
+""".strip(),
+            3,
+            'Aborted')
+
+    @unittest.skipIf(sys.platform == 'win32',
+                     "SIGFPE cannot be caught on Windows")
+    def test_sigfpe(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._sigfpe()
+""".strip(),
+            3,
+            'Floating point exception')
+
+    @unittest.skipIf(not hasattr(faulthandler, '_sigbus'),
+                     "need faulthandler._sigbus()")
+    def test_sigbus(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._sigbus()
+""".strip(),
+            3,
+            'Bus error')
+
+    @unittest.skipIf(not hasattr(faulthandler, '_sigill'),
+                     "need faulthandler._sigill()")
+    def test_sigill(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._sigill()
+""".strip(),
+            3,
+            'Illegal instruction')
+
+    def test_fatal_error(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler._fatal_error(b'xyz')
+""".strip(),
+            2,
+            'xyz')
+
+    @unittest.skipIf(sys.platform.startswith('openbsd') and HAVE_THREADS,
+                     "Issue #12868: sigaltstack() doesn't work on "
+                     "OpenBSD if Python is compiled with pthread")
+    @unittest.skipIf(not hasattr(faulthandler, '_stack_overflow'),
+                     'need faulthandler._stack_overflow()')
+    def test_stack_overflow(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._stack_overflow()
+""".strip(),
+            3,
+            '(?:Segmentation fault|Bus error)',
+            other_regex='unable to raise a stack overflow')
+
+    def test_gil_released(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable()
+faulthandler._read_null(True)
+""".strip(),
+            3,
+            '(?:Segmentation fault|Bus error|Illegal instruction)')
+
+    def test_enable_file(self):
+        with temporary_filename() as filename:
+            self.check_fatal_error("""
+import faulthandler
+output = open({filename}, 'wb')
+faulthandler.enable(output)
+faulthandler._read_null()
+""".strip().format(filename=repr(filename)),
+                4,
+                '(?:Segmentation fault|Bus error|Illegal instruction)',
+                filename=filename)
+
+    def test_enable_single_thread(self):
+        self.check_fatal_error("""
+import faulthandler
+faulthandler.enable(all_threads=False)
+faulthandler._read_null()
+""".strip(),
+            3,
+            '(?:Segmentation fault|Bus error|Illegal instruction)',
+            all_threads=False)
+
+    def test_disable(self):
+        code = """
+import faulthandler
+faulthandler.enable()
+faulthandler.disable()
+faulthandler._read_null()
+""".strip()
+        not_expected = 'Fatal Python error'
+        with support.suppress_crash_popup():
+            stderr, exitcode = self.get_output(code)
+        stder = '\n'.join(stderr)
+        self.assertTrue(not_expected not in stderr,
+                     "%r is present in %r" % (not_expected, stderr))
+        self.assertNotEqual(exitcode, 0)
+
+    def test_is_enabled(self):
+        orig_stderr = sys.stderr
+        try:
+            # regrtest may replace sys.stderr by io.StringIO object, but
+            # faulthandler.enable() requires that sys.stderr has a fileno()
+            # method
+            sys.stderr = sys.__stderr__
+
+            was_enabled = faulthandler.is_enabled()
+            try:
+                faulthandler.enable()
+                self.assertTrue(faulthandler.is_enabled())
+                faulthandler.disable()
+                self.assertFalse(faulthandler.is_enabled())
+            finally:
+                if was_enabled:
+                    faulthandler.enable()
+                else:
+                    faulthandler.disable()
+        finally:
+            sys.stderr = orig_stderr
+
+    def test_disabled_by_default(self):
+        # By default, the module should be disabled
+        code = "import faulthandler; print(faulthandler.is_enabled())"
+        rc, stdout, stderr = assert_python_ok("-c", code)
+        stdout = (stdout + stderr).strip()
+        self.assertEqual(stdout, b"False")
+
+    def test_sys_xoptions(self):
+        # Test python -X faulthandler
+        code = "import faulthandler; print(faulthandler.is_enabled())"
+        rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code)
+        stdout = (stdout + stderr).strip()
+        self.assertEqual(stdout, b"True")
+
+    def check_dump_traceback(self, filename):
+        """
+        Explicitly call dump_traceback() function and check its output.
+        Raise an error if the output doesn't match the expected format.
+        """
+        code = """
+import faulthandler
+
+def funcB():
+    if {has_filename}:
+        with open({filename}, "wb") as fp:
+            faulthandler.dump_traceback(fp, all_threads=False)
+    else:
+        faulthandler.dump_traceback(all_threads=False)
+
+def funcA():
+    funcB()
+
+funcA()
+""".strip()
+        code = code.format(
+            filename=repr(filename),
+            has_filename=bool(filename),
+        )
+        if filename:
+            lineno = 6
+        else:
+            lineno = 8
+        expected = [
+            'Traceback (most recent call first):',
+            '  File "<string>", line %s in funcB' % lineno,
+            '  File "<string>", line 11 in funcA',
+            '  File "<string>", line 13 in <module>'
+        ]
+        trace, exitcode = self.get_output(code, filename)
+        self.assertEqual(trace, expected)
+        self.assertEqual(exitcode, 0)
+
+    def test_dump_traceback(self):
+        self.check_dump_traceback(None)
+
+    def test_dump_traceback_file(self):
+        with temporary_filename() as filename:
+            self.check_dump_traceback(filename)
+
+    def test_truncate(self):
+        maxlen = 500
+        func_name = 'x' * (maxlen + 50)
+        truncated = 'x' * maxlen + '...'
+        code = """
+import faulthandler
+
+def {func_name}():
+    faulthandler.dump_traceback(all_threads=False)
+
+{func_name}()
+""".strip()
+        code = code.format(
+            func_name=func_name,
+        )
+        expected = [
+            'Traceback (most recent call first):',
+            '  File "<string>", line 4 in %s' % truncated,
+            '  File "<string>", line 6 in <module>'
+        ]
+        trace, exitcode = self.get_output(code)
+        self.assertEqual(trace, expected)
+        self.assertEqual(exitcode, 0)
+
+    @unittest.skipIf(not HAVE_THREADS, 'need threads')
+    def check_dump_traceback_threads(self, filename):
+        """
+        Call explicitly dump_traceback(all_threads=True) and check the output.
+        Raise an error if the output doesn't match the expected format.
+        """
+        code = """
+import faulthandler
+from threading import Thread, Event
+import time
+
+def dump():
+    if {filename}:
+        with open({filename}, "wb") as fp:
+            faulthandler.dump_traceback(fp, all_threads=True)
+    else:
+        faulthandler.dump_traceback(all_threads=True)
+
+class Waiter(Thread):
+    # avoid blocking if the main thread raises an exception.
+    daemon = True
+
+    def __init__(self):
+        Thread.__init__(self)
+        self.running = Event()
+        self.stop = Event()
+
+    def run(self):
+        self.running.set()
+        self.stop.wait()
+
+waiter = Waiter()
+waiter.start()
+waiter.running.wait()
+dump()
+waiter.stop.set()
+waiter.join()
+""".strip()
+        code = code.format(filename=repr(filename))
+        output, exitcode = self.get_output(code, filename)
+        output = '\n'.join(output)
+        if filename:
+            lineno = 8
+        else:
+            lineno = 10
+        regex = """
+^Thread 0x[0-9a-f]+:
+(?:  File ".*threading.py", line [0-9]+ in [_a-z]+
+){{1,3}}  File "<string>", line 23 in run
+  File ".*threading.py", line [0-9]+ in _bootstrap_inner
+  File ".*threading.py", line [0-9]+ in _bootstrap
+
+Current thread XXX:
+  File "<string>", line {lineno} in dump
+  File "<string>", line 28 in <module>$
+""".strip()
+        regex = regex.format(lineno=lineno)
+        self.assertRegex(output, regex)
+        self.assertEqual(exitcode, 0)
+
+    def test_dump_traceback_threads(self):
+        self.check_dump_traceback_threads(None)
+
+    def test_dump_traceback_threads_file(self):
+        with temporary_filename() as filename:
+            self.check_dump_traceback_threads(filename)
+
+    def _check_dump_traceback_later(self, repeat, cancel, filename, loops):
+        """
+        Check how many times the traceback is written in timeout x 2.5 seconds,
+        or timeout x 3.5 seconds if cancel is True: 1, 2 or 3 times depending
+        on repeat and cancel options.
+
+        Raise an error if the output doesn't match the expect format.
+        """
+        timeout_str = str(datetime.timedelta(seconds=TIMEOUT))
+        code = """
+import faulthandler
+import time
+
+def func(timeout, repeat, cancel, file, loops):
+    for loop in range(loops):
+        faulthandler.dump_traceback_later(timeout, repeat=repeat, file=file)
+        if cancel:
+            faulthandler.cancel_dump_traceback_later()
+        time.sleep(timeout * 5)
+        faulthandler.cancel_dump_traceback_later()
+
+timeout = {timeout}
+repeat = {repeat}
+cancel = {cancel}
+loops = {loops}
+if {has_filename}:
+    file = open({filename}, "wb")
+else:
+    file = None
+func(timeout, repeat, cancel, file, loops)
+if file is not None:
+    file.close()
+""".strip()
+        code = code.format(
+            timeout=TIMEOUT,
+            repeat=repeat,
+            cancel=cancel,
+            loops=loops,
+            has_filename=bool(filename),
+            filename=repr(filename),
+        )
+        trace, exitcode = self.get_output(code, filename)
+        trace = '\n'.join(trace)
+
+        if not cancel:
+            count = loops
+            if repeat:
+                count *= 2
+            header = r'Timeout \(%s\)!\nThread 0x[0-9a-f]+:\n' % timeout_str
+            regex = expected_traceback(9, 20, header, min_count=count)
+            self.assertRegex(trace, regex)
+        else:
+            self.assertEqual(trace, '')
+        self.assertEqual(exitcode, 0)
+
+    @unittest.skipIf(not hasattr(faulthandler, 'dump_traceback_later'),
+                     'need faulthandler.dump_traceback_later()')
+    def check_dump_traceback_later(self, repeat=False, cancel=False,
+                                    file=False, twice=False):
+        if twice:
+            loops = 2
+        else:
+            loops = 1
+        if file:
+            with temporary_filename() as filename:
+                self._check_dump_traceback_later(repeat, cancel,
+                                                  filename, loops)
+        else:
+            self._check_dump_traceback_later(repeat, cancel, None, loops)
+
+    def test_dump_traceback_later(self):
+        self.check_dump_traceback_later()
+
+    def test_dump_traceback_later_repeat(self):
+        self.check_dump_traceback_later(repeat=True)
+
+    def test_dump_traceback_later_cancel(self):
+        self.check_dump_traceback_later(cancel=True)
+
+    def test_dump_traceback_later_file(self):
+        self.check_dump_traceback_later(file=True)
+
+    def test_dump_traceback_later_twice(self):
+        self.check_dump_traceback_later(twice=True)
+
+    @unittest.skipIf(not hasattr(faulthandler, "register"),
+                     "need faulthandler.register")
+    def check_register(self, filename=False, all_threads=False,
+                       unregister=False, chain=False):
+        """
+        Register a handler displaying the traceback on a user signal. Raise the
+        signal and check the written traceback.
+
+        If chain is True, check that the previous signal handler is called.
+
+        Raise an error if the output doesn't match the expected format.
+        """
+        signum = signal.SIGUSR1
+        code = """
+import faulthandler
+import os
+import signal
+import sys
+
+def func(signum):
+    os.kill(os.getpid(), signum)
+
+def handler(signum, frame):
+    handler.called = True
+handler.called = False
+
+exitcode = 0
+signum = {signum}
+unregister = {unregister}
+chain = {chain}
+
+if {has_filename}:
+    file = open({filename}, "wb")
+else:
+    file = None
+if chain:
+    signal.signal(signum, handler)
+faulthandler.register(signum, file=file,
+                      all_threads={all_threads}, chain={chain})
+if unregister:
+    faulthandler.unregister(signum)
+func(signum)
+if chain and not handler.called:
+    if file is not None:
+        output = file
+    else:
+        output = sys.stderr
+    print("Error: signal handler not called!", file=output)
+    exitcode = 1
+if file is not None:
+    file.close()
+sys.exit(exitcode)
+""".strip()
+        code = code.format(
+            filename=repr(filename),
+            has_filename=bool(filename),
+            all_threads=all_threads,
+            signum=signum,
+            unregister=unregister,
+            chain=chain,
+        )
+        trace, exitcode = self.get_output(code, filename)
+        trace = '\n'.join(trace)
+        if not unregister:
+            if all_threads:
+                regex = 'Current thread XXX:\n'
+            else:
+                regex = 'Traceback \(most recent call first\):\n'
+            regex = expected_traceback(7, 28, regex)
+            self.assertRegex(trace, regex)
+        else:
+            self.assertEqual(trace, '')
+        if unregister:
+            self.assertNotEqual(exitcode, 0)
+        else:
+            self.assertEqual(exitcode, 0)
+
+    def test_register(self):
+        self.check_register()
+
+    def test_unregister(self):
+        self.check_register(unregister=True)
+
+    def test_register_file(self):
+        with temporary_filename() as filename:
+            self.check_register(filename=filename)
+
+    def test_register_threads(self):
+        self.check_register(all_threads=True)
+
+    def test_register_chain(self):
+        self.check_register(chain=True)
+
+
+def test_main():
+    support.run_unittest(FaultHandlerTests)
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py
index bb0da792f5..a78ddf3d3c 100644
--- a/Lib/test/test_file.py
+++ b/Lib/test/test_file.py
@@ -10,7 +10,7 @@ import _pyio as pyio
 from test.support import TESTFN, run_unittest
 from collections import UserList
 
-class AutoFileTests(unittest.TestCase):
+class AutoFileTests:
     # file tests for which a test file is automatically set up
 
     def setUp(self):
@@ -128,14 +128,14 @@ class AutoFileTests(unittest.TestCase):
     def testReadWhenWriting(self):
         self.assertRaises(IOError, self.f.read)
 
-class CAutoFileTests(AutoFileTests):
+class CAutoFileTests(AutoFileTests, unittest.TestCase):
     open = io.open
 
-class PyAutoFileTests(AutoFileTests):
+class PyAutoFileTests(AutoFileTests, unittest.TestCase):
     open = staticmethod(pyio.open)
 
 
-class OtherFileTests(unittest.TestCase):
+class OtherFileTests:
 
     def testModeStrings(self):
         # check invalid mode strings
@@ -322,22 +322,18 @@ class OtherFileTests(unittest.TestCase):
         finally:
             os.unlink(TESTFN)
 
-class COtherFileTests(OtherFileTests):
+class COtherFileTests(OtherFileTests, unittest.TestCase):
     open = io.open
 
-class PyOtherFileTests(OtherFileTests):
+class PyOtherFileTests(OtherFileTests, unittest.TestCase):
     open = staticmethod(pyio.open)
 
 
-def test_main():
+def tearDownModule():
     # Historically, these tests have been sloppy about removing TESTFN.
     # So get rid of it no matter what.
-    try:
-        run_unittest(CAutoFileTests, PyAutoFileTests,
-                     COtherFileTests, PyOtherFileTests)
-    finally:
-        if os.path.exists(TESTFN):
-            os.unlink(TESTFN)
+    if os.path.exists(TESTFN):
+        os.unlink(TESTFN)
 
 if __name__ == '__main__':
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py
index f312882b2a..1e70641150 100644
--- a/Lib/test/test_fileinput.py
+++ b/Lib/test/test_fileinput.py
@@ -2,18 +2,33 @@
 Tests for fileinput module.
 Nick Mathewson
 '''
-
+import os
+import sys
+import re
+import fileinput
+import collections
+import builtins
 import unittest
-from test.support import verbose, TESTFN, run_unittest
-from test.support import unlink as safe_unlink
-import sys, re
+
+try:
+    import bz2
+except ImportError:
+    bz2 = None
+try:
+    import gzip
+except ImportError:
+    gzip = None
+
 from io import StringIO
 from fileinput import FileInput, hook_encoded
 
+from test.support import verbose, TESTFN, run_unittest
+from test.support import unlink as safe_unlink
+
+
 # The fileinput module has 2 interfaces: the FileInput class which does
 # all the work, and a few functions (input, etc.) that use a global _state
-# variable.  We only test the FileInput class, since the other functions
-# only provide a thin facade over FileInput.
+# variable.
 
 # Write lines (a list of lines) to temp file number i, and return the
 # temp file's name.
@@ -121,7 +136,16 @@ class BufferSizesTests(unittest.TestCase):
             self.assertEqual(int(m.group(1)), fi.filelineno())
         fi.close()
 
+class UnconditionallyRaise:
+    def __init__(self, exception_type):
+        self.exception_type = exception_type
+        self.invoked = False
+    def __call__(self, *args, **kwargs):
+        self.invoked = True
+        raise self.exception_type()
+
 class FileInputTests(unittest.TestCase):
+
     def test_zero_byte_files(self):
         t1 = t2 = t3 = t4 = None
         try:
@@ -219,17 +243,20 @@ class FileInputTests(unittest.TestCase):
             self.fail("FileInput should check openhook for being callable")
         except ValueError:
             pass
-        # XXX The rot13 codec was removed.
-        #     So this test needs to be changed to use something else.
-        #     (Or perhaps the API needs to change so we can just pass
-        #     an encoding rather than using a hook?)
-##         try:
-##             t1 = writeTmp(1, ["A\nB"], mode="wb")
-##             fi = FileInput(files=t1, openhook=hook_encoded("rot13"))
-##             lines = list(fi)
-##             self.assertEqual(lines, ["N\n", "O"])
-##         finally:
-##             remove_tempfiles(t1)
+
+        class CustomOpenHook:
+            def __init__(self):
+                self.invoked = False
+            def __call__(self, *args):
+                self.invoked = True
+                return open(*args)
+
+        t = writeTmp(1, ["\n"])
+        self.addCleanup(remove_tempfiles, t)
+        custom_open_hook = CustomOpenHook()
+        with FileInput([t], openhook=custom_open_hook) as fi:
+            fi.readline()
+        self.assertTrue(custom_open_hook.invoked, "openhook not invoked")
 
     def test_context_manager(self):
         try:
@@ -254,9 +281,576 @@ class FileInputTests(unittest.TestCase):
         finally:
             remove_tempfiles(t1)
 
+    def test_empty_files_list_specified_to_constructor(self):
+        with FileInput(files=[]) as fi:
+            self.assertEqual(fi._files, ('-',))
+
+    def test__getitem__(self):
+        """Tests invoking FileInput.__getitem__() with the current
+           line number"""
+        t = writeTmp(1, ["line1\n", "line2\n"])
+        self.addCleanup(remove_tempfiles, t)
+        with FileInput(files=[t]) as fi:
+            retval1 = fi[0]
+            self.assertEqual(retval1, "line1\n")
+            retval2 = fi[1]
+            self.assertEqual(retval2, "line2\n")
+
+    def test__getitem__invalid_key(self):
+        """Tests invoking FileInput.__getitem__() with an index unequal to
+           the line number"""
+        t = writeTmp(1, ["line1\n", "line2\n"])
+        self.addCleanup(remove_tempfiles, t)
+        with FileInput(files=[t]) as fi:
+            with self.assertRaises(RuntimeError) as cm:
+                fi[1]
+        self.assertEqual(cm.exception.args, ("accessing lines out of order",))
+
+    def test__getitem__eof(self):
+        """Tests invoking FileInput.__getitem__() with the line number but at
+           end-of-input"""
+        t = writeTmp(1, [])
+        self.addCleanup(remove_tempfiles, t)
+        with FileInput(files=[t]) as fi:
+            with self.assertRaises(IndexError) as cm:
+                fi[0]
+        self.assertEqual(cm.exception.args, ("end of input reached",))
+
+    def test_nextfile_oserror_deleting_backup(self):
+        """Tests invoking FileInput.nextfile() when the attempt to delete
+           the backup file would raise OSError.  This error is expected to be
+           silently ignored"""
+
+        os_unlink_orig = os.unlink
+        os_unlink_replacement = UnconditionallyRaise(OSError)
+        try:
+            t = writeTmp(1, ["\n"])
+            self.addCleanup(remove_tempfiles, t)
+            with FileInput(files=[t], inplace=True) as fi:
+                next(fi) # make sure the file is opened
+                os.unlink = os_unlink_replacement
+                fi.nextfile()
+        finally:
+            os.unlink = os_unlink_orig
+
+        # sanity check to make sure that our test scenario was actually hit
+        self.assertTrue(os_unlink_replacement.invoked,
+                        "os.unlink() was not invoked")
+
+    def test_readline_os_fstat_raises_OSError(self):
+        """Tests invoking FileInput.readline() when os.fstat() raises OSError.
+           This exception should be silently discarded."""
+
+        os_fstat_orig = os.fstat
+        os_fstat_replacement = UnconditionallyRaise(OSError)
+        try:
+            t = writeTmp(1, ["\n"])
+            self.addCleanup(remove_tempfiles, t)
+            with FileInput(files=[t], inplace=True) as fi:
+                os.fstat = os_fstat_replacement
+                fi.readline()
+        finally:
+            os.fstat = os_fstat_orig
+
+        # sanity check to make sure that our test scenario was actually hit
+        self.assertTrue(os_fstat_replacement.invoked,
+                        "os.fstat() was not invoked")
+
+    @unittest.skipIf(not hasattr(os, "chmod"), "os.chmod does not exist")
+    def test_readline_os_chmod_raises_OSError(self):
+        """Tests invoking FileInput.readline() when os.chmod() raises OSError.
+           This exception should be silently discarded."""
+
+        os_chmod_orig = os.chmod
+        os_chmod_replacement = UnconditionallyRaise(OSError)
+        try:
+            t = writeTmp(1, ["\n"])
+            self.addCleanup(remove_tempfiles, t)
+            with FileInput(files=[t], inplace=True) as fi:
+                os.chmod = os_chmod_replacement
+                fi.readline()
+        finally:
+            os.chmod = os_chmod_orig
+
+        # sanity check to make sure that our test scenario was actually hit
+        self.assertTrue(os_chmod_replacement.invoked,
+                        "os.fstat() was not invoked")
+
+    def test_fileno_when_ValueError_raised(self):
+        class FilenoRaisesValueError(UnconditionallyRaise):
+            def __init__(self):
+                UnconditionallyRaise.__init__(self, ValueError)
+            def fileno(self):
+                self.__call__()
+
+        unconditionally_raise_ValueError = FilenoRaisesValueError()
+        t = writeTmp(1, ["\n"])
+        self.addCleanup(remove_tempfiles, t)
+        with FileInput(files=[t]) as fi:
+            file_backup = fi._file
+            try:
+                fi._file = unconditionally_raise_ValueError
+                result = fi.fileno()
+            finally:
+                fi._file = file_backup # make sure the file gets cleaned up
+
+        # sanity check to make sure that our test scenario was actually hit
+        self.assertTrue(unconditionally_raise_ValueError.invoked,
+                        "_file.fileno() was not invoked")
+
+        self.assertEqual(result, -1, "fileno() should return -1")
+
+class MockFileInput:
+    """A class that mocks out fileinput.FileInput for use during unit tests"""
+
+    def __init__(self, files=None, inplace=False, backup="", bufsize=0,
+                 mode="r", openhook=None):
+        self.files = files
+        self.inplace = inplace
+        self.backup = backup
+        self.bufsize = bufsize
+        self.mode = mode
+        self.openhook = openhook
+        self._file = None
+        self.invocation_counts = collections.defaultdict(lambda: 0)
+        self.return_values = {}
+
+    def close(self):
+        self.invocation_counts["close"] += 1
+
+    def nextfile(self):
+        self.invocation_counts["nextfile"] += 1
+        return self.return_values["nextfile"]
+
+    def filename(self):
+        self.invocation_counts["filename"] += 1
+        return self.return_values["filename"]
+
+    def lineno(self):
+        self.invocation_counts["lineno"] += 1
+        return self.return_values["lineno"]
+
+    def filelineno(self):
+        self.invocation_counts["filelineno"] += 1
+        return self.return_values["filelineno"]
+
+    def fileno(self):
+        self.invocation_counts["fileno"] += 1
+        return self.return_values["fileno"]
+
+    def isfirstline(self):
+        self.invocation_counts["isfirstline"] += 1
+        return self.return_values["isfirstline"]
+
+    def isstdin(self):
+        self.invocation_counts["isstdin"] += 1
+        return self.return_values["isstdin"]
+
+class BaseFileInputGlobalMethodsTest(unittest.TestCase):
+    """Base class for unit tests for the global function of
+       the fileinput module."""
+
+    def setUp(self):
+        self._orig_state = fileinput._state
+        self._orig_FileInput = fileinput.FileInput
+        fileinput.FileInput = MockFileInput
+
+    def tearDown(self):
+        fileinput.FileInput = self._orig_FileInput
+        fileinput._state = self._orig_state
+
+    def assertExactlyOneInvocation(self, mock_file_input, method_name):
+        # assert that the method with the given name was invoked once
+        actual_count = mock_file_input.invocation_counts[method_name]
+        self.assertEqual(actual_count, 1, method_name)
+        # assert that no other unexpected methods were invoked
+        actual_total_count = len(mock_file_input.invocation_counts)
+        self.assertEqual(actual_total_count, 1)
+
+class Test_fileinput_input(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.input()"""
+
+    def test_state_is_not_None_and_state_file_is_not_None(self):
+        """Tests invoking fileinput.input() when fileinput._state is not None
+           and its _file attribute is also not None.  Expect RuntimeError to
+           be raised with a meaningful error message and for fileinput._state
+           to *not* be modified."""
+        instance = MockFileInput()
+        instance._file = object()
+        fileinput._state = instance
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.input()
+        self.assertEqual(("input() already active",), cm.exception.args)
+        self.assertIs(instance, fileinput._state, "fileinput._state")
+
+    def test_state_is_not_None_and_state_file_is_None(self):
+        """Tests invoking fileinput.input() when fileinput._state is not None
+           but its _file attribute *is* None.  Expect it to create and return
+           a new fileinput.FileInput object with all method parameters passed
+           explicitly to the __init__() method; also ensure that
+           fileinput._state is set to the returned instance."""
+        instance = MockFileInput()
+        instance._file = None
+        fileinput._state = instance
+        self.do_test_call_input()
+
+    def test_state_is_None(self):
+        """Tests invoking fileinput.input() when fileinput._state is None
+           Expect it to create and return a new fileinput.FileInput object
+           with all method parameters passed explicitly to the __init__()
+           method; also ensure that fileinput._state is set to the returned
+           instance."""
+        fileinput._state = None
+        self.do_test_call_input()
+
+    def do_test_call_input(self):
+        """Tests that fileinput.input() creates a new fileinput.FileInput
+           object, passing the given parameters unmodified to
+           fileinput.FileInput.__init__().  Note that this test depends on the
+           monkey patching of fileinput.FileInput done by setUp()."""
+        files = object()
+        inplace = object()
+        backup = object()
+        bufsize = object()
+        mode = object()
+        openhook = object()
+
+        # call fileinput.input() with different values for each argument
+        result = fileinput.input(files=files, inplace=inplace, backup=backup,
+                                 bufsize=bufsize,
+            mode=mode, openhook=openhook)
+
+        # ensure fileinput._state was set to the returned object
+        self.assertIs(result, fileinput._state, "fileinput._state")
+
+        # ensure the parameters to fileinput.input() were passed directly
+        # to FileInput.__init__()
+        self.assertIs(files, result.files, "files")
+        self.assertIs(inplace, result.inplace, "inplace")
+        self.assertIs(backup, result.backup, "backup")
+        self.assertIs(bufsize, result.bufsize, "bufsize")
+        self.assertIs(mode, result.mode, "mode")
+        self.assertIs(openhook, result.openhook, "openhook")
+
+class Test_fileinput_close(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.close()"""
+
+    def test_state_is_None(self):
+        """Tests that fileinput.close() does nothing if fileinput._state
+           is None"""
+        fileinput._state = None
+        fileinput.close()
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests that fileinput.close() invokes close() on fileinput._state
+           and sets _state=None"""
+        instance = MockFileInput()
+        fileinput._state = instance
+        fileinput.close()
+        self.assertExactlyOneInvocation(instance, "close")
+        self.assertIsNone(fileinput._state)
+
+class Test_fileinput_nextfile(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.nextfile()"""
+
+    def test_state_is_None(self):
+        """Tests fileinput.nextfile() when fileinput._state is None.
+           Ensure that it raises RuntimeError with a meaningful error message
+           and does not modify fileinput._state"""
+        fileinput._state = None
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.nextfile()
+        self.assertEqual(("no active input()",), cm.exception.args)
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests fileinput.nextfile() when fileinput._state is not None.
+           Ensure that it invokes fileinput._state.nextfile() exactly once,
+           returns whatever it returns, and does not modify fileinput._state
+           to point to a different object."""
+        nextfile_retval = object()
+        instance = MockFileInput()
+        instance.return_values["nextfile"] = nextfile_retval
+        fileinput._state = instance
+        retval = fileinput.nextfile()
+        self.assertExactlyOneInvocation(instance, "nextfile")
+        self.assertIs(retval, nextfile_retval)
+        self.assertIs(fileinput._state, instance)
+
+class Test_fileinput_filename(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.filename()"""
+
+    def test_state_is_None(self):
+        """Tests fileinput.filename() when fileinput._state is None.
+           Ensure that it raises RuntimeError with a meaningful error message
+           and does not modify fileinput._state"""
+        fileinput._state = None
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.filename()
+        self.assertEqual(("no active input()",), cm.exception.args)
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests fileinput.filename() when fileinput._state is not None.
+           Ensure that it invokes fileinput._state.filename() exactly once,
+           returns whatever it returns, and does not modify fileinput._state
+           to point to a different object."""
+        filename_retval = object()
+        instance = MockFileInput()
+        instance.return_values["filename"] = filename_retval
+        fileinput._state = instance
+        retval = fileinput.filename()
+        self.assertExactlyOneInvocation(instance, "filename")
+        self.assertIs(retval, filename_retval)
+        self.assertIs(fileinput._state, instance)
+
+class Test_fileinput_lineno(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.lineno()"""
+
+    def test_state_is_None(self):
+        """Tests fileinput.lineno() when fileinput._state is None.
+           Ensure that it raises RuntimeError with a meaningful error message
+           and does not modify fileinput._state"""
+        fileinput._state = None
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.lineno()
+        self.assertEqual(("no active input()",), cm.exception.args)
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests fileinput.lineno() when fileinput._state is not None.
+           Ensure that it invokes fileinput._state.lineno() exactly once,
+           returns whatever it returns, and does not modify fileinput._state
+           to point to a different object."""
+        lineno_retval = object()
+        instance = MockFileInput()
+        instance.return_values["lineno"] = lineno_retval
+        fileinput._state = instance
+        retval = fileinput.lineno()
+        self.assertExactlyOneInvocation(instance, "lineno")
+        self.assertIs(retval, lineno_retval)
+        self.assertIs(fileinput._state, instance)
+
+class Test_fileinput_filelineno(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.filelineno()"""
+
+    def test_state_is_None(self):
+        """Tests fileinput.filelineno() when fileinput._state is None.
+           Ensure that it raises RuntimeError with a meaningful error message
+           and does not modify fileinput._state"""
+        fileinput._state = None
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.filelineno()
+        self.assertEqual(("no active input()",), cm.exception.args)
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests fileinput.filelineno() when fileinput._state is not None.
+           Ensure that it invokes fileinput._state.filelineno() exactly once,
+           returns whatever it returns, and does not modify fileinput._state
+           to point to a different object."""
+        filelineno_retval = object()
+        instance = MockFileInput()
+        instance.return_values["filelineno"] = filelineno_retval
+        fileinput._state = instance
+        retval = fileinput.filelineno()
+        self.assertExactlyOneInvocation(instance, "filelineno")
+        self.assertIs(retval, filelineno_retval)
+        self.assertIs(fileinput._state, instance)
+
+class Test_fileinput_fileno(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.fileno()"""
+
+    def test_state_is_None(self):
+        """Tests fileinput.fileno() when fileinput._state is None.
+           Ensure that it raises RuntimeError with a meaningful error message
+           and does not modify fileinput._state"""
+        fileinput._state = None
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.fileno()
+        self.assertEqual(("no active input()",), cm.exception.args)
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests fileinput.fileno() when fileinput._state is not None.
+           Ensure that it invokes fileinput._state.fileno() exactly once,
+           returns whatever it returns, and does not modify fileinput._state
+           to point to a different object."""
+        fileno_retval = object()
+        instance = MockFileInput()
+        instance.return_values["fileno"] = fileno_retval
+        instance.fileno_retval = fileno_retval
+        fileinput._state = instance
+        retval = fileinput.fileno()
+        self.assertExactlyOneInvocation(instance, "fileno")
+        self.assertIs(retval, fileno_retval)
+        self.assertIs(fileinput._state, instance)
+
+class Test_fileinput_isfirstline(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.isfirstline()"""
+
+    def test_state_is_None(self):
+        """Tests fileinput.isfirstline() when fileinput._state is None.
+           Ensure that it raises RuntimeError with a meaningful error message
+           and does not modify fileinput._state"""
+        fileinput._state = None
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.isfirstline()
+        self.assertEqual(("no active input()",), cm.exception.args)
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests fileinput.isfirstline() when fileinput._state is not None.
+           Ensure that it invokes fileinput._state.isfirstline() exactly once,
+           returns whatever it returns, and does not modify fileinput._state
+           to point to a different object."""
+        isfirstline_retval = object()
+        instance = MockFileInput()
+        instance.return_values["isfirstline"] = isfirstline_retval
+        fileinput._state = instance
+        retval = fileinput.isfirstline()
+        self.assertExactlyOneInvocation(instance, "isfirstline")
+        self.assertIs(retval, isfirstline_retval)
+        self.assertIs(fileinput._state, instance)
+
+class Test_fileinput_isstdin(BaseFileInputGlobalMethodsTest):
+    """Unit tests for fileinput.isstdin()"""
+
+    def test_state_is_None(self):
+        """Tests fileinput.isstdin() when fileinput._state is None.
+           Ensure that it raises RuntimeError with a meaningful error message
+           and does not modify fileinput._state"""
+        fileinput._state = None
+        with self.assertRaises(RuntimeError) as cm:
+            fileinput.isstdin()
+        self.assertEqual(("no active input()",), cm.exception.args)
+        self.assertIsNone(fileinput._state)
+
+    def test_state_is_not_None(self):
+        """Tests fileinput.isstdin() when fileinput._state is not None.
+           Ensure that it invokes fileinput._state.isstdin() exactly once,
+           returns whatever it returns, and does not modify fileinput._state
+           to point to a different object."""
+        isstdin_retval = object()
+        instance = MockFileInput()
+        instance.return_values["isstdin"] = isstdin_retval
+        fileinput._state = instance
+        retval = fileinput.isstdin()
+        self.assertExactlyOneInvocation(instance, "isstdin")
+        self.assertIs(retval, isstdin_retval)
+        self.assertIs(fileinput._state, instance)
+
+class InvocationRecorder:
+    def __init__(self):
+        self.invocation_count = 0
+    def __call__(self, *args, **kwargs):
+        self.invocation_count += 1
+        self.last_invocation = (args, kwargs)
+
+class Test_hook_compressed(unittest.TestCase):
+    """Unit tests for fileinput.hook_compressed()"""
+
+    def setUp(self):
+        self.fake_open = InvocationRecorder()
+
+    def test_empty_string(self):
+        self.do_test_use_builtin_open("", 1)
+
+    def test_no_ext(self):
+        self.do_test_use_builtin_open("abcd", 2)
+
+    @unittest.skipUnless(gzip, "Requires gzip and zlib")
+    def test_gz_ext_fake(self):
+        original_open = gzip.open
+        gzip.open = self.fake_open
+        try:
+            result = fileinput.hook_compressed("test.gz", 3)
+        finally:
+            gzip.open = original_open
+
+        self.assertEqual(self.fake_open.invocation_count, 1)
+        self.assertEqual(self.fake_open.last_invocation, (("test.gz", 3), {}))
+
+    @unittest.skipUnless(bz2, "Requires bz2")
+    def test_bz2_ext_fake(self):
+        original_open = bz2.BZ2File
+        bz2.BZ2File = self.fake_open
+        try:
+            result = fileinput.hook_compressed("test.bz2", 4)
+        finally:
+            bz2.BZ2File = original_open
+
+        self.assertEqual(self.fake_open.invocation_count, 1)
+        self.assertEqual(self.fake_open.last_invocation, (("test.bz2", 4), {}))
+
+    def test_blah_ext(self):
+        self.do_test_use_builtin_open("abcd.blah", 5)
+
+    def test_gz_ext_builtin(self):
+        self.do_test_use_builtin_open("abcd.Gz", 6)
+
+    def test_bz2_ext_builtin(self):
+        self.do_test_use_builtin_open("abcd.Bz2", 7)
+
+    def do_test_use_builtin_open(self, filename, mode):
+        original_open = self.replace_builtin_open(self.fake_open)
+        try:
+            result = fileinput.hook_compressed(filename, mode)
+        finally:
+            self.replace_builtin_open(original_open)
+
+        self.assertEqual(self.fake_open.invocation_count, 1)
+        self.assertEqual(self.fake_open.last_invocation,
+                         ((filename, mode), {}))
+
+    @staticmethod
+    def replace_builtin_open(new_open_func):
+        original_open = builtins.open
+        builtins.open = new_open_func
+        return original_open
+
+class Test_hook_encoded(unittest.TestCase):
+    """Unit tests for fileinput.hook_encoded()"""
+
+    def test(self):
+        encoding = object()
+        result = fileinput.hook_encoded(encoding)
+
+        fake_open = InvocationRecorder()
+        original_open = builtins.open
+        builtins.open = fake_open
+        try:
+            filename = object()
+            mode = object()
+            open_result = result(filename, mode)
+        finally:
+            builtins.open = original_open
+
+        self.assertEqual(fake_open.invocation_count, 1)
+
+        args, kwargs = fake_open.last_invocation
+        self.assertIs(args[0], filename)
+        self.assertIs(args[1], mode)
+        self.assertIs(kwargs.pop('encoding'), encoding)
+        self.assertFalse(kwargs)
 
 def test_main():
-    run_unittest(BufferSizesTests, FileInputTests)
+    run_unittest(
+        BufferSizesTests,
+        FileInputTests,
+        Test_fileinput_input,
+        Test_fileinput_close,
+        Test_fileinput_nextfile,
+        Test_fileinput_filename,
+        Test_fileinput_lineno,
+        Test_fileinput_filelineno,
+        Test_fileinput_fileno,
+        Test_fileinput_isfirstline,
+        Test_fileinput_isstdin,
+        Test_hook_compressed,
+        Test_hook_encoded,
+    )
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py
index 8267d9eb99..19737d9775 100644
--- a/Lib/test/test_fileio.py
+++ b/Lib/test/test_fileio.py
@@ -2,6 +2,7 @@
 
 import sys
 import os
+import io
 import errno
 import unittest
 from array import array
@@ -373,10 +374,10 @@ class OtherFileTests(unittest.TestCase):
         self.assertEqual(f.tell(), 10)
         f.truncate(5)
         self.assertEqual(f.tell(), 10)
-        self.assertEqual(f.seek(0, os.SEEK_END), 5)
+        self.assertEqual(f.seek(0, io.SEEK_END), 5)
         f.truncate(15)
         self.assertEqual(f.tell(), 5)
-        self.assertEqual(f.seek(0, os.SEEK_END), 15)
+        self.assertEqual(f.seek(0, io.SEEK_END), 15)
         f.close()
 
     def testTruncateOnWindows(self):
diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py
index 4d7bbbae17..502292f615 100644
--- a/Lib/test/test_float.py
+++ b/Lib/test/test_float.py
@@ -88,7 +88,7 @@ class GeneralFloatCases(unittest.TestCase):
         self.assertRaises(ValueError, float, "  -0x3.p-1  ")
         self.assertRaises(ValueError, float, "  +0x3.p-1  ")
         self.assertEqual(float("  25.e-1  "), 2.5)
-        self.assertEqual(support.fcmp(float("  .25e-1  "), .025), 0)
+        self.assertAlmostEqual(float("  .25e-1  "), .025)
 
     def test_floatconversion(self):
         # Make sure that calls to __float__() work properly
@@ -860,15 +860,18 @@ class InfNanTest(unittest.TestCase):
         self.assertEqual(str(1e300 * 1e300 * 0), "nan")
         self.assertEqual(str(-1e300 * 1e300 * 0), "nan")
 
-    def notest_float_nan(self):
-        self.assertTrue(NAN.is_nan())
-        self.assertFalse(INF.is_nan())
-        self.assertFalse((0.).is_nan())
+    def test_inf_signs(self):
+        self.assertEqual(copysign(1.0, float('inf')), 1.0)
+        self.assertEqual(copysign(1.0, float('-inf')), -1.0)
+
+    @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short',
+                         "applies only when using short float repr style")
+    def test_nan_signs(self):
+        # When using the dtoa.c code, the sign of float('nan') should
+        # be predictable.
+        self.assertEqual(copysign(1.0, float('nan')), 1.0)
+        self.assertEqual(copysign(1.0, float('-nan')), -1.0)
 
-    def notest_float_inf(self):
-        self.assertTrue(INF.is_inf())
-        self.assertFalse(NAN.is_inf())
-        self.assertFalse((0.).is_inf())
 
 fromHex = float.fromhex
 toHex = float.hex
diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py
index 44f1bb72ec..f8f5420040 100644
--- a/Lib/test/test_format.py
+++ b/Lib/test/test_format.py
@@ -1,4 +1,5 @@
 from test.support import verbose, TestFailed
+import locale
 import sys
 import test.support as support
 import unittest
@@ -263,6 +264,51 @@ class FormatTest(unittest.TestCase):
             else:
                 raise TestFailed('"%*d"%(maxsize, -127) should fail')
 
+    def test_non_ascii(self):
+        testformat("\u20ac=%f", (1.0,), "\u20ac=1.000000")
+
+        self.assertEqual(format("abc", "\u2007<5"), "abc\u2007\u2007")
+        self.assertEqual(format(123, "\u2007<5"), "123\u2007\u2007")
+        self.assertEqual(format(12.3, "\u2007<6"), "12.3\u2007\u2007")
+        self.assertEqual(format(0j, "\u2007<4"), "0j\u2007\u2007")
+        self.assertEqual(format(1+2j, "\u2007<8"), "(1+2j)\u2007\u2007")
+
+        self.assertEqual(format("abc", "\u2007>5"), "\u2007\u2007abc")
+        self.assertEqual(format(123, "\u2007>5"), "\u2007\u2007123")
+        self.assertEqual(format(12.3, "\u2007>6"), "\u2007\u200712.3")
+        self.assertEqual(format(1+2j, "\u2007>8"), "\u2007\u2007(1+2j)")
+        self.assertEqual(format(0j, "\u2007>4"), "\u2007\u20070j")
+
+        self.assertEqual(format("abc", "\u2007^5"), "\u2007abc\u2007")
+        self.assertEqual(format(123, "\u2007^5"), "\u2007123\u2007")
+        self.assertEqual(format(12.3, "\u2007^6"), "\u200712.3\u2007")
+        self.assertEqual(format(1+2j, "\u2007^8"), "\u2007(1+2j)\u2007")
+        self.assertEqual(format(0j, "\u2007^4"), "\u20070j\u2007")
+
+    def test_locale(self):
+        try:
+            oldloc = locale.setlocale(locale.LC_ALL)
+            locale.setlocale(locale.LC_ALL, '')
+        except locale.Error as err:
+            self.skipTest("Cannot set locale: {}".format(err))
+        try:
+            localeconv = locale.localeconv()
+            sep = localeconv['thousands_sep']
+            point = localeconv['decimal_point']
+
+            text = format(123456789, "n")
+            self.assertIn(sep, text)
+            self.assertEqual(text.replace(sep, ''), '123456789')
+
+            text = format(1234.5, "n")
+            self.assertIn(sep, text)
+            self.assertIn(point, text)
+            self.assertEqual(text.replace(sep, ''), '1234' + point + '5')
+        finally:
+            locale.setlocale(locale.LC_ALL, oldloc)
+
+
+
 def test_main():
     support.run_unittest(FormatTest)
 
diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py
index 084ae0cce5..1fad9215c0 100644
--- a/Lib/test/test_fractions.py
+++ b/Lib/test/test_fractions.py
@@ -401,14 +401,10 @@ class FractionTest(unittest.TestCase):
 
     def testMixingWithDecimal(self):
         # Decimal refuses mixed arithmetic (but not mixed comparisons)
-        self.assertRaisesMessage(
-            TypeError,
-            "unsupported operand type(s) for +: 'Fraction' and 'Decimal'",
-            operator.add, F(3,11), Decimal('3.1415926'))
-        self.assertRaisesMessage(
-            TypeError,
-            "unsupported operand type(s) for +: 'Decimal' and 'Fraction'",
-            operator.add, Decimal('3.1415926'), F(3,11))
+        self.assertRaises(TypeError, operator.add,
+                          F(3,11), Decimal('3.1415926'))
+        self.assertRaises(TypeError, operator.add,
+                          Decimal('3.1415926'), F(3,11))
 
     def testComparisons(self):
         self.assertTrue(F(1, 2) < F(2, 3))
diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py
index 5243ebb165..fd6761c651 100644
--- a/Lib/test/test_frozen.py
+++ b/Lib/test/test_frozen.py
@@ -5,6 +5,12 @@ import unittest
 import sys
 
 class FrozenTests(unittest.TestCase):
+
+    module_attrs = frozenset(['__builtins__', '__cached__', '__doc__',
+                              '__loader__', '__name__',
+                              '__package__'])
+    package_attrs = frozenset(list(module_attrs) + ['__path__'])
+
     def test_frozen(self):
         with captured_stdout() as stdout:
             try:
@@ -12,7 +18,9 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __hello__ failed:" + str(x))
             self.assertEqual(__hello__.initialized, True)
-            self.assertEqual(len(dir(__hello__)), 7, dir(__hello__))
+            expect = set(self.module_attrs)
+            expect.add('initialized')
+            self.assertEqual(set(dir(__hello__)), expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         with captured_stdout() as stdout:
@@ -21,10 +29,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__ failed:" + str(x))
             self.assertEqual(__phello__.initialized, True)
+            expect = set(self.package_attrs)
+            expect.add('initialized')
             if not "__phello__.spam" in sys.modules:
-                self.assertEqual(len(dir(__phello__)), 8, dir(__phello__))
+                self.assertEqual(set(dir(__phello__)), expect)
             else:
-                self.assertEqual(len(dir(__phello__)), 9, dir(__phello__))
+                expect.add('spam')
+                self.assertEqual(set(dir(__phello__)), expect)
             self.assertEqual(__phello__.__path__, [__phello__.__name__])
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
@@ -34,8 +45,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__.spam failed:" + str(x))
             self.assertEqual(__phello__.spam.initialized, True)
-            self.assertEqual(len(dir(__phello__.spam)), 7)
-            self.assertEqual(len(dir(__phello__)), 9)
+            spam_expect = set(self.module_attrs)
+            spam_expect.add('initialized')
+            self.assertEqual(set(dir(__phello__.spam)), spam_expect)
+            phello_expect = set(self.package_attrs)
+            phello_expect.add('initialized')
+            phello_expect.add('spam')
+            self.assertEqual(set(dir(__phello__)), phello_expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         try:
diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
index 71bc23e615..824b7c123b 100644
--- a/Lib/test/test_ftplib.py
+++ b/Lib/test/test_ftplib.py
@@ -22,10 +22,25 @@ from test.support import HOST
 threading = support.import_module('threading')
 
 # the dummy data returned by server over the data channel when
-# RETR, LIST and NLST commands are issued
+# RETR, LIST, NLST, MLSD commands are issued
 RETR_DATA = 'abcde12345\r\n' * 1000
 LIST_DATA = 'foo\r\nbar\r\n'
 NLST_DATA = 'foo\r\nbar\r\n'
+MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n"
+             "type=pdir;perm=e;unique==keVO1+d?3; ..\r\n"
+             "type=OS.unix=slink:/foobar;perm=;unique==keVO1+4G4; foobar\r\n"
+             "type=OS.unix=chr-13/29;perm=;unique==keVO1+5G4; device\r\n"
+             "type=OS.unix=blk-11/108;perm=;unique==keVO1+6G4; block\r\n"
+             "type=file;perm=awr;unique==keVO1+8G4; writable\r\n"
+             "type=dir;perm=cpmel;unique==keVO1+7G4; promiscuous\r\n"
+             "type=dir;perm=;unique==keVO1+1t2; no-exec\r\n"
+             "type=file;perm=r;unique==keVO1+EG4; two words\r\n"
+             "type=file;perm=r;unique==keVO1+IH4;  leading space\r\n"
+             "type=file;perm=r;unique==keVO1+1G4; file1\r\n"
+             "type=dir;perm=cpmel;unique==keVO1+7G4; incoming\r\n"
+             "type=file;perm=r;unique==keVO1+1G4; file2\r\n"
+             "type=file;perm=r;unique==keVO1+1G4; file3\r\n"
+             "type=file;perm=r;unique==keVO1+1G4; file4\r\n")
 
 
 class DummyDTPHandler(asynchat.async_chat):
@@ -49,6 +64,11 @@ class DummyDTPHandler(asynchat.async_chat):
             self.dtp_conn_closed = True
 
     def push(self, what):
+        if self.baseclass.next_data is not None:
+            what = self.baseclass.next_data
+            self.baseclass.next_data = None
+        if not what:
+            return self.close_when_done()
         super(DummyDTPHandler, self).push(what.encode('ascii'))
 
     def handle_error(self):
@@ -69,6 +89,7 @@ class DummyFTPHandler(asynchat.async_chat):
         self.last_received_cmd = None
         self.last_received_data = ''
         self.next_response = ''
+        self.next_data = None
         self.rest = None
         self.push('220 welcome')
 
@@ -104,7 +125,7 @@ class DummyFTPHandler(asynchat.async_chat):
         addr = list(map(int, arg.split(',')))
         ip = '%d.%d.%d.%d' %tuple(addr[:4])
         port = (addr[4] * 256) + addr[5]
-        s = socket.create_connection((ip, port), timeout=10)
+        s = socket.create_connection((ip, port), timeout=2)
         self.dtp = self.dtp_handler(s, baseclass=self)
         self.push('200 active data connection established')
 
@@ -122,7 +143,7 @@ class DummyFTPHandler(asynchat.async_chat):
     def cmd_eprt(self, arg):
         af, ip, port = arg.split(arg[0])[1:-1]
         port = int(port)
-        s = socket.create_connection((ip, port), timeout=10)
+        s = socket.create_connection((ip, port), timeout=2)
         self.dtp = self.dtp_handler(s, baseclass=self)
         self.push('200 active data connection established')
 
@@ -213,6 +234,14 @@ class DummyFTPHandler(asynchat.async_chat):
         self.dtp.push(NLST_DATA)
         self.dtp.close_when_done()
 
+    def cmd_opts(self, arg):
+        self.push('200 opts ok')
+
+    def cmd_mlsd(self, arg):
+        self.push('125 mlsd ok')
+        self.dtp.push(MLSD_DATA)
+        self.dtp.close_when_done()
+
 
 class DummyFTPServer(asyncore.dispatcher, threading.Thread):
 
@@ -274,11 +303,11 @@ if ssl is not None:
         _ssl_closing = False
 
         def secure_connection(self):
-            self.del_channel()
             socket = ssl.wrap_socket(self.socket, suppress_ragged_eofs=False,
                                      certfile=CERTFILE, server_side=True,
                                      do_handshake_on_connect=False,
                                      ssl_version=ssl.PROTOCOL_SSLv23)
+            self.del_channel()
             self.set_socket(socket)
             self._ssl_accepting = True
 
@@ -313,7 +342,10 @@ if ssl is not None:
                 # http://www.mail-archive.com/openssl-users@openssl.org/msg60710.html
                 pass
             self._ssl_closing = False
-            super(SSLConnection, self).close()
+            if getattr(self, '_ccc', False) is False:
+                super(SSLConnection, self).close()
+            else:
+                pass
 
         def handle_read_event(self):
             if self._ssl_accepting:
@@ -381,12 +413,18 @@ if ssl is not None:
         def __init__(self, conn):
             DummyFTPHandler.__init__(self, conn)
             self.secure_data_channel = False
+            self._ccc = False
 
         def cmd_auth(self, line):
             """Set up secure control channel."""
             self.push('234 AUTH TLS successful')
             self.secure_connection()
 
+        def cmd_ccc(self, line):
+            self.push('220 Reverting back to clear-text')
+            self._ccc = True
+            self._do_ssl_shutdown()
+
         def cmd_pbsz(self, line):
             """Negotiate size of buffer for secure data transfer.
             For TLS/SSL the only valid value for the parameter is '0'.
@@ -416,13 +454,17 @@ class TestFTPClass(TestCase):
     def setUp(self):
         self.server = DummyFTPServer((HOST, 0))
         self.server.start()
-        self.client = ftplib.FTP(timeout=10)
+        self.client = ftplib.FTP(timeout=2)
         self.client.connect(self.server.host, self.server.port)
 
     def tearDown(self):
         self.client.close()
         self.server.stop()
 
+    def check_data(self, received, expected):
+        self.assertEqual(len(received), len(expected))
+        self.assertEqual(received, expected)
+
     def test_getwelcome(self):
         self.assertEqual(self.client.getwelcome(), '220 welcome')
 
@@ -504,7 +546,7 @@ class TestFTPClass(TestCase):
             received.append(data.decode('ascii'))
         received = []
         self.client.retrbinary('retr', callback)
-        self.assertEqual(''.join(received), RETR_DATA)
+        self.check_data(''.join(received), RETR_DATA)
 
     def test_retrbinary_rest(self):
         def callback(data):
@@ -512,20 +554,17 @@ class TestFTPClass(TestCase):
         for rest in (0, 10, 20):
             received = []
             self.client.retrbinary('retr', callback, rest=rest)
-            self.assertEqual(''.join(received), RETR_DATA[rest:],
-                             msg='rest test case %d %d %d' % (rest,
-                                                              len(''.join(received)),
-                                                              len(RETR_DATA[rest:])))
+            self.check_data(''.join(received), RETR_DATA[rest:])
 
     def test_retrlines(self):
         received = []
         self.client.retrlines('retr', received.append)
-        self.assertEqual(''.join(received), RETR_DATA.replace('\r\n', ''))
+        self.check_data(''.join(received), RETR_DATA.replace('\r\n', ''))
 
     def test_storbinary(self):
         f = io.BytesIO(RETR_DATA.encode('ascii'))
         self.client.storbinary('stor', f)
-        self.assertEqual(self.server.handler_instance.last_received_data, RETR_DATA)
+        self.check_data(self.server.handler_instance.last_received_data, RETR_DATA)
         # test new callback arg
         flag = []
         f.seek(0)
@@ -542,7 +581,7 @@ class TestFTPClass(TestCase):
     def test_storlines(self):
         f = io.BytesIO(RETR_DATA.replace('\r\n', '\n').encode('ascii'))
         self.client.storlines('stor', f)
-        self.assertEqual(self.server.handler_instance.last_received_data, RETR_DATA)
+        self.check_data(self.server.handler_instance.last_received_data, RETR_DATA)
         # test new callback arg
         flag = []
         f.seek(0)
@@ -558,6 +597,64 @@ class TestFTPClass(TestCase):
         self.client.dir(lambda x: l.append(x))
         self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', ''))
 
+    def test_mlsd(self):
+        list(self.client.mlsd())
+        list(self.client.mlsd(path='/'))
+        list(self.client.mlsd(path='/', facts=['size', 'type']))
+
+        ls = list(self.client.mlsd())
+        for name, facts in ls:
+            self.assertIsInstance(name, str)
+            self.assertIsInstance(facts, dict)
+            self.assertTrue(name)
+            self.assertIn('type', facts)
+            self.assertIn('perm', facts)
+            self.assertIn('unique', facts)
+
+        def set_data(data):
+            self.server.handler_instance.next_data = data
+
+        def test_entry(line, type=None, perm=None, unique=None, name=None):
+            type = 'type' if type is None else type
+            perm = 'perm' if perm is None else perm
+            unique = 'unique' if unique is None else unique
+            name = 'name' if name is None else name
+            set_data(line)
+            _name, facts = next(self.client.mlsd())
+            self.assertEqual(_name, name)
+            self.assertEqual(facts['type'], type)
+            self.assertEqual(facts['perm'], perm)
+            self.assertEqual(facts['unique'], unique)
+
+        # plain
+        test_entry('type=type;perm=perm;unique=unique; name\r\n')
+        # "=" in fact value
+        test_entry('type=ty=pe;perm=perm;unique=unique; name\r\n', type="ty=pe")
+        test_entry('type==type;perm=perm;unique=unique; name\r\n', type="=type")
+        test_entry('type=t=y=pe;perm=perm;unique=unique; name\r\n', type="t=y=pe")
+        test_entry('type=====;perm=perm;unique=unique; name\r\n', type="====")
+        # spaces in name
+        test_entry('type=type;perm=perm;unique=unique; na me\r\n', name="na me")
+        test_entry('type=type;perm=perm;unique=unique; name \r\n', name="name ")
+        test_entry('type=type;perm=perm;unique=unique;  name\r\n', name=" name")
+        test_entry('type=type;perm=perm;unique=unique; n am  e\r\n', name="n am  e")
+        # ";" in name
+        test_entry('type=type;perm=perm;unique=unique; na;me\r\n', name="na;me")
+        test_entry('type=type;perm=perm;unique=unique; ;name\r\n', name=";name")
+        test_entry('type=type;perm=perm;unique=unique; ;name;\r\n', name=";name;")
+        test_entry('type=type;perm=perm;unique=unique; ;;;;\r\n', name=";;;;")
+        # case sensitiveness
+        set_data('Type=type;TyPe=perm;UNIQUE=unique; name\r\n')
+        _name, facts = next(self.client.mlsd())
+        for x in facts:
+            self.assertTrue(x.islower())
+        # no data (directory empty)
+        set_data('')
+        self.assertRaises(StopIteration, next, self.client.mlsd())
+        set_data('')
+        for x in self.client.mlsd():
+            self.fail("unexpected data %s" % data)
+
     def test_makeport(self):
         with self.client.makeport():
             # IPv4 is in use, just make sure send_eprt has not been used
@@ -584,7 +681,7 @@ class TestFTPClass(TestCase):
             return True
 
         # base test
-        with ftplib.FTP(timeout=10) as self.client:
+        with ftplib.FTP(timeout=2) as self.client:
             self.client.connect(self.server.host, self.server.port)
             self.client.sendcmd('noop')
             self.assertTrue(is_client_connected())
@@ -592,7 +689,7 @@ class TestFTPClass(TestCase):
         self.assertFalse(is_client_connected())
 
         # QUIT sent inside the with block
-        with ftplib.FTP(timeout=10) as self.client:
+        with ftplib.FTP(timeout=2) as self.client:
             self.client.connect(self.server.host, self.server.port)
             self.client.sendcmd('noop')
             self.client.quit()
@@ -602,7 +699,7 @@ class TestFTPClass(TestCase):
         # force a wrong response code to be sent on QUIT: error_perm
         # is expected and the connection is supposed to be closed
         try:
-            with ftplib.FTP(timeout=10) as self.client:
+            with ftplib.FTP(timeout=2) as self.client:
                 self.client.connect(self.server.host, self.server.port)
                 self.client.sendcmd('noop')
                 self.server.handler_instance.next_response = '550 error on quit'
@@ -616,6 +713,30 @@ class TestFTPClass(TestCase):
         self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit')
         self.assertFalse(is_client_connected())
 
+    def test_source_address(self):
+        self.client.quit()
+        port = support.find_unused_port()
+        try:
+            self.client.connect(self.server.host, self.server.port,
+                                source_address=(HOST, port))
+            self.assertEqual(self.client.sock.getsockname()[1], port)
+            self.client.quit()
+        except IOError as e:
+            if e.errno == errno.EADDRINUSE:
+                self.skipTest("couldn't bind to port %d" % port)
+            raise
+
+    def test_source_address_passive_connection(self):
+        port = support.find_unused_port()
+        self.client.source_address = (HOST, port)
+        try:
+            with self.client.transfercmd('list') as sock:
+                self.assertEqual(sock.getsockname()[1], port)
+        except IOError as e:
+            if e.errno == errno.EADDRINUSE:
+                self.skipTest("couldn't bind to port %d" % port)
+            raise
+
     def test_parse257(self):
         self.assertEqual(ftplib.parse257('257 "/foo/bar"'), '/foo/bar')
         self.assertEqual(ftplib.parse257('257 "/foo/bar" created'), '/foo/bar')
@@ -632,7 +753,7 @@ class TestFTPClass(TestCase):
 class TestIPv6Environment(TestCase):
 
     def setUp(self):
-        self.server = DummyFTPServer((HOST, 0), af=socket.AF_INET6)
+        self.server = DummyFTPServer(('::1', 0), af=socket.AF_INET6)
         self.server.start()
         self.client = ftplib.FTP()
         self.client.connect(self.server.host, self.server.port)
@@ -661,6 +782,7 @@ class TestIPv6Environment(TestCase):
                 received.append(data.decode('ascii'))
             received = []
             self.client.retrbinary('retr', callback)
+            self.assertEqual(len(''.join(received)), len(RETR_DATA))
             self.assertEqual(''.join(received), RETR_DATA)
         self.client.set_pasv(True)
         retr()
@@ -676,7 +798,7 @@ class TestTLS_FTPClassMixin(TestFTPClass):
     def setUp(self):
         self.server = DummyTLS_FTPServer((HOST, 0))
         self.server.start()
-        self.client = ftplib.FTP_TLS(timeout=10)
+        self.client = ftplib.FTP_TLS(timeout=2)
         self.client.connect(self.server.host, self.server.port)
         # enable TLS
         self.client.auth()
@@ -689,7 +811,7 @@ class TestTLS_FTPClass(TestCase):
     def setUp(self):
         self.server = DummyTLS_FTPServer((HOST, 0))
         self.server.start()
-        self.client = ftplib.FTP_TLS(timeout=10)
+        self.client = ftplib.FTP_TLS(timeout=2)
         self.client.connect(self.server.host, self.server.port)
 
     def tearDown(self):
@@ -749,7 +871,7 @@ class TestTLS_FTPClass(TestCase):
         self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE,
                           keyfile=CERTFILE, context=ctx)
 
-        self.client = ftplib.FTP_TLS(context=ctx, timeout=10)
+        self.client = ftplib.FTP_TLS(context=ctx, timeout=2)
         self.client.connect(self.server.host, self.server.port)
         self.assertNotIsInstance(self.client.sock, ssl.SSLSocket)
         self.client.auth()
@@ -761,45 +883,53 @@ class TestTLS_FTPClass(TestCase):
             self.assertIs(sock.context, ctx)
             self.assertIsInstance(sock, ssl.SSLSocket)
 
+    def test_ccc(self):
+        self.assertRaises(ValueError, self.client.ccc)
+        self.client.login(secure=True)
+        self.assertIsInstance(self.client.sock, ssl.SSLSocket)
+        self.client.ccc()
+        self.assertRaises(ValueError, self.client.sock.unwrap)
+
 
 class TestTimeouts(TestCase):
 
     def setUp(self):
         self.evt = threading.Event()
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.sock.settimeout(10)
+        self.sock.settimeout(20)
         self.port = support.bind_port(self.sock)
-        threading.Thread(target=self.server, args=(self.evt,self.sock)).start()
+        self.server_thread = threading.Thread(target=self.server)
+        self.server_thread.start()
         # Wait for the server to be ready.
         self.evt.wait()
         self.evt.clear()
+        self.old_port = ftplib.FTP.port
         ftplib.FTP.port = self.port
 
     def tearDown(self):
-        self.evt.wait()
-        self.sock.close()
+        ftplib.FTP.port = self.old_port
+        self.server_thread.join()
 
-    def server(self, evt, serv):
+    def server(self):
         # This method sets the evt 3 times:
         #  1) when the connection is ready to be accepted.
         #  2) when it is safe for the caller to close the connection
         #  3) when we have closed the socket
-        serv.listen(5)
+        self.sock.listen(5)
         # (1) Signal the caller that we are ready to accept the connection.
-        evt.set()
+        self.evt.set()
         try:
-            conn, addr = serv.accept()
+            conn, addr = self.sock.accept()
         except socket.timeout:
             pass
         else:
-            conn.send(b"1 Hola mundo\n")
+            conn.sendall(b"1 Hola mundo\n")
+            conn.shutdown(socket.SHUT_WR)
             # (2) Signal the caller that it is safe to close the socket.
-            evt.set()
+            self.evt.set()
             conn.close()
         finally:
-            serv.close()
-            # (3) Signal the caller that we are done.
-            evt.set()
+            self.sock.close()
 
     def testTimeoutDefault(self):
         # default -- use global socket timeout
@@ -857,13 +987,8 @@ class TestTimeouts(TestCase):
 
 def test_main():
     tests = [TestFTPClass, TestTimeouts]
-    if socket.has_ipv6:
-        try:
-            DummyFTPServer((HOST, 0), af=socket.AF_INET6)
-        except socket.error:
-            pass
-        else:
-            tests.append(TestIPv6Environment)
+    if support.IPV6_ENABLED:
+        tests.append(TestIPv6Environment)
 
     if ssl is not None:
         tests.extend([TestTLS_FTPClassMixin, TestTLS_FTPClass])
diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py
index 4d1936879e..c8ed83020e 100644
--- a/Lib/test/test_funcattrs.py
+++ b/Lib/test/test_funcattrs.py
@@ -2,6 +2,15 @@ from test import support
 import types
 import unittest
 
+
+def global_function():
+    def inner_function():
+        class LocalClass:
+            pass
+        return LocalClass
+    return lambda: inner_function
+
+
 class FuncAttrsTest(unittest.TestCase):
     def setUp(self):
         class F:
@@ -96,6 +105,24 @@ class FunctionPropertiesTest(FuncAttrsTest):
         self.assertEqual(self.fi.a.__name__, 'a')
         self.cannot_set_attr(self.fi.a, "__name__", 'a', AttributeError)
 
+    def test___qualname__(self):
+        # PEP 3155
+        self.assertEqual(self.b.__qualname__, 'FuncAttrsTest.setUp.<locals>.b')
+        self.assertEqual(FuncAttrsTest.setUp.__qualname__, 'FuncAttrsTest.setUp')
+        self.assertEqual(global_function.__qualname__, 'global_function')
+        self.assertEqual(global_function().__qualname__,
+                         'global_function.<locals>.<lambda>')
+        self.assertEqual(global_function()().__qualname__,
+                         'global_function.<locals>.inner_function')
+        self.assertEqual(global_function()()().__qualname__,
+                         'global_function.<locals>.inner_function.<locals>.LocalClass')
+        self.b.__qualname__ = 'c'
+        self.assertEqual(self.b.__qualname__, 'c')
+        self.b.__qualname__ = 'd'
+        self.assertEqual(self.b.__qualname__, 'd')
+        # __qualname__ must be a string
+        self.cannot_set_attr(self.b, '__qualname__', 7, TypeError)
+
     def test___code__(self):
         num_one, num_two = 7, 8
         def a(): pass
@@ -315,11 +342,37 @@ class StaticMethodAttrsTest(unittest.TestCase):
         self.assertTrue(s.__func__ is f)
 
 
+class BuiltinFunctionPropertiesTest(unittest.TestCase):
+    # XXX Not sure where this should really go since I can't find a
+    # test module specifically for builtin_function_or_method.
+
+    def test_builtin__qualname__(self):
+        import time
+
+        # builtin function:
+        self.assertEqual(len.__qualname__, 'len')
+        self.assertEqual(time.time.__qualname__, 'time')
+
+        # builtin classmethod:
+        self.assertEqual(dict.fromkeys.__qualname__, 'dict.fromkeys')
+        self.assertEqual(float.__getformat__.__qualname__,
+                         'float.__getformat__')
+
+        # builtin staticmethod:
+        self.assertEqual(str.maketrans.__qualname__, 'str.maketrans')
+        self.assertEqual(bytes.maketrans.__qualname__, 'bytes.maketrans')
+
+        # builtin bound instance method:
+        self.assertEqual([1, 2, 3].append.__qualname__, 'list.append')
+        self.assertEqual({'foo': 'bar'}.pop.__qualname__, 'dict.pop')
+
+
 def test_main():
     support.run_unittest(FunctionPropertiesTest, InstancemethodAttrTest,
                               ArbitraryFunctionAttrTest, FunctionDictsTest,
                               FunctionDocstringTest, CellTest,
-                              StaticMethodAttrsTest)
+                              StaticMethodAttrsTest,
+                              BuiltinFunctionPropertiesTest)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py
index 11e6e84420..db1e9348dd 100644
--- a/Lib/test/test_functools.py
+++ b/Lib/test/test_functools.py
@@ -246,6 +246,7 @@ class TestUpdateWrapper(unittest.TestCase):
         self.check_wrapper(wrapper, f)
         self.assertIs(wrapper.__wrapped__, f)
         self.assertEqual(wrapper.__name__, 'f')
+        self.assertEqual(wrapper.__qualname__, f.__qualname__)
         self.assertEqual(wrapper.attr, 'This is also a test')
         self.assertEqual(wrapper.__annotations__['a'], 'This is a new annotation')
         self.assertNotIn('b', wrapper.__annotations__)
@@ -266,6 +267,7 @@ class TestUpdateWrapper(unittest.TestCase):
         functools.update_wrapper(wrapper, f, (), ())
         self.check_wrapper(wrapper, f, (), ())
         self.assertEqual(wrapper.__name__, 'wrapper')
+        self.assertNotEqual(wrapper.__qualname__, f.__qualname__)
         self.assertEqual(wrapper.__doc__, None)
         self.assertEqual(wrapper.__annotations__, {})
         self.assertFalse(hasattr(wrapper, 'attr'))
@@ -283,6 +285,7 @@ class TestUpdateWrapper(unittest.TestCase):
         functools.update_wrapper(wrapper, f, assign, update)
         self.check_wrapper(wrapper, f, assign, update)
         self.assertEqual(wrapper.__name__, 'wrapper')
+        self.assertNotEqual(wrapper.__qualname__, f.__qualname__)
         self.assertEqual(wrapper.__doc__, None)
         self.assertEqual(wrapper.attr, 'This is a different test')
         self.assertEqual(wrapper.dict_attr, f.dict_attr)
@@ -330,17 +333,18 @@ class TestWraps(TestUpdateWrapper):
         def wrapper():
             pass
         self.check_wrapper(wrapper, f)
-        return wrapper
+        return wrapper, f
 
     def test_default_update(self):
-        wrapper = self._default_update()
+        wrapper, f = self._default_update()
         self.assertEqual(wrapper.__name__, 'f')
+        self.assertEqual(wrapper.__qualname__, f.__qualname__)
         self.assertEqual(wrapper.attr, 'This is also a test')
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")
     def test_default_update_doc(self):
-        wrapper = self._default_update()
+        wrapper, _ = self._default_update()
         self.assertEqual(wrapper.__doc__, 'This is a test')
 
     def test_no_update(self):
@@ -353,6 +357,7 @@ class TestWraps(TestUpdateWrapper):
             pass
         self.check_wrapper(wrapper, f, (), ())
         self.assertEqual(wrapper.__name__, 'wrapper')
+        self.assertNotEqual(wrapper.__qualname__, f.__qualname__)
         self.assertEqual(wrapper.__doc__, None)
         self.assertFalse(hasattr(wrapper, 'attr'))
 
@@ -372,6 +377,7 @@ class TestWraps(TestUpdateWrapper):
             pass
         self.check_wrapper(wrapper, f, assign, update)
         self.assertEqual(wrapper.__name__, 'wrapper')
+        self.assertNotEqual(wrapper.__qualname__, f.__qualname__)
         self.assertEqual(wrapper.__doc__, None)
         self.assertEqual(wrapper.attr, 'This is a different test')
         self.assertEqual(wrapper.dict_attr, f.dict_attr)
@@ -457,19 +463,82 @@ class TestReduce(unittest.TestCase):
         self.assertEqual(self.func(add, d), "".join(d.keys()))
 
 class TestCmpToKey(unittest.TestCase):
+
     def test_cmp_to_key(self):
+        def cmp1(x, y):
+            return (x > y) - (x < y)
+        key = functools.cmp_to_key(cmp1)
+        self.assertEqual(key(3), key(3))
+        self.assertGreater(key(3), key(1))
+        def cmp2(x, y):
+            return int(x) - int(y)
+        key = functools.cmp_to_key(cmp2)
+        self.assertEqual(key(4.0), key('4'))
+        self.assertLess(key(2), key('35'))
+
+    def test_cmp_to_key_arguments(self):
+        def cmp1(x, y):
+            return (x > y) - (x < y)
+        key = functools.cmp_to_key(mycmp=cmp1)
+        self.assertEqual(key(obj=3), key(obj=3))
+        self.assertGreater(key(obj=3), key(obj=1))
+        with self.assertRaises((TypeError, AttributeError)):
+            key(3) > 1    # rhs is not a K object
+        with self.assertRaises((TypeError, AttributeError)):
+            1 < key(3)    # lhs is not a K object
+        with self.assertRaises(TypeError):
+            key = functools.cmp_to_key()             # too few args
+        with self.assertRaises(TypeError):
+            key = functools.cmp_to_key(cmp1, None)   # too many args
+        key = functools.cmp_to_key(cmp1)
+        with self.assertRaises(TypeError):
+            key()                                    # too few args
+        with self.assertRaises(TypeError):
+            key(None, None)                          # too many args
+
+    def test_bad_cmp(self):
+        def cmp1(x, y):
+            raise ZeroDivisionError
+        key = functools.cmp_to_key(cmp1)
+        with self.assertRaises(ZeroDivisionError):
+            key(3) > key(1)
+
+        class BadCmp:
+            def __lt__(self, other):
+                raise ZeroDivisionError
+        def cmp1(x, y):
+            return BadCmp()
+        with self.assertRaises(ZeroDivisionError):
+            key(3) > key(1)
+
+    def test_obj_field(self):
+        def cmp1(x, y):
+            return (x > y) - (x < y)
+        key = functools.cmp_to_key(mycmp=cmp1)
+        self.assertEqual(key(50).obj, 50)
+
+    def test_sort_int(self):
         def mycmp(x, y):
             return y - x
         self.assertEqual(sorted(range(5), key=functools.cmp_to_key(mycmp)),
                          [4, 3, 2, 1, 0])
 
+    def test_sort_int_str(self):
+        def mycmp(x, y):
+            x, y = int(x), int(y)
+            return (x > y) - (x < y)
+        values = [5, '3', 7, 2, '0', '1', 4, '10', 1]
+        values = sorted(values, key=functools.cmp_to_key(mycmp))
+        self.assertEqual([int(value) for value in values],
+                         [0, 1, 1, 2, 3, 4, 5, 7, 10])
+
     def test_hash(self):
         def mycmp(x, y):
             return y - x
         key = functools.cmp_to_key(mycmp)
         k = key(10)
         self.assertRaises(TypeError, hash, k)
-        self.assertFalse(isinstance(k, collections.Hashable))
+        self.assertNotIsInstance(k, collections.Hashable)
 
 class TestTotalOrdering(unittest.TestCase):
 
@@ -692,6 +761,47 @@ class TestLRU(unittest.TestCase):
             with self.assertRaises(IndexError):
                 func(15)
 
+    def test_lru_with_types(self):
+        for maxsize in (None, 100):
+            @functools.lru_cache(maxsize=maxsize, typed=True)
+            def square(x):
+                return x * x
+            self.assertEqual(square(3), 9)
+            self.assertEqual(type(square(3)), type(9))
+            self.assertEqual(square(3.0), 9.0)
+            self.assertEqual(type(square(3.0)), type(9.0))
+            self.assertEqual(square(x=3), 9)
+            self.assertEqual(type(square(x=3)), type(9))
+            self.assertEqual(square(x=3.0), 9.0)
+            self.assertEqual(type(square(x=3.0)), type(9.0))
+            self.assertEqual(square.cache_info().hits, 4)
+            self.assertEqual(square.cache_info().misses, 4)
+
+    def test_need_for_rlock(self):
+        # This will deadlock on an LRU cache that uses a regular lock
+
+        @functools.lru_cache(maxsize=10)
+        def test_func(x):
+            'Used to demonstrate a reentrant lru_cache call within a single thread'
+            return x
+
+        class DoubleEq:
+            'Demonstrate a reentrant lru_cache call within a single thread'
+            def __init__(self, x):
+                self.x = x
+            def __hash__(self):
+                return self.x
+            def __eq__(self, other):
+                if self.x == 2:
+                    test_func(DoubleEq(1))
+                return self.x == other.x
+
+        test_func(DoubleEq(1))                      # Load the cache
+        test_func(DoubleEq(2))                      # Load the cache
+        self.assertEqual(test_func(DoubleEq(2)),    # Trigger a re-entrant __eq__ call
+                         DoubleEq(2))               # Verify the correct return value
+
+
 def test_main(verbose=None):
     test_classes = (
         TestPartial,
diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py
index c6689a1a18..a0c156f5f7 100644
--- a/Lib/test/test_future.py
+++ b/Lib/test/test_future.py
@@ -13,18 +13,18 @@ def get_error_location(msg):
 class FutureTest(unittest.TestCase):
 
     def test_future1(self):
-        support.unload('test_future1')
-        from test import test_future1
-        self.assertEqual(test_future1.result, 6)
+        with support.CleanImport('future_test1'):
+            from test import future_test1
+            self.assertEqual(future_test1.result, 6)
 
     def test_future2(self):
-        support.unload('test_future2')
-        from test import test_future2
-        self.assertEqual(test_future2.result, 6)
+        with support.CleanImport('future_test2'):
+            from test import future_test2
+            self.assertEqual(future_test2.result, 6)
 
     def test_future3(self):
-        support.unload('test_future3')
-        from test import test_future3
+        with support.CleanImport('test_future3'):
+            from test import test_future3
 
     def test_badfuture3(self):
         try:
@@ -103,8 +103,8 @@ class FutureTest(unittest.TestCase):
             self.fail("syntax error didn't occur")
 
     def test_multiple_features(self):
-        support.unload("test.test_future5")
-        from test import test_future5
+        with support.CleanImport("test.test_future5"):
+            from test import test_future5
 
     def test_unicode_literals_exec(self):
         scope = {}
@@ -112,8 +112,6 @@ class FutureTest(unittest.TestCase):
         self.assertIsInstance(scope["x"], str)
 
 
-def test_main():
-    support.run_unittest(FutureTest)
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_future3.py b/Lib/test/test_future3.py
index b1552a5e8e..09f1c78fa3 100644
--- a/Lib/test/test_future3.py
+++ b/Lib/test/test_future3.py
@@ -2,7 +2,6 @@ from __future__ import nested_scopes
 from __future__ import division
 
 import unittest
-from test import support
 
 x = 2
 def nester():
@@ -23,8 +22,5 @@ class TestFuture(unittest.TestCase):
     def test_nested_scopes(self):
         self.assertEqual(nester(), 3)
 
-def test_main():
-    support.run_unittest(TestFuture)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_future4.py b/Lib/test/test_future4.py
index c32f27f616..413dd4d96b 100644
--- a/Lib/test/test_future4.py
+++ b/Lib/test/test_future4.py
@@ -1,10 +1,6 @@
 from __future__ import unicode_literals
 
 import unittest
-from test import support
-
-def test_main():
-    pass
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_future5.py b/Lib/test/test_future5.py
index ed03a07037..b44b97e63e 100644
--- a/Lib/test/test_future5.py
+++ b/Lib/test/test_future5.py
@@ -17,5 +17,5 @@ class TestMultipleFeatures(unittest.TestCase):
         self.assertEqual(s.getvalue(), "foo\n")
 
 
-def test_main():
-    support.run_unittest(TestMultipleFeatures)
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py
index e1c124d160..c59b72eacf 100644
--- a/Lib/test/test_gc.py
+++ b/Lib/test/test_gc.py
@@ -1,5 +1,6 @@
 import unittest
-from test.support import verbose, run_unittest, strip_python_stderr
+from test.support import (verbose, refcount_test, run_unittest,
+                            strip_python_stderr)
 import sys
 import time
 import gc
@@ -37,6 +38,20 @@ class GC_Detector(object):
         # gc collects it.
         self.wr = weakref.ref(C1055820(666), it_happened)
 
+class Uncollectable(object):
+    """Create a reference cycle with multiple __del__ methods.
+
+    An object in a reference cycle will never have zero references,
+    and so must be garbage collected.  If one or more objects in the
+    cycle have __del__ methods, the gc refuses to guess an order,
+    and leaves the cycle uncollected."""
+    def __init__(self, partner=None):
+        if partner is None:
+            self.partner = Uncollectable(partner=self)
+        else:
+            self.partner = partner
+    def __del__(self):
+        pass
 
 ### Tests
 ###############################################################################
@@ -181,6 +196,7 @@ class GCTests(unittest.TestCase):
         del d
         self.assertEqual(gc.collect(), 2)
 
+    @refcount_test
     def test_frame(self):
         def f():
             frame = sys._getframe()
@@ -248,6 +264,7 @@ class GCTests(unittest.TestCase):
     # For example, disposed tuples are not freed, but reused.
     # To minimize variations, though, we first store the get_count() results
     # and check them at the end.
+    @refcount_test
     def test_get_count(self):
         gc.collect()
         a, b, c = gc.get_count()
@@ -261,6 +278,7 @@ class GCTests(unittest.TestCase):
         # created (the list).
         self.assertGreater(d, a)
 
+    @refcount_test
     def test_collect_generations(self):
         gc.collect()
         # This object will "trickle" into generation N + 1 after
@@ -593,6 +611,127 @@ class GCTests(unittest.TestCase):
         self.assertNotIn(b"uncollectable objects at shutdown", stderr)
 
 
+class GCCallbackTests(unittest.TestCase):
+    def setUp(self):
+        # Save gc state and disable it.
+        self.enabled = gc.isenabled()
+        gc.disable()
+        self.debug = gc.get_debug()
+        gc.set_debug(0)
+        gc.callbacks.append(self.cb1)
+        gc.callbacks.append(self.cb2)
+        self.othergarbage = []
+
+    def tearDown(self):
+        # Restore gc state
+        del self.visit
+        gc.callbacks.remove(self.cb1)
+        gc.callbacks.remove(self.cb2)
+        gc.set_debug(self.debug)
+        if self.enabled:
+            gc.enable()
+        # destroy any uncollectables
+        gc.collect()
+        for obj in gc.garbage:
+            if isinstance(obj, Uncollectable):
+                obj.partner = None
+        del gc.garbage[:]
+        del self.othergarbage
+        gc.collect()
+
+    def preclean(self):
+        # Remove all fluff from the system.  Invoke this function
+        # manually rather than through self.setUp() for maximum
+        # safety.
+        self.visit = []
+        gc.collect()
+        garbage, gc.garbage[:] = gc.garbage[:], []
+        self.othergarbage.append(garbage)
+        self.visit = []
+
+    def cb1(self, phase, info):
+        self.visit.append((1, phase, dict(info)))
+
+    def cb2(self, phase, info):
+        self.visit.append((2, phase, dict(info)))
+        if phase == "stop" and hasattr(self, "cleanup"):
+            # Clean Uncollectable from garbage
+            uc = [e for e in gc.garbage if isinstance(e, Uncollectable)]
+            gc.garbage[:] = [e for e in gc.garbage
+                             if not isinstance(e, Uncollectable)]
+            for e in uc:
+                e.partner = None
+
+    def test_collect(self):
+        self.preclean()
+        gc.collect()
+        # Algorithmically verify the contents of self.visit
+        # because it is long and tortuous.
+
+        # Count the number of visits to each callback
+        n = [v[0] for v in self.visit]
+        n1 = [i for i in n if i == 1]
+        n2 = [i for i in n if i == 2]
+        self.assertEqual(n1, [1]*2)
+        self.assertEqual(n2, [2]*2)
+
+        # Count that we got the right number of start and stop callbacks.
+        n = [v[1] for v in self.visit]
+        n1 = [i for i in n if i == "start"]
+        n2 = [i for i in n if i == "stop"]
+        self.assertEqual(n1, ["start"]*2)
+        self.assertEqual(n2, ["stop"]*2)
+
+        # Check that we got the right info dict for all callbacks
+        for v in self.visit:
+            info = v[2]
+            self.assertTrue("generation" in info)
+            self.assertTrue("collected" in info)
+            self.assertTrue("uncollectable" in info)
+
+    def test_collect_generation(self):
+        self.preclean()
+        gc.collect(2)
+        for v in self.visit:
+            info = v[2]
+            self.assertEqual(info["generation"], 2)
+
+    def test_collect_garbage(self):
+        self.preclean()
+        # Each of these cause four objects to be garbage: Two
+        # Uncolectables and their instance dicts.
+        Uncollectable()
+        Uncollectable()
+        C1055820(666)
+        gc.collect()
+        for v in self.visit:
+            if v[1] != "stop":
+                continue
+            info = v[2]
+            self.assertEqual(info["collected"], 2)
+            self.assertEqual(info["uncollectable"], 8)
+
+        # We should now have the Uncollectables in gc.garbage
+        self.assertEqual(len(gc.garbage), 4)
+        for e in gc.garbage:
+            self.assertIsInstance(e, Uncollectable)
+
+        # Now, let our callback handle the Uncollectable instances
+        self.cleanup=True
+        self.visit = []
+        gc.garbage[:] = []
+        gc.collect()
+        for v in self.visit:
+            if v[1] != "stop":
+                continue
+            info = v[2]
+            self.assertEqual(info["collected"], 0)
+            self.assertEqual(info["uncollectable"], 4)
+
+        # Uncollectables should be gone
+        self.assertEqual(len(gc.garbage), 0)
+
+
 class GCTogglingTests(unittest.TestCase):
     def setUp(self):
         gc.enable()
@@ -746,7 +885,7 @@ def test_main():
 
     try:
         gc.collect() # Delete 2nd generation garbage
-        run_unittest(GCTests, GCTogglingTests)
+        run_unittest(GCTests, GCTogglingTests, GCCallbackTests)
     finally:
         gc.set_debug(debug)
         # test gc.enable() even if GC is disabled by default
diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py
index 6d96550a9b..9713dc9c06 100644
--- a/Lib/test/test_gdb.py
+++ b/Lib/test/test_gdb.py
@@ -7,9 +7,16 @@ import os
 import re
 import subprocess
 import sys
+import sysconfig
 import unittest
 import locale
 
+# Is this Python configured to support threads?
+try:
+    import _thread
+except ImportError:
+    _thread = None
+
 from test.support import run_unittest, findfile, python_is_optimized
 
 try:
@@ -26,6 +33,9 @@ if gdb_major_version < 7:
     raise unittest.SkipTest("gdb versions before 7.0 didn't support python embedding"
                             " Saw:\n" + gdb_version.decode('ascii', 'replace'))
 
+if not sysconfig.is_python_build():
+    raise unittest.SkipTest("test_gdb only works on source builds at the moment.")
+
 # Location of custom hooks file in a repository checkout.
 checkout_hook_path = os.path.join(os.path.dirname(sys.executable),
                                   'python-gdb.py')
@@ -53,9 +63,7 @@ gdbpy_version, _ = run_gdb("--eval-command=python import sys; print sys.version_
 if not gdbpy_version:
     raise unittest.SkipTest("gdb not built with embedded python support")
 
-# Verify that "gdb" can load our custom hooks.  In theory this should never
-# fail, but we don't handle the case of the hooks file not existing if the
-# tests are run from an installed Python (we'll produce failures in that case).
+# Verify that "gdb" can load our custom hooks. In theory this should never fail.
 cmd = ['--args', sys.executable]
 _, gdbpy_errors = run_gdb('--args', sys.executable)
 if "auto-loading has been declined" in gdbpy_errors:
@@ -160,7 +168,6 @@ class DebuggerTests(unittest.TestCase):
 
         # Ensure no unexpected error messages:
         self.assertEqual(err, '')
-
         return out
 
     def get_gdb_repr(self, source,
@@ -181,7 +188,7 @@ class DebuggerTests(unittest.TestCase):
         # gdb can insert additional '\n' and space characters in various places
         # in its output, depending on the width of the terminal it's connected
         # to (using its "wrap_here" function)
-        m = re.match('.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+Python/bltinmodule.c.*',
+        m = re.match('.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+\S*Python/bltinmodule.c.*',
                      gdb_output, re.DOTALL)
         if not m:
             self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output))
@@ -343,7 +350,7 @@ class Foo:
 foo = Foo()
 foo.an_int = 42
 id(foo)''')
-        m = re.match(r'<Foo\(an_int=42\) at remote 0x[0-9a-f]+>', gdb_repr)
+        m = re.match(r'<Foo\(an_int=42\) at remote 0x-?[0-9a-f]+>', gdb_repr)
         self.assertTrue(m,
                         msg='Unexpected new-style class rendering %r' % gdb_repr)
 
@@ -356,7 +363,7 @@ foo = Foo()
 foo += [1, 2, 3]
 foo.an_int = 42
 id(foo)''')
-        m = re.match(r'<Foo\(an_int=42\) at remote 0x[0-9a-f]+>', gdb_repr)
+        m = re.match(r'<Foo\(an_int=42\) at remote 0x-?[0-9a-f]+>', gdb_repr)
 
         self.assertTrue(m,
                         msg='Unexpected new-style class rendering %r' % gdb_repr)
@@ -371,7 +378,7 @@ class Foo(tuple):
 foo = Foo((1, 2, 3))
 foo.an_int = 42
 id(foo)''')
-        m = re.match(r'<Foo\(an_int=42\) at remote 0x[0-9a-f]+>', gdb_repr)
+        m = re.match(r'<Foo\(an_int=42\) at remote 0x-?[0-9a-f]+>', gdb_repr)
 
         self.assertTrue(m,
                         msg='Unexpected new-style class rendering %r' % gdb_repr)
@@ -398,7 +405,7 @@ id(foo)''')
 
         # Match anything for the type name; 0xDEADBEEF could point to
         # something arbitrary (see  http://bugs.python.org/issue8330)
-        pattern = '<.* at remote 0x[0-9a-f]+>'
+        pattern = '<.* at remote 0x-?[0-9a-f]+>'
 
         m = re.match(pattern, gdb_repr)
         if not m:
@@ -444,7 +451,7 @@ id(foo)''')
         #  http://bugs.python.org/issue8032#msg100537 )
         gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True)
 
-        m = re.match(r'<_Helper at remote 0x[0-9a-f]+>', gdb_repr)
+        m = re.match(r'<_Helper at remote 0x-?[0-9a-f]+>', gdb_repr)
         self.assertTrue(m,
                         msg='Unexpected rendering %r' % gdb_repr)
 
@@ -475,7 +482,7 @@ class Foo:
 foo = Foo()
 foo.an_attr = foo
 id(foo)''')
-        self.assertTrue(re.match('<Foo\(an_attr=<\.\.\.>\) at remote 0x[0-9a-f]+>',
+        self.assertTrue(re.match('<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>',
                                  gdb_repr),
                         'Unexpected gdb representation: %r\n%s' % \
                             (gdb_repr, gdb_output))
@@ -488,7 +495,7 @@ class Foo(object):
 foo = Foo()
 foo.an_attr = foo
 id(foo)''')
-        self.assertTrue(re.match('<Foo\(an_attr=<\.\.\.>\) at remote 0x[0-9a-f]+>',
+        self.assertTrue(re.match('<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>',
                                  gdb_repr),
                         'Unexpected gdb representation: %r\n%s' % \
                             (gdb_repr, gdb_output))
@@ -502,7 +509,7 @@ b = Foo()
 a.an_attr = b
 b.an_attr = a
 id(a)''')
-        self.assertTrue(re.match('<Foo\(an_attr=<Foo\(an_attr=<\.\.\.>\) at remote 0x[0-9a-f]+>\) at remote 0x[0-9a-f]+>',
+        self.assertTrue(re.match('<Foo\(an_attr=<Foo\(an_attr=<\.\.\.>\) at remote 0x-?[0-9a-f]+>\) at remote 0x-?[0-9a-f]+>',
                                  gdb_repr),
                         'Unexpected gdb representation: %r\n%s' % \
                             (gdb_repr, gdb_output))
@@ -537,7 +544,7 @@ id(a)''')
 
     def test_builtin_method(self):
         gdb_repr, gdb_output = self.get_gdb_repr('import sys; id(sys.stdout.readlines)')
-        self.assertTrue(re.match('<built-in method readlines of _io.TextIOWrapper object at remote 0x[0-9a-f]+>',
+        self.assertTrue(re.match('<built-in method readlines of _io.TextIOWrapper object at remote 0x-?[0-9a-f]+>',
                                  gdb_repr),
                         'Unexpected gdb representation: %r\n%s' % \
                             (gdb_repr, gdb_output))
@@ -552,7 +559,7 @@ id(foo.__code__)''',
                                           breakpoint='builtin_id',
                                           cmds_after_breakpoint=['print (PyFrameObject*)(((PyCodeObject*)v)->co_zombieframe)']
                                           )
-        self.assertTrue(re.match('.*\s+\$1 =\s+Frame 0x[0-9a-f]+, for file <string>, line 3, in foo \(\)\s+.*',
+        self.assertTrue(re.match('.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file <string>, line 3, in foo \(\)\s+.*',
                                  gdb_output,
                                  re.DOTALL),
                         'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output))
@@ -609,7 +616,7 @@ class StackNavigationTests(DebuggerTests):
                                   cmds_after_breakpoint=['py-up'])
         self.assertMultilineMatches(bt,
                                     r'''^.*
-#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\)
     baz\(a, b, c\)
 $''')
 
@@ -638,9 +645,9 @@ $''')
                                   cmds_after_breakpoint=['py-up', 'py-down'])
         self.assertMultilineMatches(bt,
                                     r'''^.*
-#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\)
     baz\(a, b, c\)
-#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 10, in baz \(args=\(1, 2, 3\)\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 10, in baz \(args=\(1, 2, 3\)\)
     id\(42\)
 $''')
 
@@ -672,14 +679,106 @@ Traceback \(most recent call first\):
                                   cmds_after_breakpoint=['py-bt-full'])
         self.assertMultilineMatches(bt,
                                     r'''^.*
-#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\)
     baz\(a, b, c\)
-#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\)
     bar\(a, b, c\)
-#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 12, in <module> \(\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 12, in <module> \(\)
     foo\(1, 2, 3\)
 ''')
 
+    @unittest.skipUnless(_thread,
+                         "Python was compiled without thread support")
+    def test_threads(self):
+        'Verify that "py-bt" indicates threads that are waiting for the GIL'
+        cmd = '''
+from threading import Thread
+
+class TestThread(Thread):
+    # These threads would run forever, but we'll interrupt things with the
+    # debugger
+    def run(self):
+        i = 0
+        while 1:
+             i += 1
+
+t = {}
+for i in range(4):
+   t[i] = TestThread()
+   t[i].start()
+
+# Trigger a breakpoint on the main thread
+id(42)
+
+'''
+        # Verify with "py-bt":
+        gdb_output = self.get_stack_trace(cmd,
+                                          cmds_after_breakpoint=['thread apply all py-bt'])
+        self.assertIn('Waiting for the GIL', gdb_output)
+
+        # Verify with "py-bt-full":
+        gdb_output = self.get_stack_trace(cmd,
+                                          cmds_after_breakpoint=['thread apply all py-bt-full'])
+        self.assertIn('Waiting for the GIL', gdb_output)
+
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
+    # Some older versions of gdb will fail with
+    #  "Cannot find new threads: generic error"
+    # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround
+    @unittest.skipUnless(_thread,
+                         "Python was compiled without thread support")
+    def test_gc(self):
+        'Verify that "py-bt" indicates if a thread is garbage-collecting'
+        cmd = ('from gc import collect\n'
+               'id(42)\n'
+               'def foo():\n'
+               '    collect()\n'
+               'def bar():\n'
+               '    foo()\n'
+               'bar()\n')
+        # Verify with "py-bt":
+        gdb_output = self.get_stack_trace(cmd,
+                                          cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt'],
+                                          )
+        self.assertIn('Garbage-collecting', gdb_output)
+
+        # Verify with "py-bt-full":
+        gdb_output = self.get_stack_trace(cmd,
+                                          cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt-full'],
+                                          )
+        self.assertIn('Garbage-collecting', gdb_output)
+
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
+    # Some older versions of gdb will fail with
+    #  "Cannot find new threads: generic error"
+    # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround
+    @unittest.skipUnless(_thread,
+                         "Python was compiled without thread support")
+    def test_pycfunction(self):
+        'Verify that "py-bt" displays invocations of PyCFunction instances'
+        cmd = ('from time import sleep\n'
+               'def foo():\n'
+               '    sleep(1)\n'
+               'def bar():\n'
+               '    foo()\n'
+               'bar()\n')
+        # Verify with "py-bt":
+        gdb_output = self.get_stack_trace(cmd,
+                                          breakpoint='time_sleep',
+                                          cmds_after_breakpoint=['bt', 'py-bt'],
+                                          )
+        self.assertIn('<built-in method sleep', gdb_output)
+
+        # Verify with "py-bt-full":
+        gdb_output = self.get_stack_trace(cmd,
+                                          breakpoint='time_sleep',
+                                          cmds_after_breakpoint=['py-bt-full'],
+                                          )
+        self.assertIn('#0 <built-in method sleep', gdb_output)
+
+
 class PyPrintTests(DebuggerTests):
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")
@@ -713,7 +812,7 @@ class PyPrintTests(DebuggerTests):
         bt = self.get_stack_trace(script=self.get_sample_script(),
                                   cmds_after_breakpoint=['py-print len'])
         self.assertMultilineMatches(bt,
-                                    r".*\nbuiltin 'len' = <built-in method len of module object at remote 0x[0-9a-f]+>\n.*")
+                                    r".*\nbuiltin 'len' = <built-in method len of module object at remote 0x-?[0-9a-f]+>\n.*")
 
 class PyLocalsTests(DebuggerTests):
     @unittest.skipIf(python_is_optimized(),
diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py
index 2c88373831..958054aef5 100644
--- a/Lib/test/test_generators.py
+++ b/Lib/test/test_generators.py
@@ -729,29 +729,6 @@ Ye olde Fibonacci generator, tee style.
 
 syntax_tests = """
 
->>> def f():
-...     return 22
-...     yield 1
-Traceback (most recent call last):
-  ..
-SyntaxError: 'return' with argument inside generator
-
->>> def f():
-...     yield 1
-...     return 22
-Traceback (most recent call last):
-  ..
-SyntaxError: 'return' with argument inside generator
-
-"return None" is not the same as "return" in a generator:
-
->>> def f():
-...     yield 1
-...     return None
-Traceback (most recent call last):
-  ..
-SyntaxError: 'return' with argument inside generator
-
 These are fine:
 
 >>> def f():
@@ -867,20 +844,6 @@ These are fine:
 >>> type(f())
 <class 'generator'>
 
-
->>> def f():
-...     if 0:
-...         lambda x:  x        # shouldn't trigger here
-...         return              # or here
-...         def f(i):
-...             return 2*i      # or here
-...         if 0:
-...             return 3        # but *this* sucks (line 8)
-...     if 0:
-...         yield 2             # because it's a generator (line 10)
-Traceback (most recent call last):
-SyntaxError: 'return' with argument inside generator
-
 This one caused a crash (see SF bug 567538):
 
 >>> def f():
@@ -1567,11 +1530,6 @@ Traceback (most recent call last):
   ...
 SyntaxError: 'yield' outside function
 
->>> def f(): return lambda x=(yield): 1
-Traceback (most recent call last):
-  ...
-SyntaxError: 'return' with argument inside generator
-
 >>> def f(): x = yield = y
 Traceback (most recent call last):
   ...
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
index 60209235cf..fd8bc577ca 100644
--- a/Lib/test/test_genericpath.py
+++ b/Lib/test/test_genericpath.py
@@ -2,11 +2,12 @@
 Tests common to genericpath, macpath, ntpath and posixpath
 """
 
-import unittest
-from test import support
-import os
 import genericpath
+import os
 import sys
+import unittest
+import warnings
+from test import support
 
 
 def safe_rmdir(dirname):
@@ -16,9 +17,7 @@ def safe_rmdir(dirname):
         pass
 
 
-class GenericTest(unittest.TestCase):
-    # The path module to be tested
-    pathmodule = genericpath
+class GenericTest:
     common_attributes = ['commonprefix', 'getsize', 'getatime', 'getctime',
                          'getmtime', 'exists', 'isdir', 'isfile']
     attributes = []
@@ -145,6 +144,16 @@ class GenericTest(unittest.TestCase):
                 f.close()
             support.unlink(support.TESTFN)
 
+    @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
+    def test_exists_fd(self):
+        r, w = os.pipe()
+        try:
+            self.assertTrue(self.pathmodule.exists(r))
+        finally:
+            os.close(r)
+            os.close(w)
+        self.assertFalse(self.pathmodule.exists(r))
+
     def test_isdir(self):
         self.assertIs(self.pathmodule.isdir(support.TESTFN), False)
         f = open(support.TESTFN, "wb")
@@ -179,13 +188,16 @@ class GenericTest(unittest.TestCase):
             support.unlink(support.TESTFN)
             safe_rmdir(support.TESTFN)
 
+class TestGenericTest(GenericTest, unittest.TestCase):
+    # Issue 16852: GenericTest can't inherit from unittest.TestCase
+    # for test discovery purposes; CommonTest inherits from GenericTest
+    # and is only meant to be inherited by others.
+    pathmodule = genericpath
 
 # Following TestCase is not supposed to be run from test_genericpath.
 # It is inherited by other test modules (macpath, ntpath, posixpath).
 
 class CommonTest(GenericTest):
-    # The path module to be tested
-    pathmodule = None
     common_attributes = GenericTest.common_attributes + [
         # Properties
         'curdir', 'pardir', 'extsep', 'sep',
@@ -258,15 +270,21 @@ class CommonTest(GenericTest):
 
     def test_abspath(self):
         self.assertIn("foo", self.pathmodule.abspath("foo"))
-        self.assertIn(b"foo", self.pathmodule.abspath(b"foo"))
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertIn(b"foo", self.pathmodule.abspath(b"foo"))
 
         # Abspath returns bytes when the arg is bytes
-        for path in (b'', b'foo', b'f\xf2\xf2', b'/foo', b'C:\\'):
-            self.assertIsInstance(self.pathmodule.abspath(path), bytes)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            for path in (b'', b'foo', b'f\xf2\xf2', b'/foo', b'C:\\'):
+                self.assertIsInstance(self.pathmodule.abspath(path), bytes)
 
     def test_realpath(self):
         self.assertIn("foo", self.pathmodule.realpath("foo"))
-        self.assertIn(b"foo", self.pathmodule.realpath(b"foo"))
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertIn(b"foo", self.pathmodule.realpath(b"foo"))
 
     def test_normpath_issue5827(self):
         # Make sure normpath preserves unicode
@@ -282,8 +300,7 @@ class CommonTest(GenericTest):
 
         unicwd = '\xe7w\xf0'
         try:
-            fsencoding = support.TESTFN_ENCODING or "ascii"
-            unicwd.encode(fsencoding)
+            os.fsencode(unicwd)
         except (AttributeError, UnicodeEncodeError):
             # FS encoding is probably ASCII
             pass
@@ -305,13 +322,12 @@ class CommonTest(GenericTest):
         else:
             self.skipTest("need support.TESTFN_NONASCII")
 
-        with support.temp_cwd(name):
-            self.test_abspath()
-
-
-def test_main():
-    support.run_unittest(GenericTest)
+        # Test non-ASCII, non-UTF8 bytes in the path.
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            with support.temp_cwd(name):
+                self.test_abspath()
 
 
 if __name__=="__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py
index d8eb550b02..203b336fde 100644
--- a/Lib/test/test_genexps.py
+++ b/Lib/test/test_genexps.py
@@ -258,11 +258,15 @@ Verify that genexps are weakly referencable
 
 """
 
+import sys
 
-__test__ = {'doctests' : doctests}
+# Trace function can throw off the tuple reuse test.
+if hasattr(sys, 'gettrace') and sys.gettrace():
+    __test__ = {}
+else:
+    __test__ = {'doctests' : doctests}
 
 def test_main(verbose=None):
-    import sys
     from test import support
     from test import test_genexps
     support.run_doctest(test_genexps, verbose)
diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py
index 3d9c06a460..48ca94ee38 100644
--- a/Lib/test/test_getargs2.py
+++ b/Lib/test/test_getargs2.py
@@ -1,6 +1,6 @@
 import unittest
 from test import support
-from _testcapi import getargs_keywords
+from _testcapi import getargs_keywords, getargs_keyword_only
 
 """
 > How about the following counterproposal. This also changes some of
@@ -214,6 +214,36 @@ class LongLong_TestCase(unittest.TestCase):
 
         self.assertEqual(VERY_LARGE & ULLONG_MAX, getargs_K(VERY_LARGE))
 
+class Paradox:
+    "This statement is false."
+    def __bool__(self):
+        raise NotImplementedError
+
+class Boolean_TestCase(unittest.TestCase):
+    def test_p(self):
+        from _testcapi import getargs_p
+        self.assertEqual(0, getargs_p(False))
+        self.assertEqual(0, getargs_p(None))
+        self.assertEqual(0, getargs_p(0))
+        self.assertEqual(0, getargs_p(0.0))
+        self.assertEqual(0, getargs_p(0j))
+        self.assertEqual(0, getargs_p(''))
+        self.assertEqual(0, getargs_p(()))
+        self.assertEqual(0, getargs_p([]))
+        self.assertEqual(0, getargs_p({}))
+
+        self.assertEqual(1, getargs_p(True))
+        self.assertEqual(1, getargs_p(1))
+        self.assertEqual(1, getargs_p(1.0))
+        self.assertEqual(1, getargs_p(1j))
+        self.assertEqual(1, getargs_p('x'))
+        self.assertEqual(1, getargs_p((1,)))
+        self.assertEqual(1, getargs_p([1]))
+        self.assertEqual(1, getargs_p({1:2}))
+        self.assertEqual(1, getargs_p(unittest.TestCase))
+
+        self.assertRaises(NotImplementedError, getargs_p, Paradox())
+
 
 class Tuple_TestCase(unittest.TestCase):
     def test_tuple(self):
@@ -293,7 +323,87 @@ class Keywords_TestCase(unittest.TestCase):
         else:
             self.fail('TypeError should have been raised')
 
+class KeywordOnly_TestCase(unittest.TestCase):
+    def test_positional_args(self):
+        # using all possible positional args
+        self.assertEqual(
+            getargs_keyword_only(1, 2),
+            (1, 2, -1)
+            )
+
+    def test_mixed_args(self):
+        # positional and keyword args
+        self.assertEqual(
+            getargs_keyword_only(1, 2, keyword_only=3),
+            (1, 2, 3)
+            )
+
+    def test_keyword_args(self):
+        # all keywords
+        self.assertEqual(
+            getargs_keyword_only(required=1, optional=2, keyword_only=3),
+            (1, 2, 3)
+            )
+
+    def test_optional_args(self):
+        # missing optional keyword args, skipping tuples
+        self.assertEqual(
+            getargs_keyword_only(required=1, optional=2),
+            (1, 2, -1)
+            )
+        self.assertEqual(
+            getargs_keyword_only(required=1, keyword_only=3),
+            (1, -1, 3)
+            )
+
+    def test_required_args(self):
+        self.assertEqual(
+            getargs_keyword_only(1),
+            (1, -1, -1)
+            )
+        self.assertEqual(
+            getargs_keyword_only(required=1),
+            (1, -1, -1)
+            )
+        # required arg missing
+        with self.assertRaisesRegex(TypeError,
+            "Required argument 'required' \(pos 1\) not found"):
+            getargs_keyword_only(optional=2)
+
+        with self.assertRaisesRegex(TypeError,
+            "Required argument 'required' \(pos 1\) not found"):
+            getargs_keyword_only(keyword_only=3)
+
+    def test_too_many_args(self):
+        with self.assertRaisesRegex(TypeError,
+            "Function takes at most 2 positional arguments \(3 given\)"):
+            getargs_keyword_only(1, 2, 3)
+
+        with self.assertRaisesRegex(TypeError,
+            "function takes at most 3 arguments \(4 given\)"):
+            getargs_keyword_only(1, 2, 3, keyword_only=5)
+
+    def test_invalid_keyword(self):
+        # extraneous keyword arg
+        with self.assertRaisesRegex(TypeError,
+            "'monster' is an invalid keyword argument for this function"):
+            getargs_keyword_only(1, 2, monster=666)
+
+    def test_surrogate_keyword(self):
+        with self.assertRaisesRegex(TypeError,
+            "'\udc80' is an invalid keyword argument for this function"):
+            getargs_keyword_only(1, 2, **{'\uDC80': 10})
+
 class Bytes_TestCase(unittest.TestCase):
+    def test_c(self):
+        from _testcapi import getargs_c
+        self.assertRaises(TypeError, getargs_c, b'abc')  # len > 1
+        self.assertEqual(getargs_c(b'a'), b'a')
+        self.assertEqual(getargs_c(bytearray(b'a')), b'a')
+        self.assertRaises(TypeError, getargs_c, memoryview(b'a'))
+        self.assertRaises(TypeError, getargs_c, 's')
+        self.assertRaises(TypeError, getargs_c, None)
+
     def test_s(self):
         from _testcapi import getargs_s
         self.assertEqual(getargs_s('abc\xe9'), b'abc\xc3\xa9')
@@ -430,8 +540,10 @@ def test_main():
     tests = [
         Signed_TestCase,
         Unsigned_TestCase,
+        Boolean_TestCase,
         Tuple_TestCase,
         Keywords_TestCase,
+        KeywordOnly_TestCase,
         Bytes_TestCase,
         Unicode_TestCase,
     ]
diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py
index 1cdc24acf2..eb9aeb5776 100644
--- a/Lib/test/test_glob.py
+++ b/Lib/test/test_glob.py
@@ -4,7 +4,8 @@ import shutil
 import sys
 import unittest
 
-from test.support import run_unittest, TESTFN, skip_unless_symlink, can_symlink
+from test.support import (run_unittest, TESTFN, skip_unless_symlink,
+                          can_symlink, create_empty_file)
 
 
 class GlobTests(unittest.TestCase):
@@ -17,8 +18,7 @@ class GlobTests(unittest.TestCase):
         base, file = os.path.split(filename)
         if not os.path.exists(base):
             os.makedirs(base)
-        f = open(filename, 'w')
-        f.close()
+        create_empty_file(filename)
 
     def setUp(self):
         self.tempdir = TESTFN + "_dir"
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 268a633b2c..6b326bdaa1 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -10,7 +10,7 @@ from sys import *
 
 class TokenTests(unittest.TestCase):
 
-    def testBackslash(self):
+    def test_backslash(self):
         # Backslash means line continuation:
         x = 1 \
         + 1
@@ -20,7 +20,7 @@ class TokenTests(unittest.TestCase):
         x = 0
         self.assertEqual(x, 0, 'backslash ending comment')
 
-    def testPlainIntegers(self):
+    def test_plain_integers(self):
         self.assertEqual(type(000), type(0))
         self.assertEqual(0xff, 255)
         self.assertEqual(0o377, 255)
@@ -56,7 +56,7 @@ class TokenTests(unittest.TestCase):
         else:
             self.fail('Weird maxsize value %r' % maxsize)
 
-    def testLongIntegers(self):
+    def test_long_integers(self):
         x = 0
         x = 0xffffffffffffffff
         x = 0Xffffffffffffffff
@@ -66,7 +66,7 @@ class TokenTests(unittest.TestCase):
         x = 0b100000000000000000000000000000000000000000000000000000000000000000000
         x = 0B111111111111111111111111111111111111111111111111111111111111111111111
 
-    def testFloats(self):
+    def test_floats(self):
         x = 3.14
         x = 314.
         x = 0.314
@@ -80,7 +80,7 @@ class TokenTests(unittest.TestCase):
         x = .3e14
         x = 3.1e4
 
-    def testStringLiterals(self):
+    def test_string_literals(self):
         x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y)
         x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39)
         x = '"'; y = "\""; self.assertTrue(len(x) == 1 and x == y and ord(x) == 34)
@@ -120,11 +120,18 @@ the \'lazy\' dog.\n\
 '
         self.assertEqual(x, y)
 
-    def testEllipsis(self):
+    def test_ellipsis(self):
         x = ...
         self.assertTrue(x is Ellipsis)
         self.assertRaises(SyntaxError, eval, ".. .")
 
+    def test_eof_error(self):
+        samples = ("def foo(", "\ndef foo(", "def foo(\n")
+        for s in samples:
+            with self.assertRaises(SyntaxError) as cm:
+                compile(s, "<test>", "exec")
+            self.assertIn("unexpected EOF", str(cm.exception))
+
 class GrammarTests(unittest.TestCase):
 
     # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
@@ -136,11 +143,11 @@ class GrammarTests(unittest.TestCase):
     # expr_input: testlist NEWLINE
     # XXX Hard to test -- used only in calls to input()
 
-    def testEvalInput(self):
+    def test_eval_input(self):
         # testlist ENDMARKER
         x = eval('1, 0 or 1')
 
-    def testFuncdef(self):
+    def test_funcdef(self):
         ### [decorators] 'def' NAME parameters ['->' test] ':' suite
         ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
         ### decorators: decorator+
@@ -324,7 +331,7 @@ class GrammarTests(unittest.TestCase):
         check_syntax_error(self, "f(*g(1=2))")
         check_syntax_error(self, "f(**g(1=2))")
 
-    def testLambdef(self):
+    def test_lambdef(self):
         ### lambdef: 'lambda' [varargslist] ':' test
         l1 = lambda : 0
         self.assertEqual(l1(), 0)
@@ -346,7 +353,7 @@ class GrammarTests(unittest.TestCase):
     ### stmt: simple_stmt | compound_stmt
     # Tested below
 
-    def testSimpleStmt(self):
+    def test_simple_stmt(self):
         ### simple_stmt: small_stmt (';' small_stmt)* [';']
         x = 1; pass; del x
         def foo():
@@ -357,7 +364,7 @@ class GrammarTests(unittest.TestCase):
     ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
     # Tested below
 
-    def testExprStmt(self):
+    def test_expr_stmt(self):
         # (exprlist '=')* exprlist
         1
         1, 2, 3
@@ -370,7 +377,7 @@ class GrammarTests(unittest.TestCase):
         check_syntax_error(self, "x + 1 = 1")
         check_syntax_error(self, "a + 1 = b + 2")
 
-    def testDelStmt(self):
+    def test_del_stmt(self):
         # 'del' exprlist
         abc = [1,2,3]
         x, y, z = abc
@@ -379,18 +386,18 @@ class GrammarTests(unittest.TestCase):
         del abc
         del x, y, (z, xyz)
 
-    def testPassStmt(self):
+    def test_pass_stmt(self):
         # 'pass'
         pass
 
     # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
     # Tested below
 
-    def testBreakStmt(self):
+    def test_break_stmt(self):
         # 'break'
         while 1: break
 
-    def testContinueStmt(self):
+    def test_continue_stmt(self):
         # 'continue'
         i = 1
         while i: i = 0; continue
@@ -442,7 +449,7 @@ class GrammarTests(unittest.TestCase):
                 self.fail("continue then break in try/except in loop broken!")
         test_inner()
 
-    def testReturn(self):
+    def test_return(self):
         # 'return' [testlist]
         def g1(): return
         def g2(): return 1
@@ -450,17 +457,49 @@ class GrammarTests(unittest.TestCase):
         x = g2()
         check_syntax_error(self, "class foo:return 1")
 
-    def testYield(self):
+    def test_yield(self):
+        # Allowed as standalone statement
+        def g(): yield 1
+        def g(): yield from ()
+        # Allowed as RHS of assignment
+        def g(): x = yield 1
+        def g(): x = yield from ()
+        # Ordinary yield accepts implicit tuples
+        def g(): yield 1, 1
+        def g(): x = yield 1, 1
+        # 'yield from' does not
+        check_syntax_error(self, "def g(): yield from (), 1")
+        check_syntax_error(self, "def g(): x = yield from (), 1")
+        # Requires parentheses as subexpression
+        def g(): 1, (yield 1)
+        def g(): 1, (yield from ())
+        check_syntax_error(self, "def g(): 1, yield 1")
+        check_syntax_error(self, "def g(): 1, yield from ()")
+        # Requires parentheses as call argument
+        def g(): f((yield 1))
+        def g(): f((yield 1), 1)
+        def g(): f((yield from ()))
+        def g(): f((yield from ()), 1)
+        check_syntax_error(self, "def g(): f(yield 1)")
+        check_syntax_error(self, "def g(): f(yield 1, 1)")
+        check_syntax_error(self, "def g(): f(yield from ())")
+        check_syntax_error(self, "def g(): f(yield from (), 1)")
+        # Not allowed at top level
+        check_syntax_error(self, "yield")
+        check_syntax_error(self, "yield from")
+        # Not allowed at class scope
         check_syntax_error(self, "class foo:yield 1")
+        check_syntax_error(self, "class foo:yield from ()")
+
 
-    def testRaise(self):
+    def test_raise(self):
         # 'raise' test [',' test]
         try: raise RuntimeError('just testing')
         except RuntimeError: pass
         try: raise KeyboardInterrupt
         except KeyboardInterrupt: pass
 
-    def testImport(self):
+    def test_import(self):
         # 'import' dotted_as_names
         import sys
         import time, sys
@@ -473,13 +512,13 @@ class GrammarTests(unittest.TestCase):
         from sys import (path, argv)
         from sys import (path, argv,)
 
-    def testGlobal(self):
+    def test_global(self):
         # 'global' NAME (',' NAME)*
         global a
         global a, b
         global one, two, three, four, five, six, seven, eight, nine, ten
 
-    def testNonlocal(self):
+    def test_nonlocal(self):
         # 'nonlocal' NAME (',' NAME)*
         x = 0
         y = 0
@@ -487,7 +526,7 @@ class GrammarTests(unittest.TestCase):
             nonlocal x
             nonlocal x, y
 
-    def testAssert(self):
+    def test_assert(self):
         # assertTruestmt: 'assert' test [',' test]
         assert 1
         assert 1, 1
@@ -526,7 +565,7 @@ class GrammarTests(unittest.TestCase):
     ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
     # Tested below
 
-    def testIf(self):
+    def test_if(self):
         # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
         if 1: pass
         if 1: pass
@@ -539,7 +578,7 @@ class GrammarTests(unittest.TestCase):
         elif 0: pass
         else: pass
 
-    def testWhile(self):
+    def test_while(self):
         # 'while' test ':' suite ['else' ':' suite]
         while 0: pass
         while 0: pass
@@ -554,7 +593,7 @@ class GrammarTests(unittest.TestCase):
             x = 2
         self.assertEqual(x, 2)
 
-    def testFor(self):
+    def test_for(self):
         # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
         for i in 1, 2, 3: pass
         for i, j, k in (): pass
@@ -581,7 +620,7 @@ class GrammarTests(unittest.TestCase):
             result.append(x)
         self.assertEqual(result, [1, 2, 3])
 
-    def testTry(self):
+    def test_try(self):
         ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
         ###         | 'try' ':' suite 'finally' ':' suite
         ### except_clause: 'except' [expr ['as' expr]]
@@ -604,7 +643,7 @@ class GrammarTests(unittest.TestCase):
         try: pass
         finally: pass
 
-    def testSuite(self):
+    def test_suite(self):
         # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
         if 1: pass
         if 1:
@@ -619,7 +658,7 @@ class GrammarTests(unittest.TestCase):
             pass
             #
 
-    def testTest(self):
+    def test_test(self):
         ### and_test ('or' and_test)*
         ### and_test: not_test ('and' not_test)*
         ### not_test: 'not' not_test | comparison
@@ -630,7 +669,7 @@ class GrammarTests(unittest.TestCase):
         if not 1 and 1 and 1: pass
         if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
 
-    def testComparison(self):
+    def test_comparison(self):
         ### comparison: expr (comp_op expr)*
         ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
         if 1: pass
@@ -647,36 +686,36 @@ class GrammarTests(unittest.TestCase):
         if 1 not in (): pass
         if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
 
-    def testBinaryMaskOps(self):
+    def test_binary_mask_ops(self):
         x = 1 & 1
         x = 1 ^ 1
         x = 1 | 1
 
-    def testShiftOps(self):
+    def test_shift_ops(self):
         x = 1 << 1
         x = 1 >> 1
         x = 1 << 1 >> 1
 
-    def testAdditiveOps(self):
+    def test_additive_ops(self):
         x = 1
         x = 1 + 1
         x = 1 - 1 - 1
         x = 1 - 1 + 1 - 1 + 1
 
-    def testMultiplicativeOps(self):
+    def test_multiplicative_ops(self):
         x = 1 * 1
         x = 1 / 1
         x = 1 % 1
         x = 1 / 1 * 1 % 1
 
-    def testUnaryOps(self):
+    def test_unary_ops(self):
         x = +1
         x = -1
         x = ~1
         x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
         x = -1*1/1 + 1*1 - ---1*1
 
-    def testSelectors(self):
+    def test_selectors(self):
         ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
         ### subscript: expr | [expr] ':' [expr]
 
@@ -706,7 +745,7 @@ class GrammarTests(unittest.TestCase):
         L.sort(key=lambda x: x if isinstance(x, tuple) else ())
         self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
 
-    def testAtoms(self):
+    def test_atoms(self):
         ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
         ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
 
@@ -741,7 +780,7 @@ class GrammarTests(unittest.TestCase):
     ### testlist: test (',' test)* [',']
     # These have been exercised enough above
 
-    def testClassdef(self):
+    def test_classdef(self):
         # 'class' NAME ['(' [testlist] ')'] ':' suite
         class B: pass
         class B2(): pass
@@ -760,14 +799,14 @@ class GrammarTests(unittest.TestCase):
         @class_decorator
         class G: pass
 
-    def testDictcomps(self):
+    def test_dictcomps(self):
         # dictorsetmaker: ( (test ':' test (comp_for |
         #                                   (',' test ':' test)* [','])) |
         #                   (test (comp_for | (',' test)* [','])) )
         nums = [1, 2, 3]
         self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
 
-    def testListcomps(self):
+    def test_listcomps(self):
         # list comprehension tests
         nums = [1, 2, 3, 4, 5]
         strs = ["Apple", "Banana", "Coconut"]
@@ -830,7 +869,7 @@ class GrammarTests(unittest.TestCase):
         self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
                              ('Macdonalds', 'Cheeseburger')])
 
-    def testGenexps(self):
+    def test_genexps(self):
         # generator expression tests
         g = ([x for x in range(10)] for x in range(1))
         self.assertEqual(next(g), [x for x in range(10)])
@@ -865,7 +904,7 @@ class GrammarTests(unittest.TestCase):
         check_syntax_error(self, "foo(x for x in range(10), 100)")
         check_syntax_error(self, "foo(100, x for x in range(10))")
 
-    def testComprehensionSpecials(self):
+    def test_comprehension_specials(self):
         # test for outmost iterable precomputation
         x = 10; g = (i for i in range(x)); x = 5
         self.assertEqual(len(list(g)), 10)
@@ -904,7 +943,7 @@ class GrammarTests(unittest.TestCase):
         with manager() as x, manager():
             pass
 
-    def testIfElseExpr(self):
+    def test_if_else_expr(self):
         # Test ifelse expressions in various cases
         def _checkeval(msg, ret):
             "helper to check that evaluation of expressions is done correctly"
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
index ba9d7dac8b..ebd4c43855 100644
--- a/Lib/test/test_gzip.py
+++ b/Lib/test/test_gzip.py
@@ -33,7 +33,7 @@ class UnseekableIO(io.BytesIO):
         raise io.UnsupportedOperation
 
 
-class TestGzip(unittest.TestCase):
+class BaseTest(unittest.TestCase):
     filename = support.TESTFN
 
     def setUp(self):
@@ -43,6 +43,7 @@ class TestGzip(unittest.TestCase):
         support.unlink(self.filename)
 
 
+class TestGzip(BaseTest):
     def test_write(self):
         with gzip.GzipFile(self.filename, 'wb') as f:
             f.write(data1 * 50)
@@ -64,6 +65,21 @@ class TestGzip(unittest.TestCase):
             d = f.read()
         self.assertEqual(d, data1*50)
 
+    def test_read1(self):
+        self.test_write()
+        blocks = []
+        nread = 0
+        with gzip.GzipFile(self.filename, 'r') as f:
+            while True:
+                d = f.read1()
+                if not d:
+                    break
+                blocks.append(d)
+                nread += len(d)
+                # Check that position was updated correctly (see issue10791).
+                self.assertEqual(f.tell(), nread)
+        self.assertEqual(b''.join(blocks), data1 * 50)
+
     def test_io_on_closed_object(self):
         # Test that I/O operations on closed GzipFile objects raise a
         # ValueError, just like the corresponding functions on file objects.
@@ -100,14 +116,14 @@ class TestGzip(unittest.TestCase):
         # Bug #1074261 was triggered when reading a file that contained
         # many, many members.  Create such a file and verify that reading it
         # works.
-        with gzip.open(self.filename, 'wb', 9) as f:
+        with gzip.GzipFile(self.filename, 'wb', 9) as f:
             f.write(b'a')
         for i in range(0, 200):
-            with gzip.open(self.filename, "ab", 9) as f: # append
+            with gzip.GzipFile(self.filename, "ab", 9) as f: # append
                 f.write(b'a')
 
         # Try reading the file
-        with gzip.open(self.filename, "rb") as zgfile:
+        with gzip.GzipFile(self.filename, "rb") as zgfile:
             contents = b""
             while 1:
                 ztxt = zgfile.read(8192)
@@ -124,7 +140,7 @@ class TestGzip(unittest.TestCase):
             with io.BufferedReader(f) as r:
                 lines = [line for line in r]
 
-        self.assertEqual(lines, 50 * data1.splitlines(True))
+        self.assertEqual(lines, 50 * data1.splitlines(keepends=True))
 
     def test_readline(self):
         self.test_write()
@@ -323,6 +339,14 @@ class TestGzip(unittest.TestCase):
             self.assertEqual(f.read(100), b'')
             self.assertEqual(nread, len(uncompressed))
 
+    def test_textio_readlines(self):
+        # Issue #10791: TextIOWrapper.readlines() fails when wrapping GzipFile.
+        lines = (data1 * 50).decode("ascii").splitlines(keepends=True)
+        self.test_write()
+        with gzip.GzipFile(self.filename, 'r') as f:
+            with io.TextIOWrapper(f, encoding="ascii") as t:
+                self.assertEqual(t.readlines(), lines)
+
     def test_fileobj_from_fdopen(self):
         # Issue #13781: Opening a GzipFile for writing fails when using a
         # fileobj created with os.fdopen().
@@ -380,8 +404,108 @@ class TestGzip(unittest.TestCase):
                 self.assertRaises(EOFError, f.read, 1)
 
 
+class TestOpen(BaseTest):
+    def test_binary_modes(self):
+        uncompressed = data1 * 50
+        with gzip.open(self.filename, "wb") as f:
+            f.write(uncompressed)
+        with open(self.filename, "rb") as f:
+            file_data = gzip.decompress(f.read())
+            self.assertEqual(file_data, uncompressed)
+        with gzip.open(self.filename, "rb") as f:
+            self.assertEqual(f.read(), uncompressed)
+        with gzip.open(self.filename, "ab") as f:
+            f.write(uncompressed)
+        with open(self.filename, "rb") as f:
+            file_data = gzip.decompress(f.read())
+            self.assertEqual(file_data, uncompressed * 2)
+
+    def test_implicit_binary_modes(self):
+        # Test implicit binary modes (no "b" or "t" in mode string).
+        uncompressed = data1 * 50
+        with gzip.open(self.filename, "w") as f:
+            f.write(uncompressed)
+        with open(self.filename, "rb") as f:
+            file_data = gzip.decompress(f.read())
+            self.assertEqual(file_data, uncompressed)
+        with gzip.open(self.filename, "r") as f:
+            self.assertEqual(f.read(), uncompressed)
+        with gzip.open(self.filename, "a") as f:
+            f.write(uncompressed)
+        with open(self.filename, "rb") as f:
+            file_data = gzip.decompress(f.read())
+            self.assertEqual(file_data, uncompressed * 2)
+
+    def test_text_modes(self):
+        uncompressed = data1.decode("ascii") * 50
+        uncompressed_raw = uncompressed.replace("\n", os.linesep)
+        with gzip.open(self.filename, "wt") as f:
+            f.write(uncompressed)
+        with open(self.filename, "rb") as f:
+            file_data = gzip.decompress(f.read()).decode("ascii")
+            self.assertEqual(file_data, uncompressed_raw)
+        with gzip.open(self.filename, "rt") as f:
+            self.assertEqual(f.read(), uncompressed)
+        with gzip.open(self.filename, "at") as f:
+            f.write(uncompressed)
+        with open(self.filename, "rb") as f:
+            file_data = gzip.decompress(f.read()).decode("ascii")
+            self.assertEqual(file_data, uncompressed_raw * 2)
+
+    def test_fileobj(self):
+        uncompressed_bytes = data1 * 50
+        uncompressed_str = uncompressed_bytes.decode("ascii")
+        compressed = gzip.compress(uncompressed_bytes)
+        with gzip.open(io.BytesIO(compressed), "r") as f:
+            self.assertEqual(f.read(), uncompressed_bytes)
+        with gzip.open(io.BytesIO(compressed), "rb") as f:
+            self.assertEqual(f.read(), uncompressed_bytes)
+        with gzip.open(io.BytesIO(compressed), "rt") as f:
+            self.assertEqual(f.read(), uncompressed_str)
+
+    def test_bad_params(self):
+        # Test invalid parameter combinations.
+        with self.assertRaises(TypeError):
+            gzip.open(123.456)
+        with self.assertRaises(ValueError):
+            gzip.open(self.filename, "wbt")
+        with self.assertRaises(ValueError):
+            gzip.open(self.filename, "rb", encoding="utf-8")
+        with self.assertRaises(ValueError):
+            gzip.open(self.filename, "rb", errors="ignore")
+        with self.assertRaises(ValueError):
+            gzip.open(self.filename, "rb", newline="\n")
+
+    def test_encoding(self):
+        # Test non-default encoding.
+        uncompressed = data1.decode("ascii") * 50
+        uncompressed_raw = uncompressed.replace("\n", os.linesep)
+        with gzip.open(self.filename, "wt", encoding="utf-16") as f:
+            f.write(uncompressed)
+        with open(self.filename, "rb") as f:
+            file_data = gzip.decompress(f.read()).decode("utf-16")
+            self.assertEqual(file_data, uncompressed_raw)
+        with gzip.open(self.filename, "rt", encoding="utf-16") as f:
+            self.assertEqual(f.read(), uncompressed)
+
+    def test_encoding_error_handler(self):
+        # Test with non-default encoding error handler.
+        with gzip.open(self.filename, "wb") as f:
+            f.write(b"foo\xffbar")
+        with gzip.open(self.filename, "rt", encoding="ascii", errors="ignore") \
+                as f:
+            self.assertEqual(f.read(), "foobar")
+
+    def test_newline(self):
+        # Test with explicit newline (universal newline mode disabled).
+        uncompressed = data1.decode("ascii") * 50
+        with gzip.open(self.filename, "wt", newline="\n") as f:
+            f.write(uncompressed)
+        with gzip.open(self.filename, "rt", newline="\r") as f:
+            self.assertEqual(f.readlines(), [uncompressed])
+
 def test_main(verbose=None):
-    support.run_unittest(TestGzip)
+    support.run_unittest(TestGzip, TestOpen)
 
 if __name__ == "__main__":
     test_main(verbose=True)
diff --git a/Lib/test/test_hash.py b/Lib/test/test_hash.py
index c0dd34dc7f..e3ab6e4385 100644
--- a/Lib/test/test_hash.py
+++ b/Lib/test/test_hash.py
@@ -7,7 +7,6 @@ import datetime
 import os
 import sys
 import unittest
-from test import support
 from test.script_helper import assert_python_ok
 from collections import Hashable
 
@@ -45,6 +44,16 @@ class HashEqualityTestCase(unittest.TestCase):
         self.same_hash(int(1.23e300), float(1.23e300))
         self.same_hash(float(0.5), complex(0.5, 0.0))
 
+    def test_unaligned_buffers(self):
+        # The hash function for bytes-like objects shouldn't have
+        # alignment-dependent results (example in issue #16427).
+        b = b"123456789abcdefghijklmnopqrstuvwxyz" * 128
+        for i in range(16):
+            for j in range(16):
+                aligned = b[i:128+j]
+                unaligned = memoryview(b)[i:128+j]
+                self.assertEqual(hash(aligned), hash(unaligned))
+
 
 _default_hash = object.__hash__
 class DefaultHash(object): pass
@@ -113,8 +122,7 @@ class DefaultIterSeq(object):
         return self.seq[index]
 
 class HashBuiltinsTestCase(unittest.TestCase):
-    hashes_to_check = [range(10),
-                       enumerate(range(10)),
+    hashes_to_check = [enumerate(range(10)),
                        iter(DefaultIterSeq()),
                        iter(lambda: 0, 0),
                       ]
@@ -124,7 +132,7 @@ class HashBuiltinsTestCase(unittest.TestCase):
         for obj in self.hashes_to_check:
             self.assertEqual(hash(obj), _default_hash(obj))
 
-class HashRandomizationTests(unittest.TestCase):
+class HashRandomizationTests:
 
     # Each subclass should define a field "repr_", containing the repr() of
     # an object to be tested
@@ -160,8 +168,8 @@ class StringlikeHashRandomizationTests(HashRandomizationTests):
         else:
             known_hash_of_obj = -1600925533
 
-        # Randomization is disabled by default:
-        self.assertEqual(self.get_hash(self.repr_), known_hash_of_obj)
+        # Randomization is enabled by default:
+        self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
 
         # It can also be disabled by setting the seed to 0:
         self.assertEqual(self.get_hash(self.repr_, seed=0), known_hash_of_obj)
@@ -181,42 +189,40 @@ class StringlikeHashRandomizationTests(HashRandomizationTests):
                 h = -1024014457
         self.assertEqual(self.get_hash(self.repr_, seed=42), h)
 
-class StrHashRandomizationTests(StringlikeHashRandomizationTests):
+class StrHashRandomizationTests(StringlikeHashRandomizationTests,
+                                unittest.TestCase):
     repr_ = repr('abc')
 
     def test_empty_string(self):
         self.assertEqual(hash(""), 0)
 
-class BytesHashRandomizationTests(StringlikeHashRandomizationTests):
+class BytesHashRandomizationTests(StringlikeHashRandomizationTests,
+                                  unittest.TestCase):
     repr_ = repr(b'abc')
 
     def test_empty_string(self):
         self.assertEqual(hash(b""), 0)
 
+class MemoryviewHashRandomizationTests(StringlikeHashRandomizationTests,
+                                       unittest.TestCase):
+    repr_ = "memoryview(b'abc')"
+
+    def test_empty_string(self):
+        self.assertEqual(hash(memoryview(b"")), 0)
+
 class DatetimeTests(HashRandomizationTests):
     def get_hash_command(self, repr_):
         return 'import datetime; print(hash(%s))' % repr_
 
-class DatetimeDateTests(DatetimeTests):
+class DatetimeDateTests(DatetimeTests, unittest.TestCase):
     repr_ = repr(datetime.date(1066, 10, 14))
 
-class DatetimeDatetimeTests(DatetimeTests):
+class DatetimeDatetimeTests(DatetimeTests, unittest.TestCase):
     repr_ = repr(datetime.datetime(1, 2, 3, 4, 5, 6, 7))
 
-class DatetimeTimeTests(DatetimeTests):
+class DatetimeTimeTests(DatetimeTests, unittest.TestCase):
     repr_ = repr(datetime.time(0))
 
 
-def test_main():
-    support.run_unittest(HashEqualityTestCase,
-                         HashInheritanceTestCase,
-                         HashBuiltinsTestCase,
-                         StrHashRandomizationTests,
-                         BytesHashRandomizationTests,
-                         DatetimeDateTests,
-                         DatetimeDatetimeTests,
-                         DatetimeTimeTests)
-
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 29d3a1cc44..32f85e9b48 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -9,6 +9,7 @@
 import array
 import hashlib
 import itertools
+import os
 import sys
 try:
     import threading
@@ -37,7 +38,8 @@ class HashLibTestCase(unittest.TestCase):
                              'sha224', 'SHA224', 'sha256', 'SHA256',
                              'sha384', 'SHA384', 'sha512', 'SHA512' )
 
-    _warn_on_extension_import = COMPILED_WITH_PYDEBUG
+    # Issue #14693: fallback modules are always compiled under POSIX
+    _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG
 
     def _conditional_import_module(self, module_name):
         """Import a module and return a reference to it or None on failure."""
diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py
index 20fb89c000..b48ca68461 100644
--- a/Lib/test/test_heapq.py
+++ b/Lib/test/test_heapq.py
@@ -2,6 +2,7 @@
 
 import sys
 import random
+import unittest
 
 from test import support
 from unittest import TestCase, skipUnless
@@ -25,8 +26,7 @@ class TestModules(TestCase):
             self.assertEqual(getattr(c_heapq, fname).__module__, '_heapq')
 
 
-class TestHeap(TestCase):
-    module = None
+class TestHeap:
 
     def test_push_pop(self):
         # 1) Push 256 random numbers and pop them off, verifying all's OK.
@@ -214,12 +214,12 @@ class TestHeap(TestCase):
         self.assertRaises(TypeError, data, LE)
 
 
-class TestHeapPython(TestHeap):
+class TestHeapPython(TestHeap, TestCase):
     module = py_heapq
 
 
 @skipUnless(c_heapq, 'requires _heapq')
-class TestHeapC(TestHeap):
+class TestHeapC(TestHeap, TestCase):
     module = c_heapq
 
 
@@ -329,8 +329,7 @@ class SideEffectLT:
         return self.value < other.value
 
 
-class TestErrorHandling(TestCase):
-    module = None
+class TestErrorHandling:
 
     def test_non_sequence(self):
         for f in (self.module.heapify, self.module.heappop):
@@ -397,31 +396,13 @@ class TestErrorHandling(TestCase):
             self.module.heappop(heap)
 
 
-class TestErrorHandlingPython(TestErrorHandling):
+class TestErrorHandlingPython(TestErrorHandling, TestCase):
     module = py_heapq
 
 @skipUnless(c_heapq, 'requires _heapq')
-class TestErrorHandlingC(TestErrorHandling):
+class TestErrorHandlingC(TestErrorHandling, TestCase):
     module = c_heapq
 
 
-#==============================================================================
-
-
-def test_main(verbose=None):
-    test_classes = [TestModules, TestHeapPython, TestHeapC,
-                    TestErrorHandlingPython, TestErrorHandlingC]
-    support.run_unittest(*test_classes)
-
-    # verify reference counting
-    if verbose and hasattr(sys, "gettotalrefcount"):
-        import gc
-        counts = [None] * 5
-        for i in range(len(counts)):
-            support.run_unittest(*test_classes)
-            gc.collect()
-            counts[i] = sys.gettotalrefcount()
-        print(counts)
-
 if __name__ == "__main__":
-    test_main(verbose=True)
+    unittest.main()
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index 4de0620e1a..4ca7cec44c 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -234,6 +234,18 @@ class ConstructorTestCase(unittest.TestCase):
         except:
             self.fail("Standard constructor call raised exception.")
 
+    def test_with_str_key(self):
+        # Pass a key of type str, which is an error, because it expects a key
+        # of type bytes
+        with self.assertRaises(TypeError):
+            h = hmac.HMAC("key")
+
+    def test_dot_new_with_str_key(self):
+        # Pass a key of type str, which is an error, because it expects a key
+        # of type bytes
+        with self.assertRaises(TypeError):
+            h = hmac.new("key")
+
     def test_withtext(self):
         # Constructor call with text.
         try:
@@ -302,12 +314,121 @@ class CopyTestCase(unittest.TestCase):
         self.assertEqual(h1.hexdigest(), h2.hexdigest(),
             "Hexdigest of copy doesn't match original hexdigest.")
 
+class CompareDigestTestCase(unittest.TestCase):
+
+    def test_compare_digest(self):
+        # Testing input type exception handling
+        a, b = 100, 200
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = 100, b"foobar"
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = b"foobar", 200
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = "foobar", b"foobar"
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = b"foobar", "foobar"
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+
+        # Testing bytes of different lengths
+        a, b = b"foobar", b"foo"
+        self.assertFalse(hmac.compare_digest(a, b))
+        a, b = b"\xde\xad\xbe\xef", b"\xde\xad"
+        self.assertFalse(hmac.compare_digest(a, b))
+
+        # Testing bytes of same lengths, different values
+        a, b = b"foobar", b"foobaz"
+        self.assertFalse(hmac.compare_digest(a, b))
+        a, b = b"\xde\xad\xbe\xef", b"\xab\xad\x1d\xea"
+        self.assertFalse(hmac.compare_digest(a, b))
+
+        # Testing bytes of same lengths, same values
+        a, b = b"foobar", b"foobar"
+        self.assertTrue(hmac.compare_digest(a, b))
+        a, b = b"\xde\xad\xbe\xef", b"\xde\xad\xbe\xef"
+        self.assertTrue(hmac.compare_digest(a, b))
+
+        # Testing bytearrays of same lengths, same values
+        a, b = bytearray(b"foobar"), bytearray(b"foobar")
+        self.assertTrue(hmac.compare_digest(a, b))
+
+        # Testing bytearrays of diffeent lengths
+        a, b = bytearray(b"foobar"), bytearray(b"foo")
+        self.assertFalse(hmac.compare_digest(a, b))
+
+        # Testing bytearrays of same lengths, different values
+        a, b = bytearray(b"foobar"), bytearray(b"foobaz")
+        self.assertFalse(hmac.compare_digest(a, b))
+
+        # Testing byte and bytearray of same lengths, same values
+        a, b = bytearray(b"foobar"), b"foobar"
+        self.assertTrue(hmac.compare_digest(a, b))
+        self.assertTrue(hmac.compare_digest(b, a))
+
+        # Testing byte bytearray of diffeent lengths
+        a, b = bytearray(b"foobar"), b"foo"
+        self.assertFalse(hmac.compare_digest(a, b))
+        self.assertFalse(hmac.compare_digest(b, a))
+
+        # Testing byte and bytearray of same lengths, different values
+        a, b = bytearray(b"foobar"), b"foobaz"
+        self.assertFalse(hmac.compare_digest(a, b))
+        self.assertFalse(hmac.compare_digest(b, a))
+
+        # Testing str of same lengths
+        a, b = "foobar", "foobar"
+        self.assertTrue(hmac.compare_digest(a, b))
+
+        # Testing str of diffeent lengths
+        a, b = "foo", "foobar"
+        self.assertFalse(hmac.compare_digest(a, b))
+
+        # Testing bytes of same lengths, different values
+        a, b = "foobar", "foobaz"
+        self.assertFalse(hmac.compare_digest(a, b))
+
+        # Testing error cases
+        a, b = "foobar", b"foobar"
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = b"foobar", "foobar"
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = b"foobar", 1
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = 100, 200
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+        a, b = "fooä", "fooä"
+        self.assertRaises(TypeError, hmac.compare_digest, a, b)
+
+        # subclasses are supported by ignore __eq__
+        class mystr(str):
+            def __eq__(self, other):
+                return False
+
+        a, b = mystr("foobar"), mystr("foobar")
+        self.assertTrue(hmac.compare_digest(a, b))
+        a, b = mystr("foobar"), "foobar"
+        self.assertTrue(hmac.compare_digest(a, b))
+        a, b = mystr("foobar"), mystr("foobaz")
+        self.assertFalse(hmac.compare_digest(a, b))
+
+        class mybytes(bytes):
+            def __eq__(self, other):
+                return False
+
+        a, b = mybytes(b"foobar"), mybytes(b"foobar")
+        self.assertTrue(hmac.compare_digest(a, b))
+        a, b = mybytes(b"foobar"), b"foobar"
+        self.assertTrue(hmac.compare_digest(a, b))
+        a, b = mybytes(b"foobar"), mybytes(b"foobaz")
+        self.assertFalse(hmac.compare_digest(a, b))
+
+
 def test_main():
     support.run_unittest(
         TestVectorsTestCase,
         ConstructorTestCase,
         SanityTestCase,
-        CopyTestCase
+        CopyTestCase,
+        CompareDigestTestCase
     )
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py
index c4f80cca30..c5d878dca5 100644
--- a/Lib/test/test_htmlparser.py
+++ b/Lib/test/test_htmlparser.py
@@ -102,7 +102,8 @@ class TestCaseBase(unittest.TestCase):
 class HTMLParserStrictTestCase(TestCaseBase):
 
     def get_collector(self):
-        return EventCollector(strict=True)
+        with support.check_warnings(("", DeprecationWarning), quite=False):
+            return EventCollector(strict=True)
 
     def test_processing_instruction_only(self):
         self._run_check("<?processing instruction>", [
@@ -455,7 +456,7 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
         self._run_check('<form action="/xxx.php?a=1&amp;b=2&amp", '
                         'method="post">', [
                             ('starttag', 'form',
-                                [('action', '/xxx.php?a=1&b=2&amp'),
+                                [('action', '/xxx.php?a=1&b=2&'),
                                  (',', None), ('method', 'post')])])
 
     def test_weird_chars_in_unquoted_attribute_values(self):
@@ -540,6 +541,11 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
         self.assertEqual(p.unescape('&#0038;'),'&')
         # see #12888
         self.assertEqual(p.unescape('&#123; ' * 1050), '{ ' * 1050)
+        # see #15156
+        self.assertEqual(p.unescape('&Eacuteric&Eacute;ric'
+                                    '&alphacentauri&alpha;centauri'),
+                                    'ÉricÉric&alphacentauriαcentauri')
+        self.assertEqual(p.unescape('&co;'), '&co;')
 
     def test_broken_comments(self):
         html = ('<! not really a comment >'
@@ -594,7 +600,8 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
 class AttributesStrictTestCase(TestCaseBase):
 
     def get_collector(self):
-        return EventCollector(strict=True)
+        with support.check_warnings(("", DeprecationWarning), quite=False):
+            return EventCollector(strict=True)
 
     def test_attr_syntax(self):
         output = [
diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py
index 41e0dfde33..a35ec95a69 100644
--- a/Lib/test/test_http_cookiejar.py
+++ b/Lib/test/test_http_cookiejar.py
@@ -248,18 +248,19 @@ class FileCookieJarTests(unittest.TestCase):
         self.assertEqual(c._cookies["www.acme.com"]["/"]["boo"].value, None)
 
     def test_bad_magic(self):
-        # IOErrors (eg. file doesn't exist) are allowed to propagate
+        # OSErrors (eg. file doesn't exist) are allowed to propagate
         filename = test.support.TESTFN
         for cookiejar_class in LWPCookieJar, MozillaCookieJar:
             c = cookiejar_class()
             try:
                 c.load(filename="for this test to work, a file with this "
                                 "filename should not exist")
-            except IOError as exc:
-                # exactly IOError, not LoadError
-                self.assertIs(exc.__class__, IOError)
+            except OSError as exc:
+                # an OSError subclass (likely FileNotFoundError), but not
+                # LoadError
+                self.assertIsNot(exc.__class__, LoadError)
             else:
-                self.fail("expected IOError for invalid filename")
+                self.fail("expected OSError for invalid filename")
         # Invalid contents of cookies file (eg. bad magic string)
         # causes a LoadError.
         try:
diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py
index 1f1ca5852e..e8327e59eb 100644
--- a/Lib/test/test_http_cookies.py
+++ b/Lib/test/test_http_cookies.py
@@ -34,6 +34,15 @@ class CookieTests(unittest.TestCase):
              'dict': {'keebler' : 'E=mc2'},
              'repr': "<SimpleCookie: keebler='E=mc2'>",
              'output': 'Set-Cookie: keebler=E=mc2'},
+
+            # Cookies with ':' character in their name. Though not mentioned in
+            # RFC, servers / browsers allow it.
+
+             {'data': 'key:term=value:term',
+             'dict': {'key:term' : 'value:term'},
+             'repr': "<SimpleCookie: key:term='value:term'>",
+             'output': 'Set-Cookie: key:term=value:term'},
+
         ]
 
         for case in cases:
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index 420302c332..db123dcb56 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -192,6 +192,26 @@ class BasicTest(TestCase):
         resp.close()
         self.assertTrue(resp.closed)
 
+    def test_partial_readintos(self):
+        # if we have a length, the system knows when to close itself
+        # same behaviour than when we read the whole thing with read()
+        body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText"
+        sock = FakeSocket(body)
+        resp = client.HTTPResponse(sock)
+        resp.begin()
+        b = bytearray(2)
+        n = resp.readinto(b)
+        self.assertEqual(n, 2)
+        self.assertEqual(bytes(b), b'Te')
+        self.assertFalse(resp.isclosed())
+        n = resp.readinto(b)
+        self.assertEqual(n, 2)
+        self.assertEqual(bytes(b), b'xt')
+        self.assertTrue(resp.isclosed())
+        self.assertFalse(resp.closed)
+        resp.close()
+        self.assertTrue(resp.closed)
+
     def test_partial_reads_no_content_length(self):
         # when no length is present, the socket should be gracefully closed when
         # all data was read
@@ -208,6 +228,25 @@ class BasicTest(TestCase):
         resp.close()
         self.assertTrue(resp.closed)
 
+    def test_partial_readintos_no_content_length(self):
+        # when no length is present, the socket should be gracefully closed when
+        # all data was read
+        body = "HTTP/1.1 200 Ok\r\n\r\nText"
+        sock = FakeSocket(body)
+        resp = client.HTTPResponse(sock)
+        resp.begin()
+        b = bytearray(2)
+        n = resp.readinto(b)
+        self.assertEqual(n, 2)
+        self.assertEqual(bytes(b), b'Te')
+        self.assertFalse(resp.isclosed())
+        n = resp.readinto(b)
+        self.assertEqual(n, 2)
+        self.assertEqual(bytes(b), b'xt')
+        n = resp.readinto(b)
+        self.assertEqual(n, 0)
+        self.assertTrue(resp.isclosed())
+
     def test_partial_reads_incomplete_body(self):
         # if the server shuts down the connection before the whole
         # content-length is delivered, the socket is gracefully closed
@@ -220,6 +259,25 @@ class BasicTest(TestCase):
         self.assertEqual(resp.read(2), b'xt')
         self.assertEqual(resp.read(1), b'')
         self.assertTrue(resp.isclosed())
+
+    def test_partial_readintos_incomplete_body(self):
+        # if the server shuts down the connection before the whole
+        # content-length is delivered, the socket is gracefully closed
+        body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText"
+        sock = FakeSocket(body)
+        resp = client.HTTPResponse(sock)
+        resp.begin()
+        b = bytearray(2)
+        n = resp.readinto(b)
+        self.assertEqual(n, 2)
+        self.assertEqual(bytes(b), b'Te')
+        self.assertFalse(resp.isclosed())
+        n = resp.readinto(b)
+        self.assertEqual(n, 2)
+        self.assertEqual(bytes(b), b'xt')
+        n = resp.readinto(b)
+        self.assertEqual(n, 0)
+        self.assertTrue(resp.isclosed())
         self.assertFalse(resp.closed)
         resp.close()
         self.assertTrue(resp.closed)
@@ -272,6 +330,21 @@ class BasicTest(TestCase):
         if resp.read():
             self.fail("Did not expect response from HEAD request")
 
+    def test_readinto_head(self):
+        # Test that the library doesn't attempt to read any data
+        # from a HEAD request.  (Tickles SF bug #622042.)
+        sock = FakeSocket(
+            'HTTP/1.1 200 OK\r\n'
+            'Content-Length: 14432\r\n'
+            '\r\n',
+            NoEOFStringIO)
+        resp = client.HTTPResponse(sock, method="HEAD")
+        resp.begin()
+        b = bytearray(5)
+        if resp.readinto(b) != 0:
+            self.fail("Did not expect response from HEAD request")
+        self.assertEqual(bytes(b), b'\x00'*5)
+
     def test_send_file(self):
         expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n'
                     b'Accept-Encoding: identity\r\nContent-Length:')
@@ -327,15 +400,28 @@ class BasicTest(TestCase):
             'Transfer-Encoding: chunked\r\n\r\n'
             'a\r\n'
             'hello worl\r\n'
-            '1\r\n'
-            'd\r\n'
+            '3\r\n'
+            'd! \r\n'
+            '8\r\n'
+            'and now \r\n'
+            '22\r\n'
+            'for something completely different\r\n'
         )
+        expected = b'hello world! and now for something completely different'
         sock = FakeSocket(chunked_start + '0\r\n')
         resp = client.HTTPResponse(sock, method="GET")
         resp.begin()
-        self.assertEqual(resp.read(), b'hello world')
+        self.assertEqual(resp.read(), expected)
         resp.close()
 
+        # Various read sizes
+        for n in range(1, 12):
+            sock = FakeSocket(chunked_start + '0\r\n')
+            resp = client.HTTPResponse(sock, method="GET")
+            resp.begin()
+            self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected)
+            resp.close()
+
         for x in ('', 'foo\r\n'):
             sock = FakeSocket(chunked_start + x)
             resp = client.HTTPResponse(sock, method="GET")
@@ -343,9 +429,64 @@ class BasicTest(TestCase):
             try:
                 resp.read()
             except client.IncompleteRead as i:
-                self.assertEqual(i.partial, b'hello world')
-                self.assertEqual(repr(i),'IncompleteRead(11 bytes read)')
-                self.assertEqual(str(i),'IncompleteRead(11 bytes read)')
+                self.assertEqual(i.partial, expected)
+                expected_message = 'IncompleteRead(%d bytes read)' % len(expected)
+                self.assertEqual(repr(i), expected_message)
+                self.assertEqual(str(i), expected_message)
+            else:
+                self.fail('IncompleteRead expected')
+            finally:
+                resp.close()
+
+    def test_readinto_chunked(self):
+        chunked_start = (
+            'HTTP/1.1 200 OK\r\n'
+            'Transfer-Encoding: chunked\r\n\r\n'
+            'a\r\n'
+            'hello worl\r\n'
+            '3\r\n'
+            'd! \r\n'
+            '8\r\n'
+            'and now \r\n'
+            '22\r\n'
+            'for something completely different\r\n'
+        )
+        expected = b'hello world! and now for something completely different'
+        nexpected = len(expected)
+        b = bytearray(128)
+
+        sock = FakeSocket(chunked_start + '0\r\n')
+        resp = client.HTTPResponse(sock, method="GET")
+        resp.begin()
+        n = resp.readinto(b)
+        self.assertEqual(b[:nexpected], expected)
+        self.assertEqual(n, nexpected)
+        resp.close()
+
+        # Various read sizes
+        for n in range(1, 12):
+            sock = FakeSocket(chunked_start + '0\r\n')
+            resp = client.HTTPResponse(sock, method="GET")
+            resp.begin()
+            m = memoryview(b)
+            i = resp.readinto(m[0:n])
+            i += resp.readinto(m[i:n + i])
+            i += resp.readinto(m[i:])
+            self.assertEqual(b[:nexpected], expected)
+            self.assertEqual(i, nexpected)
+            resp.close()
+
+        for x in ('', 'foo\r\n'):
+            sock = FakeSocket(chunked_start + x)
+            resp = client.HTTPResponse(sock, method="GET")
+            resp.begin()
+            try:
+                n = resp.readinto(b)
+            except client.IncompleteRead as i:
+                self.assertEqual(i.partial, expected)
+                expected_message = 'IncompleteRead(%d bytes read)' % len(expected)
+                self.assertEqual(repr(i), expected_message)
+                self.assertEqual(str(i), expected_message)
             else:
                 self.fail('IncompleteRead expected')
             finally:
@@ -371,6 +512,29 @@ class BasicTest(TestCase):
         resp.close()
         self.assertTrue(resp.closed)
 
+    def test_readinto_chunked_head(self):
+        chunked_start = (
+            'HTTP/1.1 200 OK\r\n'
+            'Transfer-Encoding: chunked\r\n\r\n'
+            'a\r\n'
+            'hello world\r\n'
+            '1\r\n'
+            'd\r\n'
+        )
+        sock = FakeSocket(chunked_start + '0\r\n')
+        resp = client.HTTPResponse(sock, method="HEAD")
+        resp.begin()
+        b = bytearray(5)
+        n = resp.readinto(b)
+        self.assertEqual(n, 0)
+        self.assertEqual(bytes(b), b'\x00'*5)
+        self.assertEqual(resp.status, 200)
+        self.assertEqual(resp.reason, 'OK')
+        self.assertTrue(resp.isclosed())
+        self.assertFalse(resp.closed)
+        resp.close()
+        self.assertTrue(resp.closed)
+
     def test_negative_content_length(self):
         sock = FakeSocket(
             'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n')
@@ -588,8 +752,7 @@ class HTTPSTest(TestCase):
     def test_local_good_hostname(self):
         # The (valid) cert validates the HTTP hostname
         import ssl
-        from test.ssl_servers import make_https_server
-        server = make_https_server(self, CERT_localhost)
+        server = self.make_server(CERT_localhost)
         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = ssl.CERT_REQUIRED
         context.load_verify_locations(CERT_localhost)
@@ -597,12 +760,12 @@ class HTTPSTest(TestCase):
         h.request('GET', '/nonexistent')
         resp = h.getresponse()
         self.assertEqual(resp.status, 404)
+        del server
 
     def test_local_bad_hostname(self):
         # The (valid) cert doesn't validate the HTTP hostname
         import ssl
-        from test.ssl_servers import make_https_server
-        server = make_https_server(self, CERT_fakehostname)
+        server = self.make_server(CERT_fakehostname)
         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = ssl.CERT_REQUIRED
         context.load_verify_locations(CERT_fakehostname)
@@ -620,6 +783,7 @@ class HTTPSTest(TestCase):
         h.request('GET', '/nonexistent')
         resp = h.getresponse()
         self.assertEqual(resp.status, 404)
+        del server
 
     @unittest.skipIf(not hasattr(client, 'HTTPSConnection'),
                      'http.client.HTTPSConnection not available')
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
index d71da1a83d..03c0776ce5 100644
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -470,6 +470,23 @@ class RejectingSocketlessRequestHandler(SocketlessRequestHandler):
         self.send_error(417)
         return False
 
+
+class AuditableBytesIO:
+
+    def __init__(self):
+        self.datas = []
+
+    def write(self, data):
+        self.datas.append(data)
+
+    def getData(self):
+        return b''.join(self.datas)
+
+    @property
+    def numWrites(self):
+        return len(self.datas)
+
+
 class BaseHTTPRequestHandlerTestCase(unittest.TestCase):
     """Test the functionality of the BaseHTTPServer.
 
@@ -536,27 +553,49 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase):
         self.verify_get_called()
         self.assertEqual(result[-1], b'<html><body>Data</body></html>\r\n')
 
-    def test_header_buffering(self):
+    def test_header_buffering_of_send_error(self):
 
-        def _readAndReseek(f):
-            pos = f.tell()
-            f.seek(0)
-            data = f.read()
-            f.seek(pos)
-            return data
+        input = BytesIO(b'GET / HTTP/1.1\r\n\r\n')
+        output = AuditableBytesIO()
+        handler = SocketlessRequestHandler()
+        handler.rfile = input
+        handler.wfile = output
+        handler.request_version = 'HTTP/1.1'
+        handler.requestline = ''
+        handler.command = None
+
+        handler.send_error(418)
+        self.assertEqual(output.numWrites, 2)
+
+    def test_header_buffering_of_send_response_only(self):
 
         input = BytesIO(b'GET / HTTP/1.1\r\n\r\n')
-        output = BytesIO()
-        self.handler.rfile = input
-        self.handler.wfile = output
-        self.handler.request_version = 'HTTP/1.1'
+        output = AuditableBytesIO()
+        handler = SocketlessRequestHandler()
+        handler.rfile = input
+        handler.wfile = output
+        handler.request_version = 'HTTP/1.1'
 
-        self.handler.send_header('Foo', 'foo')
-        self.handler.send_header('bar', 'bar')
-        self.assertEqual(_readAndReseek(output), b'')
-        self.handler.end_headers()
-        self.assertEqual(_readAndReseek(output),
-                         b'Foo: foo\r\nbar: bar\r\n\r\n')
+        handler.send_response_only(418)
+        self.assertEqual(output.numWrites, 0)
+        handler.end_headers()
+        self.assertEqual(output.numWrites, 1)
+
+    def test_header_buffering_of_send_header(self):
+
+        input = BytesIO(b'GET / HTTP/1.1\r\n\r\n')
+        output = AuditableBytesIO()
+        handler = SocketlessRequestHandler()
+        handler.rfile = input
+        handler.wfile = output
+        handler.request_version = 'HTTP/1.1'
+
+        handler.send_header('Foo', 'foo')
+        handler.send_header('bar', 'bar')
+        self.assertEqual(output.numWrites, 0)
+        handler.end_headers()
+        self.assertEqual(output.getData(), b'Foo: foo\r\nbar: bar\r\n\r\n')
+        self.assertEqual(output.numWrites, 1)
 
     def test_header_unbuffered_when_continue(self):
 
diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 6b29943ebd..7db3f7dddc 100644
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -11,9 +11,9 @@ import socketserver
 import time
 import calendar
 
-from test.support import reap_threads, verbose, transient_internet, run_with_tz
+from test.support import reap_threads, verbose, transient_internet, run_with_tz, run_with_locale
 import unittest
-
+from datetime import datetime, timezone, timedelta
 try:
     import ssl
 except ImportError:
@@ -43,14 +43,30 @@ class TestImaplib(unittest.TestCase):
                             imaplib.Internaldate2tuple(
             b'25 (INTERNALDATE "02-Apr-2000 03:30:00 +0000")'))
 
-    def test_that_Time2Internaldate_returns_a_result(self):
-        # We can check only that it successfully produces a result,
-        # not the correctness of the result itself, since the result
-        # depends on the timezone the machine is in.
-        timevalues = [2000000000, 2000000000.0, time.localtime(2000000000),
-                      '"18-May-2033 05:33:20 +0200"']
 
-        for t in timevalues:
+
+    def timevalues(self):
+        return [2000000000, 2000000000.0, time.localtime(2000000000),
+                (2033, 5, 18, 5, 33, 20, -1, -1, -1),
+                (2033, 5, 18, 5, 33, 20, -1, -1, 1),
+                datetime.fromtimestamp(2000000000,
+                                       timezone(timedelta(0, 2*60*60))),
+                '"18-May-2033 05:33:20 +0200"']
+
+    @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
+    @run_with_tz('STD-1DST')
+    def test_Time2Internaldate(self):
+        expected = '"18-May-2033 05:33:20 +0200"'
+
+        for t in self.timevalues():
+            internal = imaplib.Time2Internaldate(t)
+            self.assertEqual(internal, expected)
+
+    def test_that_Time2Internaldate_returns_a_result(self):
+        # Without tzset, we can check only that it successfully
+        # produces a result, not the correctness of the result itself,
+        # since the result depends on the timezone the machine is in.
+        for t in self.timevalues():
             imaplib.Time2Internaldate(t)
 
 
@@ -374,14 +390,61 @@ class RemoteIMAP_SSLTest(RemoteIMAPTest):
     port = 993
     imap_class = IMAP4_SSL
 
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def create_ssl_context(self):
+        ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ssl_context.load_cert_chain(CERTFILE)
+        return ssl_context
+
+    def check_logincapa(self, server):
+        try:
+            for cap in server.capabilities:
+                self.assertIsInstance(cap, str)
+            self.assertNotIn('LOGINDISABLED', server.capabilities)
+            self.assertIn('AUTH=PLAIN', server.capabilities)
+            rs = server.login(self.username, self.password)
+            self.assertEqual(rs[0], 'OK')
+        finally:
+            server.logout()
+
     def test_logincapa(self):
-        for cap in self.server.capabilities:
-            self.assertIsInstance(cap, str)
-        self.assertNotIn('LOGINDISABLED', self.server.capabilities)
-        self.assertIn('AUTH=PLAIN', self.server.capabilities)
+        with transient_internet(self.host):
+            _server = self.imap_class(self.host, self.port)
+            self.check_logincapa(_server)
+
+    def test_logincapa_with_client_certfile(self):
+        with transient_internet(self.host):
+            _server = self.imap_class(self.host, self.port, certfile=CERTFILE)
+            self.check_logincapa(_server)
+
+    def test_logincapa_with_client_ssl_context(self):
+        with transient_internet(self.host):
+            _server = self.imap_class(self.host, self.port, ssl_context=self.create_ssl_context())
+            self.check_logincapa(_server)
+
+    def test_logout(self):
+        with transient_internet(self.host):
+            _server = self.imap_class(self.host, self.port)
+            rs = _server.logout()
+            self.assertEqual(rs[0], 'BYE')
+
+    def test_ssl_context_certfile_exclusive(self):
+        with transient_internet(self.host):
+            self.assertRaises(ValueError, self.imap_class, self.host, self.port,
+                              certfile=CERTFILE, ssl_context=self.create_ssl_context())
+
+    def test_ssl_context_keyfile_exclusive(self):
+        with transient_internet(self.host):
+            self.assertRaises(ValueError, self.imap_class, self.host, self.port,
+                              keyfile=CERTFILE, ssl_context=self.create_ssl_context())
 
 
-def test_main():
+def load_tests(*args):
     tests = [TestImaplib]
 
     if support.is_resource_enabled('network'):
@@ -396,9 +459,9 @@ def test_main():
             RemoteIMAPTest, RemoteIMAP_SSLTest, RemoteIMAP_STARTTLSTest,
         ])
 
-    support.run_unittest(*tests)
+    return unittest.TestSuite([unittest.makeSuite(test) for test in tests])
 
 
 if __name__ == "__main__":
     support.use_resources = ['network']
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index 551ad1b85c..65c9f25c83 100644
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -1,11 +1,12 @@
 import imp
+import importlib
 import os
 import os.path
 import shutil
 import sys
-import unittest
 from test import support
-import importlib
+import unittest
+import warnings
 
 class LockTests(unittest.TestCase):
 
@@ -58,6 +59,10 @@ class ImportTests(unittest.TestCase):
             with imp.find_module('module_' + mod, self.test_path)[0] as fd:
                 self.assertEqual(fd.encoding, encoding)
 
+        path = [os.path.dirname(__file__)]
+        with self.assertRaises(SyntaxError):
+            imp.find_module('badsyntax_pep3120', path)
+
     def test_issue1267(self):
         for mod, encoding, _ in self.test_strings:
             fp, filename, info  = imp.find_module('module_' + mod,
@@ -150,18 +155,24 @@ class ImportTests(unittest.TestCase):
                 mod = imp.load_module(temp_mod_name, file, filename, info)
                 self.assertEqual(mod.a, 1)
 
-            mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
             self.assertEqual(mod.a, 1)
 
-            mod = imp.load_compiled(
-                temp_mod_name, imp.cache_from_source(temp_mod_name + '.py'))
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                mod = imp.load_compiled(
+                    temp_mod_name, imp.cache_from_source(temp_mod_name + '.py'))
             self.assertEqual(mod.a, 1)
 
             if not os.path.exists(test_package_name):
                 os.mkdir(test_package_name)
             with open(init_file_name, 'w') as file:
                 file.write('b = 2\n')
-            package = imp.load_package(test_package_name, test_package_name)
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                package = imp.load_package(test_package_name, test_package_name)
             self.assertEqual(package.b, 2)
         finally:
             del sys.path[0]
@@ -175,6 +186,48 @@ class ImportTests(unittest.TestCase):
         self.assertRaises(SyntaxError,
                           imp.find_module, "badsyntax_pep3120", [path])
 
+    def test_load_from_source(self):
+        # Verify that the imp module can correctly load and find .py files
+        # XXX (ncoghlan): It would be nice to use support.CleanImport
+        # here, but that breaks because the os module registers some
+        # handlers in copy_reg on import. Since CleanImport doesn't
+        # revert that registration, the module is left in a broken
+        # state after reversion. Reinitialising the module contents
+        # and just reverting os.environ to its previous state is an OK
+        # workaround
+        orig_path = os.path
+        orig_getenv = os.getenv
+        with support.EnvironmentVarGuard():
+            x = imp.find_module("os")
+            self.addCleanup(x[0].close)
+            new_os = imp.load_module("os", *x)
+            self.assertIs(os, new_os)
+            self.assertIs(orig_path, new_os.path)
+            self.assertIsNot(orig_getenv, new_os.getenv)
+
+    @support.cpython_only
+    def test_issue15828_load_extensions(self):
+        # Issue 15828 picked up that the adapter between the old imp API
+        # and importlib couldn't handle C extensions
+        example = "_heapq"
+        x = imp.find_module(example)
+        file_ = x[0]
+        if file_ is not None:
+            self.addCleanup(file_.close)
+        mod = imp.load_module(example, *x)
+        self.assertEqual(mod.__name__, example)
+
+    def test_load_dynamic_ImportError_path(self):
+        # Issue #1559549 added `name` and `path` attributes to ImportError
+        # in order to provide better detail. Issue #10854 implemented those
+        # attributes on import failures of extensions on Windows.
+        path = 'bogus file path'
+        name = 'extension'
+        with self.assertRaises(ImportError) as err:
+            imp.load_dynamic(name, path)
+        self.assertIn(path, err.exception.path)
+        self.assertEqual(name, err.exception.name)
+
 
 class ReloadTests(unittest.TestCase):
 
@@ -209,72 +262,83 @@ class PEP3147Tests(unittest.TestCase):
 
     tag = imp.get_tag()
 
+    @unittest.skipUnless(sys.implementation.cache_tag is not None,
+                         'requires sys.implementation.cache_tag not be None')
     def test_cache_from_source(self):
         # Given the path to a .py file, return the path to its PEP 3147
         # defined .pyc file (i.e. under __pycache__).
-        self.assertEqual(
-            imp.cache_from_source('/foo/bar/baz/qux.py', True),
-            '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag))
+        path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+        expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+                              'qux.{}.pyc'.format(self.tag))
+        self.assertEqual(imp.cache_from_source(path, True), expect)
+
+    def test_cache_from_source_no_cache_tag(self):
+        # Non cache tag means NotImplementedError.
+        with support.swap_attr(sys.implementation, 'cache_tag', None):
+            with self.assertRaises(NotImplementedError):
+                imp.cache_from_source('whatever.py')
+
+    def test_cache_from_source_no_dot(self):
+        # Directory with a dot, filename without dot.
+        path = os.path.join('foo.bar', 'file')
+        expect = os.path.join('foo.bar', '__pycache__',
+                              'file{}.pyc'.format(self.tag))
+        self.assertEqual(imp.cache_from_source(path, True), expect)
 
     def test_cache_from_source_optimized(self):
         # Given the path to a .py file, return the path to its PEP 3147
         # defined .pyo file (i.e. under __pycache__).
-        self.assertEqual(
-            imp.cache_from_source('/foo/bar/baz/qux.py', False),
-            '/foo/bar/baz/__pycache__/qux.{}.pyo'.format(self.tag))
+        path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+        expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+                              'qux.{}.pyo'.format(self.tag))
+        self.assertEqual(imp.cache_from_source(path, False), expect)
 
     def test_cache_from_source_cwd(self):
-        self.assertEqual(imp.cache_from_source('foo.py', True),
-                         os.sep.join(('__pycache__',
-                                      'foo.{}.pyc'.format(self.tag))))
+        path = 'foo.py'
+        expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag))
+        self.assertEqual(imp.cache_from_source(path, True), expect)
 
     def test_cache_from_source_override(self):
         # When debug_override is not None, it can be any true-ish or false-ish
         # value.
-        self.assertEqual(
-            imp.cache_from_source('/foo/bar/baz.py', []),
-            '/foo/bar/__pycache__/baz.{}.pyo'.format(self.tag))
-        self.assertEqual(
-            imp.cache_from_source('/foo/bar/baz.py', [17]),
-            '/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag))
+        path = os.path.join('foo', 'bar', 'baz.py')
+        partial_expect = os.path.join('foo', 'bar', '__pycache__',
+                                      'baz.{}.py'.format(self.tag))
+        self.assertEqual(imp.cache_from_source(path, []), partial_expect + 'o')
+        self.assertEqual(imp.cache_from_source(path, [17]),
+                         partial_expect + 'c')
         # However if the bool-ishness can't be determined, the exception
         # propagates.
         class Bearish:
             def __bool__(self): raise RuntimeError
-        self.assertRaises(
-            RuntimeError,
-            imp.cache_from_source, '/foo/bar/baz.py', Bearish())
-
-    @unittest.skipIf(os.altsep is None,
-                     'test meaningful only where os.altsep is defined')
-    def test_altsep_cache_from_source(self):
-        # Windows path and PEP 3147.
-        self.assertEqual(
-            imp.cache_from_source('\\foo\\bar\\baz\\qux.py', True),
-            '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+        with self.assertRaises(RuntimeError):
+            imp.cache_from_source('/foo/bar/baz.py', Bearish())
 
-    @unittest.skipIf(os.altsep is None,
-                     'test meaningful only where os.altsep is defined')
-    def test_altsep_and_sep_cache_from_source(self):
-        # Windows path and PEP 3147 where altsep is right of sep.
-        self.assertEqual(
-            imp.cache_from_source('\\foo\\bar/baz\\qux.py', True),
-            '\\foo\\bar/baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
-
-    @unittest.skipIf(os.altsep is None,
+    @unittest.skipUnless(os.sep == '\\' and os.altsep == '/',
                      'test meaningful only where os.altsep is defined')
     def test_sep_altsep_and_sep_cache_from_source(self):
         # Windows path and PEP 3147 where sep is right of altsep.
         self.assertEqual(
             imp.cache_from_source('\\foo\\bar\\baz/qux.py', True),
-            '\\foo\\bar\\baz/__pycache__/qux.{}.pyc'.format(self.tag))
+            '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
 
+    @unittest.skipUnless(sys.implementation.cache_tag is not None,
+                         'requires sys.implementation.cache_tag to not be '
+                         'None')
     def test_source_from_cache(self):
         # Given the path to a PEP 3147 defined .pyc file, return the path to
         # its source.  This tests the good path.
-        self.assertEqual(imp.source_from_cache(
-            '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)),
-            '/foo/bar/baz/qux.py')
+        path = os.path.join('foo', 'bar', 'baz', '__pycache__',
+                            'qux.{}.pyc'.format(self.tag))
+        expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+        self.assertEqual(imp.source_from_cache(path), expect)
+
+    def test_source_from_cache_no_cache_tag(self):
+        # If sys.implementation.cache_tag is None, raise NotImplementedError.
+        path = os.path.join('blah', '__pycache__', 'whatever.pyc')
+        with support.swap_attr(sys.implementation, 'cache_tag', None):
+            with self.assertRaises(NotImplementedError):
+                imp.source_from_cache(path)
 
     def test_source_from_cache_bad_path(self):
         # When the path to a pyc file is not in PEP 3147 format, a ValueError
@@ -305,6 +369,12 @@ class PEP3147Tests(unittest.TestCase):
             '/foo/bar/foo.cpython-32.foo.pyc')
 
     def test_package___file__(self):
+        try:
+            m = __import__('pep3147')
+        except ImportError:
+            pass
+        else:
+            self.fail("pep3147 module already exists: %r" % (m,))
         # Test that a package's __file__ points to the right source directory.
         os.mkdir('pep3147')
         sys.path.insert(0, os.curdir)
@@ -314,14 +384,16 @@ class PEP3147Tests(unittest.TestCase):
             shutil.rmtree('pep3147')
         self.addCleanup(cleanup)
         # Touch the __init__.py file.
-        with open('pep3147/__init__.py', 'w'):
-            pass
+        support.create_empty_file('pep3147/__init__.py')
+        importlib.invalidate_caches()
+        expected___file__ = os.sep.join(('.', 'pep3147', '__init__.py'))
         m = __import__('pep3147')
+        self.assertEqual(m.__file__, expected___file__, (m.__file__, m.__path__, sys.path, sys.path_importer_cache))
         # Ensure we load the pyc file.
-        support.forget('pep3147')
+        support.unload('pep3147')
         m = __import__('pep3147')
-        self.assertEqual(m.__file__,
-                         os.sep.join(('.', 'pep3147', '__init__.py')))
+        support.unload('pep3147')
+        self.assertEqual(m.__file__, expected___file__, (m.__file__, m.__path__, sys.path, sys.path_importer_cache))
 
 
 class NullImporterTests(unittest.TestCase):
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
index b10f350e1e..b0e5a95b23 100644
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -1,7 +1,8 @@
+# We import importlib *ASAP* in order to test #15386
+import importlib
 import builtins
 import imp
-from importlib.test.import_ import test_relative_imports
-from importlib.test.import_ import util as importlib_util
+from test.test_importlib.import_ import util as importlib_util
 import marshal
 import os
 import platform
@@ -12,39 +13,50 @@ import sys
 import unittest
 import textwrap
 import errno
+import shutil
+import contextlib
 
+import test.support
 from test.support import (
     EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython,
     make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask,
-    unlink, unload)
+    unlink, unload, create_empty_file, cpython_only)
 from test import script_helper
 
 
-def _files(name):
-    return (name + os.extsep + "py",
-            name + os.extsep + "pyc",
-            name + os.extsep + "pyo",
-            name + os.extsep + "pyw",
-            name + "$py.class")
-
-def chmod_files(name):
-    for f in _files(name):
-        try:
-            os.chmod(f, 0o600)
-        except OSError as exc:
-            if exc.errno != errno.ENOENT:
-                raise
-
 def remove_files(name):
-    for f in _files(name):
+    for f in (name + ".py",
+              name + ".pyc",
+              name + ".pyo",
+              name + ".pyw",
+              name + "$py.class"):
         unlink(f)
     rmtree('__pycache__')
 
 
+@contextlib.contextmanager
+def _ready_to_import(name=None, source=""):
+    # sets up a temporary directory and removes it
+    # creates the module file
+    # temporarily clears the module from sys.modules (if any)
+    name = name or "spam"
+    with script_helper.temp_dir() as tempdir:
+        path = script_helper.make_script(tempdir, name, source)
+        old_module = sys.modules.pop(name, None)
+        try:
+            sys.path.insert(0, tempdir)
+            yield name, path
+            sys.path.remove(tempdir)
+        finally:
+            if old_module is not None:
+                sys.modules[name] = old_module
+
+
 class ImportTests(unittest.TestCase):
 
     def setUp(self):
         remove_files(TESTFN)
+        importlib.invalidate_caches()
 
     def tearDown(self):
         unload(TESTFN)
@@ -82,6 +94,7 @@ class ImportTests(unittest.TestCase):
 
             if TESTFN in sys.modules:
                 del sys.modules[TESTFN]
+            importlib.invalidate_caches()
             try:
                 try:
                     mod = __import__(TESTFN)
@@ -107,90 +120,6 @@ class ImportTests(unittest.TestCase):
         finally:
             del sys.path[0]
 
-    @unittest.skipUnless(os.name == 'posix',
-                         "test meaningful only on posix systems")
-    def test_execute_bit_not_copied(self):
-        # Issue 6070: under posix .pyc files got their execute bit set if
-        # the .py file had the execute bit set, but they aren't executable.
-        with temp_umask(0o022):
-            sys.path.insert(0, os.curdir)
-            try:
-                fname = TESTFN + os.extsep + "py"
-                open(fname, 'w').close()
-                os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
-                                 stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
-                fn = imp.cache_from_source(fname)
-                unlink(fn)
-                __import__(TESTFN)
-                if not os.path.exists(fn):
-                    self.fail("__import__ did not result in creation of "
-                              "either a .pyc or .pyo file")
-                s = os.stat(fn)
-                self.assertEqual(stat.S_IMODE(s.st_mode),
-                                 stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
-            finally:
-                del sys.path[0]
-                remove_files(TESTFN)
-                unload(TESTFN)
-
-    def test_rewrite_pyc_with_read_only_source(self):
-        # Issue 6074: a long time ago on posix, and more recently on Windows,
-        # a read only source file resulted in a read only pyc file, which
-        # led to problems with updating it later
-        sys.path.insert(0, os.curdir)
-        fname = TESTFN + os.extsep + "py"
-        try:
-            # Write a Python file, make it read-only and import it
-            with open(fname, 'w') as f:
-                f.write("x = 'original'\n")
-            # Tweak the mtime of the source to ensure pyc gets updated later
-            s = os.stat(fname)
-            os.utime(fname, (s.st_atime, s.st_mtime-100000000))
-            os.chmod(fname, 0o400)
-            m1 = __import__(TESTFN)
-            self.assertEqual(m1.x, 'original')
-            # Change the file and then reimport it
-            os.chmod(fname, 0o600)
-            with open(fname, 'w') as f:
-                f.write("x = 'rewritten'\n")
-            unload(TESTFN)
-            m2 = __import__(TESTFN)
-            self.assertEqual(m2.x, 'rewritten')
-            # Now delete the source file and check the pyc was rewritten
-            unlink(fname)
-            unload(TESTFN)
-            if __debug__:
-                bytecode_name = fname + "c"
-            else:
-                bytecode_name = fname + "o"
-            os.rename(imp.cache_from_source(fname), bytecode_name)
-            m3 = __import__(TESTFN)
-            self.assertEqual(m3.x, 'rewritten')
-        finally:
-            chmod_files(TESTFN)
-            remove_files(TESTFN)
-            unload(TESTFN)
-            del sys.path[0]
-
-    def test_imp_module(self):
-        # Verify that the imp module can correctly load and find .py files
-        # XXX (ncoghlan): It would be nice to use support.CleanImport
-        # here, but that breaks because the os module registers some
-        # handlers in copy_reg on import. Since CleanImport doesn't
-        # revert that registration, the module is left in a broken
-        # state after reversion. Reinitialising the module contents
-        # and just reverting os.environ to its previous state is an OK
-        # workaround
-        orig_path = os.path
-        orig_getenv = os.getenv
-        with EnvironmentVarGuard():
-            x = imp.find_module("os")
-            self.addCleanup(x[0].close)
-            new_os = imp.load_module("os", *x)
-            self.assertIs(os, new_os)
-            self.assertIs(orig_path, new_os.path)
-            self.assertIsNot(orig_getenv, new_os.getenv)
-
     def test_bug7732(self):
         source = TESTFN + '.py'
         os.mkdir(source)
@@ -220,6 +149,7 @@ class ImportTests(unittest.TestCase):
 
         # Need to be able to load from current dir.
         sys.path.append('')
+        importlib.invalidate_caches()
 
         try:
             make_legacy_pyc(filename)
@@ -239,6 +169,7 @@ class ImportTests(unittest.TestCase):
         # New in 2.4, we shouldn't be able to import that no matter how often
         # we try.
         sys.path.insert(0, os.curdir)
+        importlib.invalidate_caches()
         if TESTFN in sys.modules:
             del sys.modules[TESTFN]
         try:
@@ -312,6 +243,7 @@ class ImportTests(unittest.TestCase):
             os.remove(source)
             del sys.modules[TESTFN]
             make_legacy_pyc(source)
+            importlib.invalidate_caches()
             mod = __import__(TESTFN)
             base, ext = os.path.splitext(mod.__file__)
             self.assertIn(ext, ('.pyc', '.pyo'))
@@ -332,12 +264,6 @@ class ImportTests(unittest.TestCase):
         import test.support as y
         self.assertIs(y, test.support, y.__name__)
 
-    def test_import_initless_directory_warning(self):
-        with check_warnings(('', ImportWarning)):
-            # Just a random non-package directory we always expect to be
-            # somewhere in sys.path...
-            self.assertRaises(ImportError, __import__, "site-packages")
-
     def test_import_by_filename(self):
         path = os.path.abspath(TESTFN)
         encoding = sys.getfilesystemencoding()
@@ -347,8 +273,6 @@ class ImportTests(unittest.TestCase):
             self.skipTest('path is not encodable to {}'.format(encoding))
         with self.assertRaises(ImportError) as c:
             __import__(path)
-        self.assertEqual("Import by filename is not supported.",
-                         c.exception.args[0])
 
     def test_import_in_del_does_not_crash(self):
         # Issue 4236
@@ -385,6 +309,98 @@ class ImportTests(unittest.TestCase):
             del sys.path[0]
             remove_files(TESTFN)
 
+    def test_bogus_fromlist(self):
+        try:
+            __import__('http', fromlist=['blah'])
+        except ImportError:
+            self.fail("fromlist must allow bogus names")
+
+
+class FilePermissionTests(unittest.TestCase):
+    # tests for file mode on cached .pyc/.pyo files
+
+    @unittest.skipUnless(os.name == 'posix',
+                         "test meaningful only on posix systems")
+    def test_creation_mode(self):
+        mask = 0o022
+        with temp_umask(mask), _ready_to_import() as (name, path):
+            cached_path = imp.cache_from_source(path)
+            module = __import__(name)
+            if not os.path.exists(cached_path):
+                self.fail("__import__ did not result in creation of "
+                          "either a .pyc or .pyo file")
+            stat_info = os.stat(cached_path)
+
+        # Check that the umask is respected, and the executable bits
+        # aren't set.
+        self.assertEqual(oct(stat.S_IMODE(stat_info.st_mode)),
+                         oct(0o666 & ~mask))
+
+    @unittest.skipUnless(os.name == 'posix',
+                         "test meaningful only on posix systems")
+    def test_cached_mode_issue_2051(self):
+        # permissions of .pyc should match those of .py, regardless of mask
+        mode = 0o600
+        with temp_umask(0o022), _ready_to_import() as (name, path):
+            cached_path = imp.cache_from_source(path)
+            os.chmod(path, mode)
+            __import__(name)
+            if not os.path.exists(cached_path):
+                self.fail("__import__ did not result in creation of "
+                          "either a .pyc or .pyo file")
+            stat_info = os.stat(cached_path)
+
+        self.assertEqual(oct(stat.S_IMODE(stat_info.st_mode)), oct(mode))
+
+    @unittest.skipUnless(os.name == 'posix',
+                         "test meaningful only on posix systems")
+    def test_cached_readonly(self):
+        mode = 0o400
+        with temp_umask(0o022), _ready_to_import() as (name, path):
+            cached_path = imp.cache_from_source(path)
+            os.chmod(path, mode)
+            __import__(name)
+            if not os.path.exists(cached_path):
+                self.fail("__import__ did not result in creation of "
+                          "either a .pyc or .pyo file")
+            stat_info = os.stat(cached_path)
+
+        expected = mode | 0o200 # Account for fix for issue #6074
+        self.assertEqual(oct(stat.S_IMODE(stat_info.st_mode)), oct(expected))
+
+    def test_pyc_always_writable(self):
+        # Initially read-only .pyc files on Windows used to cause problems
+        # with later updates, see issue #6074 for details
+        with _ready_to_import() as (name, path):
+            # Write a Python file, make it read-only and import it
+            with open(path, 'w') as f:
+                f.write("x = 'original'\n")
+            # Tweak the mtime of the source to ensure pyc gets updated later
+            s = os.stat(path)
+            os.utime(path, (s.st_atime, s.st_mtime-100000000))
+            os.chmod(path, 0o400)
+            m = __import__(name)
+            self.assertEqual(m.x, 'original')
+            # Change the file and then reimport it
+            os.chmod(path, 0o600)
+            with open(path, 'w') as f:
+                f.write("x = 'rewritten'\n")
+            unload(name)
+            importlib.invalidate_caches()
+            m = __import__(name)
+            self.assertEqual(m.x, 'rewritten')
+            # Now delete the source file and check the pyc was rewritten
+            unlink(path)
+            unload(name)
+            importlib.invalidate_caches()
+            if __debug__:
+                bytecode_only = path + "c"
+            else:
+                bytecode_only = path + "o"
+            os.rename(imp.cache_from_source(path), bytecode_only)
+            m = __import__(name)
+            self.assertEqual(m.x, 'rewritten')
+
 
 class PycRewritingTests(unittest.TestCase):
     # Test that the `co_filename` attribute on code objects always points
@@ -412,6 +428,7 @@ func_filename = func.__code__.co_filename
         with open(self.file_name, "w") as f:
             f.write(self.module_source)
         sys.path.insert(0, self.dir_name)
+        importlib.invalidate_caches()
 
     def tearDown(self):
         sys.path[:] = self.sys_path
@@ -451,6 +468,7 @@ func_filename = func.__code__.co_filename
         py_compile.compile(self.file_name, dfile=target)
         os.remove(self.file_name)
         pyc_file = make_legacy_pyc(self.file_name)
+        importlib.invalidate_caches()
         mod = self.import_module()
         self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
@@ -459,7 +477,7 @@ func_filename = func.__code__.co_filename
     def test_foreign_code(self):
         py_compile.compile(self.file_name)
         with open(self.compiled_name, "rb") as f:
-            header = f.read(8)
+            header = f.read(12)
             code = marshal.load(f)
         constants = list(code.co_consts)
         foreign_code = test_main.__code__
@@ -501,9 +519,11 @@ class PathsTests(unittest.TestCase):
         unload("test_trailing_slash")
 
     # Regression test for http://bugs.python.org/issue3677.
-    def _test_UNC_path(self):
-        with open(os.path.join(self.path, 'test_trailing_slash.py'), 'w') as f:
-            f.write("testdata = 'test_trailing_slash'")
+    @unittest.skipUnless(sys.platform == 'win32', 'Windows-specific')
+    def test_UNC_path(self):
+        with open(os.path.join(self.path, 'test_unc_path.py'), 'w') as f:
+            f.write("testdata = 'test_unc_path'")
+        importlib.invalidate_caches()
         # Create the UNC path, like \\myhost\c$\foo\bar.
         path = os.path.abspath(self.path)
         import socket
@@ -518,13 +538,15 @@ class PathsTests(unittest.TestCase):
                 # See issue #15338
                 self.skipTest("cannot access administrative share %r" % (unc,))
             raise
-        sys.path.append(path)
-        mod = __import__("test_trailing_slash")
-        self.assertEqual(mod.testdata, 'test_trailing_slash')
-        unload("test_trailing_slash")
-
-    if sys.platform == "win32":
-        test_UNC_path = _test_UNC_path
+        sys.path.insert(0, unc)
+        try:
+            mod = __import__("test_unc_path")
+        except ImportError as e:
+            self.fail("could not import 'test_unc_path' from %r: %r"
+                      % (unc, e))
+        self.assertEqual(mod.testdata, 'test_unc_path')
+        self.assertTrue(mod.__file__.startswith(unc), mod.__file__)
+        unload("test_unc_path")
 
 
 class RelativeImportTests(unittest.TestCase):
@@ -565,7 +587,7 @@ class RelativeImportTests(unittest.TestCase):
 
         # Check relative import fails with package set to a non-string
         ns = dict(__package__=object())
-        self.assertRaises(ValueError, check_relative)
+        self.assertRaises(TypeError, check_relative)
 
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
@@ -613,6 +635,7 @@ class PycacheTests(unittest.TestCase):
         with open(self.source, 'w') as fp:
             print('# This is a test file written by test_import.py', file=fp)
         sys.path.insert(0, os.curdir)
+        importlib.invalidate_caches()
 
     def tearDown(self):
         assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]'
@@ -660,6 +683,7 @@ class PycacheTests(unittest.TestCase):
         pyc_file = make_legacy_pyc(self.source)
         os.remove(self.source)
         unload(TESTFN)
+        importlib.invalidate_caches()
         m = __import__(TESTFN)
         self.assertEqual(m.__file__,
                          os.path.join(os.curdir, os.path.relpath(pyc_file)))
@@ -680,6 +704,7 @@ class PycacheTests(unittest.TestCase):
         pyc_file = make_legacy_pyc(self.source)
         os.remove(self.source)
         unload(TESTFN)
+        importlib.invalidate_caches()
         m = __import__(TESTFN)
         self.assertEqual(m.__cached__,
                          os.path.join(os.curdir, os.path.relpath(pyc_file)))
@@ -688,6 +713,8 @@ class PycacheTests(unittest.TestCase):
         # Like test___cached__ but for packages.
         def cleanup():
             rmtree('pep3147')
+            unload('pep3147.foo')
+            unload('pep3147')
         os.mkdir('pep3147')
         self.addCleanup(cleanup)
         # Touch the __init__.py
@@ -695,8 +722,7 @@ class PycacheTests(unittest.TestCase):
             pass
         with open(os.path.join('pep3147', 'foo.py'), 'w'):
             pass
-        unload('pep3147.foo')
-        unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
@@ -710,18 +736,20 @@ class PycacheTests(unittest.TestCase):
         # PEP 3147 pyc file.
         def cleanup():
             rmtree('pep3147')
+            unload('pep3147.foo')
+            unload('pep3147')
         os.mkdir('pep3147')
         self.addCleanup(cleanup)
-        unload('pep3147.foo')
-        unload('pep3147')
         # Touch the __init__.py
         with open(os.path.join('pep3147', '__init__.py'), 'w'):
             pass
         with open(os.path.join('pep3147', 'foo.py'), 'w'):
             pass
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
@@ -730,22 +758,256 @@ class PycacheTests(unittest.TestCase):
         self.assertEqual(sys.modules['pep3147.foo'].__cached__,
                          os.path.join(os.curdir, foo_pyc))
 
+    def test_recompute_pyc_same_second(self):
+        # Even when the source file doesn't change timestamp, a change in
+        # source size is enough to trigger recomputation of the pyc file.
+        __import__(TESTFN)
+        unload(TESTFN)
+        with open(self.source, 'a') as fp:
+            print("x = 5", file=fp)
+        m = __import__(TESTFN)
+        self.assertEqual(m.x, 5)
+
 
-class RelativeImportFromImportlibTests(test_relative_imports.RelativeImports):
+class TestSymbolicallyLinkedPackage(unittest.TestCase):
+    package_name = 'sample'
+    tagged = package_name + '-tagged'
 
     def setUp(self):
-        self._importlib_util_flag = importlib_util.using___import__
-        importlib_util.using___import__ = True
+        test.support.rmtree(self.tagged)
+        test.support.rmtree(self.package_name)
+        self.orig_sys_path = sys.path[:]
+
+        # create a sample package; imagine you have a package with a tag and
+        #  you want to symbolically link it from its untagged name.
+        os.mkdir(self.tagged)
+        self.addCleanup(test.support.rmtree, self.tagged)
+        init_file = os.path.join(self.tagged, '__init__.py')
+        test.support.create_empty_file(init_file)
+        assert os.path.exists(init_file)
+
+        # now create a symlink to the tagged package
+        # sample -> sample-tagged
+        os.symlink(self.tagged, self.package_name, target_is_directory=True)
+        self.addCleanup(test.support.unlink, self.package_name)
+        importlib.invalidate_caches()
+
+        self.assertEqual(os.path.isdir(self.package_name), True)
+
+        assert os.path.isfile(os.path.join(self.package_name, '__init__.py'))
 
     def tearDown(self):
-        importlib_util.using___import__ = self._importlib_util_flag
+        sys.path[:] = self.orig_sys_path
+
+    # regression test for issue6727
+    @unittest.skipUnless(
+        not hasattr(sys, 'getwindowsversion')
+        or sys.getwindowsversion() >= (6, 0),
+        "Windows Vista or later required")
+    @test.support.skip_unless_symlink
+    def test_symlinked_dir_importable(self):
+        # make sure sample can only be imported from the current directory.
+        sys.path[:] = ['.']
+        assert os.path.exists(self.package_name)
+        assert os.path.exists(os.path.join(self.package_name, '__init__.py'))
+
+        # Try to import the package
+        importlib.import_module(self.package_name)
+
+
+@cpython_only
+class ImportlibBootstrapTests(unittest.TestCase):
+    # These tests check that importlib is bootstrapped.
+
+    def test_frozen_importlib(self):
+        mod = sys.modules['_frozen_importlib']
+        self.assertTrue(mod)
+
+    def test_frozen_importlib_is_bootstrap(self):
+        from importlib import _bootstrap
+        mod = sys.modules['_frozen_importlib']
+        self.assertIs(mod, _bootstrap)
+        self.assertEqual(mod.__name__, 'importlib._bootstrap')
+        self.assertEqual(mod.__package__, 'importlib')
+        self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__)
+
+    def test_there_can_be_only_one(self):
+        # Issue #15386 revealed a tricky loophole in the bootstrapping
+        # This test is technically redundant, since the bug caused importing
+        # this test module to crash completely, but it helps prove the point
+        from importlib import machinery
+        mod = sys.modules['_frozen_importlib']
+        self.assertIs(machinery.FileFinder, mod.FileFinder)
+        self.assertIs(imp.new_module, mod.new_module)
+
+
+class ImportTracebackTests(unittest.TestCase):
+
+    def setUp(self):
+        os.mkdir(TESTFN)
+        self.old_path = sys.path[:]
+        sys.path.insert(0, TESTFN)
+
+    def tearDown(self):
+        sys.path[:] = self.old_path
+        rmtree(TESTFN)
+
+    def create_module(self, mod, contents, ext=".py"):
+        fname = os.path.join(TESTFN, mod + ext)
+        with open(fname, "w") as f:
+            f.write(contents)
+        self.addCleanup(unload, mod)
+        importlib.invalidate_caches()
+        return fname
+
+    def assert_traceback(self, tb, files):
+        deduped_files = []
+        while tb:
+            code = tb.tb_frame.f_code
+            fn = code.co_filename
+            if not deduped_files or fn != deduped_files[-1]:
+                deduped_files.append(fn)
+            tb = tb.tb_next
+        self.assertEqual(len(deduped_files), len(files), deduped_files)
+        for fn, pat in zip(deduped_files, files):
+            self.assertIn(pat, fn)
+
+    def test_nonexistent_module(self):
+        try:
+            # assertRaises() clears __traceback__
+            import nonexistent_xyzzy
+        except ImportError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ImportError should have been raised")
+        self.assert_traceback(tb, [__file__])
+
+    def test_nonexistent_module_nested(self):
+        self.create_module("foo", "import nonexistent_xyzzy")
+        try:
+            import foo
+        except ImportError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ImportError should have been raised")
+        self.assert_traceback(tb, [__file__, 'foo.py'])
+
+    def test_exec_failure(self):
+        self.create_module("foo", "1/0")
+        try:
+            import foo
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, 'foo.py'])
+
+    def test_exec_failure_nested(self):
+        self.create_module("foo", "import bar")
+        self.create_module("bar", "1/0")
+        try:
+            import foo
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, 'foo.py', 'bar.py'])
+
+    # A few more examples from issue #15425
+    def test_syntax_error(self):
+        self.create_module("foo", "invalid syntax is invalid")
+        try:
+            import foo
+        except SyntaxError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("SyntaxError should have been raised")
+        self.assert_traceback(tb, [__file__])
+
+    def _setup_broken_package(self, parent, child):
+        pkg_name = "_parent_foo"
+        self.addCleanup(unload, pkg_name)
+        pkg_path = os.path.join(TESTFN, pkg_name)
+        os.mkdir(pkg_path)
+        # Touch the __init__.py
+        init_path = os.path.join(pkg_path, '__init__.py')
+        with open(init_path, 'w') as f:
+            f.write(parent)
+        bar_path = os.path.join(pkg_path, 'bar.py')
+        with open(bar_path, 'w') as f:
+            f.write(child)
+        importlib.invalidate_caches()
+        return init_path, bar_path
+
+    def test_broken_submodule(self):
+        init_path, bar_path = self._setup_broken_package("", "1/0")
+        try:
+            import _parent_foo.bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, bar_path])
+
+    def test_broken_from(self):
+        init_path, bar_path = self._setup_broken_package("", "1/0")
+        try:
+            from _parent_foo import bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ImportError should have been raised")
+        self.assert_traceback(tb, [__file__, bar_path])
+
+    def test_broken_parent(self):
+        init_path, bar_path = self._setup_broken_package("1/0", "")
+        try:
+            import _parent_foo.bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, init_path])
+
+    def test_broken_parent_from(self):
+        init_path, bar_path = self._setup_broken_package("1/0", "")
+        try:
+            from _parent_foo import bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, init_path])
+
+    @cpython_only
+    def test_import_bug(self):
+        # We simulate a bug in importlib and check that it's not stripped
+        # away from the traceback.
+        self.create_module("foo", "")
+        importlib = sys.modules['_frozen_importlib']
+        old_load_module = importlib.SourceLoader.load_module
+        try:
+            def load_module(*args):
+                1/0
+            importlib.SourceLoader.load_module = load_module
+            try:
+                import foo
+            except ZeroDivisionError as e:
+                tb = e.__traceback__
+            else:
+                self.fail("ZeroDivisionError should have been raised")
+            self.assert_traceback(tb, [__file__, '<frozen importlib', __file__])
+        finally:
+            importlib.SourceLoader.load_module = old_load_module
 
 
 def test_main(verbose=None):
-    run_unittest(ImportTests, PycacheTests,
+    run_unittest(ImportTests, PycacheTests, FilePermissionTests,
                  PycRewritingTests, PathsTests, RelativeImportTests,
                  OverridingImportBuiltinTests,
-                 RelativeImportFromImportlibTests)
+                 ImportlibBootstrapTests,
+                 TestSymbolicallyLinkedPackage,
+                 ImportTracebackTests)
 
 
 if __name__ == '__main__':
diff --git a/Lib/test/test_importhooks.py b/Lib/test/test_importhooks.py
index ec6730e9ba..2a22d1a186 100644
--- a/Lib/test/test_importhooks.py
+++ b/Lib/test/test_importhooks.py
@@ -51,7 +51,7 @@ class TestImporter:
 
     def __init__(self, path=test_path):
         if path != test_path:
-            # if out class is on sys.path_hooks, we must raise
+            # if our class is on sys.path_hooks, we must raise
             # ImportError for any path item that we can't handle.
             raise ImportError
         self.path = path
@@ -215,7 +215,7 @@ class ImportHooksTestCase(ImportHooksBaseTestCase):
         self.doTestImports(i)
 
     def testPathHook(self):
-        sys.path_hooks.append(PathImporter)
+        sys.path_hooks.insert(0, PathImporter)
         sys.path.append(test_path)
         self.doTestImports()
 
@@ -228,8 +228,10 @@ class ImportHooksTestCase(ImportHooksBaseTestCase):
     def testImpWrapper(self):
         i = ImpWrapper()
         sys.meta_path.append(i)
-        sys.path_hooks.append(ImpWrapper)
-        mnames = ("colorsys", "urllib.parse", "distutils.core")
+        sys.path_hooks.insert(0, ImpWrapper)
+        mnames = (
+            "colorsys", "urllib.parse", "distutils.core", "sys",
+        )
         for mname in mnames:
             parent = mname.split(".")[0]
             for n in list(sys.modules):
@@ -237,7 +239,8 @@ class ImportHooksTestCase(ImportHooksBaseTestCase):
                     del sys.modules[n]
         for mname in mnames:
             m = __import__(mname, globals(), locals(), ["__dummy__"])
-            m.__loader__  # to make sure we actually handled the import
+            # to make sure we actually handled the import
+            self.assertTrue(hasattr(m, "__loader__"))
 
 
 def test_main():
diff --git a/Lib/test/test_importlib.py b/Lib/test/test_importlib.py
deleted file mode 100644
index 6ed05859c9..0000000000
--- a/Lib/test/test_importlib.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from importlib.test.__main__ import test_main
-
-
-if __name__ == '__main__':
-    test_main()
diff --git a/Lib/test/test_importlib/__init__.py b/Lib/test/test_importlib/__init__.py
new file mode 100644
index 0000000000..0e345cdc2d
--- /dev/null
+++ b/Lib/test/test_importlib/__init__.py
@@ -0,0 +1,33 @@
+import os
+import sys
+from test import support
+import unittest
+
+def test_suite(package=__package__, directory=os.path.dirname(__file__)):
+    suite = unittest.TestSuite()
+    for name in os.listdir(directory):
+        if name.startswith(('.', '__')):
+            continue
+        path = os.path.join(directory, name)
+        if (os.path.isfile(path) and name.startswith('test_') and
+                name.endswith('.py')):
+            submodule_name = os.path.splitext(name)[0]
+            module_name = "{0}.{1}".format(package, submodule_name)
+            __import__(module_name, level=0)
+            module_tests = unittest.findTestCases(sys.modules[module_name])
+            suite.addTest(module_tests)
+        elif os.path.isdir(path):
+            package_name = "{0}.{1}".format(package, name)
+            __import__(package_name, level=0)
+            package_tests = getattr(sys.modules[package_name], 'test_suite')()
+            suite.addTest(package_tests)
+        else:
+            continue
+    return suite
+
+
+def test_main():
+    start_dir = os.path.dirname(__file__)
+    top_dir = os.path.dirname(os.path.dirname(start_dir))
+    test_loader = unittest.TestLoader()
+    support.run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir))
diff --git a/Lib/test/test_importlib/__main__.py b/Lib/test/test_importlib/__main__.py
new file mode 100644
index 0000000000..c39712871f
--- /dev/null
+++ b/Lib/test/test_importlib/__main__.py
@@ -0,0 +1,20 @@
+"""Run importlib's test suite.
+
+Specifying the ``--builtin`` flag will run tests, where applicable, with
+builtins.__import__ instead of importlib.__import__.
+
+"""
+from . import test_main
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Execute the importlib test '
+                                                  'suite')
+    parser.add_argument('-b', '--builtin', action='store_true', default=False,
+                        help='use builtins.__import__() instead of importlib')
+    args = parser.parse_args()
+    if args.builtin:
+        util.using___import__ = True
+    test_main()
diff --git a/Lib/importlib/test/abc.py b/Lib/test/test_importlib/abc.py
index 2c17ac329b..2c17ac329b 100644
--- a/Lib/importlib/test/abc.py
+++ b/Lib/test/test_importlib/abc.py
diff --git a/Lib/importlib/test/builtin/__init__.py b/Lib/test/test_importlib/builtin/__init__.py
index 31a3b5f7d4..15c0ade207 100644
--- a/Lib/importlib/test/builtin/__init__.py
+++ b/Lib/test/test_importlib/builtin/__init__.py
@@ -1,10 +1,10 @@
-import importlib.test
+from .. import test_suite
 import os
 
 
 def test_suite():
     directory = os.path.dirname(__file__)
-    return importlib.test.test_suite('importlib.test.builtin', directory)
+    return test_suite('importlib.test.builtin', directory)
 
 
 if __name__ == '__main__':
diff --git a/Lib/importlib/test/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py
index 40f690e4af..146538d891 100644
--- a/Lib/importlib/test/builtin/test_finder.py
+++ b/Lib/test/test_importlib/builtin/test_finder.py
@@ -35,14 +35,14 @@ class FinderTests(abc.FinderTests):
     def test_failure(self):
         assert 'importlib' not in sys.builtin_module_names
         loader = machinery.BuiltinImporter.find_module('importlib')
-        self.assertTrue(loader is None)
+        self.assertIsNone(loader)
 
     def test_ignore_path(self):
         # The value for 'path' should always trigger a failed import.
         with util.uncache(builtin_util.NAME):
             loader = machinery.BuiltinImporter.find_module(builtin_util.NAME,
                                                             ['pkg'])
-            self.assertTrue(loader is None)
+            self.assertIsNone(loader)
 
 
 
diff --git a/Lib/importlib/test/builtin/test_loader.py b/Lib/test/test_importlib/builtin/test_loader.py
index 1a8539b1e8..8e186e7156 100644
--- a/Lib/importlib/test/builtin/test_loader.py
+++ b/Lib/test/test_importlib/builtin/test_loader.py
@@ -18,10 +18,10 @@ class LoaderTests(abc.LoaderTests):
 
     def verify(self, module):
         """Verify that the module matches against what it should have."""
-        self.assertTrue(isinstance(module, types.ModuleType))
+        self.assertIsInstance(module, types.ModuleType)
         for attr, value in self.verification.items():
             self.assertEqual(getattr(module, attr), value)
-        self.assertTrue(module.__name__ in sys.modules)
+        self.assertIn(module.__name__, sys.modules)
 
     load_module = staticmethod(lambda name:
                                 machinery.BuiltinImporter.load_module(name))
@@ -49,20 +49,22 @@ class LoaderTests(abc.LoaderTests):
         with util.uncache(builtin_util.NAME):
             module1 = self.load_module(builtin_util.NAME)
             module2 = self.load_module(builtin_util.NAME)
-            self.assertTrue(module1 is module2)
+            self.assertIs(module1, module2)
 
     def test_unloadable(self):
         name = 'dssdsdfff'
         assert name not in sys.builtin_module_names
-        with self.assertRaises(ImportError):
+        with self.assertRaises(ImportError) as cm:
             self.load_module(name)
+        self.assertEqual(cm.exception.name, name)
 
     def test_already_imported(self):
         # Using the name of a module already imported but not a built-in should
         # still fail.
         assert hasattr(importlib, '__file__')  # Not a built-in.
-        with self.assertRaises(ImportError):
+        with self.assertRaises(ImportError) as cm:
             self.load_module('importlib')
+        self.assertEqual(cm.exception.name, 'importlib')
 
 
 class InspectLoaderTests(unittest.TestCase):
@@ -72,12 +74,12 @@ class InspectLoaderTests(unittest.TestCase):
     def test_get_code(self):
         # There is no code object.
         result = machinery.BuiltinImporter.get_code(builtin_util.NAME)
-        self.assertTrue(result is None)
+        self.assertIsNone(result)
 
     def test_get_source(self):
         # There is no source.
         result = machinery.BuiltinImporter.get_source(builtin_util.NAME)
-        self.assertTrue(result is None)
+        self.assertIsNone(result)
 
     def test_is_package(self):
         # Cannot be a package.
@@ -88,8 +90,9 @@ class InspectLoaderTests(unittest.TestCase):
         # Modules not built-in should raise ImportError.
         for meth_name in ('get_code', 'get_source', 'is_package'):
             method = getattr(machinery.BuiltinImporter, meth_name)
-        with self.assertRaises(ImportError):
+        with self.assertRaises(ImportError) as cm:
             method(builtin_util.BAD_NAME)
+        self.assertRaises(builtin_util.BAD_NAME)
 
 
 
diff --git a/Lib/importlib/test/builtin/util.py b/Lib/test/test_importlib/builtin/util.py
index 5704699ee2..5704699ee2 100644
--- a/Lib/importlib/test/builtin/util.py
+++ b/Lib/test/test_importlib/builtin/util.py
diff --git a/Lib/importlib/test/frozen/__init__.py b/Lib/test/test_importlib/extension/__init__.py
index 2945eeb0bc..c0339236fa 100644
--- a/Lib/importlib/test/frozen/__init__.py
+++ b/Lib/test/test_importlib/extension/__init__.py
@@ -1,11 +1,11 @@
-import importlib.test
+from .. import test_suite
 import os.path
 import unittest
 
 
 def test_suite():
     directory = os.path.dirname(__file__)
-    return importlib.test.test_suite('importlib.test.frozen', directory)
+    return test_suite('importlib.test.extension', directory)
 
 
 if __name__ == '__main__':
diff --git a/Lib/importlib/test/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py
index e062fb6597..76c53e4fd6 100644
--- a/Lib/importlib/test/extension/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py
@@ -1,3 +1,4 @@
+import imp
 import sys
 from test import support
 import unittest
@@ -13,19 +14,26 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
         good_name = ext_util.NAME
         bad_name = good_name.upper()
         assert good_name != bad_name
-        finder = _bootstrap._FileFinder(ext_util.PATH,
-                                        _bootstrap._ExtensionFinderDetails())
+        finder = _bootstrap.FileFinder(ext_util.PATH,
+                                        (_bootstrap.ExtensionFileLoader,
+                                         _bootstrap.EXTENSION_SUFFIXES))
         return finder.find_module(bad_name)
 
     def test_case_sensitive(self):
         with support.EnvironmentVarGuard() as env:
             env.unset('PYTHONCASEOK')
+            if b'PYTHONCASEOK' in _bootstrap._os.environ:
+                self.skipTest('os.environ changes not reflected in '
+                              '_os.environ')
             loader = self.find_module()
             self.assertIsNone(loader)
 
     def test_case_insensitivity(self):
         with support.EnvironmentVarGuard() as env:
             env.set('PYTHONCASEOK', '1')
+            if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+                self.skipTest('os.environ changes not reflected in '
+                              '_os.environ')
             loader = self.find_module()
             self.assertTrue(hasattr(loader, 'load_module'))
 
diff --git a/Lib/importlib/test/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py
index ea97483317..a63cfdb9b4 100644
--- a/Lib/importlib/test/extension/test_finder.py
+++ b/Lib/test/test_importlib/extension/test_finder.py
@@ -1,4 +1,4 @@
-from importlib import _bootstrap
+from importlib import machinery
 from .. import abc
 from . import util
 
@@ -9,15 +9,16 @@ class FinderTests(abc.FinderTests):
     """Test the finder for extension modules."""
 
     def find_module(self, fullname):
-        importer = _bootstrap._FileFinder(util.PATH,
-                                          _bootstrap._ExtensionFinderDetails())
+        importer = machinery.FileFinder(util.PATH,
+                                        (machinery.ExtensionFileLoader,
+                                         machinery.EXTENSION_SUFFIXES))
         return importer.find_module(fullname)
 
     def test_module(self):
         self.assertTrue(self.find_module(util.NAME))
 
     def test_package(self):
-        # Extension modules cannot be an __init__ for a package.
+        # No extension module as an __init__ available for testing.
         pass
 
     def test_module_in_package(self):
@@ -25,7 +26,7 @@ class FinderTests(abc.FinderTests):
         pass
 
     def test_package_in_package(self):
-        # Extension modules cannot be an __init__ for a package.
+        # No extension module as an __init__ available for testing.
         pass
 
     def test_package_over_module(self):
@@ -33,9 +34,7 @@ class FinderTests(abc.FinderTests):
         pass
 
     def test_failure(self):
-        self.assertTrue(self.find_module('asdfjkl;') is None)
-
-    # XXX Raise an exception if someone tries to use the 'path' argument?
+        self.assertIsNone(self.find_module('asdfjkl;'))
 
 
 def test_main():
diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py
new file mode 100644
index 0000000000..ca5af201c4
--- /dev/null
+++ b/Lib/test/test_importlib/extension/test_loader.py
@@ -0,0 +1,79 @@
+from importlib import machinery
+from . import util as ext_util
+from .. import abc
+from .. import util
+
+import os.path
+import sys
+import unittest
+
+
+class LoaderTests(abc.LoaderTests):
+
+    """Test load_module() for extension modules."""
+
+    def setUp(self):
+        self.loader = machinery.ExtensionFileLoader(ext_util.NAME,
+                                                     ext_util.FILEPATH)
+
+    def load_module(self, fullname):
+        return self.loader.load_module(fullname)
+
+    def test_load_module_API(self):
+        # Test the default argument for load_module().
+        self.loader.load_module()
+        self.loader.load_module(None)
+        with self.assertRaises(ImportError):
+            self.load_module('XXX')
+
+
+    def test_module(self):
+        with util.uncache(ext_util.NAME):
+            module = self.load_module(ext_util.NAME)
+            for attr, value in [('__name__', ext_util.NAME),
+                                ('__file__', ext_util.FILEPATH),
+                                ('__package__', '')]:
+                self.assertEqual(getattr(module, attr), value)
+            self.assertIn(ext_util.NAME, sys.modules)
+            self.assertIsInstance(module.__loader__,
+                                  machinery.ExtensionFileLoader)
+
+    def test_package(self):
+        # No extension module as __init__ available for testing.
+        pass
+
+    def test_lacking_parent(self):
+        # No extension module in a package available for testing.
+        pass
+
+    def test_module_reuse(self):
+        with util.uncache(ext_util.NAME):
+            module1 = self.load_module(ext_util.NAME)
+            module2 = self.load_module(ext_util.NAME)
+            self.assertIs(module1, module2)
+
+    def test_state_after_failure(self):
+        # No easy way to trigger a failure after a successful import.
+        pass
+
+    def test_unloadable(self):
+        name = 'asdfjkl;'
+        with self.assertRaises(ImportError) as cm:
+            self.load_module(name)
+        self.assertEqual(cm.exception.name, name)
+
+    def test_is_package(self):
+        self.assertFalse(self.loader.is_package(ext_util.NAME))
+        for suffix in machinery.EXTENSION_SUFFIXES:
+            path = os.path.join('some', 'path', 'pkg', '__init__' + suffix)
+            loader = machinery.ExtensionFileLoader('pkg', path)
+            self.assertTrue(loader.is_package('pkg'))
+
+
+def test_main():
+    from test.support import run_unittest
+    run_unittest(LoaderTests)
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/importlib/test/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py
index 4610420d29..1d969a1b28 100644
--- a/Lib/importlib/test/extension/test_path_hook.py
+++ b/Lib/test/test_importlib/extension/test_path_hook.py
@@ -1,4 +1,4 @@
-from importlib import _bootstrap
+from importlib import machinery
 from . import util
 
 import collections
@@ -14,7 +14,8 @@ class PathHookTests(unittest.TestCase):
     # XXX Should it only work for directories containing an extension module?
 
     def hook(self, entry):
-        return _bootstrap._file_path_hook(entry)
+        return machinery.FileFinder.path_hook((machinery.ExtensionFileLoader,
+            machinery.EXTENSION_SUFFIXES))(entry)
 
     def test_success(self):
         # Path hook should handle a directory where a known extension module
diff --git a/Lib/importlib/test/extension/util.py b/Lib/test/test_importlib/extension/util.py
index d149169748..a266dd98c8 100644
--- a/Lib/importlib/test/extension/util.py
+++ b/Lib/test/test_importlib/extension/util.py
@@ -1,4 +1,5 @@
 import imp
+from importlib import machinery
 import os
 import sys
 
@@ -6,10 +7,9 @@ PATH = None
 EXT = None
 FILENAME = None
 NAME = '_testcapi'
-_file_exts = [x[0] for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION]
 try:
     for PATH in sys.path:
-        for EXT in _file_exts:
+        for EXT in machinery.EXTENSION_SUFFIXES:
             FILENAME = NAME + EXT
             FILEPATH = os.path.join(PATH, FILENAME)
             if os.path.exists(os.path.join(PATH, FILENAME)):
@@ -18,4 +18,3 @@ try:
         PATH = EXT = FILENAME = FILEPATH = None
 except StopIteration:
     pass
-del _file_exts
diff --git a/Lib/importlib/test/source/__init__.py b/Lib/test/test_importlib/frozen/__init__.py
index 8d7c49dc9c..9ef103bce7 100644
--- a/Lib/importlib/test/source/__init__.py
+++ b/Lib/test/test_importlib/frozen/__init__.py
@@ -1,11 +1,11 @@
-import importlib.test
+from .. import test_suite
 import os.path
 import unittest
 
 
 def test_suite():
     directory = os.path.dirname(__file__)
-    return importlib.test.test_suite('importlib.test.source', directory)
+    return test_suite('importlib.test.frozen', directory)
 
 
 if __name__ == '__main__':
diff --git a/Lib/importlib/test/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py
index db88379d12..fa0c2a037e 100644
--- a/Lib/importlib/test/frozen/test_finder.py
+++ b/Lib/test/test_importlib/frozen/test_finder.py
@@ -1,4 +1,4 @@
-from ... import machinery
+from importlib import machinery
 from .. import abc
 
 import unittest
@@ -35,7 +35,7 @@ class FinderTests(abc.FinderTests):
 
     def test_failure(self):
         loader = self.find('<not real>')
-        self.assertTrue(loader is None)
+        self.assertIsNone(loader)
 
 
 def test_main():
diff --git a/Lib/importlib/test/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py
index b685ef5708..4b8ec15554 100644
--- a/Lib/importlib/test/frozen/test_loader.py
+++ b/Lib/test/test_importlib/frozen/test_loader.py
@@ -10,55 +10,70 @@ class LoaderTests(abc.LoaderTests):
     def test_module(self):
         with util.uncache('__hello__'), captured_stdout() as stdout:
             module = machinery.FrozenImporter.load_module('__hello__')
-            check = {'__name__': '__hello__', '__file__': '<frozen>',
-                    '__package__': '', '__loader__': machinery.FrozenImporter}
+            check = {'__name__': '__hello__',
+                    '__package__': '',
+                    '__loader__': machinery.FrozenImporter,
+                    }
             for attr, value in check.items():
                 self.assertEqual(getattr(module, attr), value)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
 
     def test_package(self):
         with util.uncache('__phello__'),  captured_stdout() as stdout:
             module = machinery.FrozenImporter.load_module('__phello__')
-            check = {'__name__': '__phello__', '__file__': '<frozen>',
-                     '__package__': '__phello__', '__path__': ['__phello__'],
-                     '__loader__': machinery.FrozenImporter}
+            check = {'__name__': '__phello__',
+                     '__package__': '__phello__',
+                     '__path__': ['__phello__'],
+                     '__loader__': machinery.FrozenImporter,
+                     }
             for attr, value in check.items():
                 attr_value = getattr(module, attr)
                 self.assertEqual(attr_value, value,
                                  "for __phello__.%s, %r != %r" %
                                  (attr, attr_value, value))
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
 
     def test_lacking_parent(self):
         with util.uncache('__phello__', '__phello__.spam'), \
              captured_stdout() as stdout:
             module = machinery.FrozenImporter.load_module('__phello__.spam')
-            check = {'__name__': '__phello__.spam', '__file__': '<frozen>',
+            check = {'__name__': '__phello__.spam',
                     '__package__': '__phello__',
-                    '__loader__': machinery.FrozenImporter}
+                    '__loader__': machinery.FrozenImporter,
+                    }
             for attr, value in check.items():
                 attr_value = getattr(module, attr)
                 self.assertEqual(attr_value, value,
                                  "for __phello__.spam.%s, %r != %r" %
                                  (attr, attr_value, value))
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
 
     def test_module_reuse(self):
         with util.uncache('__hello__'), captured_stdout() as stdout:
             module1 = machinery.FrozenImporter.load_module('__hello__')
             module2 = machinery.FrozenImporter.load_module('__hello__')
-            self.assertTrue(module1 is module2)
+            self.assertIs(module1, module2)
             self.assertEqual(stdout.getvalue(),
                              'Hello world!\nHello world!\n')
 
+    def test_module_repr(self):
+        with util.uncache('__hello__'), captured_stdout():
+            module = machinery.FrozenImporter.load_module('__hello__')
+            self.assertEqual(repr(module),
+                             "<module '__hello__' (frozen)>")
+
     def test_state_after_failure(self):
         # No way to trigger an error in a frozen module.
         pass
 
     def test_unloadable(self):
         assert machinery.FrozenImporter.find_module('_not_real') is None
-        with self.assertRaises(ImportError):
+        with self.assertRaises(ImportError) as cm:
             machinery.FrozenImporter.load_module('_not_real')
+        self.assertEqual(cm.exception.name, '_not_real')
 
 
 class InspectLoaderTests(unittest.TestCase):
@@ -78,7 +93,7 @@ class InspectLoaderTests(unittest.TestCase):
     def test_get_source(self):
         # Should always return None.
         result = machinery.FrozenImporter.get_source('__hello__')
-        self.assertTrue(result is None)
+        self.assertIsNone(result)
 
     def test_is_package(self):
         # Should be able to tell what is a package.
@@ -86,14 +101,15 @@ class InspectLoaderTests(unittest.TestCase):
                     ('__phello__.spam', False))
         for name, is_package in test_for:
             result = machinery.FrozenImporter.is_package(name)
-            self.assertTrue(bool(result) == is_package)
+            self.assertEqual(bool(result), is_package)
 
     def test_failure(self):
         # Raise ImportError for modules that are not frozen.
         for meth_name in ('get_code', 'get_source', 'is_package'):
             method = getattr(machinery.FrozenImporter, meth_name)
-            with self.assertRaises(ImportError):
+            with self.assertRaises(ImportError) as cm:
                 method('importlib')
+            self.assertEqual(cm.exception.name, 'importlib')
 
 
 def test_main():
diff --git a/Lib/importlib/test/import_/__init__.py b/Lib/test/test_importlib/import_/__init__.py
index fdf7661dc0..366e531333 100644
--- a/Lib/importlib/test/import_/__init__.py
+++ b/Lib/test/test_importlib/import_/__init__.py
@@ -1,11 +1,11 @@
-import importlib.test
+from .. import test_suite
 import os.path
 import unittest
 
 
 def test_suite():
     directory = os.path.dirname(__file__)
-    return importlib.test.test_suite('importlib.test.import_', directory)
+    return test_suite('importlib.test.import_', directory)
 
 
 if __name__ == '__main__':
diff --git a/Lib/importlib/test/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py
index 5056ae59cc..783cde1729 100644
--- a/Lib/importlib/test/import_/test___package__.py
+++ b/Lib/test/test_importlib/import_/test___package__.py
@@ -67,7 +67,7 @@ class Using__package__(unittest.TestCase):
 
     def test_bunk__package__(self):
         globals = {'__package__': 42}
-        with self.assertRaises(ValueError):
+        with self.assertRaises(TypeError):
             import_util.import_('', globals, {}, ['relimport'], 1)
 
 
diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py
new file mode 100644
index 0000000000..3d4cd94889
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_api.py
@@ -0,0 +1,67 @@
+from .. import util as importlib_test_util
+from . import util
+import imp
+import sys
+import unittest
+
+
+class BadLoaderFinder:
+    bad = 'fine.bogus'
+    @classmethod
+    def find_module(cls, fullname, path):
+        if fullname == cls.bad:
+            return cls
+    @classmethod
+    def load_module(cls, fullname):
+        if fullname == cls.bad:
+            raise ImportError('I cannot be loaded!')
+
+
+class APITest(unittest.TestCase):
+
+    """Test API-specific details for __import__ (e.g. raising the right
+    exception when passing in an int for the module name)."""
+
+    def test_name_requires_rparition(self):
+        # Raise TypeError if a non-string is passed in for the module name.
+        with self.assertRaises(TypeError):
+            util.import_(42)
+
+    def test_negative_level(self):
+        # Raise ValueError when a negative level is specified.
+        # PEP 328 did away with sys.module None entries and the ambiguity of
+        # absolute/relative imports.
+        with self.assertRaises(ValueError):
+            util.import_('os', globals(), level=-1)
+
+    def test_nonexistent_fromlist_entry(self):
+        # If something in fromlist doesn't exist, that's okay.
+        # issue15715
+        mod = imp.new_module('fine')
+        mod.__path__ = ['XXX']
+        with importlib_test_util.import_state(meta_path=[BadLoaderFinder]):
+            with importlib_test_util.uncache('fine'):
+                sys.modules['fine'] = mod
+                util.import_('fine', fromlist=['not here'])
+
+    def test_fromlist_load_error_propagates(self):
+        # If something in fromlist triggers an exception not related to not
+        # existing, let that exception propagate.
+        # issue15316
+        mod = imp.new_module('fine')
+        mod.__path__ = ['XXX']
+        with importlib_test_util.import_state(meta_path=[BadLoaderFinder]):
+            with importlib_test_util.uncache('fine'):
+                sys.modules['fine'] = mod
+                with self.assertRaises(ImportError):
+                    util.import_('fine', fromlist=['bogus'])
+
+
+
+def test_main():
+    from test.support import run_unittest
+    run_unittest(APITest)
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/importlib/test/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py
index 48dc64311a..bf680277d6 100644
--- a/Lib/importlib/test/import_/test_caching.py
+++ b/Lib/test/test_importlib/import_/test_caching.py
@@ -34,8 +34,9 @@ class UseCache(unittest.TestCase):
         name = 'using_None'
         with util.uncache(name):
             sys.modules[name] = None
-            with self.assertRaises(ImportError):
+            with self.assertRaises(ImportError) as cm:
                 import_util.import_(name)
+            self.assertEqual(cm.exception.name, name)
 
     def create_mock(self, *names, return_=None):
         mock = util.mock_modules(*names)
@@ -64,7 +65,8 @@ class UseCache(unittest.TestCase):
             with util.import_state(meta_path=[importer]):
                 module = import_util.import_('pkg.module')
                 self.assertTrue(hasattr(module, 'module'))
-                self.assertTrue(id(module.module), id(sys.modules['pkg.module']))
+                self.assertEqual(id(module.module),
+                                 id(sys.modules['pkg.module']))
 
     # See test_using_cache_after_loader() for reasoning.
     @import_util.importlib_only
diff --git a/Lib/importlib/test/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py
index 7ecde037ae..c16c33710f 100644
--- a/Lib/importlib/test/import_/test_fromlist.py
+++ b/Lib/test/test_importlib/import_/test_fromlist.py
@@ -1,6 +1,7 @@
 """Test that the semantics relating to the 'fromlist' argument are correct."""
 from .. import util
 from . import util as import_util
+import imp
 import unittest
 
 class ReturnValue(unittest.TestCase):
@@ -38,11 +39,9 @@ class HandlingFromlist(unittest.TestCase):
     [object case]. This is even true if the object does not exist [bad object].
 
     If a package is being imported, then what is listed in fromlist may be
-    treated as a module to be imported [module]. But once again, even if
-    something in fromlist does not exist as a module, no error is raised
-    [no module]. And this extends to what is contained in __all__ when '*' is
-    imported [using *]. And '*' does not need to be the only name in the
-    fromlist [using * with others].
+    treated as a module to be imported [module]. And this extends to what is
+    contained in __all__ when '*' is imported [using *]. And '*' does not need
+    to be the only name in the fromlist [using * with others].
 
     """
 
@@ -53,7 +52,7 @@ class HandlingFromlist(unittest.TestCase):
                 module = import_util.import_('module', fromlist=['attr'])
                 self.assertEqual(module.__name__, 'module')
 
-    def test_unexistent_object(self):
+    def test_nonexistent_object(self):
         # [bad object]
         with util.mock_modules('module') as importer:
             with util.import_state(meta_path=[importer]):
@@ -70,13 +69,18 @@ class HandlingFromlist(unittest.TestCase):
                 self.assertTrue(hasattr(module, 'module'))
                 self.assertEqual(module.module.__name__, 'pkg.module')
 
-    def test_no_module_from_package(self):
-        # [no module]
-        with util.mock_modules('pkg.__init__') as importer:
+    def test_module_from_package_triggers_ImportError(self):
+        # If a submodule causes an ImportError because it tries to import
+        # a module which doesn't exist, that should let the ImportError
+        # propagate.
+        def module_code():
+            import i_do_not_exist
+        with util.mock_modules('pkg.__init__', 'pkg.mod',
+                               module_code={'pkg.mod': module_code}) as importer:
             with util.import_state(meta_path=[importer]):
-                module = import_util.import_('pkg', fromlist='non_existent')
-                self.assertEqual(module.__name__, 'pkg')
-                self.assertTrue(not hasattr(module, 'non_existent'))
+                with self.assertRaises(ImportError) as exc:
+                    import_util.import_('pkg', fromlist=['mod'])
+                self.assertEqual('i_do_not_exist', exc.exception.name)
 
     def test_empty_string(self):
         with util.mock_modules('pkg.__init__', 'pkg.mod') as importer:
diff --git a/Lib/importlib/test/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py
index 3b130c9a13..4d85f80e67 100644
--- a/Lib/importlib/test/import_/test_meta_path.py
+++ b/Lib/test/test_importlib/import_/test_meta_path.py
@@ -1,7 +1,10 @@
 from .. import util
 from . import util as import_util
+import importlib._bootstrap
+import sys
 from types import MethodType
 import unittest
+import warnings
 
 
 class CallingOrder(unittest.TestCase):
@@ -33,6 +36,21 @@ class CallingOrder(unittest.TestCase):
             with util.import_state(meta_path=[first, second]):
                 self.assertEqual(import_util.import_(mod_name), 42)
 
+    def test_empty(self):
+        # Raise an ImportWarning if sys.meta_path is empty.
+        module_name = 'nothing'
+        try:
+            del sys.modules[module_name]
+        except KeyError:
+            pass
+        with util.import_state(meta_path=[]):
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter('always')
+                self.assertIsNone(importlib._bootstrap._find_module('nothing',
+                                                                    None))
+                self.assertEqual(len(w), 1)
+                self.assertTrue(issubclass(w[-1].category, ImportWarning))
+
 
 class CallSignature(unittest.TestCase):
 
@@ -64,7 +82,7 @@ class CallSignature(unittest.TestCase):
                 self.assertEqual(len(args), 2)
                 self.assertEqual(len(kwargs), 0)
                 self.assertEqual(args[0], mod_name)
-                self.assertTrue(args[1] is None)
+                self.assertIsNone(args[1])
 
     def test_with_path(self):
         # [path set]
@@ -84,7 +102,7 @@ class CallSignature(unittest.TestCase):
                 # Assuming all arguments are positional.
                 self.assertTrue(not kwargs)
                 self.assertEqual(args[0], mod_name)
-                self.assertTrue(args[1] is path)
+                self.assertIs(args[1], path)
 
 
 
diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py
new file mode 100644
index 0000000000..bfa18dc217
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_packages.py
@@ -0,0 +1,112 @@
+from .. import util
+from . import util as import_util
+import sys
+import unittest
+import importlib
+from test import support
+
+
+class ParentModuleTests(unittest.TestCase):
+
+    """Importing a submodule should import the parent modules."""
+
+    def test_import_parent(self):
+        with util.mock_modules('pkg.__init__', 'pkg.module') as mock:
+            with util.import_state(meta_path=[mock]):
+                module = import_util.import_('pkg.module')
+                self.assertIn('pkg', sys.modules)
+
+    def test_bad_parent(self):
+        with util.mock_modules('pkg.module') as mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises(ImportError) as cm:
+                    import_util.import_('pkg.module')
+                self.assertEqual(cm.exception.name, 'pkg')
+
+    def test_raising_parent_after_importing_child(self):
+        def __init__():
+            import pkg.module
+            1/0
+        mock = util.mock_modules('pkg.__init__', 'pkg.module',
+                                 module_code={'pkg': __init__})
+        with mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises(ZeroDivisionError):
+                    import_util.import_('pkg')
+                self.assertNotIn('pkg', sys.modules)
+                self.assertIn('pkg.module', sys.modules)
+                with self.assertRaises(ZeroDivisionError):
+                    import_util.import_('pkg.module')
+                self.assertNotIn('pkg', sys.modules)
+                self.assertIn('pkg.module', sys.modules)
+
+    def test_raising_parent_after_relative_importing_child(self):
+        def __init__():
+            from . import module
+            1/0
+        mock = util.mock_modules('pkg.__init__', 'pkg.module',
+                                 module_code={'pkg': __init__})
+        with mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    # This raises ImportError on the "from . import module"
+                    # line, not sure why.
+                    import_util.import_('pkg')
+                self.assertNotIn('pkg', sys.modules)
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    import_util.import_('pkg.module')
+                self.assertNotIn('pkg', sys.modules)
+                # XXX False
+                #self.assertIn('pkg.module', sys.modules)
+
+    def test_raising_parent_after_double_relative_importing_child(self):
+        def __init__():
+            from ..subpkg import module
+            1/0
+        mock = util.mock_modules('pkg.__init__', 'pkg.subpkg.__init__',
+                                 'pkg.subpkg.module',
+                                 module_code={'pkg.subpkg': __init__})
+        with mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    # This raises ImportError on the "from ..subpkg import module"
+                    # line, not sure why.
+                    import_util.import_('pkg.subpkg')
+                self.assertNotIn('pkg.subpkg', sys.modules)
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    import_util.import_('pkg.subpkg.module')
+                self.assertNotIn('pkg.subpkg', sys.modules)
+                # XXX False
+                #self.assertIn('pkg.subpkg.module', sys.modules)
+
+    def test_module_not_package(self):
+        # Try to import a submodule from a non-package should raise ImportError.
+        assert not hasattr(sys, '__path__')
+        with self.assertRaises(ImportError) as cm:
+            import_util.import_('sys.no_submodules_here')
+        self.assertEqual(cm.exception.name, 'sys.no_submodules_here')
+
+    def test_module_not_package_but_side_effects(self):
+        # If a module injects something into sys.modules as a side-effect, then
+        # pick up on that fact.
+        name = 'mod'
+        subname = name + '.b'
+        def module_injection():
+            sys.modules[subname] = 'total bunk'
+        mock_modules = util.mock_modules('mod',
+                                         module_code={'mod': module_injection})
+        with mock_modules as mock:
+            with util.import_state(meta_path=[mock]):
+                try:
+                    submodule = import_util.import_(subname)
+                finally:
+                    support.unload(subname)
+
+
+def test_main():
+    from test.support import run_unittest
+    run_unittest(ParentModuleTests)
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py
new file mode 100644
index 0000000000..d82b7f6b0c
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_path.py
@@ -0,0 +1,120 @@
+from importlib import _bootstrap
+from importlib import machinery
+from importlib import import_module
+from .. import util
+from . import util as import_util
+import os
+import sys
+from types import ModuleType
+import unittest
+import warnings
+import zipimport
+
+
+class FinderTests(unittest.TestCase):
+
+    """Tests for PathFinder."""
+
+    def test_failure(self):
+        # Test None returned upon not finding a suitable finder.
+        module = '<test module>'
+        with util.import_state():
+            self.assertIsNone(machinery.PathFinder.find_module(module))
+
+    def test_sys_path(self):
+        # Test that sys.path is used when 'path' is None.
+        # Implicitly tests that sys.path_importer_cache is used.
+        module = '<test module>'
+        path = '<test path>'
+        importer = util.mock_modules(module)
+        with util.import_state(path_importer_cache={path: importer},
+                               path=[path]):
+            loader = machinery.PathFinder.find_module(module)
+            self.assertIs(loader, importer)
+
+    def test_path(self):
+        # Test that 'path' is used when set.
+        # Implicitly tests that sys.path_importer_cache is used.
+        module = '<test module>'
+        path = '<test path>'
+        importer = util.mock_modules(module)
+        with util.import_state(path_importer_cache={path: importer}):
+            loader = machinery.PathFinder.find_module(module, [path])
+            self.assertIs(loader, importer)
+
+    def test_empty_list(self):
+        # An empty list should not count as asking for sys.path.
+        module = 'module'
+        path = '<test path>'
+        importer = util.mock_modules(module)
+        with util.import_state(path_importer_cache={path: importer},
+                               path=[path]):
+            self.assertIsNone(machinery.PathFinder.find_module('module', []))
+
+    def test_path_hooks(self):
+        # Test that sys.path_hooks is used.
+        # Test that sys.path_importer_cache is set.
+        module = '<test module>'
+        path = '<test path>'
+        importer = util.mock_modules(module)
+        hook = import_util.mock_path_hook(path, importer=importer)
+        with util.import_state(path_hooks=[hook]):
+            loader = machinery.PathFinder.find_module(module, [path])
+            self.assertIs(loader, importer)
+            self.assertIn(path, sys.path_importer_cache)
+            self.assertIs(sys.path_importer_cache[path], importer)
+
+    def test_empty_path_hooks(self):
+        # Test that if sys.path_hooks is empty a warning is raised,
+        # sys.path_importer_cache gets None set, and PathFinder returns None.
+        path_entry = 'bogus_path'
+        with util.import_state(path_importer_cache={}, path_hooks=[],
+                               path=[path_entry]):
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter('always')
+                self.assertIsNone(machinery.PathFinder.find_module('os'))
+                self.assertIsNone(sys.path_importer_cache[path_entry])
+                self.assertEqual(len(w), 1)
+                self.assertTrue(issubclass(w[-1].category, ImportWarning))
+
+    def test_path_importer_cache_empty_string(self):
+        # The empty string should create a finder using the cwd.
+        path = ''
+        module = '<test module>'
+        importer = util.mock_modules(module)
+        hook = import_util.mock_path_hook(os.curdir, importer=importer)
+        with util.import_state(path=[path], path_hooks=[hook]):
+            loader = machinery.PathFinder.find_module(module)
+            self.assertIs(loader, importer)
+            self.assertIn(os.curdir, sys.path_importer_cache)
+
+    def test_None_on_sys_path(self):
+        # Putting None in sys.path[0] caused an import regression from Python
+        # 3.2: http://bugs.python.org/issue16514
+        new_path = sys.path[:]
+        new_path.insert(0, None)
+        new_path_importer_cache = sys.path_importer_cache.copy()
+        new_path_importer_cache.pop(None, None)
+        new_path_hooks = [zipimport.zipimporter,
+                          _bootstrap.FileFinder.path_hook(
+                              *_bootstrap._get_supported_file_loaders())]
+        missing = object()
+        email = sys.modules.pop('email', missing)
+        try:
+            with util.import_state(meta_path=sys.meta_path[:],
+                                   path=new_path,
+                                   path_importer_cache=new_path_importer_cache,
+                                   path_hooks=new_path_hooks):
+                module = import_module('email')
+                self.assertIsInstance(module, ModuleType)
+        finally:
+            if email is not missing:
+                sys.modules['email'] = email
+
+
+def test_main():
+    from test.support import run_unittest
+    run_unittest(FinderTests)
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/importlib/test/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py
index a0f6b2d827..4569c26424 100644
--- a/Lib/importlib/test/import_/test_relative_imports.py
+++ b/Lib/test/test_importlib/import_/test_relative_imports.py
@@ -193,6 +193,20 @@ class RelativeImports(unittest.TestCase):
             self.assertEqual(module.__name__, '__runpy_pkg__.uncle.cousin')
         self.relative_import_test(create, globals_, callback)
 
+    def test_import_relative_import_no_fromlist(self):
+        # Import a relative module w/ no fromlist.
+        create = ['crash.__init__', 'crash.mod']
+        globals_ = [{'__package__': 'crash', '__name__': 'crash'}]
+        def callback(global_):
+            import_util.import_('crash')
+            mod = import_util.import_('mod', global_, {}, [], 1)
+            self.assertEqual(mod.__name__, 'crash.mod')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_relative_import_no_globals(self):
+        # No globals for a relative import is an error.
+        with self.assertRaises(KeyError):
+            import_util.import_('sys', level=1)
 
 
 def test_main():
diff --git a/Lib/importlib/test/import_/util.py b/Lib/test/test_importlib/import_/util.py
index 649c5ed27c..86ac065e64 100644
--- a/Lib/importlib/test/import_/util.py
+++ b/Lib/test/test_importlib/import_/util.py
@@ -1,6 +1,5 @@
 import functools
 import importlib
-import importlib._bootstrap
 import unittest
 
 
diff --git a/Lib/test/test_importlib/regrtest.py b/Lib/test/test_importlib/regrtest.py
new file mode 100644
index 0000000000..a5be11fd4e
--- /dev/null
+++ b/Lib/test/test_importlib/regrtest.py
@@ -0,0 +1,17 @@
+"""Run Python's standard test suite using importlib.__import__.
+
+Tests known to fail because of assumptions that importlib (properly)
+invalidates are automatically skipped if the entire test suite is run.
+Otherwise all command-line options valid for test.regrtest are also valid for
+this script.
+
+"""
+import importlib
+import sys
+from test import regrtest
+
+if __name__ == '__main__':
+    __builtins__.__import__ = importlib.__import__
+    sys.path_importer_cache.clear()
+
+    regrtest.main(quiet=True, verbose2=True)
diff --git a/Lib/importlib/test/extension/__init__.py b/Lib/test/test_importlib/source/__init__.py
index 2ec584072d..3ef97f3aa0 100644
--- a/Lib/importlib/test/extension/__init__.py
+++ b/Lib/test/test_importlib/source/__init__.py
@@ -1,11 +1,11 @@
-import importlib.test
+from .. import test_suite
 import os.path
 import unittest
 
 
 def test_suite():
     directory = os.path.dirname(__file__)
-    return importlib.test.test_suite('importlib.test.extension', directory)
+    return test.test_suite('importlib.test.source', directory)
 
 
 if __name__ == '__main__':
diff --git a/Lib/importlib/test/source/test_abc_loader.py b/Lib/test/test_importlib/source/test_abc_loader.py
index 32459074a0..0d912b6469 100644
--- a/Lib/importlib/test/source/test_abc_loader.py
+++ b/Lib/test/test_importlib/source/test_abc_loader.py
@@ -32,6 +32,9 @@ class SourceOnlyLoaderMock(abc.SourceLoader):
     def get_filename(self, fullname):
         return self.path
 
+    def module_repr(self, module):
+        return '<module>'
+
 
 class SourceLoaderMock(SourceOnlyLoaderMock):
 
@@ -40,8 +43,10 @@ class SourceLoaderMock(SourceOnlyLoaderMock):
     def __init__(self, path, magic=imp.get_magic()):
         super().__init__(path)
         self.bytecode_path = imp.cache_from_source(self.path)
+        self.source_size = len(self.source)
         data = bytearray(magic)
-        data.extend(marshal._w_long(self.source_mtime))
+        data.extend(importlib._w_long(self.source_mtime))
+        data.extend(importlib._w_long(self.source_size))
         code_object = compile(self.source, self.path, 'exec',
                                 dont_inherit=True)
         data.extend(marshal.dumps(code_object))
@@ -56,9 +61,9 @@ class SourceLoaderMock(SourceOnlyLoaderMock):
         else:
             raise IOError
 
-    def path_mtime(self, path):
+    def path_stats(self, path):
         assert path == self.path
-        return self.source_mtime
+        return {'mtime': self.source_mtime, 'size': self.source_size}
 
     def set_data(self, path, data):
         self.written[path] = bytes(data)
@@ -102,9 +107,12 @@ class PyLoaderMock(abc.PyLoader):
             warnings.simplefilter("always")
             path = super().get_filename(name)
             assert len(w) == 1
-            assert issubclass(w[0].category, PendingDeprecationWarning)
+            assert issubclass(w[0].category, DeprecationWarning)
             return path
 
+    def module_repr(self):
+        return '<module>'
+
 
 class PyLoaderCompatMock(PyLoaderMock):
 
@@ -146,11 +154,12 @@ class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock):
             self.bytecode_to_path[name] = data['path']
             magic = data.get('magic', imp.get_magic())
             mtime = importlib._w_long(data.get('mtime', self.default_mtime))
+            source_size = importlib._w_long(len(self.source) & 0xFFFFFFFF)
             if 'bc' in data:
                 bc = data['bc']
             else:
                 bc = self.compile_bc(name)
-            self.module_bytecode[name] = magic + mtime + bc
+            self.module_bytecode[name] = magic + mtime + source_size + bc
 
     def compile_bc(self, name):
         source_path = self.module_paths.get(name, '<test>') or '<test>'
@@ -198,7 +207,7 @@ class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock):
             warnings.simplefilter("always")
             code_object = super().get_code(name)
             assert len(w) == 1
-            assert issubclass(w[0].category, PendingDeprecationWarning)
+            assert issubclass(w[0].category, DeprecationWarning)
             return code_object
 
 class PyLoaderTests(testing_abc.LoaderTests):
@@ -219,7 +228,7 @@ class PyLoaderTests(testing_abc.LoaderTests):
         mock = self.mocker({name: path})
         with util.uncache(name):
             module = mock.load_module(name)
-            self.assertTrue(name in sys.modules)
+            self.assertIn(name, sys.modules)
         self.eq_attrs(module, __name__=name, __file__=path, __package__='',
                         __loader__=mock)
         self.assertTrue(not hasattr(module, '__path__'))
@@ -231,7 +240,7 @@ class PyLoaderTests(testing_abc.LoaderTests):
         mock = self.mocker({name: path})
         with util.uncache(name):
             module = mock.load_module(name)
-            self.assertTrue(name in sys.modules)
+            self.assertIn(name, sys.modules)
         self.eq_attrs(module, __name__=name, __file__=path,
                 __path__=[os.path.dirname(path)], __package__=name,
                 __loader__=mock)
@@ -257,8 +266,8 @@ class PyLoaderTests(testing_abc.LoaderTests):
         with util.uncache(name):
             sys.modules[name] = module
             loaded_module = mock.load_module(name)
-            self.assertTrue(loaded_module is module)
-            self.assertTrue(sys.modules[name] is module)
+            self.assertIs(loaded_module, module)
+            self.assertIs(sys.modules[name], module)
         return mock, name
 
     def test_state_after_failure(self):
@@ -271,7 +280,7 @@ class PyLoaderTests(testing_abc.LoaderTests):
             sys.modules[name] = module
             with self.assertRaises(ZeroDivisionError):
                 mock.load_module(name)
-            self.assertTrue(sys.modules[name] is module)
+            self.assertIs(sys.modules[name], module)
             self.assertTrue(hasattr(module, 'blah'))
         return mock
 
@@ -282,7 +291,7 @@ class PyLoaderTests(testing_abc.LoaderTests):
         with util.uncache(name):
             with self.assertRaises(ZeroDivisionError):
                 mock.load_module(name)
-            self.assertTrue(name not in sys.modules)
+            self.assertNotIn(name, sys.modules)
         return mock
 
 
@@ -342,7 +351,10 @@ class PyPycLoaderTests(PyLoaderTests):
         self.assertEqual(magic, imp.get_magic())
         mtime = importlib._r_long(mock.module_bytecode[name][4:8])
         self.assertEqual(mtime, 1)
-        bc = mock.module_bytecode[name][8:]
+        source_size = mock.module_bytecode[name][8:12]
+        self.assertEqual(len(mock.source) & 0xFFFFFFFF,
+                         importlib._r_long(source_size))
+        bc = mock.module_bytecode[name][12:]
         self.assertEqual(bc, mock.compile_bc(name))
 
     def test_module(self):
@@ -412,8 +424,7 @@ class SkipWritingBytecodeTests(unittest.TestCase):
         sys.dont_write_bytecode = dont_write_bytecode
         with util.uncache(name):
             mock.load_module(name)
-        self.assertTrue((name in mock.module_bytecode) is not
-                        dont_write_bytecode)
+        self.assertIsNot(name in mock.module_bytecode, dont_write_bytecode)
 
     def test_no_bytecode_written(self):
         self.run_test(True)
@@ -438,7 +449,7 @@ class RegeneratedBytecodeTests(unittest.TestCase):
                                         'magic': bad_magic}})
         with util.uncache(name):
             mock.load_module(name)
-        self.assertTrue(name in mock.module_bytecode)
+        self.assertIn(name, mock.module_bytecode)
         magic = mock.module_bytecode[name][:4]
         self.assertEqual(magic, imp.get_magic())
 
@@ -451,7 +462,7 @@ class RegeneratedBytecodeTests(unittest.TestCase):
                 {name: {'path': 'path/to/mod.bytecode', 'mtime': old_mtime}})
         with util.uncache(name):
             mock.load_module(name)
-        self.assertTrue(name in mock.module_bytecode)
+        self.assertIn(name, mock.module_bytecode)
         mtime = importlib._r_long(mock.module_bytecode[name][4:8])
         self.assertEqual(mtime, PyPycLoaderMock.default_mtime)
 
@@ -469,8 +480,9 @@ class BadBytecodeFailureTests(unittest.TestCase):
                 {'path': os.path.join('path', 'to', 'mod'),
                  'magic': bad_magic}}
         mock = PyPycLoaderMock({name: None}, bc)
-        with util.uncache(name), self.assertRaises(ImportError):
+        with util.uncache(name), self.assertRaises(ImportError) as cm:
             mock.load_module(name)
+        self.assertEqual(cm.exception.name, name)
 
     def test_no_bytecode(self):
         # Missing code object bytecode should lead to an EOFError.
@@ -514,8 +526,9 @@ class MissingPathsTests(unittest.TestCase):
         # If all *_path methods return None, raise ImportError.
         name = 'mod'
         mock = PyPycLoaderMock({name: None})
-        with util.uncache(name), self.assertRaises(ImportError):
+        with util.uncache(name), self.assertRaises(ImportError) as cm:
             mock.load_module(name)
+        self.assertEqual(cm.exception.name, name)
 
     def test_source_path_ImportError(self):
         # An ImportError from source_path should trigger an ImportError.
@@ -531,7 +544,7 @@ class MissingPathsTests(unittest.TestCase):
         mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')})
         bad_meth = types.MethodType(raise_ImportError, mock)
         mock.bytecode_path = bad_meth
-        with util.uncache(name), self.assertRaises(ImportError):
+        with util.uncache(name), self.assertRaises(ImportError) as cm:
             mock.load_module(name)
 
 
@@ -592,15 +605,17 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness):
         def raise_IOError(path):
             raise IOError
         self.loader.get_data = raise_IOError
-        with self.assertRaises(ImportError):
+        with self.assertRaises(ImportError) as cm:
             self.loader.get_source(self.name)
+        self.assertEqual(cm.exception.name, self.name)
 
     def test_is_package(self):
         # Properly detect when loading a package.
-        self.setUp(is_package=True)
-        self.assertTrue(self.loader.is_package(self.name))
         self.setUp(is_package=False)
         self.assertFalse(self.loader.is_package(self.name))
+        self.setUp(is_package=True)
+        self.assertTrue(self.loader.is_package(self.name))
+        self.assertFalse(self.loader.is_package(self.name + '.__init__'))
 
     def test_get_code(self):
         # Verify the code object is created.
@@ -615,7 +630,7 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness):
             module = self.loader.load_module(self.name)
             self.verify_module(module)
             self.assertEqual(module.__path__, [os.path.dirname(self.path)])
-            self.assertTrue(self.name in sys.modules)
+            self.assertIn(self.name, sys.modules)
 
     def test_package_settings(self):
         # __package__ needs to be set, while __path__ is set on if the module
@@ -656,7 +671,8 @@ class SourceLoaderBytecodeTests(SourceLoaderTestHarness):
         if bytecode_written:
             self.assertIn(self.cached, self.loader.written)
             data = bytearray(imp.get_magic())
-            data.extend(marshal._w_long(self.loader.source_mtime))
+            data.extend(importlib._w_long(self.loader.source_mtime))
+            data.extend(importlib._w_long(self.loader.source_size))
             data.extend(marshal.dumps(code_object))
             self.assertEqual(self.loader.written[self.cached], bytes(data))
 
@@ -762,18 +778,32 @@ class SourceLoaderGetSourceTests(unittest.TestCase):
         expect = io.IncrementalNewlineDecoder(None, True).decode(source)
         self.assertEqual(mock.get_source(name), expect)
 
+
 class AbstractMethodImplTests(unittest.TestCase):
 
     """Test the concrete abstractmethod implementations."""
 
-    class Loader(abc.Loader):
-        def load_module(self, fullname):
-            super().load_module(fullname)
+    class MetaPathFinder(abc.MetaPathFinder):
+        def find_module(self, fullname, path):
+            super().find_module(fullname, path)
 
-    class Finder(abc.Finder):
+    class PathEntryFinder(abc.PathEntryFinder):
         def find_module(self, _):
             super().find_module(_)
 
+        def find_loader(self, _):
+            super().find_loader(_)
+
+    class Finder(abc.Finder):
+        def find_module(self, fullname, path):
+            super().find_module(fullname, path)
+
+    class Loader(abc.Loader):
+        def load_module(self, fullname):
+            super().load_module(fullname)
+        def module_repr(self, module):
+            super().module_repr(module)
+
     class ResourceLoader(Loader, abc.ResourceLoader):
         def get_data(self, _):
             super().get_data(_)
@@ -847,7 +877,7 @@ class AbstractMethodImplTests(unittest.TestCase):
         # Required abstractmethods.
         self.raises_NotImplementedError(ins, 'get_filename', 'get_data')
         # Optional abstractmethods.
-        self.raises_NotImplementedError(ins,'path_mtime', 'set_data')
+        self.raises_NotImplementedError(ins,'path_stats', 'set_data')
 
     def test_PyLoader(self):
         self.raises_NotImplementedError(self.PyLoader(), 'source_path',
diff --git a/Lib/importlib/test/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py
index 73777de4ba..241173fb44 100644
--- a/Lib/importlib/test/source/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/source/test_case_sensitivity.py
@@ -1,7 +1,9 @@
 """Test case-sensitivity (PEP 235)."""
 from importlib import _bootstrap
+from importlib import machinery
 from .. import util
 from . import util as source_util
+import imp
 import os
 import sys
 from test import support as test_support
@@ -19,9 +21,11 @@ class CaseSensitivityTest(unittest.TestCase):
     assert name != name.lower()
 
     def find(self, path):
-        finder = _bootstrap._FileFinder(path,
-                                        _bootstrap._SourceFinderDetails(),
-                                        _bootstrap._SourcelessFinderDetails())
+        finder = machinery.FileFinder(path,
+                                      (machinery.SourceFileLoader,
+                                            machinery.SOURCE_SUFFIXES),
+                                        (machinery.SourcelessFileLoader,
+                                            machinery.BYTECODE_SUFFIXES))
         return finder.find_module(self.name)
 
     def sensitivity_test(self):
@@ -37,6 +41,9 @@ class CaseSensitivityTest(unittest.TestCase):
     def test_sensitive(self):
         with test_support.EnvironmentVarGuard() as env:
             env.unset('PYTHONCASEOK')
+            if b'PYTHONCASEOK' in _bootstrap._os.environ:
+                self.skipTest('os.environ changes not reflected in '
+                              '_os.environ')
             sensitive, insensitive = self.sensitivity_test()
             self.assertTrue(hasattr(sensitive, 'load_module'))
             self.assertIn(self.name, sensitive.get_filename(self.name))
@@ -45,6 +52,9 @@ class CaseSensitivityTest(unittest.TestCase):
     def test_insensitive(self):
         with test_support.EnvironmentVarGuard() as env:
             env.set('PYTHONCASEOK', '1')
+            if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+                self.skipTest('os.environ changes not reflected in '
+                              '_os.environ')
             sensitive, insensitive = self.sensitivity_test()
             self.assertTrue(hasattr(sensitive, 'load_module'))
             self.assertIn(self.name, sensitive.get_filename(self.name))
diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index c7a7d8fbca..90f9d30129 100644
--- a/Lib/importlib/test/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -1,5 +1,6 @@
+from importlib import machinery
 import importlib
-from importlib import _bootstrap
+import importlib.abc
 from .. import abc
 from .. import util
 from . import util as source_util
@@ -24,12 +25,48 @@ class SimpleTest(unittest.TestCase):
 
     """
 
+    def test_load_module_API(self):
+        # If fullname is not specified that assume self.name is desired.
+        class TesterMixin(importlib.abc.Loader):
+            def load_module(self, fullname): return fullname
+            def module_repr(self, module): return '<module>'
+
+        class Tester(importlib.abc.FileLoader, TesterMixin):
+            def get_code(self, _): pass
+            def get_source(self, _): pass
+            def is_package(self, _): pass
+
+        name = 'mod_name'
+        loader = Tester(name, 'some_path')
+        self.assertEqual(name, loader.load_module())
+        self.assertEqual(name, loader.load_module(None))
+        self.assertEqual(name, loader.load_module(name))
+        with self.assertRaises(ImportError):
+            loader.load_module(loader.name + 'XXX')
+
+    def test_get_filename_API(self):
+        # If fullname is not set then assume self.path is desired.
+        class Tester(importlib.abc.FileLoader):
+            def get_code(self, _): pass
+            def get_source(self, _): pass
+            def is_package(self, _): pass
+            def module_repr(self, _): pass
+
+        path = 'some_path'
+        name = 'some_name'
+        loader = Tester(name, path)
+        self.assertEqual(path, loader.get_filename(name))
+        self.assertEqual(path, loader.get_filename())
+        self.assertEqual(path, loader.get_filename(None))
+        with self.assertRaises(ImportError):
+            loader.get_filename(name + 'XXX')
+
     # [basic]
     def test_module(self):
         with source_util.create_modules('_temp') as mapping:
-            loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
+            loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
             module = loader.load_module('_temp')
-            self.assertTrue('_temp' in sys.modules)
+            self.assertIn('_temp', sys.modules)
             check = {'__name__': '_temp', '__file__': mapping['_temp'],
                      '__package__': ''}
             for attr, value in check.items():
@@ -37,10 +74,10 @@ class SimpleTest(unittest.TestCase):
 
     def test_package(self):
         with source_util.create_modules('_pkg.__init__') as mapping:
-            loader = _bootstrap._SourceFileLoader('_pkg',
+            loader = machinery.SourceFileLoader('_pkg',
                                                  mapping['_pkg.__init__'])
             module = loader.load_module('_pkg')
-            self.assertTrue('_pkg' in sys.modules)
+            self.assertIn('_pkg', sys.modules)
             check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'],
                      '__path__': [os.path.dirname(mapping['_pkg.__init__'])],
                      '__package__': '_pkg'}
@@ -50,10 +87,10 @@ class SimpleTest(unittest.TestCase):
 
     def test_lacking_parent(self):
         with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping:
-            loader = _bootstrap._SourceFileLoader('_pkg.mod',
+            loader = machinery.SourceFileLoader('_pkg.mod',
                                                     mapping['_pkg.mod'])
             module = loader.load_module('_pkg.mod')
-            self.assertTrue('_pkg.mod' in sys.modules)
+            self.assertIn('_pkg.mod', sys.modules)
             check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'],
                      '__package__': '_pkg'}
             for attr, value in check.items():
@@ -65,19 +102,14 @@ class SimpleTest(unittest.TestCase):
 
     def test_module_reuse(self):
         with source_util.create_modules('_temp') as mapping:
-            loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
+            loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
             module = loader.load_module('_temp')
             module_id = id(module)
             module_dict_id = id(module.__dict__)
             with open(mapping['_temp'], 'w') as file:
                 file.write("testing_var = 42\n")
-            # For filesystems where the mtime is only to a second granularity,
-            # everything that has happened above can be too fast;
-            # force an mtime on the source that is guaranteed to be different
-            # than the original mtime.
-            loader.path_mtime = self.fake_mtime(loader.path_mtime)
             module = loader.load_module('_temp')
-            self.assertTrue('testing_var' in module.__dict__,
+            self.assertIn('testing_var', module.__dict__,
                          "'testing_var' not in "
                             "{0}".format(list(module.__dict__.keys())))
             self.assertEqual(module, sys.modules['_temp'])
@@ -95,7 +127,7 @@ class SimpleTest(unittest.TestCase):
                 setattr(orig_module, attr, value)
             with open(mapping[name], 'w') as file:
                 file.write('+++ bad syntax +++')
-            loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
+            loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
             with self.assertRaises(SyntaxError):
                 loader.load_module(name)
             for attr in attributes:
@@ -106,10 +138,10 @@ class SimpleTest(unittest.TestCase):
         with source_util.create_modules('_temp') as mapping:
             with open(mapping['_temp'], 'w') as file:
                 file.write('=')
-            loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
+            loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
             with self.assertRaises(SyntaxError):
                 loader.load_module('_temp')
-            self.assertTrue('_temp' not in sys.modules)
+            self.assertNotIn('_temp', sys.modules)
 
     def test_file_from_empty_string_dir(self):
         # Loading a module found from an empty string entry on sys.path should
@@ -119,7 +151,7 @@ class SimpleTest(unittest.TestCase):
             file.write("# test file for importlib")
         try:
             with util.uncache('_temp'):
-                loader = _bootstrap._SourceFileLoader('_temp', file_path)
+                loader = machinery.SourceFileLoader('_temp', file_path)
                 mod = loader.load_module('_temp')
                 self.assertEqual(file_path, mod.__file__)
                 self.assertEqual(imp.cache_from_source(file_path),
@@ -145,7 +177,7 @@ class SimpleTest(unittest.TestCase):
                 if e.errno != getattr(errno, 'EOVERFLOW', None):
                     raise
                 self.skipTest("cannot set modification time to large integer ({})".format(e))
-            loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
+            loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
             mod = loader.load_module('_temp')
             # Sanity checks.
             self.assertEqual(mod.__cached__, compiled)
@@ -159,7 +191,7 @@ class BadBytecodeTest(unittest.TestCase):
     def import_(self, file, module_name):
         loader = self.loader(module_name, file)
         module = loader.load_module(module_name)
-        self.assertTrue(module_name in sys.modules)
+        self.assertIn(module_name, sys.modules)
 
     def manipulate_bytecode(self, name, mapping, manipulator, *,
                             del_source=False):
@@ -215,10 +247,17 @@ class BadBytecodeTest(unittest.TestCase):
                                                 del_source=del_source)
             test('_temp', mapping, bc_path)
 
+    def _test_partial_size(self, test, *, del_source=False):
+        with source_util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: bc[:11],
+                                                del_source=del_source)
+            test('_temp', mapping, bc_path)
+
     def _test_no_marshal(self, *, del_source=False):
         with source_util.create_modules('_temp') as mapping:
             bc_path = self.manipulate_bytecode('_temp', mapping,
-                                                lambda bc: bc[:8],
+                                                lambda bc: bc[:12],
                                                 del_source=del_source)
             file_path = mapping['_temp'] if not del_source else bc_path
             with self.assertRaises(EOFError):
@@ -227,16 +266,18 @@ class BadBytecodeTest(unittest.TestCase):
     def _test_non_code_marshal(self, *, del_source=False):
         with source_util.create_modules('_temp') as mapping:
             bytecode_path = self.manipulate_bytecode('_temp', mapping,
-                                    lambda bc: bc[:8] + marshal.dumps(b'abcd'),
+                                    lambda bc: bc[:12] + marshal.dumps(b'abcd'),
                                     del_source=del_source)
             file_path = mapping['_temp'] if not del_source else bytecode_path
-            with self.assertRaises(ImportError):
+            with self.assertRaises(ImportError) as cm:
                 self.import_(file_path, '_temp')
+            self.assertEqual(cm.exception.name, '_temp')
+            self.assertEqual(cm.exception.path, bytecode_path)
 
     def _test_bad_marshal(self, *, del_source=False):
         with source_util.create_modules('_temp') as mapping:
             bytecode_path = self.manipulate_bytecode('_temp', mapping,
-                                                lambda bc: bc[:8] + b'<test>',
+                                                lambda bc: bc[:12] + b'<test>',
                                                 del_source=del_source)
             file_path = mapping['_temp'] if not del_source else bytecode_path
             with self.assertRaises(EOFError):
@@ -251,7 +292,7 @@ class BadBytecodeTest(unittest.TestCase):
 
 class SourceLoaderBadBytecodeTest(BadBytecodeTest):
 
-    loader = _bootstrap._SourceFileLoader
+    loader = machinery.SourceFileLoader
 
     @source_util.writes_bytecode_files
     def test_empty_file(self):
@@ -260,7 +301,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
         def test(name, mapping, bytecode_path):
             self.import_(mapping[name], name)
             with open(bytecode_path, 'rb') as file:
-                self.assertGreater(len(file.read()), 8)
+                self.assertGreater(len(file.read()), 12)
 
         self._test_empty_file(test)
 
@@ -268,7 +309,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
         def test(name, mapping, bytecode_path):
             self.import_(mapping[name], name)
             with open(bytecode_path, 'rb') as file:
-                self.assertGreater(len(file.read()), 8)
+                self.assertGreater(len(file.read()), 12)
 
         self._test_partial_magic(test)
 
@@ -279,7 +320,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
         def test(name, mapping, bytecode_path):
             self.import_(mapping[name], name)
             with open(bytecode_path, 'rb') as file:
-                self.assertGreater(len(file.read()), 8)
+                self.assertGreater(len(file.read()), 12)
 
         self._test_magic_only(test)
 
@@ -301,11 +342,22 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
         def test(name, mapping, bc_path):
             self.import_(mapping[name], name)
             with open(bc_path, 'rb') as file:
-                self.assertGreater(len(file.read()), 8)
+                self.assertGreater(len(file.read()), 12)
 
         self._test_partial_timestamp(test)
 
     @source_util.writes_bytecode_files
+    def test_partial_size(self):
+        # When the size is partial, regenerate the .pyc, else
+        # raise EOFError.
+        def test(name, mapping, bc_path):
+            self.import_(mapping[name], name)
+            with open(bc_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 12)
+
+        self._test_partial_size(test)
+
+    @source_util.writes_bytecode_files
     def test_no_marshal(self):
         # When there is only the magic number and timestamp, raise EOFError.
         self._test_no_marshal()
@@ -364,19 +416,23 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
 
 class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
 
-    loader = _bootstrap._SourcelessFileLoader
+    loader = machinery.SourcelessFileLoader
 
     def test_empty_file(self):
         def test(name, mapping, bytecode_path):
-            with self.assertRaises(ImportError):
+            with self.assertRaises(ImportError) as cm:
                 self.import_(bytecode_path, name)
+            self.assertEqual(cm.exception.name, name)
+            self.assertEqual(cm.exception.path, bytecode_path)
 
         self._test_empty_file(test, del_source=True)
 
     def test_partial_magic(self):
         def test(name, mapping, bytecode_path):
-            with self.assertRaises(ImportError):
+            with self.assertRaises(ImportError) as cm:
                 self.import_(bytecode_path, name)
+            self.assertEqual(cm.exception.name, name)
+            self.assertEqual(cm.exception.path, bytecode_path)
         self._test_partial_magic(test, del_source=True)
 
     def test_magic_only(self):
@@ -388,8 +444,10 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
 
     def test_bad_magic(self):
         def test(name, mapping, bytecode_path):
-            with self.assertRaises(ImportError):
+            with self.assertRaises(ImportError) as cm:
                 self.import_(bytecode_path, name)
+            self.assertEqual(cm.exception.name, name)
+            self.assertEqual(cm.exception.path, bytecode_path)
 
         self._test_bad_magic(test, del_source=True)
 
@@ -400,6 +458,13 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
 
         self._test_partial_timestamp(test, del_source=True)
 
+    def test_partial_size(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(EOFError):
+                self.import_(bytecode_path, name)
+
+        self._test_partial_size(test, del_source=True)
+
     def test_no_marshal(self):
         self._test_no_marshal(del_source=True)
 
diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py
index 7b9088da0c..8e4986835d 100644
--- a/Lib/importlib/test/source/test_finder.py
+++ b/Lib/test/test_importlib/source/test_finder.py
@@ -1,10 +1,15 @@
-from importlib import _bootstrap
 from .. import abc
 from . import util as source_util
-from test.support import make_legacy_pyc
-import os
+
+from importlib import machinery
 import errno
+import imp
+import os
 import py_compile
+import stat
+import sys
+import tempfile
+from test.support import make_legacy_pyc
 import unittest
 import warnings
 
@@ -33,11 +38,15 @@ class FinderTests(abc.FinderTests):
 
     """
 
+    def get_finder(self, root):
+        loader_details = [(machinery.SourceFileLoader,
+                            machinery.SOURCE_SUFFIXES),
+                          (machinery.SourcelessFileLoader,
+                            machinery.BYTECODE_SUFFIXES)]
+        return machinery.FileFinder(root, *loader_details)
+
     def import_(self, root, module):
-        finder = _bootstrap._FileFinder(root,
-                                        _bootstrap._SourceFinderDetails(),
-                                        _bootstrap._SourcelessFinderDetails())
-        return finder.find_module(module)
+        return self.get_finder(root).find_module(module)
 
     def run_test(self, test, create=None, *, compile_=None, unlink=None):
         """Test the finding of 'test' with the creation of modules listed in
@@ -102,39 +111,21 @@ class FinderTests(abc.FinderTests):
             loader = self.import_(pkg_dir, 'pkg.sub')
             self.assertTrue(hasattr(loader, 'load_module'))
 
-    # [sub empty]
-    def test_empty_sub_directory(self):
-        context = source_util.create_modules('pkg.__init__', 'pkg.sub.__init__')
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", ImportWarning)
-            with context as mapping:
-                os.unlink(mapping['pkg.sub.__init__'])
-                pkg_dir = os.path.dirname(mapping['pkg.__init__'])
-                with self.assertRaises(ImportWarning):
-                    self.import_(pkg_dir, 'pkg.sub')
-
     # [package over modules]
     def test_package_over_module(self):
         name = '_temp'
         loader = self.run_test(name, {'{0}.__init__'.format(name), name})
-        self.assertTrue('__init__' in loader.get_filename(name))
-
+        self.assertIn('__init__', loader.get_filename(name))
 
     def test_failure(self):
         with source_util.create_modules('blah') as mapping:
             nothing = self.import_(mapping['.root'], 'sdfsadsadf')
-            self.assertTrue(nothing is None)
-
-    # [empty dir]
-    def test_empty_dir(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", ImportWarning)
-            with self.assertRaises(ImportWarning):
-                self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'})
+            self.assertIsNone(nothing)
 
     def test_empty_string_for_dir(self):
         # The empty string from sys.path means to search in the cwd.
-        finder = _bootstrap._FileFinder('', _bootstrap._SourceFinderDetails())
+        finder = machinery.FileFinder('', (machinery.SourceFileLoader,
+            machinery.SOURCE_SUFFIXES))
         with open('mod.py', 'w') as file:
             file.write("# test file for importlib")
         try:
@@ -143,6 +134,53 @@ class FinderTests(abc.FinderTests):
         finally:
             os.unlink('mod.py')
 
+    def test_invalidate_caches(self):
+        # invalidate_caches() should reset the mtime.
+        finder = machinery.FileFinder('', (machinery.SourceFileLoader,
+            machinery.SOURCE_SUFFIXES))
+        finder._path_mtime = 42
+        finder.invalidate_caches()
+        self.assertEqual(finder._path_mtime, -1)
+
+    # Regression test for http://bugs.python.org/issue14846
+    def test_dir_removal_handling(self):
+        mod = 'mod'
+        with source_util.create_modules(mod) as mapping:
+            finder = self.get_finder(mapping['.root'])
+            self.assertIsNotNone(finder.find_module(mod))
+        self.assertIsNone(finder.find_module(mod))
+
+    @unittest.skipUnless(sys.platform != 'win32',
+            'os.chmod() does not support the needed arguments under Windows')
+    def test_no_read_directory(self):
+        # Issue #16730
+        tempdir = tempfile.TemporaryDirectory()
+        original_mode = os.stat(tempdir.name).st_mode
+        def cleanup(tempdir):
+            """Cleanup function for the temporary directory.
+
+            Since we muck with the permissions, we want to set them back to
+            their original values to make sure the directory can be properly
+            cleaned up.
+
+            """
+            os.chmod(tempdir.name, original_mode)
+            # If this is not explicitly called then the __del__ method is used,
+            # but since already mucking around might as well explicitly clean
+            # up.
+            tempdir.__exit__(None, None, None)
+        self.addCleanup(cleanup, tempdir)
+        os.chmod(tempdir.name, stat.S_IWUSR | stat.S_IXUSR)
+        finder = self.get_finder(tempdir.name)
+        self.assertEqual((None, []), finder.find_loader('doesnotexist'))
+
+    def test_ignore_file(self):
+        # If a directory got changed to a file from underneath us, then don't
+        # worry about looking for submodules.
+        with tempfile.NamedTemporaryFile() as file_obj:
+            finder = self.get_finder(file_obj.name)
+            self.assertEqual((None, []), finder.find_loader('doesnotexist'))
+
 
 def test_main():
     from test.support import run_unittest
diff --git a/Lib/importlib/test/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py
index 374f7b6ad3..6a78792f07 100644
--- a/Lib/importlib/test/source/test_path_hook.py
+++ b/Lib/test/test_importlib/source/test_path_hook.py
@@ -1,5 +1,7 @@
-from importlib import _bootstrap
 from . import util as source_util
+
+from importlib import machinery
+import imp
 import unittest
 
 
@@ -7,14 +9,18 @@ class PathHookTest(unittest.TestCase):
 
     """Test the path hook for source."""
 
+    def path_hook(self):
+        return machinery.FileFinder.path_hook((machinery.SourceFileLoader,
+            machinery.SOURCE_SUFFIXES))
+
     def test_success(self):
         with source_util.create_modules('dummy') as mapping:
-            self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']),
+            self.assertTrue(hasattr(self.path_hook()(mapping['.root']),
                                  'find_module'))
 
     def test_empty_string(self):
         # The empty string represents the cwd.
-        self.assertTrue(hasattr(_bootstrap._file_path_hook(''), 'find_module'))
+        self.assertTrue(hasattr(self.path_hook()(''), 'find_module'))
 
 
 def test_main():
diff --git a/Lib/importlib/test/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py
index 794a3df246..0ca5195439 100644
--- a/Lib/importlib/test/source/test_source_encoding.py
+++ b/Lib/test/test_importlib/source/test_source_encoding.py
@@ -1,6 +1,6 @@
-from importlib import _bootstrap
 from . import util as source_util
 
+from importlib import _bootstrap
 import codecs
 import re
 import sys
@@ -35,8 +35,8 @@ class EncodingTest(unittest.TestCase):
         with source_util.create_modules(self.module_name) as mapping:
             with open(mapping[self.module_name], 'wb') as file:
                 file.write(source)
-            loader = _bootstrap._SourceFileLoader(self.module_name,
-                                       mapping[self.module_name])
+            loader = _bootstrap.SourceFileLoader(self.module_name,
+                                                  mapping[self.module_name])
             return loader.load_module(self.module_name)
 
     def create_source(self, encoding):
@@ -97,7 +97,7 @@ class LineEndingTest(unittest.TestCase):
         with source_util.create_modules(module_name) as mapping:
             with open(mapping[module_name], 'wb') as file:
                 file.write(source)
-            loader = _bootstrap._SourceFileLoader(module_name,
+            loader = _bootstrap.SourceFileLoader(module_name,
                                                  mapping[module_name])
             return loader.load_module(module_name)
 
diff --git a/Lib/importlib/test/source/util.py b/Lib/test/test_importlib/source/util.py
index ae65663a67..ae65663a67 100644
--- a/Lib/importlib/test/source/util.py
+++ b/Lib/test/test_importlib/source/util.py
diff --git a/Lib/importlib/test/test_abc.py b/Lib/test/test_importlib/test_abc.py
index 0ecbe390ad..c620c3771b 100644
--- a/Lib/importlib/test/test_abc.py
+++ b/Lib/test/test_importlib/test_abc.py
@@ -30,10 +30,17 @@ class InheritanceTests:
                "{0} is not a superclass of {1}".format(superclass, self.__test))
 
 
-class Finder(InheritanceTests, unittest.TestCase):
+class MetaPathFinder(InheritanceTests, unittest.TestCase):
 
+    superclasses = [abc.Finder]
     subclasses = [machinery.BuiltinImporter, machinery.FrozenImporter,
-                    machinery.PathFinder]
+                    machinery.PathFinder, machinery.WindowsRegistryFinder]
+
+
+class PathEntryFinder(InheritanceTests, unittest.TestCase):
+
+    superclasses = [abc.Finder]
+    subclasses = [machinery.FileFinder]
 
 
 class Loader(InheritanceTests, unittest.TestCase):
@@ -50,7 +57,7 @@ class InspectLoader(InheritanceTests, unittest.TestCase):
 
     superclasses = [abc.Loader]
     subclasses = [abc.PyLoader, machinery.BuiltinImporter,
-                    machinery.FrozenImporter]
+                    machinery.FrozenImporter, machinery.ExtensionFileLoader]
 
 
 class ExecutionLoader(InheritanceTests, unittest.TestCase):
@@ -59,9 +66,16 @@ class ExecutionLoader(InheritanceTests, unittest.TestCase):
     subclasses = [abc.PyLoader]
 
 
+class FileLoader(InheritanceTests, unittest.TestCase):
+
+    superclasses = [abc.ResourceLoader, abc.ExecutionLoader]
+    subclasses = [machinery.SourceFileLoader, machinery.SourcelessFileLoader]
+
+
 class SourceLoader(InheritanceTests, unittest.TestCase):
 
     superclasses = [abc.ResourceLoader, abc.ExecutionLoader]
+    subclasses = [machinery.SourceFileLoader]
 
 
 class PyLoader(InheritanceTests, unittest.TestCase):
diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
new file mode 100644
index 0000000000..b1a58944f3
--- /dev/null
+++ b/Lib/test/test_importlib/test_api.py
@@ -0,0 +1,202 @@
+from . import util
+import imp
+import importlib
+from importlib import machinery
+import sys
+from test import support
+import types
+import unittest
+
+
+class ImportModuleTests(unittest.TestCase):
+
+    """Test importlib.import_module."""
+
+    def test_module_import(self):
+        # Test importing a top-level module.
+        with util.mock_modules('top_level') as mock:
+            with util.import_state(meta_path=[mock]):
+                module = importlib.import_module('top_level')
+                self.assertEqual(module.__name__, 'top_level')
+
+    def test_absolute_package_import(self):
+        # Test importing a module from a package with an absolute name.
+        pkg_name = 'pkg'
+        pkg_long_name = '{0}.__init__'.format(pkg_name)
+        name = '{0}.mod'.format(pkg_name)
+        with util.mock_modules(pkg_long_name, name) as mock:
+            with util.import_state(meta_path=[mock]):
+                module = importlib.import_module(name)
+                self.assertEqual(module.__name__, name)
+
+    def test_shallow_relative_package_import(self):
+        # Test importing a module from a package through a relative import.
+        pkg_name = 'pkg'
+        pkg_long_name = '{0}.__init__'.format(pkg_name)
+        module_name = 'mod'
+        absolute_name = '{0}.{1}'.format(pkg_name, module_name)
+        relative_name = '.{0}'.format(module_name)
+        with util.mock_modules(pkg_long_name, absolute_name) as mock:
+            with util.import_state(meta_path=[mock]):
+                importlib.import_module(pkg_name)
+                module = importlib.import_module(relative_name, pkg_name)
+                self.assertEqual(module.__name__, absolute_name)
+
+    def test_deep_relative_package_import(self):
+        modules = ['a.__init__', 'a.b.__init__', 'a.c']
+        with util.mock_modules(*modules) as mock:
+            with util.import_state(meta_path=[mock]):
+                importlib.import_module('a')
+                importlib.import_module('a.b')
+                module = importlib.import_module('..c', 'a.b')
+                self.assertEqual(module.__name__, 'a.c')
+
+    def test_absolute_import_with_package(self):
+        # Test importing a module from a package with an absolute name with
+        # the 'package' argument given.
+        pkg_name = 'pkg'
+        pkg_long_name = '{0}.__init__'.format(pkg_name)
+        name = '{0}.mod'.format(pkg_name)
+        with util.mock_modules(pkg_long_name, name) as mock:
+            with util.import_state(meta_path=[mock]):
+                importlib.import_module(pkg_name)
+                module = importlib.import_module(name, pkg_name)
+                self.assertEqual(module.__name__, name)
+
+    def test_relative_import_wo_package(self):
+        # Relative imports cannot happen without the 'package' argument being
+        # set.
+        with self.assertRaises(TypeError):
+            importlib.import_module('.support')
+
+
+    def test_loaded_once(self):
+        # Issue #13591: Modules should only be loaded once when
+        # initializing the parent package attempts to import the
+        # module currently being imported.
+        b_load_count = 0
+        def load_a():
+            importlib.import_module('a.b')
+        def load_b():
+            nonlocal b_load_count
+            b_load_count += 1
+        code = {'a': load_a, 'a.b': load_b}
+        modules = ['a.__init__', 'a.b']
+        with util.mock_modules(*modules, module_code=code) as mock:
+            with util.import_state(meta_path=[mock]):
+                importlib.import_module('a.b')
+        self.assertEqual(b_load_count, 1)
+
+
+class FindLoaderTests(unittest.TestCase):
+
+    class FakeMetaFinder:
+        @staticmethod
+        def find_module(name, path=None): return name, path
+
+    def test_sys_modules(self):
+        # If a module with __loader__ is in sys.modules, then return it.
+        name = 'some_mod'
+        with util.uncache(name):
+            module = imp.new_module(name)
+            loader = 'a loader!'
+            module.__loader__ = loader
+            sys.modules[name] = module
+            found = importlib.find_loader(name)
+            self.assertEqual(loader, found)
+
+    def test_sys_modules_loader_is_None(self):
+        # If sys.modules[name].__loader__ is None, raise ValueError.
+        name = 'some_mod'
+        with util.uncache(name):
+            module = imp.new_module(name)
+            module.__loader__ = None
+            sys.modules[name] = module
+            with self.assertRaises(ValueError):
+                importlib.find_loader(name)
+
+    def test_success(self):
+        # Return the loader found on sys.meta_path.
+        name = 'some_mod'
+        with util.uncache(name):
+            with util.import_state(meta_path=[self.FakeMetaFinder]):
+                self.assertEqual((name, None), importlib.find_loader(name))
+
+    def test_success_path(self):
+        # Searching on a path should work.
+        name = 'some_mod'
+        path = 'path to some place'
+        with util.uncache(name):
+            with util.import_state(meta_path=[self.FakeMetaFinder]):
+                self.assertEqual((name, path),
+                                 importlib.find_loader(name, path))
+
+    def test_nothing(self):
+        # None is returned upon failure to find a loader.
+        self.assertIsNone(importlib.find_loader('nevergoingtofindthismodule'))
+
+
+class InvalidateCacheTests(unittest.TestCase):
+
+    def test_method_called(self):
+        # If defined the method should be called.
+        class InvalidatingNullFinder:
+            def __init__(self, *ignored):
+                self.called = False
+            def find_module(self, *args):
+                return None
+            def invalidate_caches(self):
+                self.called = True
+
+        key = 'gobledeegook'
+        meta_ins = InvalidatingNullFinder()
+        path_ins = InvalidatingNullFinder()
+        sys.meta_path.insert(0, meta_ins)
+        self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
+        sys.path_importer_cache[key] = path_ins
+        self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
+        importlib.invalidate_caches()
+        self.assertTrue(meta_ins.called)
+        self.assertTrue(path_ins.called)
+
+    def test_method_lacking(self):
+        # There should be no issues if the method is not defined.
+        key = 'gobbledeegook'
+        sys.path_importer_cache[key] = imp.NullImporter('abc')
+        self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
+        importlib.invalidate_caches()  # Shouldn't trigger an exception.
+
+
+class FrozenImportlibTests(unittest.TestCase):
+
+    def test_no_frozen_importlib(self):
+        # Should be able to import w/o _frozen_importlib being defined.
+        module = support.import_fresh_module('importlib', blocked=['_frozen_importlib'])
+        self.assertFalse(isinstance(module.__loader__,
+                                    machinery.FrozenImporter))
+
+
+class StartupTests(unittest.TestCase):
+
+    def test_everyone_has___loader__(self):
+        # Issue #17098: all modules should have __loader__ defined.
+        for name, module in sys.modules.items():
+            if isinstance(module, types.ModuleType):
+                if name in sys.builtin_module_names:
+                    self.assertEqual(importlib.machinery.BuiltinImporter,
+                                     module.__loader__)
+                elif imp.is_frozen(name):
+                    self.assertEqual(importlib.machinery.FrozenImporter,
+                                     module.__loader__)
+
+def test_main():
+    from test.support import run_unittest
+    run_unittest(ImportModuleTests,
+                 FindLoaderTests,
+                 InvalidateCacheTests,
+                 FrozenImportlibTests,
+                 StartupTests)
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py
new file mode 100644
index 0000000000..c373b11256
--- /dev/null
+++ b/Lib/test/test_importlib/test_locks.py
@@ -0,0 +1,129 @@
+from importlib import _bootstrap
+import sys
+import time
+import unittest
+import weakref
+
+from test import support
+
+try:
+    import threading
+except ImportError:
+    threading = None
+else:
+    from test import lock_tests
+
+
+LockType = _bootstrap._ModuleLock
+DeadlockError = _bootstrap._DeadlockError
+
+
+if threading is not None:
+    class ModuleLockAsRLockTests(lock_tests.RLockTests):
+        locktype = staticmethod(lambda: LockType("some_lock"))
+
+        # _is_owned() unsupported
+        test__is_owned = None
+        # acquire(blocking=False) unsupported
+        test_try_acquire = None
+        test_try_acquire_contended = None
+        # `with` unsupported
+        test_with = None
+        # acquire(timeout=...) unsupported
+        test_timeout = None
+        # _release_save() unsupported
+        test_release_save_unacquired = None
+
+else:
+    class ModuleLockAsRLockTests(unittest.TestCase):
+        pass
+
+
+@unittest.skipUnless(threading, "threads needed for this test")
+class DeadlockAvoidanceTests(unittest.TestCase):
+
+    def setUp(self):
+        try:
+            self.old_switchinterval = sys.getswitchinterval()
+            sys.setswitchinterval(0.000001)
+        except AttributeError:
+            self.old_switchinterval = None
+
+    def tearDown(self):
+        if self.old_switchinterval is not None:
+            sys.setswitchinterval(self.old_switchinterval)
+
+    def run_deadlock_avoidance_test(self, create_deadlock):
+        NLOCKS = 10
+        locks = [LockType(str(i)) for i in range(NLOCKS)]
+        pairs = [(locks[i], locks[(i+1)%NLOCKS]) for i in range(NLOCKS)]
+        if create_deadlock:
+            NTHREADS = NLOCKS
+        else:
+            NTHREADS = NLOCKS - 1
+        barrier = threading.Barrier(NTHREADS)
+        results = []
+        def _acquire(lock):
+            """Try to acquire the lock. Return True on success, False on deadlock."""
+            try:
+                lock.acquire()
+            except DeadlockError:
+                return False
+            else:
+                return True
+        def f():
+            a, b = pairs.pop()
+            ra = _acquire(a)
+            barrier.wait()
+            rb = _acquire(b)
+            results.append((ra, rb))
+            if rb:
+                b.release()
+            if ra:
+                a.release()
+        lock_tests.Bunch(f, NTHREADS).wait_for_finished()
+        self.assertEqual(len(results), NTHREADS)
+        return results
+
+    def test_deadlock(self):
+        results = self.run_deadlock_avoidance_test(True)
+        # At least one of the threads detected a potential deadlock on its
+        # second acquire() call.  It may be several of them, because the
+        # deadlock avoidance mechanism is conservative.
+        nb_deadlocks = results.count((True, False))
+        self.assertGreaterEqual(nb_deadlocks, 1)
+        self.assertEqual(results.count((True, True)), len(results) - nb_deadlocks)
+
+    def test_no_deadlock(self):
+        results = self.run_deadlock_avoidance_test(False)
+        self.assertEqual(results.count((True, False)), 0)
+        self.assertEqual(results.count((True, True)), len(results))
+
+
+class LifetimeTests(unittest.TestCase):
+
+    def test_lock_lifetime(self):
+        name = "xyzzy"
+        self.assertNotIn(name, _bootstrap._module_locks)
+        lock = _bootstrap._get_module_lock(name)
+        self.assertIn(name, _bootstrap._module_locks)
+        wr = weakref.ref(lock)
+        del lock
+        support.gc_collect()
+        self.assertNotIn(name, _bootstrap._module_locks)
+        self.assertIsNone(wr())
+
+    def test_all_locks(self):
+        support.gc_collect()
+        self.assertEqual(0, len(_bootstrap._module_locks), _bootstrap._module_locks)
+
+
+@support.reap_threads
+def test_main():
+    support.run_unittest(ModuleLockAsRLockTests,
+                         DeadlockAvoidanceTests,
+                         LifetimeTests)
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/importlib/test/test_util.py b/Lib/test/test_importlib/test_util.py
index 602447f09e..efc8977fb4 100644
--- a/Lib/importlib/test/test_util.py
+++ b/Lib/test/test_importlib/test_util.py
@@ -29,8 +29,8 @@ class ModuleForLoaderTests(unittest.TestCase):
         module_name = 'a.b.c'
         with test_util.uncache(module_name):
             module = self.return_module(module_name)
-            self.assertTrue(module_name in sys.modules)
-        self.assertTrue(isinstance(module, types.ModuleType))
+            self.assertIn(module_name, sys.modules)
+        self.assertIsInstance(module, types.ModuleType)
         self.assertEqual(module.__name__, module_name)
 
     def test_reload(self):
@@ -48,7 +48,7 @@ class ModuleForLoaderTests(unittest.TestCase):
         name = 'a.b.c'
         with test_util.uncache(name):
             self.raise_exception(name)
-            self.assertTrue(name not in sys.modules)
+            self.assertNotIn(name, sys.modules)
 
     def test_reload_failure(self):
         # Test that a failure on reload leaves the module in-place.
@@ -59,10 +59,57 @@ class ModuleForLoaderTests(unittest.TestCase):
             self.raise_exception(name)
             self.assertIs(module, sys.modules[name])
 
+    def test_decorator_attrs(self):
+        def fxn(self, module): pass
+        wrapped = util.module_for_loader(fxn)
+        self.assertEqual(wrapped.__name__, fxn.__name__)
+        self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
 
-class SetPackageTests(unittest.TestCase):
+    def test_false_module(self):
+        # If for some odd reason a module is considered false, still return it
+        # from sys.modules.
+        class FalseModule(types.ModuleType):
+            def __bool__(self): return False
+
+        name = 'mod'
+        module = FalseModule(name)
+        with test_util.uncache(name):
+            self.assertFalse(module)
+            sys.modules[name] = module
+            given = self.return_module(name)
+            self.assertIs(given, module)
+
+    def test_attributes_set(self):
+        # __name__, __loader__, and __package__ should be set (when
+        # is_package() is defined; undefined implicitly tested elsewhere).
+        class FakeLoader:
+            def __init__(self, is_package):
+                self._pkg = is_package
+            def is_package(self, name):
+                return self._pkg
+            @util.module_for_loader
+            def load_module(self, module):
+                return module
+
+        name = 'pkg.mod'
+        with test_util.uncache(name):
+            loader = FakeLoader(False)
+            module = loader.load_module(name)
+            self.assertEqual(module.__name__, name)
+            self.assertIs(module.__loader__, loader)
+            self.assertEqual(module.__package__, 'pkg')
+
+        name = 'pkg.sub'
+        with test_util.uncache(name):
+            loader = FakeLoader(True)
+            module = loader.load_module(name)
+            self.assertEqual(module.__name__, name)
+            self.assertIs(module.__loader__, loader)
+            self.assertEqual(module.__package__, name)
 
 
+class SetPackageTests(unittest.TestCase):
+
     """Tests for importlib.util.set_package."""
 
     def verify(self, module, expect):
@@ -108,10 +155,53 @@ class SetPackageTests(unittest.TestCase):
             module.__package__ = value
             self.verify(module, value)
 
+    def test_decorator_attrs(self):
+        def fxn(module): pass
+        wrapped = util.set_package(fxn)
+        self.assertEqual(wrapped.__name__, fxn.__name__)
+        self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
+
+
+class ResolveNameTests(unittest.TestCase):
+
+    """Tests importlib.util.resolve_name()."""
+
+    def test_absolute(self):
+        # bacon
+        self.assertEqual('bacon', util.resolve_name('bacon', None))
+
+    def test_aboslute_within_package(self):
+        # bacon in spam
+        self.assertEqual('bacon', util.resolve_name('bacon', 'spam'))
+
+    def test_no_package(self):
+        # .bacon in ''
+        with self.assertRaises(ValueError):
+            util.resolve_name('.bacon', '')
+
+    def test_in_package(self):
+        # .bacon in spam
+        self.assertEqual('spam.eggs.bacon',
+                         util.resolve_name('.bacon', 'spam.eggs'))
+
+    def test_other_package(self):
+        # ..bacon in spam.bacon
+        self.assertEqual('spam.bacon',
+                         util.resolve_name('..bacon', 'spam.eggs'))
+
+    def test_escape(self):
+        # ..bacon in spam
+        with self.assertRaises(ValueError):
+            util.resolve_name('..bacon', 'spam')
+
 
 def test_main():
     from test import support
-    support.run_unittest(ModuleForLoaderTests, SetPackageTests)
+    support.run_unittest(
+            ModuleForLoaderTests,
+            SetPackageTests,
+            ResolveNameTests
+        )
 
 
 if __name__ == '__main__':
diff --git a/Lib/importlib/test/util.py b/Lib/test/test_importlib/util.py
index 93b7cd2861..ef32f7d690 100644
--- a/Lib/importlib/test/util.py
+++ b/Lib/test/test_importlib/util.py
@@ -35,7 +35,7 @@ def uncache(*names):
     for name in names:
         if name in ('sys', 'marshal', 'imp'):
             raise ValueError(
-                "cannot uncache {0} as it will break _importlib".format(name))
+                "cannot uncache {0}".format(name))
         try:
             del sys.modules[name]
         except KeyError:
@@ -124,7 +124,11 @@ class mock_modules:
         else:
             sys.modules[fullname] = self.modules[fullname]
             if fullname in self.module_code:
-                self.module_code[fullname]()
+                try:
+                    self.module_code[fullname]()
+                except Exception:
+                    del sys.modules[fullname]
+                    raise
             return self.modules[fullname]
 
     def __enter__(self):
diff --git a/Lib/test/test_index.py b/Lib/test/test_index.py
index 7a94af1b0a..66eedaaed7 100644
--- a/Lib/test/test_index.py
+++ b/Lib/test/test_index.py
@@ -56,7 +56,7 @@ class BaseTestCase(unittest.TestCase):
         self.assertRaises(TypeError, slice(self.n).indices, 0)
 
 
-class SeqTestCase(unittest.TestCase):
+class SeqTestCase:
     # This test case isn't run directly. It just defines common tests
     # to the different sequence types below
     def setUp(self):
@@ -126,7 +126,7 @@ class SeqTestCase(unittest.TestCase):
         self.assertRaises(TypeError, sliceobj, self.n, self)
 
 
-class ListTestCase(SeqTestCase):
+class ListTestCase(SeqTestCase, unittest.TestCase):
     seq = [0,10,20,30,40,50]
 
     def test_setdelitem(self):
@@ -182,19 +182,19 @@ class NewSeq:
         return self._list[index]
 
 
-class TupleTestCase(SeqTestCase):
+class TupleTestCase(SeqTestCase, unittest.TestCase):
     seq = (0,10,20,30,40,50)
 
-class ByteArrayTestCase(SeqTestCase):
+class ByteArrayTestCase(SeqTestCase, unittest.TestCase):
     seq = bytearray(b"this is a test")
 
-class BytesTestCase(SeqTestCase):
+class BytesTestCase(SeqTestCase, unittest.TestCase):
     seq = b"this is a test"
 
-class StringTestCase(SeqTestCase):
+class StringTestCase(SeqTestCase, unittest.TestCase):
     seq = "this is a test"
 
-class NewSeqTestCase(SeqTestCase):
+class NewSeqTestCase(SeqTestCase, unittest.TestCase):
     seq = NewSeq((0,10,20,30,40,50))
 
 
@@ -237,18 +237,5 @@ class OverflowTestCase(unittest.TestCase):
         self.assertRaises(OverflowError, lambda: "a" * self.neg)
 
 
-def test_main():
-    support.run_unittest(
-        BaseTestCase,
-        ListTestCase,
-        TupleTestCase,
-        BytesTestCase,
-        ByteArrayTestCase,
-        StringTestCase,
-        NewSeqTestCase,
-        RangeTestCase,
-        OverflowTestCase,
-    )
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 4b7ee4ed68..6e3f04e68a 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -41,11 +41,6 @@ def revise(filename, *args):
 
 import builtins
 
-try:
-    1/0
-except:
-    tb = sys.exc_info()[2]
-
 git = mod.StupidGit()
 
 class IsTestBase(unittest.TestCase):
@@ -79,23 +74,31 @@ class TestPredicates(IsTestBase):
 
 
     def test_excluding_predicates(self):
+        global tb
         self.istest(inspect.isbuiltin, 'sys.exit')
         self.istest(inspect.isbuiltin, '[].append')
         self.istest(inspect.iscode, 'mod.spam.__code__')
-        self.istest(inspect.isframe, 'tb.tb_frame')
+        try:
+            1/0
+        except:
+            tb = sys.exc_info()[2]
+            self.istest(inspect.isframe, 'tb.tb_frame')
+            self.istest(inspect.istraceback, 'tb')
+            if hasattr(types, 'GetSetDescriptorType'):
+                self.istest(inspect.isgetsetdescriptor,
+                            'type(tb.tb_frame).f_locals')
+            else:
+                self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals))
+        finally:
+            # Clear traceback and all the frames and local variables hanging to it.
+            tb = None
         self.istest(inspect.isfunction, 'mod.spam')
         self.istest(inspect.isfunction, 'mod.StupidGit.abuse')
         self.istest(inspect.ismethod, 'git.argue')
         self.istest(inspect.ismodule, 'mod')
-        self.istest(inspect.istraceback, 'tb')
         self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory')
         self.istest(inspect.isgenerator, '(x for x in range(2))')
         self.istest(inspect.isgeneratorfunction, 'generator_function_example')
-        if hasattr(types, 'GetSetDescriptorType'):
-            self.istest(inspect.isgetsetdescriptor,
-                        'type(tb.tb_frame).f_locals')
-        else:
-            self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals))
         if hasattr(types, 'MemberDescriptorType'):
             self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days')
         else:
@@ -282,7 +285,10 @@ class TestRetrievingSourceCode(GetSourceBase):
         co = compile("None", fn, "exec")
         self.assertEqual(inspect.getsourcefile(co), None)
         linecache.cache[co.co_filename] = (1, None, "None", co.co_filename)
-        self.assertEqual(normcase(inspect.getsourcefile(co)), fn)
+        try:
+            self.assertEqual(normcase(inspect.getsourcefile(co)), fn)
+        finally:
+            del linecache.cache[co.co_filename]
 
     def test_getfile(self):
         self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__)
@@ -304,7 +310,7 @@ class TestRetrievingSourceCode(GetSourceBase):
         getlines = linecache.getlines
         def monkey(filename, module_globals=None):
             if filename == fn:
-                return source.splitlines(True)
+                return source.splitlines(keepends=True)
             else:
                 return getlines(filename, module_globals)
         linecache.getlines = monkey
@@ -404,8 +410,11 @@ class TestBuggyCases(GetSourceBase):
         self.assertRaises(IOError, inspect.findsource, co)
         self.assertRaises(IOError, inspect.getsource, co)
         linecache.cache[co.co_filename] = (1, None, lines, co.co_filename)
-        self.assertEqual(inspect.findsource(co), (lines,0))
-        self.assertEqual(inspect.getsource(co), lines[0])
+        try:
+            self.assertEqual(inspect.findsource(co), (lines,0))
+            self.assertEqual(inspect.getsource(co), lines[0])
+        finally:
+            del linecache.cache[co.co_filename]
 
 class TestNoEOL(GetSourceBase):
     def __init__(self, *args, **kwargs):
@@ -662,6 +671,129 @@ class TestClassesAndFunctions(unittest.TestCase):
         self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod))
 
 
+_global_ref = object()
+class TestGetClosureVars(unittest.TestCase):
+
+    def test_name_resolution(self):
+        # Basic test of the 4 different resolution mechanisms
+        def f(nonlocal_ref):
+            def g(local_ref):
+                print(local_ref, nonlocal_ref, _global_ref, unbound_ref)
+            return g
+        _arg = object()
+        nonlocal_vars = {"nonlocal_ref": _arg}
+        global_vars = {"_global_ref": _global_ref}
+        builtin_vars = {"print": print}
+        unbound_names = {"unbound_ref"}
+        expected = inspect.ClosureVars(nonlocal_vars, global_vars,
+                                       builtin_vars, unbound_names)
+        self.assertEqual(inspect.getclosurevars(f(_arg)), expected)
+
+    def test_generator_closure(self):
+        def f(nonlocal_ref):
+            def g(local_ref):
+                print(local_ref, nonlocal_ref, _global_ref, unbound_ref)
+                yield
+            return g
+        _arg = object()
+        nonlocal_vars = {"nonlocal_ref": _arg}
+        global_vars = {"_global_ref": _global_ref}
+        builtin_vars = {"print": print}
+        unbound_names = {"unbound_ref"}
+        expected = inspect.ClosureVars(nonlocal_vars, global_vars,
+                                       builtin_vars, unbound_names)
+        self.assertEqual(inspect.getclosurevars(f(_arg)), expected)
+
+    def test_method_closure(self):
+        class C:
+            def f(self, nonlocal_ref):
+                def g(local_ref):
+                    print(local_ref, nonlocal_ref, _global_ref, unbound_ref)
+                return g
+        _arg = object()
+        nonlocal_vars = {"nonlocal_ref": _arg}
+        global_vars = {"_global_ref": _global_ref}
+        builtin_vars = {"print": print}
+        unbound_names = {"unbound_ref"}
+        expected = inspect.ClosureVars(nonlocal_vars, global_vars,
+                                       builtin_vars, unbound_names)
+        self.assertEqual(inspect.getclosurevars(C().f(_arg)), expected)
+
+    def test_nonlocal_vars(self):
+        # More complex tests of nonlocal resolution
+        def _nonlocal_vars(f):
+            return inspect.getclosurevars(f).nonlocals
+
+        def make_adder(x):
+            def add(y):
+                return x + y
+            return add
+
+        def curry(func, arg1):
+            return lambda arg2: func(arg1, arg2)
+
+        def less_than(a, b):
+            return a < b
+
+        # The infamous Y combinator.
+        def Y(le):
+            def g(f):
+                return le(lambda x: f(f)(x))
+            Y.g_ref = g
+            return g(g)
+
+        def check_y_combinator(func):
+            self.assertEqual(_nonlocal_vars(func), {'f': Y.g_ref})
+
+        inc = make_adder(1)
+        add_two = make_adder(2)
+        greater_than_five = curry(less_than, 5)
+
+        self.assertEqual(_nonlocal_vars(inc), {'x': 1})
+        self.assertEqual(_nonlocal_vars(add_two), {'x': 2})
+        self.assertEqual(_nonlocal_vars(greater_than_five),
+                         {'arg1': 5, 'func': less_than})
+        self.assertEqual(_nonlocal_vars((lambda x: lambda y: x + y)(3)),
+                         {'x': 3})
+        Y(check_y_combinator)
+
+    def test_getclosurevars_empty(self):
+        def foo(): pass
+        _empty = inspect.ClosureVars({}, {}, {}, set())
+        self.assertEqual(inspect.getclosurevars(lambda: True), _empty)
+        self.assertEqual(inspect.getclosurevars(foo), _empty)
+
+    def test_getclosurevars_error(self):
+        class T: pass
+        self.assertRaises(TypeError, inspect.getclosurevars, 1)
+        self.assertRaises(TypeError, inspect.getclosurevars, list)
+        self.assertRaises(TypeError, inspect.getclosurevars, {})
+
+    def _private_globals(self):
+        code = """def f(): print(path)"""
+        ns = {}
+        exec(code, ns)
+        return ns["f"], ns
+
+    def test_builtins_fallback(self):
+        f, ns = self._private_globals()
+        ns.pop("__builtins__", None)
+        expected = inspect.ClosureVars({}, {}, {"print":print}, {"path"})
+        self.assertEqual(inspect.getclosurevars(f), expected)
+
+    def test_builtins_as_dict(self):
+        f, ns = self._private_globals()
+        ns["__builtins__"] = {"path":1}
+        expected = inspect.ClosureVars({}, {}, {"path":1}, {"print"})
+        self.assertEqual(inspect.getclosurevars(f), expected)
+
+    def test_builtins_as_module(self):
+        f, ns = self._private_globals()
+        ns["__builtins__"] = os
+        expected = inspect.ClosureVars({}, {}, {"path":os.path}, {"print"})
+        self.assertEqual(inspect.getclosurevars(f), expected)
+
+
 class TestGetcallargsFunctions(unittest.TestCase):
 
     def assertEqualCallArgs(self, func, call_params_string, locs=None):
@@ -1169,6 +1301,982 @@ class TestGetGeneratorState(unittest.TestCase):
             self.assertIn(name, repr(state))
             self.assertIn(name, str(state))
 
+    def test_getgeneratorlocals(self):
+        def each(lst, a=None):
+            b=(1, 2, 3)
+            for v in lst:
+                if v == 3:
+                    c = 12
+                yield v
+
+        numbers = each([1, 2, 3])
+        self.assertEqual(inspect.getgeneratorlocals(numbers),
+                         {'a': None, 'lst': [1, 2, 3]})
+        next(numbers)
+        self.assertEqual(inspect.getgeneratorlocals(numbers),
+                         {'a': None, 'lst': [1, 2, 3], 'v': 1,
+                          'b': (1, 2, 3)})
+        next(numbers)
+        self.assertEqual(inspect.getgeneratorlocals(numbers),
+                         {'a': None, 'lst': [1, 2, 3], 'v': 2,
+                          'b': (1, 2, 3)})
+        next(numbers)
+        self.assertEqual(inspect.getgeneratorlocals(numbers),
+                         {'a': None, 'lst': [1, 2, 3], 'v': 3,
+                          'b': (1, 2, 3), 'c': 12})
+        try:
+            next(numbers)
+        except StopIteration:
+            pass
+        self.assertEqual(inspect.getgeneratorlocals(numbers), {})
+
+    def test_getgeneratorlocals_empty(self):
+        def yield_one():
+            yield 1
+        one = yield_one()
+        self.assertEqual(inspect.getgeneratorlocals(one), {})
+        try:
+            next(one)
+        except StopIteration:
+            pass
+        self.assertEqual(inspect.getgeneratorlocals(one), {})
+
+    def test_getgeneratorlocals_error(self):
+        self.assertRaises(TypeError, inspect.getgeneratorlocals, 1)
+        self.assertRaises(TypeError, inspect.getgeneratorlocals, lambda x: True)
+        self.assertRaises(TypeError, inspect.getgeneratorlocals, set)
+        self.assertRaises(TypeError, inspect.getgeneratorlocals, (2,3))
+
+
+class TestSignatureObject(unittest.TestCase):
+    @staticmethod
+    def signature(func):
+        sig = inspect.signature(func)
+        return (tuple((param.name,
+                       (... if param.default is param.empty else param.default),
+                       (... if param.annotation is param.empty
+                                                        else param.annotation),
+                       str(param.kind).lower())
+                                    for param in sig.parameters.values()),
+                (... if sig.return_annotation is sig.empty
+                                            else sig.return_annotation))
+
+    def test_signature_object(self):
+        S = inspect.Signature
+        P = inspect.Parameter
+
+        self.assertEqual(str(S()), '()')
+
+        def test(po, pk, *args, ko, **kwargs):
+            pass
+        sig = inspect.signature(test)
+        po = sig.parameters['po'].replace(kind=P.POSITIONAL_ONLY)
+        pk = sig.parameters['pk']
+        args = sig.parameters['args']
+        ko = sig.parameters['ko']
+        kwargs = sig.parameters['kwargs']
+
+        S((po, pk, args, ko, kwargs))
+
+        with self.assertRaisesRegex(ValueError, 'wrong parameter order'):
+            S((pk, po, args, ko, kwargs))
+
+        with self.assertRaisesRegex(ValueError, 'wrong parameter order'):
+            S((po, args, pk, ko, kwargs))
+
+        with self.assertRaisesRegex(ValueError, 'wrong parameter order'):
+            S((args, po, pk, ko, kwargs))
+
+        with self.assertRaisesRegex(ValueError, 'wrong parameter order'):
+            S((po, pk, args, kwargs, ko))
+
+        kwargs2 = kwargs.replace(name='args')
+        with self.assertRaisesRegex(ValueError, 'duplicate parameter name'):
+            S((po, pk, args, kwargs2, ko))
+
+    def test_signature_immutability(self):
+        def test(a):
+            pass
+        sig = inspect.signature(test)
+
+        with self.assertRaises(AttributeError):
+            sig.foo = 'bar'
+
+        with self.assertRaises(TypeError):
+            sig.parameters['a'] = None
+
+    def test_signature_on_noarg(self):
+        def test():
+            pass
+        self.assertEqual(self.signature(test), ((), ...))
+
+    def test_signature_on_wargs(self):
+        def test(a, b:'foo') -> 123:
+            pass
+        self.assertEqual(self.signature(test),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('b', ..., 'foo', "positional_or_keyword")),
+                          123))
+
+    def test_signature_on_wkwonly(self):
+        def test(*, a:float, b:str) -> int:
+            pass
+        self.assertEqual(self.signature(test),
+                         ((('a', ..., float, "keyword_only"),
+                           ('b', ..., str, "keyword_only")),
+                           int))
+
+    def test_signature_on_complex_args(self):
+        def test(a, b:'foo'=10, *args:'bar', spam:'baz', ham=123, **kwargs:int):
+            pass
+        self.assertEqual(self.signature(test),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('b', 10, 'foo', "positional_or_keyword"),
+                           ('args', ..., 'bar', "var_positional"),
+                           ('spam', ..., 'baz', "keyword_only"),
+                           ('ham', 123, ..., "keyword_only"),
+                           ('kwargs', ..., int, "var_keyword")),
+                          ...))
+
+    def test_signature_on_builtin_function(self):
+        with self.assertRaisesRegex(ValueError, 'not supported by signature'):
+            inspect.signature(type)
+        with self.assertRaisesRegex(ValueError, 'not supported by signature'):
+            # support for 'wrapper_descriptor'
+            inspect.signature(type.__call__)
+        with self.assertRaisesRegex(ValueError, 'not supported by signature'):
+            # support for 'method-wrapper'
+            inspect.signature(min.__call__)
+        with self.assertRaisesRegex(ValueError,
+                                     'no signature found for builtin function'):
+            # support for 'method-wrapper'
+            inspect.signature(min)
+
+    def test_signature_on_non_function(self):
+        with self.assertRaisesRegex(TypeError, 'is not a callable object'):
+            inspect.signature(42)
+
+        with self.assertRaisesRegex(TypeError, 'is not a Python function'):
+            inspect.Signature.from_function(42)
+
+    def test_signature_on_method(self):
+        class Test:
+            def foo(self, arg1, arg2=1) -> int:
+                pass
+
+        meth = Test().foo
+
+        self.assertEqual(self.signature(meth),
+                         ((('arg1', ..., ..., "positional_or_keyword"),
+                           ('arg2', 1, ..., "positional_or_keyword")),
+                          int))
+
+    def test_signature_on_classmethod(self):
+        class Test:
+            @classmethod
+            def foo(cls, arg1, *, arg2=1):
+                pass
+
+        meth = Test().foo
+        self.assertEqual(self.signature(meth),
+                         ((('arg1', ..., ..., "positional_or_keyword"),
+                           ('arg2', 1, ..., "keyword_only")),
+                          ...))
+
+        meth = Test.foo
+        self.assertEqual(self.signature(meth),
+                         ((('arg1', ..., ..., "positional_or_keyword"),
+                           ('arg2', 1, ..., "keyword_only")),
+                          ...))
+
+    def test_signature_on_staticmethod(self):
+        class Test:
+            @staticmethod
+            def foo(cls, *, arg):
+                pass
+
+        meth = Test().foo
+        self.assertEqual(self.signature(meth),
+                         ((('cls', ..., ..., "positional_or_keyword"),
+                           ('arg', ..., ..., "keyword_only")),
+                          ...))
+
+        meth = Test.foo
+        self.assertEqual(self.signature(meth),
+                         ((('cls', ..., ..., "positional_or_keyword"),
+                           ('arg', ..., ..., "keyword_only")),
+                          ...))
+
+    def test_signature_on_partial(self):
+        from functools import partial
+
+        def test():
+            pass
+
+        self.assertEqual(self.signature(partial(test)), ((), ...))
+
+        with self.assertRaisesRegex(ValueError, "has incorrect arguments"):
+            inspect.signature(partial(test, 1))
+
+        with self.assertRaisesRegex(ValueError, "has incorrect arguments"):
+            inspect.signature(partial(test, a=1))
+
+        def test(a, b, *, c, d):
+            pass
+
+        self.assertEqual(self.signature(partial(test)),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('b', ..., ..., "positional_or_keyword"),
+                           ('c', ..., ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, 1)),
+                         ((('b', ..., ..., "positional_or_keyword"),
+                           ('c', ..., ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, 1, c=2)),
+                         ((('b', ..., ..., "positional_or_keyword"),
+                           ('c', 2, ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, b=1, c=2)),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('b', 1, ..., "positional_or_keyword"),
+                           ('c', 2, ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, 0, b=1, c=2)),
+                         ((('b', 1, ..., "positional_or_keyword"),
+                           ('c', 2, ..., "keyword_only"),
+                           ('d', ..., ..., "keyword_only"),),
+                          ...))
+
+        def test(a, *args, b, **kwargs):
+            pass
+
+        self.assertEqual(self.signature(partial(test, 1)),
+                         ((('args', ..., ..., "var_positional"),
+                           ('b', ..., ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, 1, 2, 3)),
+                         ((('args', ..., ..., "var_positional"),
+                           ('b', ..., ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
+
+        self.assertEqual(self.signature(partial(test, 1, 2, 3, test=True)),
+                         ((('args', ..., ..., "var_positional"),
+                           ('b', ..., ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, 1, 2, 3, test=1, b=0)),
+                         ((('args', ..., ..., "var_positional"),
+                           ('b', 0, ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, b=0)),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('args', ..., ..., "var_positional"),
+                           ('b', 0, ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
+        self.assertEqual(self.signature(partial(test, b=0, test=1)),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('args', ..., ..., "var_positional"),
+                           ('b', 0, ..., "keyword_only"),
+                           ('kwargs', ..., ..., "var_keyword")),
+                          ...))
+
+        def test(a, b, c:int) -> 42:
+            pass
+
+        sig = test.__signature__ = inspect.signature(test)
+
+        self.assertEqual(self.signature(partial(partial(test, 1))),
+                         ((('b', ..., ..., "positional_or_keyword"),
+                           ('c', ..., int, "positional_or_keyword")),
+                          42))
+
+        self.assertEqual(self.signature(partial(partial(test, 1), 2)),
+                         ((('c', ..., int, "positional_or_keyword"),),
+                          42))
+
+        psig = inspect.signature(partial(partial(test, 1), 2))
+
+        def foo(a):
+            return a
+        _foo = partial(partial(foo, a=10), a=20)
+        self.assertEqual(self.signature(_foo),
+                         ((('a', 20, ..., "positional_or_keyword"),),
+                          ...))
+        # check that we don't have any side-effects in signature(),
+        # and the partial object is still functioning
+        self.assertEqual(_foo(), 20)
+
+        def foo(a, b, c):
+            return a, b, c
+        _foo = partial(partial(foo, 1, b=20), b=30)
+        self.assertEqual(self.signature(_foo),
+                         ((('b', 30, ..., "positional_or_keyword"),
+                           ('c', ..., ..., "positional_or_keyword")),
+                          ...))
+        self.assertEqual(_foo(c=10), (1, 30, 10))
+        _foo = partial(_foo, 2) # now 'b' has two values -
+                                # positional and keyword
+        with self.assertRaisesRegex(ValueError, "has incorrect arguments"):
+            inspect.signature(_foo)
+
+        def foo(a, b, c, *, d):
+            return a, b, c, d
+        _foo = partial(partial(foo, d=20, c=20), b=10, d=30)
+        self.assertEqual(self.signature(_foo),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('b', 10, ..., "positional_or_keyword"),
+                           ('c', 20, ..., "positional_or_keyword"),
+                           ('d', 30, ..., "keyword_only")),
+                          ...))
+        ba = inspect.signature(_foo).bind(a=200, b=11)
+        self.assertEqual(_foo(*ba.args, **ba.kwargs), (200, 11, 20, 30))
+
+        def foo(a=1, b=2, c=3):
+            return a, b, c
+        _foo = partial(foo, a=10, c=13)
+        ba = inspect.signature(_foo).bind(11)
+        self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 2, 13))
+        ba = inspect.signature(_foo).bind(11, 12)
+        self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 12, 13))
+        ba = inspect.signature(_foo).bind(11, b=12)
+        self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 12, 13))
+        ba = inspect.signature(_foo).bind(b=12)
+        self.assertEqual(_foo(*ba.args, **ba.kwargs), (10, 12, 13))
+        _foo = partial(_foo, b=10)
+        ba = inspect.signature(_foo).bind(12, 14)
+        self.assertEqual(_foo(*ba.args, **ba.kwargs), (12, 14, 13))
+
+    def test_signature_on_decorated(self):
+        import functools
+
+        def decorator(func):
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs) -> int:
+                return func(*args, **kwargs)
+            return wrapper
+
+        class Foo:
+            @decorator
+            def bar(self, a, b):
+                pass
+
+        self.assertEqual(self.signature(Foo.bar),
+                         ((('self', ..., ..., "positional_or_keyword"),
+                           ('a', ..., ..., "positional_or_keyword"),
+                           ('b', ..., ..., "positional_or_keyword")),
+                          ...))
+
+        self.assertEqual(self.signature(Foo().bar),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('b', ..., ..., "positional_or_keyword")),
+                          ...))
+
+        # Test that we handle method wrappers correctly
+        def decorator(func):
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs) -> int:
+                return func(42, *args, **kwargs)
+            sig = inspect.signature(func)
+            new_params = tuple(sig.parameters.values())[1:]
+            wrapper.__signature__ = sig.replace(parameters=new_params)
+            return wrapper
+
+        class Foo:
+            @decorator
+            def __call__(self, a, b):
+                pass
+
+        self.assertEqual(self.signature(Foo.__call__),
+                         ((('a', ..., ..., "positional_or_keyword"),
+                           ('b', ..., ..., "positional_or_keyword")),
+                          ...))
+
+        self.assertEqual(self.signature(Foo().__call__),
+                         ((('b', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+    def test_signature_on_class(self):
+        class C:
+            def __init__(self, a):
+                pass
+
+        self.assertEqual(self.signature(C),
+                         ((('a', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+        class CM(type):
+            def __call__(cls, a):
+                pass
+        class C(metaclass=CM):
+            def __init__(self, b):
+                pass
+
+        self.assertEqual(self.signature(C),
+                         ((('a', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+        class CM(type):
+            def __new__(mcls, name, bases, dct, *, foo=1):
+                return super().__new__(mcls, name, bases, dct)
+        class C(metaclass=CM):
+            def __init__(self, b):
+                pass
+
+        self.assertEqual(self.signature(C),
+                         ((('b', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+        self.assertEqual(self.signature(CM),
+                         ((('name', ..., ..., "positional_or_keyword"),
+                           ('bases', ..., ..., "positional_or_keyword"),
+                           ('dct', ..., ..., "positional_or_keyword"),
+                           ('foo', 1, ..., "keyword_only")),
+                          ...))
+
+        class CMM(type):
+            def __new__(mcls, name, bases, dct, *, foo=1):
+                return super().__new__(mcls, name, bases, dct)
+            def __call__(cls, nm, bs, dt):
+                return type(nm, bs, dt)
+        class CM(type, metaclass=CMM):
+            def __new__(mcls, name, bases, dct, *, bar=2):
+                return super().__new__(mcls, name, bases, dct)
+        class C(metaclass=CM):
+            def __init__(self, b):
+                pass
+
+        self.assertEqual(self.signature(CMM),
+                         ((('name', ..., ..., "positional_or_keyword"),
+                           ('bases', ..., ..., "positional_or_keyword"),
+                           ('dct', ..., ..., "positional_or_keyword"),
+                           ('foo', 1, ..., "keyword_only")),
+                          ...))
+
+        self.assertEqual(self.signature(CM),
+                         ((('nm', ..., ..., "positional_or_keyword"),
+                           ('bs', ..., ..., "positional_or_keyword"),
+                           ('dt', ..., ..., "positional_or_keyword")),
+                          ...))
+
+        self.assertEqual(self.signature(C),
+                         ((('b', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+        class CM(type):
+            def __init__(cls, name, bases, dct, *, bar=2):
+                return super().__init__(name, bases, dct)
+        class C(metaclass=CM):
+            def __init__(self, b):
+                pass
+
+        self.assertEqual(self.signature(CM),
+                         ((('name', ..., ..., "positional_or_keyword"),
+                           ('bases', ..., ..., "positional_or_keyword"),
+                           ('dct', ..., ..., "positional_or_keyword"),
+                           ('bar', 2, ..., "keyword_only")),
+                          ...))
+
+    def test_signature_on_callable_objects(self):
+        class Foo:
+            def __call__(self, a):
+                pass
+
+        self.assertEqual(self.signature(Foo()),
+                         ((('a', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+        class Spam:
+            pass
+        with self.assertRaisesRegex(TypeError, "is not a callable object"):
+            inspect.signature(Spam())
+
+        class Bar(Spam, Foo):
+            pass
+
+        self.assertEqual(self.signature(Bar()),
+                         ((('a', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+        class ToFail:
+            __call__ = type
+        with self.assertRaisesRegex(ValueError, "not supported by signature"):
+            inspect.signature(ToFail())
+
+
+        class Wrapped:
+            pass
+        Wrapped.__wrapped__ = lambda a: None
+        self.assertEqual(self.signature(Wrapped),
+                         ((('a', ..., ..., "positional_or_keyword"),),
+                          ...))
+
+    def test_signature_on_lambdas(self):
+        self.assertEqual(self.signature((lambda a=10: a)),
+                         ((('a', 10, ..., "positional_or_keyword"),),
+                          ...))
+
+    def test_signature_equality(self):
+        def foo(a, *, b:int) -> float: pass
+        self.assertNotEqual(inspect.signature(foo), 42)
+
+        def bar(a, *, b:int) -> float: pass
+        self.assertEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def bar(a, *, b:int) -> int: pass
+        self.assertNotEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def bar(a, *, b:int): pass
+        self.assertNotEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def bar(a, *, b:int=42) -> float: pass
+        self.assertNotEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def bar(a, *, c) -> float: pass
+        self.assertNotEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def bar(a, b:int) -> float: pass
+        self.assertNotEqual(inspect.signature(foo), inspect.signature(bar))
+        def spam(b:int, a) -> float: pass
+        self.assertNotEqual(inspect.signature(spam), inspect.signature(bar))
+
+        def foo(*, a, b, c): pass
+        def bar(*, c, b, a): pass
+        self.assertEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def foo(*, a=1, b, c): pass
+        def bar(*, c, b, a=1): pass
+        self.assertEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def foo(pos, *, a=1, b, c): pass
+        def bar(pos, *, c, b, a=1): pass
+        self.assertEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def foo(pos, *, a, b, c): pass
+        def bar(pos, *, c, b, a=1): pass
+        self.assertNotEqual(inspect.signature(foo), inspect.signature(bar))
+
+        def foo(pos, *args, a=42, b, c, **kwargs:int): pass
+        def bar(pos, *args, c, b, a=42, **kwargs:int): pass
+        self.assertEqual(inspect.signature(foo), inspect.signature(bar))
+
+    def test_signature_unhashable(self):
+        def foo(a): pass
+        sig = inspect.signature(foo)
+        with self.assertRaisesRegex(TypeError, 'unhashable type'):
+            hash(sig)
+
+    def test_signature_str(self):
+        def foo(a:int=1, *, b, c=None, **kwargs) -> 42:
+            pass
+        self.assertEqual(str(inspect.signature(foo)),
+                         '(a:int=1, *, b, c=None, **kwargs) -> 42')
+
+        def foo(a:int=1, *args, b, c=None, **kwargs) -> 42:
+            pass
+        self.assertEqual(str(inspect.signature(foo)),
+                         '(a:int=1, *args, b, c=None, **kwargs) -> 42')
+
+        def foo():
+            pass
+        self.assertEqual(str(inspect.signature(foo)), '()')
+
+    def test_signature_str_positional_only(self):
+        P = inspect.Parameter
+
+        def test(a_po, *, b, **kwargs):
+            return a_po, kwargs
+
+        sig = inspect.signature(test)
+        new_params = list(sig.parameters.values())
+        new_params[0] = new_params[0].replace(kind=P.POSITIONAL_ONLY)
+        test.__signature__ = sig.replace(parameters=new_params)
+
+        self.assertEqual(str(inspect.signature(test)),
+                         '(<a_po>, *, b, **kwargs)')
+
+        sig = inspect.signature(test)
+        new_params = list(sig.parameters.values())
+        new_params[0] = new_params[0].replace(name=None)
+        test.__signature__ = sig.replace(parameters=new_params)
+        self.assertEqual(str(inspect.signature(test)),
+                         '(<0>, *, b, **kwargs)')
+
+    def test_signature_replace_anno(self):
+        def test() -> 42:
+            pass
+
+        sig = inspect.signature(test)
+        sig = sig.replace(return_annotation=None)
+        self.assertIs(sig.return_annotation, None)
+        sig = sig.replace(return_annotation=sig.empty)
+        self.assertIs(sig.return_annotation, sig.empty)
+        sig = sig.replace(return_annotation=42)
+        self.assertEqual(sig.return_annotation, 42)
+        self.assertEqual(sig, inspect.signature(test))
+
+
+class TestParameterObject(unittest.TestCase):
+    def test_signature_parameter_kinds(self):
+        P = inspect.Parameter
+        self.assertTrue(P.POSITIONAL_ONLY < P.POSITIONAL_OR_KEYWORD < \
+                        P.VAR_POSITIONAL < P.KEYWORD_ONLY < P.VAR_KEYWORD)
+
+        self.assertEqual(str(P.POSITIONAL_ONLY), 'POSITIONAL_ONLY')
+        self.assertTrue('POSITIONAL_ONLY' in repr(P.POSITIONAL_ONLY))
+
+    def test_signature_parameter_object(self):
+        p = inspect.Parameter('foo', default=10,
+                              kind=inspect.Parameter.POSITIONAL_ONLY)
+        self.assertEqual(p.name, 'foo')
+        self.assertEqual(p.default, 10)
+        self.assertIs(p.annotation, p.empty)
+        self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY)
+
+        with self.assertRaisesRegex(ValueError, 'invalid value'):
+            inspect.Parameter('foo', default=10, kind='123')
+
+        with self.assertRaisesRegex(ValueError, 'not a valid parameter name'):
+            inspect.Parameter('1', kind=inspect.Parameter.VAR_KEYWORD)
+
+        with self.assertRaisesRegex(ValueError,
+                                     'non-positional-only parameter'):
+            inspect.Parameter(None, kind=inspect.Parameter.VAR_KEYWORD)
+
+        with self.assertRaisesRegex(ValueError, 'cannot have default values'):
+            inspect.Parameter('a', default=42,
+                              kind=inspect.Parameter.VAR_KEYWORD)
+
+        with self.assertRaisesRegex(ValueError, 'cannot have default values'):
+            inspect.Parameter('a', default=42,
+                              kind=inspect.Parameter.VAR_POSITIONAL)
+
+        p = inspect.Parameter('a', default=42,
+                              kind=inspect.Parameter.POSITIONAL_OR_KEYWORD)
+        with self.assertRaisesRegex(ValueError, 'cannot have default values'):
+            p.replace(kind=inspect.Parameter.VAR_POSITIONAL)
+
+        self.assertTrue(repr(p).startswith('<Parameter'))
+
+    def test_signature_parameter_equality(self):
+        P = inspect.Parameter
+        p = P('foo', default=42, kind=inspect.Parameter.KEYWORD_ONLY)
+
+        self.assertEqual(p, p)
+        self.assertNotEqual(p, 42)
+
+        self.assertEqual(p, P('foo', default=42,
+                              kind=inspect.Parameter.KEYWORD_ONLY))
+
+    def test_signature_parameter_unhashable(self):
+        p = inspect.Parameter('foo', default=42,
+                              kind=inspect.Parameter.KEYWORD_ONLY)
+
+        with self.assertRaisesRegex(TypeError, 'unhashable type'):
+            hash(p)
+
+    def test_signature_parameter_replace(self):
+        p = inspect.Parameter('foo', default=42,
+                              kind=inspect.Parameter.KEYWORD_ONLY)
+
+        self.assertIsNot(p, p.replace())
+        self.assertEqual(p, p.replace())
+
+        p2 = p.replace(annotation=1)
+        self.assertEqual(p2.annotation, 1)
+        p2 = p2.replace(annotation=p2.empty)
+        self.assertEqual(p, p2)
+
+        p2 = p2.replace(name='bar')
+        self.assertEqual(p2.name, 'bar')
+        self.assertNotEqual(p2, p)
+
+        with self.assertRaisesRegex(ValueError, 'not a valid parameter name'):
+            p2 = p2.replace(name=p2.empty)
+
+        p2 = p2.replace(name='foo', default=None)
+        self.assertIs(p2.default, None)
+        self.assertNotEqual(p2, p)
+
+        p2 = p2.replace(name='foo', default=p2.empty)
+        self.assertIs(p2.default, p2.empty)
+
+
+        p2 = p2.replace(default=42, kind=p2.POSITIONAL_OR_KEYWORD)
+        self.assertEqual(p2.kind, p2.POSITIONAL_OR_KEYWORD)
+        self.assertNotEqual(p2, p)
+
+        with self.assertRaisesRegex(ValueError, 'invalid value for'):
+            p2 = p2.replace(kind=p2.empty)
+
+        p2 = p2.replace(kind=p2.KEYWORD_ONLY)
+        self.assertEqual(p2, p)
+
+    def test_signature_parameter_positional_only(self):
+        p = inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY)
+        self.assertEqual(str(p), '<>')
+
+        p = p.replace(name='1')
+        self.assertEqual(str(p), '<1>')
+
+    def test_signature_parameter_immutability(self):
+        p = inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY)
+
+        with self.assertRaises(AttributeError):
+            p.foo = 'bar'
+
+        with self.assertRaises(AttributeError):
+            p.kind = 123
+
+
+class TestSignatureBind(unittest.TestCase):
+    @staticmethod
+    def call(func, *args, **kwargs):
+        sig = inspect.signature(func)
+        ba = sig.bind(*args, **kwargs)
+        return func(*ba.args, **ba.kwargs)
+
+    def test_signature_bind_empty(self):
+        def test():
+            return 42
+
+        self.assertEqual(self.call(test), 42)
+        with self.assertRaisesRegex(TypeError, 'too many positional arguments'):
+            self.call(test, 1)
+        with self.assertRaisesRegex(TypeError, 'too many positional arguments'):
+            self.call(test, 1, spam=10)
+        with self.assertRaisesRegex(TypeError, 'too many keyword arguments'):
+            self.call(test, spam=1)
+
+    def test_signature_bind_var(self):
+        def test(*args, **kwargs):
+            return args, kwargs
+
+        self.assertEqual(self.call(test), ((), {}))
+        self.assertEqual(self.call(test, 1), ((1,), {}))
+        self.assertEqual(self.call(test, 1, 2), ((1, 2), {}))
+        self.assertEqual(self.call(test, foo='bar'), ((), {'foo': 'bar'}))
+        self.assertEqual(self.call(test, 1, foo='bar'), ((1,), {'foo': 'bar'}))
+        self.assertEqual(self.call(test, args=10), ((), {'args': 10}))
+        self.assertEqual(self.call(test, 1, 2, foo='bar'),
+                         ((1, 2), {'foo': 'bar'}))
+
+    def test_signature_bind_just_args(self):
+        def test(a, b, c):
+            return a, b, c
+
+        self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3))
+
+        with self.assertRaisesRegex(TypeError, 'too many positional arguments'):
+            self.call(test, 1, 2, 3, 4)
+
+        with self.assertRaisesRegex(TypeError, "'b' parameter lacking default"):
+            self.call(test, 1)
+
+        with self.assertRaisesRegex(TypeError, "'a' parameter lacking default"):
+            self.call(test)
+
+        def test(a, b, c=10):
+            return a, b, c
+        self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3))
+        self.assertEqual(self.call(test, 1, 2), (1, 2, 10))
+
+        def test(a=1, b=2, c=3):
+            return a, b, c
+        self.assertEqual(self.call(test, a=10, c=13), (10, 2, 13))
+        self.assertEqual(self.call(test, a=10), (10, 2, 3))
+        self.assertEqual(self.call(test, b=10), (1, 10, 3))
+
+    def test_signature_bind_varargs_order(self):
+        def test(*args):
+            return args
+
+        self.assertEqual(self.call(test), ())
+        self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3))
+
+    def test_signature_bind_args_and_varargs(self):
+        def test(a, b, c=3, *args):
+            return a, b, c, args
+
+        self.assertEqual(self.call(test, 1, 2, 3, 4, 5), (1, 2, 3, (4, 5)))
+        self.assertEqual(self.call(test, 1, 2), (1, 2, 3, ()))
+        self.assertEqual(self.call(test, b=1, a=2), (2, 1, 3, ()))
+        self.assertEqual(self.call(test, 1, b=2), (1, 2, 3, ()))
+
+        with self.assertRaisesRegex(TypeError,
+                                     "multiple values for argument 'c'"):
+            self.call(test, 1, 2, 3, c=4)
+
+    def test_signature_bind_just_kwargs(self):
+        def test(**kwargs):
+            return kwargs
+
+        self.assertEqual(self.call(test), {})
+        self.assertEqual(self.call(test, foo='bar', spam='ham'),
+                         {'foo': 'bar', 'spam': 'ham'})
+
+    def test_signature_bind_args_and_kwargs(self):
+        def test(a, b, c=3, **kwargs):
+            return a, b, c, kwargs
+
+        self.assertEqual(self.call(test, 1, 2), (1, 2, 3, {}))
+        self.assertEqual(self.call(test, 1, 2, foo='bar', spam='ham'),
+                         (1, 2, 3, {'foo': 'bar', 'spam': 'ham'}))
+        self.assertEqual(self.call(test, b=2, a=1, foo='bar', spam='ham'),
+                         (1, 2, 3, {'foo': 'bar', 'spam': 'ham'}))
+        self.assertEqual(self.call(test, a=1, b=2, foo='bar', spam='ham'),
+                         (1, 2, 3, {'foo': 'bar', 'spam': 'ham'}))
+        self.assertEqual(self.call(test, 1, b=2, foo='bar', spam='ham'),
+                         (1, 2, 3, {'foo': 'bar', 'spam': 'ham'}))
+        self.assertEqual(self.call(test, 1, b=2, c=4, foo='bar', spam='ham'),
+                         (1, 2, 4, {'foo': 'bar', 'spam': 'ham'}))
+        self.assertEqual(self.call(test, 1, 2, 4, foo='bar'),
+                         (1, 2, 4, {'foo': 'bar'}))
+        self.assertEqual(self.call(test, c=5, a=4, b=3),
+                         (4, 3, 5, {}))
+
+    def test_signature_bind_kwonly(self):
+        def test(*, foo):
+            return foo
+        with self.assertRaisesRegex(TypeError,
+                                     'too many positional arguments'):
+            self.call(test, 1)
+        self.assertEqual(self.call(test, foo=1), 1)
+
+        def test(a, *, foo=1, bar):
+            return foo
+        with self.assertRaisesRegex(TypeError,
+                                     "'bar' parameter lacking default value"):
+            self.call(test, 1)
+
+        def test(foo, *, bar):
+            return foo, bar
+        self.assertEqual(self.call(test, 1, bar=2), (1, 2))
+        self.assertEqual(self.call(test, bar=2, foo=1), (1, 2))
+
+        with self.assertRaisesRegex(TypeError,
+                                     'too many keyword arguments'):
+            self.call(test, bar=2, foo=1, spam=10)
+
+        with self.assertRaisesRegex(TypeError,
+                                     'too many positional arguments'):
+            self.call(test, 1, 2)
+
+        with self.assertRaisesRegex(TypeError,
+                                     'too many positional arguments'):
+            self.call(test, 1, 2, bar=2)
+
+        with self.assertRaisesRegex(TypeError,
+                                     'too many keyword arguments'):
+            self.call(test, 1, bar=2, spam='ham')
+
+        with self.assertRaisesRegex(TypeError,
+                                     "'bar' parameter lacking default value"):
+            self.call(test, 1)
+
+        def test(foo, *, bar, **bin):
+            return foo, bar, bin
+        self.assertEqual(self.call(test, 1, bar=2), (1, 2, {}))
+        self.assertEqual(self.call(test, foo=1, bar=2), (1, 2, {}))
+        self.assertEqual(self.call(test, 1, bar=2, spam='ham'),
+                         (1, 2, {'spam': 'ham'}))
+        self.assertEqual(self.call(test, spam='ham', foo=1, bar=2),
+                         (1, 2, {'spam': 'ham'}))
+        with self.assertRaisesRegex(TypeError,
+                                     "'foo' parameter lacking default value"):
+            self.call(test, spam='ham', bar=2)
+        self.assertEqual(self.call(test, 1, bar=2, bin=1, spam=10),
+                         (1, 2, {'bin': 1, 'spam': 10}))
+
+    def test_signature_bind_arguments(self):
+        def test(a, *args, b, z=100, **kwargs):
+            pass
+        sig = inspect.signature(test)
+        ba = sig.bind(10, 20, b=30, c=40, args=50, kwargs=60)
+        # we won't have 'z' argument in the bound arguments object, as we didn't
+        # pass it to the 'bind'
+        self.assertEqual(tuple(ba.arguments.items()),
+                         (('a', 10), ('args', (20,)), ('b', 30),
+                          ('kwargs', {'c': 40, 'args': 50, 'kwargs': 60})))
+        self.assertEqual(ba.kwargs,
+                         {'b': 30, 'c': 40, 'args': 50, 'kwargs': 60})
+        self.assertEqual(ba.args, (10, 20))
+
+    def test_signature_bind_positional_only(self):
+        P = inspect.Parameter
+
+        def test(a_po, b_po, c_po=3, foo=42, *, bar=50, **kwargs):
+            return a_po, b_po, c_po, foo, bar, kwargs
+
+        sig = inspect.signature(test)
+        new_params = collections.OrderedDict(tuple(sig.parameters.items()))
+        for name in ('a_po', 'b_po', 'c_po'):
+            new_params[name] = new_params[name].replace(kind=P.POSITIONAL_ONLY)
+        new_sig = sig.replace(parameters=new_params.values())
+        test.__signature__ = new_sig
+
+        self.assertEqual(self.call(test, 1, 2, 4, 5, bar=6),
+                         (1, 2, 4, 5, 6, {}))
+
+        with self.assertRaisesRegex(TypeError, "parameter is positional only"):
+            self.call(test, 1, 2, c_po=4)
+
+        with self.assertRaisesRegex(TypeError, "parameter is positional only"):
+            self.call(test, a_po=1, b_po=2)
+
+    def test_signature_bind_with_self_arg(self):
+        # Issue #17071: one of the parameters is named "self
+        def test(a, self, b):
+            pass
+        sig = inspect.signature(test)
+        ba = sig.bind(1, 2, 3)
+        self.assertEqual(ba.args, (1, 2, 3))
+        ba = sig.bind(1, self=2, b=3)
+        self.assertEqual(ba.args, (1, 2, 3))
+
+
+class TestBoundArguments(unittest.TestCase):
+    def test_signature_bound_arguments_unhashable(self):
+        def foo(a): pass
+        ba = inspect.signature(foo).bind(1)
+
+        with self.assertRaisesRegex(TypeError, 'unhashable type'):
+            hash(ba)
+
+    def test_signature_bound_arguments_equality(self):
+        def foo(a): pass
+        ba = inspect.signature(foo).bind(1)
+        self.assertEqual(ba, ba)
+
+        ba2 = inspect.signature(foo).bind(1)
+        self.assertEqual(ba, ba2)
+
+        ba3 = inspect.signature(foo).bind(2)
+        self.assertNotEqual(ba, ba3)
+        ba3.arguments['a'] = 1
+        self.assertEqual(ba, ba3)
+
+        def bar(b): pass
+        ba4 = inspect.signature(bar).bind(1)
+        self.assertNotEqual(ba, ba4)
+
 
 def test_main():
     run_unittest(
@@ -1176,7 +2284,8 @@ def test_main():
         TestInterpreterStack, TestClassesAndFunctions, TestPredicates,
         TestGetcallargsFunctions, TestGetcallargsMethods,
         TestGetcallargsUnboundMethods, TestGetattrStatic, TestGetGeneratorState,
-        TestNoEOL
+        TestNoEOL, TestSignatureObject, TestSignatureBind, TestParameterObject,
+        TestBoundArguments, TestGetClosureVars
     )
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py
index 671b20a584..c35a42f596 100644
--- a/Lib/test/test_int.py
+++ b/Lib/test/test_int.py
@@ -2,7 +2,6 @@ import sys
 
 import unittest
 from test import support
-from test.support import run_unittest
 
 L = [
         ('0', 0),
@@ -226,6 +225,9 @@ class IntTestCases(unittest.TestCase):
         self.assertIs(int(b'10'), 10)
         self.assertIs(int(b'-1'), -1)
 
+    def test_no_args(self):
+        self.assertEqual(int(), 0)
+
     def test_keyword_args(self):
         # Test invoking int() using keyword arguments.
         self.assertEqual(int(x=1.2), 1)
@@ -234,6 +236,27 @@ class IntTestCases(unittest.TestCase):
         self.assertRaises(TypeError, int, base=10)
         self.assertRaises(TypeError, int, base=0)
 
+    def test_non_numeric_input_types(self):
+        # Test possible non-numeric types for the argument x, including
+        # subclasses of the explicitly documented accepted types.
+        class CustomStr(str): pass
+        class CustomBytes(bytes): pass
+        class CustomByteArray(bytearray): pass
+
+        values = [b'100',
+                  bytearray(b'100'),
+                  CustomStr('100'),
+                  CustomBytes(b'100'),
+                  CustomByteArray(b'100')]
+
+        for x in values:
+            msg = 'x has type %s' % type(x).__name__
+            self.assertEqual(int(x), 100, msg=msg)
+            self.assertEqual(int(x, 2), 4, msg=msg)
+
+    def test_string_float(self):
+        self.assertRaises(ValueError, int, '1.2')
+
     def test_intconversion(self):
         # Test __int__()
         class ClassicMissingMethods:
@@ -318,6 +341,18 @@ class IntTestCases(unittest.TestCase):
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
 
+                # Regression test for bugs.python.org/issue16060.
+                class BadInt(trunc_result_base):
+                    def __int__(self):
+                        return 42.0
+
+                class TruncReturnsBadInt(base):
+                    def __trunc__(self):
+                        return BadInt()
+
+                with self.assertRaises(TypeError):
+                    int(TruncReturnsBadInt())
+
     def test_error_message(self):
         testlist = ('\xbd', '123\xbd', '  123 456  ')
         for s in testlist:
@@ -329,7 +364,7 @@ class IntTestCases(unittest.TestCase):
                 self.fail("Expected int(%r) to raise a ValueError", s)
 
 def test_main():
-    run_unittest(IntTestCases)
+    support.run_unittest(IntTestCases)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index 8ef314e55d..64c66d8867 100644
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -19,21 +19,22 @@
 # test both implementations. This file has lots of examples.
 ################################################################################
 
+import abc
+import array
+import errno
+import locale
 import os
+import pickle
+import random
+import signal
 import sys
 import time
-import array
-import random
 import unittest
-import weakref
-import abc
-import signal
-import errno
 import warnings
-import pickle
+import weakref
 import _testcapi
-from itertools import cycle, count
 from collections import deque, UserList
+from itertools import cycle, count
 from test import support
 
 import codecs
@@ -50,7 +51,7 @@ except ImportError:
 
 def _default_chunk_size():
     """Get the default TextIOWrapper chunk size"""
-    with open(__file__, "r", encoding="latin1") as f:
+    with open(__file__, "r", encoding="latin-1") as f:
         return f._CHUNK_SIZE
 
 
@@ -603,6 +604,7 @@ class IOTest(unittest.TestCase):
             raise IOError()
         f.flush = bad_flush
         self.assertRaises(IOError, f.close) # exception not swallowed
+        self.assertTrue(f.closed)
 
     def test_multi_close(self):
         f = self.open(support.TESTFN, "wb", buffering=0)
@@ -635,6 +637,15 @@ class IOTest(unittest.TestCase):
         for obj in test:
             self.assertTrue(hasattr(obj, "__dict__"))
 
+    def test_opener(self):
+        with self.open(support.TESTFN, "w") as f:
+            f.write("egg\n")
+        fd = os.open(support.TESTFN, os.O_RDONLY)
+        def opener(path, flags):
+            return fd
+        with self.open("non-existent", "r", opener=opener) as f:
+            self.assertEqual(f.read(), "egg\n")
+
     def test_fileio_closefd(self):
         # Issue #4841
         with self.open(__file__, 'rb') as f1, \
@@ -697,7 +708,7 @@ class CommonBufferedTests:
         bufio = self.tp(rawio)
         # Invalid whence
         self.assertRaises(ValueError, bufio.seek, 0, -1)
-        self.assertRaises(ValueError, bufio.seek, 0, 3)
+        self.assertRaises(ValueError, bufio.seek, 0, 9)
 
     def test_override_destructor(self):
         tp = self.tp
@@ -771,6 +782,22 @@ class CommonBufferedTests:
         raw.flush = bad_flush
         b = self.tp(raw)
         self.assertRaises(IOError, b.close) # exception not swallowed
+        self.assertTrue(b.closed)
+
+    def test_close_error_on_close(self):
+        raw = self.MockRawIO()
+        def bad_flush():
+            raise IOError('flush')
+        def bad_close():
+            raise IOError('close')
+        raw.close = bad_close
+        b = self.tp(raw)
+        b.flush = bad_flush
+        with self.assertRaises(IOError) as err: # exception not swallowed
+            b.close()
+        self.assertEqual(err.exception.args, ('close',))
+        self.assertEqual(err.exception.__context__.args, ('flush',))
+        self.assertFalse(b.closed)
 
     def test_multi_close(self):
         raw = self.MockRawIO()
@@ -863,6 +890,12 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
         self.assertEqual(b, b"gf")
         self.assertEqual(bufio.readinto(b), 0)
         self.assertEqual(b, b"gf")
+        rawio = self.MockRawIO((b"abc", None))
+        bufio = self.tp(rawio)
+        self.assertEqual(bufio.readinto(b), 2)
+        self.assertEqual(b, b"ab")
+        self.assertEqual(bufio.readinto(b), 1)
+        self.assertEqual(b, b"cb")
 
     def test_readlines(self):
         def bufio():
@@ -1283,11 +1316,20 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests):
         self.assertRaises(IOError, bufio.tell)
         self.assertRaises(IOError, bufio.write, b"abcdef")
 
-    def test_max_buffer_size_deprecation(self):
-        with support.check_warnings(("max_buffer_size is deprecated",
-                                     DeprecationWarning)):
+    def test_max_buffer_size_removal(self):
+        with self.assertRaises(TypeError):
             self.tp(self.MockRawIO(), 8, 12)
 
+    def test_write_error_on_close(self):
+        raw = self.MockRawIO()
+        def bad_write(b):
+            raise IOError()
+        raw.write = bad_write
+        b = self.tp(raw)
+        b.write(b'spam')
+        self.assertRaises(IOError, b.close) # exception not swallowed
+        self.assertTrue(b.closed)
+
 
 class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
     tp = io.BufferedWriter
@@ -1346,9 +1388,8 @@ class BufferedRWPairTest(unittest.TestCase):
         pair = self.tp(self.MockRawIO(), self.MockRawIO())
         self.assertRaises(self.UnsupportedOperation, pair.detach)
 
-    def test_constructor_max_buffer_size_deprecation(self):
-        with support.check_warnings(("max_buffer_size is deprecated",
-                                     DeprecationWarning)):
+    def test_constructor_max_buffer_size_removal(self):
+        with self.assertRaises(TypeError):
             self.tp(self.MockRawIO(), self.MockRawIO(), 8, 12)
 
     def test_constructor_with_not_readable(self):
@@ -1871,11 +1912,11 @@ class TextIOWrapperTest(unittest.TestCase):
         r = self.BytesIO(b"\xc3\xa9\n\n")
         b = self.BufferedReader(r, 1000)
         t = self.TextIOWrapper(b)
-        t.__init__(b, encoding="latin1", newline="\r\n")
-        self.assertEqual(t.encoding, "latin1")
+        t.__init__(b, encoding="latin-1", newline="\r\n")
+        self.assertEqual(t.encoding, "latin-1")
         self.assertEqual(t.line_buffering, False)
-        t.__init__(b, encoding="utf8", line_buffering=True)
-        self.assertEqual(t.encoding, "utf8")
+        t.__init__(b, encoding="utf-8", line_buffering=True)
+        self.assertEqual(t.encoding, "utf-8")
         self.assertEqual(t.line_buffering, True)
         self.assertEqual("\xe9\n", t.readline())
         self.assertRaises(TypeError, t.__init__, b, newline=42)
@@ -1922,6 +1963,24 @@ class TextIOWrapperTest(unittest.TestCase):
         t.write("A\rB")
         self.assertEqual(r.getvalue(), b"XY\nZA\rB")
 
+    def test_default_encoding(self):
+        old_environ = dict(os.environ)
+        try:
+            # try to get a user preferred encoding different than the current
+            # locale encoding to check that TextIOWrapper() uses the current
+            # locale encoding and not the user preferred encoding
+            for key in ('LC_ALL', 'LANG', 'LC_CTYPE'):
+                if key in os.environ:
+                    del os.environ[key]
+
+            current_locale_encoding = locale.getpreferredencoding(False)
+            b = self.BytesIO()
+            t = self.TextIOWrapper(b)
+            self.assertEqual(t.encoding, current_locale_encoding)
+        finally:
+            os.environ.clear()
+            os.environ.update(old_environ)
+
     # Issue 15989
     def test_device_encoding(self):
         b = self.BytesIO()
@@ -1933,8 +1992,8 @@ class TextIOWrapperTest(unittest.TestCase):
     def test_encoding(self):
         # Check the encoding attribute is always set, and valid
         b = self.BytesIO()
-        t = self.TextIOWrapper(b, encoding="utf8")
-        self.assertEqual(t.encoding, "utf8")
+        t = self.TextIOWrapper(b, encoding="utf-8")
+        self.assertEqual(t.encoding, "utf-8")
         t = self.TextIOWrapper(b)
         self.assertTrue(t.encoding is not None)
         codecs.lookup(t.encoding)
@@ -2027,8 +2086,8 @@ class TextIOWrapperTest(unittest.TestCase):
         testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG"
         normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
         for newline, expected in [
-            (None, normalized.decode("ascii").splitlines(True)),
-            ("", testdata.decode("ascii").splitlines(True)),
+            (None, normalized.decode("ascii").splitlines(keepends=True)),
+            ("", testdata.decode("ascii").splitlines(keepends=True)),
             ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]),
             ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]),
             ("\r",  ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]),
@@ -2113,7 +2172,7 @@ class TextIOWrapperTest(unittest.TestCase):
 
     def test_basic_io(self):
         for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65):
-            for enc in "ascii", "latin1", "utf8" :# , "utf-16-be", "utf-16-le":
+            for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le":
                 f = self.open(support.TESTFN, "w+", encoding=enc)
                 f._CHUNK_SIZE = chunksize
                 self.assertEqual(f.write("abc"), 3)
@@ -2163,7 +2222,7 @@ class TextIOWrapperTest(unittest.TestCase):
         self.assertEqual(rlines, wlines)
 
     def test_telling(self):
-        f = self.open(support.TESTFN, "w+", encoding="utf8")
+        f = self.open(support.TESTFN, "w+", encoding="utf-8")
         p0 = f.tell()
         f.write("\xff\n")
         p1 = f.tell()
@@ -2431,6 +2490,7 @@ class TextIOWrapperTest(unittest.TestCase):
         with self.open(support.TESTFN, "w", errors="replace") as f:
             self.assertEqual(f.errors, "replace")
 
+    @support.no_tracing
     @unittest.skipUnless(threading, 'Threading required for this test.')
     def test_threads_write(self):
         # Issue6750: concurrent writes could duplicate data
@@ -2459,6 +2519,7 @@ class TextIOWrapperTest(unittest.TestCase):
             raise IOError()
         txt.flush = bad_flush
         self.assertRaises(IOError, txt.close) # exception not swallowed
+        self.assertTrue(txt.closed)
 
     def test_multi_close(self):
         txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
@@ -2772,12 +2833,6 @@ class MiscIOTest(unittest.TestCase):
 
     def test_blockingioerror(self):
         # Various BlockingIOError issues
-        self.assertRaises(TypeError, self.BlockingIOError)
-        self.assertRaises(TypeError, self.BlockingIOError, 1)
-        self.assertRaises(TypeError, self.BlockingIOError, 1, 2, 3, 4)
-        self.assertRaises(TypeError, self.BlockingIOError, 1, "", None)
-        b = self.BlockingIOError(1, "")
-        self.assertEqual(b.characters_written, 0)
         class C(str):
             pass
         c = C("")
@@ -2919,6 +2974,7 @@ class MiscIOTest(unittest.TestCase):
 
                 except self.BlockingIOError as e:
                     self.assertEqual(e.args[0], errno.EAGAIN)
+                    self.assertEqual(e.args[2], e.characters_written)
                     sent[-1] = sent[-1][:e.characters_written]
                     received.append(rf.read())
                     msg = b'BLOCKED'
@@ -2931,6 +2987,7 @@ class MiscIOTest(unittest.TestCase):
                     break
                 except self.BlockingIOError as e:
                     self.assertEqual(e.args[0], errno.EAGAIN)
+                    self.assertEqual(e.args[2], e.characters_written)
                     self.assertEqual(e.characters_written, 0)
                     received.append(rf.read())
 
@@ -2941,6 +2998,24 @@ class MiscIOTest(unittest.TestCase):
         self.assertTrue(wf.closed)
         self.assertTrue(rf.closed)
 
+    def test_create_fail(self):
+        # 'x' mode fails if file is existing
+        with self.open(support.TESTFN, 'w'):
+            pass
+        self.assertRaises(FileExistsError, self.open, support.TESTFN, 'x')
+
+    def test_create_writes(self):
+        # 'x' mode opens for writing
+        with self.open(support.TESTFN, 'xb') as f:
+            f.write(b"spam")
+        with self.open(support.TESTFN, 'rb') as f:
+            self.assertEqual(b"spam", f.read())
+
+    def test_open_allargs(self):
+        # there used to be a buffer overflow in the parser for rawmode
+        self.assertRaises(ValueError, self.open, support.TESTFN, 'rwax+')
+
+
 class CMiscIOTest(MiscIOTest):
     io = io
 
@@ -2961,14 +3036,14 @@ class SignalsTest(unittest.TestCase):
         1/0
 
     @unittest.skipUnless(threading, 'Threading required for this test.')
-    @unittest.skipIf(sys.platform in ('freebsd5', 'freebsd6', 'freebsd7'),
-                     'issue #12429: skip test on FreeBSD <= 7')
     def check_interrupted_write(self, item, bytes, **fdopen_kwargs):
         """Check that a partial write, when it gets interrupted, properly
         invokes the signal handler, and bubbles up the exception raised
         in the latter."""
         read_results = []
         def _read():
+            if hasattr(signal, 'pthread_sigmask'):
+                signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM])
             s = os.read(r, 1)
             read_results.append(s)
         t = threading.Thread(target=_read)
@@ -2986,7 +3061,7 @@ class SignalsTest(unittest.TestCase):
             # The buffered IO layer must check for pending signal
             # handlers, which in this case will invoke alarm_interrupt().
             self.assertRaises(ZeroDivisionError,
-                              wio.write, item * (1024 * 1024))
+                        wio.write, item * (support.PIPE_MAX_SIZE // len(item)))
             t.join()
             # We got one byte, get another one and check that it isn't a
             # repeat of the first one.
@@ -3013,6 +3088,7 @@ class SignalsTest(unittest.TestCase):
     def test_interrupted_write_text(self):
         self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii")
 
+    @support.no_tracing
     def check_reentrant_write(self, data, **fdopen_kwargs):
         def on_alarm(*args):
             # Will be called reentrantly from the same thread
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
new file mode 100644
index 0000000000..99c54f1614
--- /dev/null
+++ b/Lib/test/test_ipaddress.py
@@ -0,0 +1,1649 @@
+# Copyright 2007 Google Inc.
+#  Licensed to PSF under a Contributor Agreement.
+
+"""Unittest for ipaddress module."""
+
+
+import unittest
+import re
+import contextlib
+import operator
+import ipaddress
+
+class BaseTestCase(unittest.TestCase):
+    # One big change in ipaddress over the original ipaddr module is
+    # error reporting that tries to assume users *don't know the rules*
+    # for what constitutes an RFC compliant IP address
+
+    # Ensuring these errors are emitted correctly in all relevant cases
+    # meant moving to a more systematic test structure that allows the
+    # test structure to map more directly to the module structure
+
+    # Note that if the constructors are refactored so that addresses with
+    # multiple problems get classified differently, that's OK - just
+    # move the affected examples to the newly appropriate test case.
+
+    # There is some duplication between the original relatively ad hoc
+    # test suite and the new systematic tests. While some redundancy in
+    # testing is considered preferable to accidentally deleting a valid
+    # test, the original test suite will likely be reduced over time as
+    # redundant tests are identified.
+
+    @property
+    def factory(self):
+        raise NotImplementedError
+
+    @contextlib.contextmanager
+    def assertCleanError(self, exc_type, details, *args):
+        """
+        Ensure exception does not display a context by default
+
+        Wraps unittest.TestCase.assertRaisesRegex
+        """
+        if args:
+            details = details % args
+        cm = self.assertRaisesRegex(exc_type, details)
+        with cm as exc:
+            yield exc
+        # Ensure we produce clean tracebacks on failure
+        if exc.exception.__context__ is not None:
+            self.assertTrue(exc.exception.__suppress_context__)
+
+    def assertAddressError(self, details, *args):
+        """Ensure a clean AddressValueError"""
+        return self.assertCleanError(ipaddress.AddressValueError,
+                                       details, *args)
+
+    def assertNetmaskError(self, details, *args):
+        """Ensure a clean NetmaskValueError"""
+        return self.assertCleanError(ipaddress.NetmaskValueError,
+                                details, *args)
+
+    def assertInstancesEqual(self, lhs, rhs):
+        """Check constructor arguments produce equivalent instances"""
+        self.assertEqual(self.factory(lhs), self.factory(rhs))
+
+class CommonTestMixin:
+
+    def test_empty_address(self):
+        with self.assertAddressError("Address cannot be empty"):
+            self.factory("")
+
+    def test_floats_rejected(self):
+        with self.assertAddressError(re.escape(repr("1.0"))):
+            self.factory(1.0)
+
+    def test_not_an_index_issue15559(self):
+        # Implementing __index__ makes for a very nasty interaction with the
+        # bytes constructor. Thus, we disallow implicit use as an integer
+        self.assertRaises(TypeError, operator.index, self.factory(1))
+        self.assertRaises(TypeError, hex, self.factory(1))
+        self.assertRaises(TypeError, bytes, self.factory(1))
+
+
+class CommonTestMixin_v4(CommonTestMixin):
+
+    def test_leading_zeros(self):
+        self.assertInstancesEqual("000.000.000.000", "0.0.0.0")
+        self.assertInstancesEqual("192.168.000.001", "192.168.0.1")
+
+    def test_int(self):
+        self.assertInstancesEqual(0, "0.0.0.0")
+        self.assertInstancesEqual(3232235521, "192.168.0.1")
+
+    def test_packed(self):
+        self.assertInstancesEqual(bytes.fromhex("00000000"), "0.0.0.0")
+        self.assertInstancesEqual(bytes.fromhex("c0a80001"), "192.168.0.1")
+
+    def test_negative_ints_rejected(self):
+        msg = "-1 (< 0) is not permitted as an IPv4 address"
+        with self.assertAddressError(re.escape(msg)):
+            self.factory(-1)
+
+    def test_large_ints_rejected(self):
+        msg = "%d (>= 2**32) is not permitted as an IPv4 address"
+        with self.assertAddressError(re.escape(msg % 2**32)):
+            self.factory(2**32)
+
+    def test_bad_packed_length(self):
+        def assertBadLength(length):
+            addr = bytes(length)
+            msg = "%r (len %d != 4) is not permitted as an IPv4 address"
+            with self.assertAddressError(re.escape(msg % (addr, length))):
+                self.factory(addr)
+
+        assertBadLength(3)
+        assertBadLength(5)
+
+class CommonTestMixin_v6(CommonTestMixin):
+
+    def test_leading_zeros(self):
+        self.assertInstancesEqual("0000::0000", "::")
+        self.assertInstancesEqual("000::c0a8:0001", "::c0a8:1")
+
+    def test_int(self):
+        self.assertInstancesEqual(0, "::")
+        self.assertInstancesEqual(3232235521, "::c0a8:1")
+
+    def test_packed(self):
+        addr = bytes(12) + bytes.fromhex("00000000")
+        self.assertInstancesEqual(addr, "::")
+        addr = bytes(12) + bytes.fromhex("c0a80001")
+        self.assertInstancesEqual(addr, "::c0a8:1")
+        addr = bytes.fromhex("c0a80001") + bytes(12)
+        self.assertInstancesEqual(addr, "c0a8:1::")
+
+    def test_negative_ints_rejected(self):
+        msg = "-1 (< 0) is not permitted as an IPv6 address"
+        with self.assertAddressError(re.escape(msg)):
+            self.factory(-1)
+
+    def test_large_ints_rejected(self):
+        msg = "%d (>= 2**128) is not permitted as an IPv6 address"
+        with self.assertAddressError(re.escape(msg % 2**128)):
+            self.factory(2**128)
+
+    def test_bad_packed_length(self):
+        def assertBadLength(length):
+            addr = bytes(length)
+            msg = "%r (len %d != 16) is not permitted as an IPv6 address"
+            with self.assertAddressError(re.escape(msg % (addr, length))):
+                self.factory(addr)
+                self.factory(addr)
+
+        assertBadLength(15)
+        assertBadLength(17)
+
+
+class AddressTestCase_v4(BaseTestCase, CommonTestMixin_v4):
+    factory = ipaddress.IPv4Address
+
+    def test_network_passed_as_address(self):
+        addr = "127.0.0.1/24"
+        with self.assertAddressError("Unexpected '/' in %r", addr):
+            ipaddress.IPv4Address(addr)
+
+    def test_bad_address_split(self):
+        def assertBadSplit(addr):
+            with self.assertAddressError("Expected 4 octets in %r", addr):
+                ipaddress.IPv4Address(addr)
+
+        assertBadSplit("127.0.1")
+        assertBadSplit("42.42.42.42.42")
+        assertBadSplit("42.42.42")
+        assertBadSplit("42.42")
+        assertBadSplit("42")
+        assertBadSplit("42..42.42.42")
+        assertBadSplit("42.42.42.42.")
+        assertBadSplit("42.42.42.42...")
+        assertBadSplit(".42.42.42.42")
+        assertBadSplit("...42.42.42.42")
+        assertBadSplit("016.016.016")
+        assertBadSplit("016.016")
+        assertBadSplit("016")
+        assertBadSplit("000")
+        assertBadSplit("0x0a.0x0a.0x0a")
+        assertBadSplit("0x0a.0x0a")
+        assertBadSplit("0x0a")
+        assertBadSplit(".")
+        assertBadSplit("bogus")
+        assertBadSplit("bogus.com")
+        assertBadSplit("1000")
+        assertBadSplit("1000000000000000")
+        assertBadSplit("192.168.0.1.com")
+
+    def test_empty_octet(self):
+        def assertBadOctet(addr):
+            with self.assertAddressError("Empty octet not permitted in %r",
+                                          addr):
+                ipaddress.IPv4Address(addr)
+
+        assertBadOctet("42..42.42")
+        assertBadOctet("...")
+
+    def test_invalid_characters(self):
+        def assertBadOctet(addr, octet):
+            msg = "Only decimal digits permitted in %r in %r" % (octet, addr)
+            with self.assertAddressError(re.escape(msg)):
+                ipaddress.IPv4Address(addr)
+
+        assertBadOctet("0x0a.0x0a.0x0a.0x0a", "0x0a")
+        assertBadOctet("0xa.0x0a.0x0a.0x0a", "0xa")
+        assertBadOctet("42.42.42.-0", "-0")
+        assertBadOctet("42.42.42.+0", "+0")
+        assertBadOctet("42.42.42.-42", "-42")
+        assertBadOctet("+1.+2.+3.4", "+1")
+        assertBadOctet("1.2.3.4e0", "4e0")
+        assertBadOctet("1.2.3.4::", "4::")
+        assertBadOctet("1.a.2.3", "a")
+
+    def test_octal_decimal_ambiguity(self):
+        def assertBadOctet(addr, octet):
+            msg = "Ambiguous (octal/decimal) value in %r not permitted in %r"
+            with self.assertAddressError(re.escape(msg % (octet, addr))):
+                ipaddress.IPv4Address(addr)
+
+        assertBadOctet("016.016.016.016", "016")
+        assertBadOctet("001.000.008.016", "008")
+
+    def test_octet_length(self):
+        def assertBadOctet(addr, octet):
+            msg = "At most 3 characters permitted in %r in %r"
+            with self.assertAddressError(re.escape(msg % (octet, addr))):
+                ipaddress.IPv4Address(addr)
+
+        assertBadOctet("0000.000.000.000", "0000")
+        assertBadOctet("12345.67899.-54321.-98765", "12345")
+
+    def test_octet_limit(self):
+        def assertBadOctet(addr, octet):
+            msg = "Octet %d (> 255) not permitted in %r" % (octet, addr)
+            with self.assertAddressError(re.escape(msg)):
+                ipaddress.IPv4Address(addr)
+
+        assertBadOctet("257.0.0.0", 257)
+        assertBadOctet("192.168.0.999", 999)
+
+
+class AddressTestCase_v6(BaseTestCase, CommonTestMixin_v6):
+    factory = ipaddress.IPv6Address
+
+    def test_network_passed_as_address(self):
+        addr = "::1/24"
+        with self.assertAddressError("Unexpected '/' in %r", addr):
+            ipaddress.IPv6Address(addr)
+
+    def test_bad_address_split_v6_not_enough_parts(self):
+        def assertBadSplit(addr):
+            msg = "At least 3 parts expected in %r"
+            with self.assertAddressError(msg, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadSplit(":")
+        assertBadSplit(":1")
+        assertBadSplit("FEDC:9878")
+
+    def test_bad_address_split_v6_too_many_colons(self):
+        def assertBadSplit(addr):
+            msg = "At most 8 colons permitted in %r"
+            with self.assertAddressError(msg, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadSplit("9:8:7:6:5:4:3::2:1")
+        assertBadSplit("10:9:8:7:6:5:4:3:2:1")
+        assertBadSplit("::8:7:6:5:4:3:2:1")
+        assertBadSplit("8:7:6:5:4:3:2:1::")
+        # A trailing IPv4 address is two parts
+        assertBadSplit("10:9:8:7:6:5:4:3:42.42.42.42")
+
+    def test_bad_address_split_v6_too_many_parts(self):
+        def assertBadSplit(addr):
+            msg = "Exactly 8 parts expected without '::' in %r"
+            with self.assertAddressError(msg, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadSplit("3ffe:0:0:0:0:0:0:0:1")
+        assertBadSplit("9:8:7:6:5:4:3:2:1")
+        assertBadSplit("7:6:5:4:3:2:1")
+        # A trailing IPv4 address is two parts
+        assertBadSplit("9:8:7:6:5:4:3:42.42.42.42")
+        assertBadSplit("7:6:5:4:3:42.42.42.42")
+
+    def test_bad_address_split_v6_too_many_parts_with_double_colon(self):
+        def assertBadSplit(addr):
+            msg = "Expected at most 7 other parts with '::' in %r"
+            with self.assertAddressError(msg, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadSplit("1:2:3:4::5:6:7:8")
+
+    def test_bad_address_split_v6_repeated_double_colon(self):
+        def assertBadSplit(addr):
+            msg = "At most one '::' permitted in %r"
+            with self.assertAddressError(msg, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadSplit("3ffe::1::1")
+        assertBadSplit("1::2::3::4:5")
+        assertBadSplit("2001::db:::1")
+        assertBadSplit("3ffe::1::")
+        assertBadSplit("::3ffe::1")
+        assertBadSplit(":3ffe::1::1")
+        assertBadSplit("3ffe::1::1:")
+        assertBadSplit(":3ffe::1::1:")
+        assertBadSplit(":::")
+        assertBadSplit('2001:db8:::1')
+
+    def test_bad_address_split_v6_leading_colon(self):
+        def assertBadSplit(addr):
+            msg = "Leading ':' only permitted as part of '::' in %r"
+            with self.assertAddressError(msg, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadSplit(":2001:db8::1")
+        assertBadSplit(":1:2:3:4:5:6:7")
+        assertBadSplit(":1:2:3:4:5:6:")
+        assertBadSplit(":6:5:4:3:2:1::")
+
+    def test_bad_address_split_v6_trailing_colon(self):
+        def assertBadSplit(addr):
+            msg = "Trailing ':' only permitted as part of '::' in %r"
+            with self.assertAddressError(msg, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadSplit("2001:db8::1:")
+        assertBadSplit("1:2:3:4:5:6:7:")
+        assertBadSplit("::1.2.3.4:")
+        assertBadSplit("::7:6:5:4:3:2:")
+
+    def test_bad_v4_part_in(self):
+        def assertBadAddressPart(addr, v4_error):
+            with self.assertAddressError("%s in %r", v4_error, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadAddressPart("3ffe::1.net", "Expected 4 octets in '1.net'")
+        assertBadAddressPart("3ffe::127.0.1",
+                             "Expected 4 octets in '127.0.1'")
+        assertBadAddressPart("::1.2.3",
+                             "Expected 4 octets in '1.2.3'")
+        assertBadAddressPart("::1.2.3.4.5",
+                             "Expected 4 octets in '1.2.3.4.5'")
+        assertBadAddressPart("3ffe::1.1.1.net",
+                             "Only decimal digits permitted in 'net' "
+                             "in '1.1.1.net'")
+
+    def test_invalid_characters(self):
+        def assertBadPart(addr, part):
+            msg = "Only hex digits permitted in %r in %r" % (part, addr)
+            with self.assertAddressError(re.escape(msg)):
+                ipaddress.IPv6Address(addr)
+
+        assertBadPart("3ffe::goog", "goog")
+        assertBadPart("3ffe::-0", "-0")
+        assertBadPart("3ffe::+0", "+0")
+        assertBadPart("3ffe::-1", "-1")
+        assertBadPart("1.2.3.4::", "1.2.3.4")
+        assertBadPart('1234:axy::b', "axy")
+
+    def test_part_length(self):
+        def assertBadPart(addr, part):
+            msg = "At most 4 characters permitted in %r in %r"
+            with self.assertAddressError(msg, part, addr):
+                ipaddress.IPv6Address(addr)
+
+        assertBadPart("::00000", "00000")
+        assertBadPart("3ffe::10000", "10000")
+        assertBadPart("02001:db8::", "02001")
+        assertBadPart('2001:888888::1', "888888")
+
+
+class NetmaskTestMixin_v4(CommonTestMixin_v4):
+    """Input validation on interfaces and networks is very similar"""
+
+    def test_split_netmask(self):
+        addr = "1.2.3.4/32/24"
+        with self.assertAddressError("Only one '/' permitted in %r" % addr):
+            self.factory(addr)
+
+    def test_address_errors(self):
+        def assertBadAddress(addr, details):
+            with self.assertAddressError(details):
+                self.factory(addr)
+
+        assertBadAddress("/", "Address cannot be empty")
+        assertBadAddress("/8", "Address cannot be empty")
+        assertBadAddress("bogus", "Expected 4 octets")
+        assertBadAddress("google.com", "Expected 4 octets")
+        assertBadAddress("10/8", "Expected 4 octets")
+        assertBadAddress("::1.2.3.4", "Only decimal digits")
+        assertBadAddress("1.2.3.256", re.escape("256 (> 255)"))
+
+    def test_netmask_errors(self):
+        def assertBadNetmask(addr, netmask):
+            msg = "%r is not a valid netmask"
+            with self.assertNetmaskError(msg % netmask):
+                self.factory("%s/%s" % (addr, netmask))
+
+        assertBadNetmask("1.2.3.4", "")
+        assertBadNetmask("1.2.3.4", "33")
+        assertBadNetmask("1.2.3.4", "254.254.255.256")
+        assertBadNetmask("1.1.1.1", "254.xyz.2.3")
+        assertBadNetmask("1.1.1.1", "240.255.0.0")
+        assertBadNetmask("1.1.1.1", "pudding")
+
+class InterfaceTestCase_v4(BaseTestCase, NetmaskTestMixin_v4):
+    factory = ipaddress.IPv4Interface
+
+class NetworkTestCase_v4(BaseTestCase, NetmaskTestMixin_v4):
+    factory = ipaddress.IPv4Network
+
+
+class NetmaskTestMixin_v6(CommonTestMixin_v6):
+    """Input validation on interfaces and networks is very similar"""
+
+    def test_split_netmask(self):
+        addr = "cafe:cafe::/128/190"
+        with self.assertAddressError("Only one '/' permitted in %r" % addr):
+            self.factory(addr)
+
+    def test_address_errors(self):
+        def assertBadAddress(addr, details):
+            with self.assertAddressError(details):
+                self.factory(addr)
+
+        assertBadAddress("/", "Address cannot be empty")
+        assertBadAddress("/8", "Address cannot be empty")
+        assertBadAddress("google.com", "At least 3 parts")
+        assertBadAddress("1.2.3.4", "At least 3 parts")
+        assertBadAddress("10/8", "At least 3 parts")
+        assertBadAddress("1234:axy::b", "Only hex digits")
+
+    def test_netmask_errors(self):
+        def assertBadNetmask(addr, netmask):
+            msg = "%r is not a valid netmask"
+            with self.assertNetmaskError(msg % netmask):
+                self.factory("%s/%s" % (addr, netmask))
+
+        assertBadNetmask("::1", "")
+        assertBadNetmask("::1", "::1")
+        assertBadNetmask("::1", "1::")
+        assertBadNetmask("::1", "129")
+        assertBadNetmask("::1", "pudding")
+
+class InterfaceTestCase_v6(BaseTestCase, NetmaskTestMixin_v6):
+    factory = ipaddress.IPv6Interface
+
+class NetworkTestCase_v6(BaseTestCase, NetmaskTestMixin_v6):
+    factory = ipaddress.IPv6Network
+
+
+class FactoryFunctionErrors(BaseTestCase):
+
+    def assertFactoryError(self, factory, kind):
+        """Ensure a clean ValueError with the expected message"""
+        addr = "camelot"
+        msg = '%r does not appear to be an IPv4 or IPv6 %s'
+        with self.assertCleanError(ValueError, msg, addr, kind):
+            factory(addr)
+
+    def test_ip_address(self):
+        self.assertFactoryError(ipaddress.ip_address, "address")
+
+    def test_ip_interface(self):
+        self.assertFactoryError(ipaddress.ip_interface, "interface")
+
+    def test_ip_network(self):
+        self.assertFactoryError(ipaddress.ip_network, "network")
+
+
+class ComparisonTests(unittest.TestCase):
+
+    v4addr = ipaddress.IPv4Address(1)
+    v4net = ipaddress.IPv4Network(1)
+    v4intf = ipaddress.IPv4Interface(1)
+    v6addr = ipaddress.IPv6Address(1)
+    v6net = ipaddress.IPv6Network(1)
+    v6intf = ipaddress.IPv6Interface(1)
+
+    v4_addresses = [v4addr, v4intf]
+    v4_objects = v4_addresses + [v4net]
+    v6_addresses = [v6addr, v6intf]
+    v6_objects = v6_addresses + [v6net]
+    objects = v4_objects + v6_objects
+
+    def test_foreign_type_equality(self):
+        # __eq__ should never raise TypeError directly
+        other = object()
+        for obj in self.objects:
+            self.assertNotEqual(obj, other)
+            self.assertFalse(obj == other)
+            self.assertEqual(obj.__eq__(other), NotImplemented)
+            self.assertEqual(obj.__ne__(other), NotImplemented)
+
+    def test_mixed_type_equality(self):
+        # Ensure none of the internal objects accidentally
+        # expose the right set of attributes to become "equal"
+        for lhs in self.objects:
+            for rhs in self.objects:
+                if lhs is rhs:
+                    continue
+                self.assertNotEqual(lhs, rhs)
+
+    def test_containment(self):
+        for obj in self.v4_addresses:
+            self.assertIn(obj, self.v4net)
+        for obj in self.v6_addresses:
+            self.assertIn(obj, self.v6net)
+        for obj in self.v4_objects + [self.v6net]:
+            self.assertNotIn(obj, self.v6net)
+        for obj in self.v6_objects + [self.v4net]:
+            self.assertNotIn(obj, self.v4net)
+
+    def test_mixed_type_ordering(self):
+        for lhs in self.objects:
+            for rhs in self.objects:
+                if isinstance(lhs, type(rhs)) or isinstance(rhs, type(lhs)):
+                    continue
+                self.assertRaises(TypeError, lambda: lhs < rhs)
+                self.assertRaises(TypeError, lambda: lhs > rhs)
+                self.assertRaises(TypeError, lambda: lhs <= rhs)
+                self.assertRaises(TypeError, lambda: lhs >= rhs)
+
+    def test_mixed_type_key(self):
+        # with get_mixed_type_key, you can sort addresses and network.
+        v4_ordered = [self.v4addr, self.v4net, self.v4intf]
+        v6_ordered = [self.v6addr, self.v6net, self.v6intf]
+        self.assertEqual(v4_ordered,
+                         sorted(self.v4_objects,
+                                key=ipaddress.get_mixed_type_key))
+        self.assertEqual(v6_ordered,
+                         sorted(self.v6_objects,
+                                key=ipaddress.get_mixed_type_key))
+        self.assertEqual(v4_ordered + v6_ordered,
+                         sorted(self.objects,
+                                key=ipaddress.get_mixed_type_key))
+        self.assertEqual(NotImplemented, ipaddress.get_mixed_type_key(object))
+
+    def test_incompatible_versions(self):
+        # These should always raise TypeError
+        v4addr = ipaddress.ip_address('1.1.1.1')
+        v4net = ipaddress.ip_network('1.1.1.1')
+        v6addr = ipaddress.ip_address('::1')
+        v6net = ipaddress.ip_address('::1')
+
+        self.assertRaises(TypeError, v4addr.__lt__, v6addr)
+        self.assertRaises(TypeError, v4addr.__gt__, v6addr)
+        self.assertRaises(TypeError, v4net.__lt__, v6net)
+        self.assertRaises(TypeError, v4net.__gt__, v6net)
+
+        self.assertRaises(TypeError, v6addr.__lt__, v4addr)
+        self.assertRaises(TypeError, v6addr.__gt__, v4addr)
+        self.assertRaises(TypeError, v6net.__lt__, v4net)
+        self.assertRaises(TypeError, v6net.__gt__, v4net)
+
+
+
+class IpaddrUnitTest(unittest.TestCase):
+
+    def setUp(self):
+        self.ipv4_address = ipaddress.IPv4Address('1.2.3.4')
+        self.ipv4_interface = ipaddress.IPv4Interface('1.2.3.4/24')
+        self.ipv4_network = ipaddress.IPv4Network('1.2.3.0/24')
+        #self.ipv4_hostmask = ipaddress.IPv4Interface('10.0.0.1/0.255.255.255')
+        self.ipv6_address = ipaddress.IPv6Interface(
+            '2001:658:22a:cafe:200:0:0:1')
+        self.ipv6_interface = ipaddress.IPv6Interface(
+            '2001:658:22a:cafe:200:0:0:1/64')
+        self.ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/64')
+
+    def testRepr(self):
+        self.assertEqual("IPv4Interface('1.2.3.4/32')",
+                         repr(ipaddress.IPv4Interface('1.2.3.4')))
+        self.assertEqual("IPv6Interface('::1/128')",
+                         repr(ipaddress.IPv6Interface('::1')))
+
+    # issue57
+    def testAddressIntMath(self):
+        self.assertEqual(ipaddress.IPv4Address('1.1.1.1') + 255,
+                         ipaddress.IPv4Address('1.1.2.0'))
+        self.assertEqual(ipaddress.IPv4Address('1.1.1.1') - 256,
+                         ipaddress.IPv4Address('1.1.0.1'))
+        self.assertEqual(ipaddress.IPv6Address('::1') + (2**16 - 2),
+                         ipaddress.IPv6Address('::ffff'))
+        self.assertEqual(ipaddress.IPv6Address('::ffff') - (2**16 - 2),
+                         ipaddress.IPv6Address('::1'))
+
+    def testInvalidIntToBytes(self):
+        self.assertRaises(ValueError, ipaddress.v4_int_to_packed, -1)
+        self.assertRaises(ValueError, ipaddress.v4_int_to_packed,
+                          2 ** ipaddress.IPV4LENGTH)
+        self.assertRaises(ValueError, ipaddress.v6_int_to_packed, -1)
+        self.assertRaises(ValueError, ipaddress.v6_int_to_packed,
+                          2 ** ipaddress.IPV6LENGTH)
+
+    def testInternals(self):
+        first, last = ipaddress._find_address_range([
+            ipaddress.IPv4Address('10.10.10.10'),
+            ipaddress.IPv4Address('10.10.10.12')])
+        self.assertEqual(first, last)
+        self.assertEqual(128, ipaddress._count_righthand_zero_bits(0, 128))
+        self.assertEqual("IPv4Network('1.2.3.0/24')", repr(self.ipv4_network))
+
+    def testMissingAddressVersion(self):
+        class Broken(ipaddress._BaseAddress):
+            pass
+        broken = Broken('127.0.0.1')
+        with self.assertRaisesRegex(NotImplementedError, "Broken.*version"):
+            broken.version
+
+    def testMissingNetworkVersion(self):
+        class Broken(ipaddress._BaseNetwork):
+            pass
+        broken = Broken('127.0.0.1')
+        with self.assertRaisesRegex(NotImplementedError, "Broken.*version"):
+            broken.version
+
+    def testMissingAddressClass(self):
+        class Broken(ipaddress._BaseNetwork):
+            pass
+        broken = Broken('127.0.0.1')
+        with self.assertRaisesRegex(NotImplementedError, "Broken.*address"):
+            broken._address_class
+
+    def testGetNetwork(self):
+        self.assertEqual(int(self.ipv4_network.network_address), 16909056)
+        self.assertEqual(str(self.ipv4_network.network_address), '1.2.3.0')
+
+        self.assertEqual(int(self.ipv6_network.network_address),
+                         42540616829182469433403647294022090752)
+        self.assertEqual(str(self.ipv6_network.network_address),
+                         '2001:658:22a:cafe::')
+        self.assertEqual(str(self.ipv6_network.hostmask),
+                         '::ffff:ffff:ffff:ffff')
+
+    def testIpFromInt(self):
+        self.assertEqual(self.ipv4_interface._ip,
+                         ipaddress.IPv4Interface(16909060)._ip)
+
+        ipv4 = ipaddress.ip_network('1.2.3.4')
+        ipv6 = ipaddress.ip_network('2001:658:22a:cafe:200:0:0:1')
+        self.assertEqual(ipv4, ipaddress.ip_network(int(ipv4.network_address)))
+        self.assertEqual(ipv6, ipaddress.ip_network(int(ipv6.network_address)))
+
+        v6_int = 42540616829182469433547762482097946625
+        self.assertEqual(self.ipv6_interface._ip,
+                         ipaddress.IPv6Interface(v6_int)._ip)
+
+        self.assertEqual(ipaddress.ip_network(self.ipv4_address._ip).version,
+                         4)
+        self.assertEqual(ipaddress.ip_network(self.ipv6_address._ip).version,
+                         6)
+
+    def testIpFromPacked(self):
+        address = ipaddress.ip_address
+        self.assertEqual(self.ipv4_interface._ip,
+                         ipaddress.ip_interface(b'\x01\x02\x03\x04')._ip)
+        self.assertEqual(address('255.254.253.252'),
+                         address(b'\xff\xfe\xfd\xfc'))
+        self.assertEqual(self.ipv6_interface.ip,
+                         ipaddress.ip_interface(
+                    b'\x20\x01\x06\x58\x02\x2a\xca\xfe'
+                    b'\x02\x00\x00\x00\x00\x00\x00\x01').ip)
+        self.assertEqual(address('ffff:2:3:4:ffff::'),
+                         address(b'\xff\xff\x00\x02\x00\x03\x00\x04' +
+                            b'\xff\xff' + b'\x00' * 6))
+        self.assertEqual(address('::'),
+                         address(b'\x00' * 16))
+
+    def testGetIp(self):
+        self.assertEqual(int(self.ipv4_interface.ip), 16909060)
+        self.assertEqual(str(self.ipv4_interface.ip), '1.2.3.4')
+
+        self.assertEqual(int(self.ipv6_interface.ip),
+                         42540616829182469433547762482097946625)
+        self.assertEqual(str(self.ipv6_interface.ip),
+                         '2001:658:22a:cafe:200::1')
+
+    def testGetNetmask(self):
+        self.assertEqual(int(self.ipv4_network.netmask), 4294967040)
+        self.assertEqual(str(self.ipv4_network.netmask), '255.255.255.0')
+        self.assertEqual(int(self.ipv6_network.netmask),
+                         340282366920938463444927863358058659840)
+        self.assertEqual(self.ipv6_network.prefixlen, 64)
+
+    def testZeroNetmask(self):
+        ipv4_zero_netmask = ipaddress.IPv4Interface('1.2.3.4/0')
+        self.assertEqual(int(ipv4_zero_netmask.network.netmask), 0)
+        self.assertTrue(ipv4_zero_netmask.network._is_valid_netmask(
+                str(0)))
+        self.assertTrue(ipv4_zero_netmask._is_valid_netmask('0'))
+        self.assertTrue(ipv4_zero_netmask._is_valid_netmask('0.0.0.0'))
+        self.assertFalse(ipv4_zero_netmask._is_valid_netmask('invalid'))
+
+        ipv6_zero_netmask = ipaddress.IPv6Interface('::1/0')
+        self.assertEqual(int(ipv6_zero_netmask.network.netmask), 0)
+        self.assertTrue(ipv6_zero_netmask.network._is_valid_netmask(
+                str(0)))
+
+    def testIPv4NetAndHostmasks(self):
+        net = self.ipv4_network
+        self.assertFalse(net._is_valid_netmask('invalid'))
+        self.assertTrue(net._is_valid_netmask('128.128.128.128'))
+        self.assertFalse(net._is_valid_netmask('128.128.128.127'))
+        self.assertFalse(net._is_valid_netmask('128.128.128.255'))
+        self.assertTrue(net._is_valid_netmask('255.128.128.128'))
+
+        self.assertFalse(net._is_hostmask('invalid'))
+        self.assertTrue(net._is_hostmask('128.255.255.255'))
+        self.assertFalse(net._is_hostmask('255.255.255.255'))
+        self.assertFalse(net._is_hostmask('1.2.3.4'))
+
+        net = ipaddress.IPv4Network('127.0.0.0/0.0.0.255')
+        self.assertEqual(24, net.prefixlen)
+
+    def testGetBroadcast(self):
+        self.assertEqual(int(self.ipv4_network.broadcast_address), 16909311)
+        self.assertEqual(str(self.ipv4_network.broadcast_address), '1.2.3.255')
+
+        self.assertEqual(int(self.ipv6_network.broadcast_address),
+                         42540616829182469451850391367731642367)
+        self.assertEqual(str(self.ipv6_network.broadcast_address),
+                         '2001:658:22a:cafe:ffff:ffff:ffff:ffff')
+
+    def testGetPrefixlen(self):
+        self.assertEqual(self.ipv4_interface.network.prefixlen, 24)
+        self.assertEqual(self.ipv6_interface.network.prefixlen, 64)
+
+    def testGetSupernet(self):
+        self.assertEqual(self.ipv4_network.supernet().prefixlen, 23)
+        self.assertEqual(str(self.ipv4_network.supernet().network_address),
+                         '1.2.2.0')
+        self.assertEqual(
+            ipaddress.IPv4Interface('0.0.0.0/0').network.supernet(),
+            ipaddress.IPv4Network('0.0.0.0/0'))
+
+        self.assertEqual(self.ipv6_network.supernet().prefixlen, 63)
+        self.assertEqual(str(self.ipv6_network.supernet().network_address),
+                         '2001:658:22a:cafe::')
+        self.assertEqual(ipaddress.IPv6Interface('::0/0').network.supernet(),
+                         ipaddress.IPv6Network('::0/0'))
+
+    def testGetSupernet3(self):
+        self.assertEqual(self.ipv4_network.supernet(3).prefixlen, 21)
+        self.assertEqual(str(self.ipv4_network.supernet(3).network_address),
+                         '1.2.0.0')
+
+        self.assertEqual(self.ipv6_network.supernet(3).prefixlen, 61)
+        self.assertEqual(str(self.ipv6_network.supernet(3).network_address),
+                         '2001:658:22a:caf8::')
+
+    def testGetSupernet4(self):
+        self.assertRaises(ValueError, self.ipv4_network.supernet,
+                          prefixlen_diff=2, new_prefix=1)
+        self.assertRaises(ValueError, self.ipv4_network.supernet,
+                          new_prefix=25)
+        self.assertEqual(self.ipv4_network.supernet(prefixlen_diff=2),
+                         self.ipv4_network.supernet(new_prefix=22))
+
+        self.assertRaises(ValueError, self.ipv6_network.supernet,
+                          prefixlen_diff=2, new_prefix=1)
+        self.assertRaises(ValueError, self.ipv6_network.supernet,
+                          new_prefix=65)
+        self.assertEqual(self.ipv6_network.supernet(prefixlen_diff=2),
+                         self.ipv6_network.supernet(new_prefix=62))
+
+    def testHosts(self):
+        hosts = list(self.ipv4_network.hosts())
+        self.assertEqual(254, len(hosts))
+        self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0])
+        self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1])
+
+        # special case where only 1 bit is left for address
+        self.assertEqual([ipaddress.IPv4Address('2.0.0.0'),
+                          ipaddress.IPv4Address('2.0.0.1')],
+                         list(ipaddress.ip_network('2.0.0.0/31').hosts()))
+
+    def testFancySubnetting(self):
+        self.assertEqual(sorted(self.ipv4_network.subnets(prefixlen_diff=3)),
+                         sorted(self.ipv4_network.subnets(new_prefix=27)))
+        self.assertRaises(ValueError, list,
+                          self.ipv4_network.subnets(new_prefix=23))
+        self.assertRaises(ValueError, list,
+                          self.ipv4_network.subnets(prefixlen_diff=3,
+                                                   new_prefix=27))
+        self.assertEqual(sorted(self.ipv6_network.subnets(prefixlen_diff=4)),
+                         sorted(self.ipv6_network.subnets(new_prefix=68)))
+        self.assertRaises(ValueError, list,
+                          self.ipv6_network.subnets(new_prefix=63))
+        self.assertRaises(ValueError, list,
+                          self.ipv6_network.subnets(prefixlen_diff=4,
+                                                   new_prefix=68))
+
+    def testGetSubnets(self):
+        self.assertEqual(list(self.ipv4_network.subnets())[0].prefixlen, 25)
+        self.assertEqual(str(list(
+                    self.ipv4_network.subnets())[0].network_address),
+                         '1.2.3.0')
+        self.assertEqual(str(list(
+                    self.ipv4_network.subnets())[1].network_address),
+                         '1.2.3.128')
+
+        self.assertEqual(list(self.ipv6_network.subnets())[0].prefixlen, 65)
+
+    def testGetSubnetForSingle32(self):
+        ip = ipaddress.IPv4Network('1.2.3.4/32')
+        subnets1 = [str(x) for x in ip.subnets()]
+        subnets2 = [str(x) for x in ip.subnets(2)]
+        self.assertEqual(subnets1, ['1.2.3.4/32'])
+        self.assertEqual(subnets1, subnets2)
+
+    def testGetSubnetForSingle128(self):
+        ip = ipaddress.IPv6Network('::1/128')
+        subnets1 = [str(x) for x in ip.subnets()]
+        subnets2 = [str(x) for x in ip.subnets(2)]
+        self.assertEqual(subnets1, ['::1/128'])
+        self.assertEqual(subnets1, subnets2)
+
+    def testSubnet2(self):
+        ips = [str(x) for x in self.ipv4_network.subnets(2)]
+        self.assertEqual(
+            ips,
+            ['1.2.3.0/26', '1.2.3.64/26', '1.2.3.128/26', '1.2.3.192/26'])
+
+        ipsv6 = [str(x) for x in self.ipv6_network.subnets(2)]
+        self.assertEqual(
+            ipsv6,
+            ['2001:658:22a:cafe::/66',
+             '2001:658:22a:cafe:4000::/66',
+             '2001:658:22a:cafe:8000::/66',
+             '2001:658:22a:cafe:c000::/66'])
+
+    def testSubnetFailsForLargeCidrDiff(self):
+        self.assertRaises(ValueError, list,
+                          self.ipv4_interface.network.subnets(9))
+        self.assertRaises(ValueError, list,
+                          self.ipv4_network.subnets(9))
+        self.assertRaises(ValueError, list,
+                          self.ipv6_interface.network.subnets(65))
+        self.assertRaises(ValueError, list,
+                          self.ipv6_network.subnets(65))
+
+    def testSupernetFailsForLargeCidrDiff(self):
+        self.assertRaises(ValueError,
+                          self.ipv4_interface.network.supernet, 25)
+        self.assertRaises(ValueError,
+                          self.ipv6_interface.network.supernet, 65)
+
+    def testSubnetFailsForNegativeCidrDiff(self):
+        self.assertRaises(ValueError, list,
+                          self.ipv4_interface.network.subnets(-1))
+        self.assertRaises(ValueError, list,
+                          self.ipv4_network.subnets(-1))
+        self.assertRaises(ValueError, list,
+                          self.ipv6_interface.network.subnets(-1))
+        self.assertRaises(ValueError, list,
+                          self.ipv6_network.subnets(-1))
+
+    def testGetNum_Addresses(self):
+        self.assertEqual(self.ipv4_network.num_addresses, 256)
+        self.assertEqual(list(self.ipv4_network.subnets())[0].num_addresses,
+                         128)
+        self.assertEqual(self.ipv4_network.supernet().num_addresses, 512)
+
+        self.assertEqual(self.ipv6_network.num_addresses, 18446744073709551616)
+        self.assertEqual(list(self.ipv6_network.subnets())[0].num_addresses,
+                         9223372036854775808)
+        self.assertEqual(self.ipv6_network.supernet().num_addresses,
+                         36893488147419103232)
+
+    def testContains(self):
+        self.assertTrue(ipaddress.IPv4Interface('1.2.3.128/25') in
+                        self.ipv4_network)
+        self.assertFalse(ipaddress.IPv4Interface('1.2.4.1/24') in
+                         self.ipv4_network)
+        # We can test addresses and string as well.
+        addr1 = ipaddress.IPv4Address('1.2.3.37')
+        self.assertTrue(addr1 in self.ipv4_network)
+        # issue 61, bad network comparison on like-ip'd network objects
+        # with identical broadcast addresses.
+        self.assertFalse(ipaddress.IPv4Network('1.1.0.0/16').__contains__(
+                ipaddress.IPv4Network('1.0.0.0/15')))
+
+    def testNth(self):
+        self.assertEqual(str(self.ipv4_network[5]), '1.2.3.5')
+        self.assertRaises(IndexError, self.ipv4_network.__getitem__, 256)
+
+        self.assertEqual(str(self.ipv6_network[5]),
+                         '2001:658:22a:cafe::5')
+
+    def testGetitem(self):
+        # http://code.google.com/p/ipaddr-py/issues/detail?id=15
+        addr = ipaddress.IPv4Network('172.31.255.128/255.255.255.240')
+        self.assertEqual(28, addr.prefixlen)
+        addr_list = list(addr)
+        self.assertEqual('172.31.255.128', str(addr_list[0]))
+        self.assertEqual('172.31.255.128', str(addr[0]))
+        self.assertEqual('172.31.255.143', str(addr_list[-1]))
+        self.assertEqual('172.31.255.143', str(addr[-1]))
+        self.assertEqual(addr_list[-1], addr[-1])
+
+    def testEqual(self):
+        self.assertTrue(self.ipv4_interface ==
+                        ipaddress.IPv4Interface('1.2.3.4/24'))
+        self.assertFalse(self.ipv4_interface ==
+                         ipaddress.IPv4Interface('1.2.3.4/23'))
+        self.assertFalse(self.ipv4_interface ==
+                         ipaddress.IPv6Interface('::1.2.3.4/24'))
+        self.assertFalse(self.ipv4_interface == '')
+        self.assertFalse(self.ipv4_interface == [])
+        self.assertFalse(self.ipv4_interface == 2)
+
+        self.assertTrue(self.ipv6_interface ==
+            ipaddress.IPv6Interface('2001:658:22a:cafe:200::1/64'))
+        self.assertFalse(self.ipv6_interface ==
+            ipaddress.IPv6Interface('2001:658:22a:cafe:200::1/63'))
+        self.assertFalse(self.ipv6_interface ==
+                         ipaddress.IPv4Interface('1.2.3.4/23'))
+        self.assertFalse(self.ipv6_interface == '')
+        self.assertFalse(self.ipv6_interface == [])
+        self.assertFalse(self.ipv6_interface == 2)
+
+    def testNotEqual(self):
+        self.assertFalse(self.ipv4_interface !=
+                         ipaddress.IPv4Interface('1.2.3.4/24'))
+        self.assertTrue(self.ipv4_interface !=
+                        ipaddress.IPv4Interface('1.2.3.4/23'))
+        self.assertTrue(self.ipv4_interface !=
+                        ipaddress.IPv6Interface('::1.2.3.4/24'))
+        self.assertTrue(self.ipv4_interface != '')
+        self.assertTrue(self.ipv4_interface != [])
+        self.assertTrue(self.ipv4_interface != 2)
+
+        self.assertTrue(self.ipv4_address !=
+                         ipaddress.IPv4Address('1.2.3.5'))
+        self.assertTrue(self.ipv4_address != '')
+        self.assertTrue(self.ipv4_address != [])
+        self.assertTrue(self.ipv4_address != 2)
+
+        self.assertFalse(self.ipv6_interface !=
+            ipaddress.IPv6Interface('2001:658:22a:cafe:200::1/64'))
+        self.assertTrue(self.ipv6_interface !=
+            ipaddress.IPv6Interface('2001:658:22a:cafe:200::1/63'))
+        self.assertTrue(self.ipv6_interface !=
+                        ipaddress.IPv4Interface('1.2.3.4/23'))
+        self.assertTrue(self.ipv6_interface != '')
+        self.assertTrue(self.ipv6_interface != [])
+        self.assertTrue(self.ipv6_interface != 2)
+
+        self.assertTrue(self.ipv6_address !=
+                        ipaddress.IPv4Address('1.2.3.4'))
+        self.assertTrue(self.ipv6_address != '')
+        self.assertTrue(self.ipv6_address != [])
+        self.assertTrue(self.ipv6_address != 2)
+
+    def testSlash32Constructor(self):
+        self.assertEqual(str(ipaddress.IPv4Interface(
+                    '1.2.3.4/255.255.255.255')), '1.2.3.4/32')
+
+    def testSlash128Constructor(self):
+        self.assertEqual(str(ipaddress.IPv6Interface('::1/128')),
+                                  '::1/128')
+
+    def testSlash0Constructor(self):
+        self.assertEqual(str(ipaddress.IPv4Interface('1.2.3.4/0.0.0.0')),
+                          '1.2.3.4/0')
+
+    def testCollapsing(self):
+        # test only IP addresses including some duplicates
+        ip1 = ipaddress.IPv4Address('1.1.1.0')
+        ip2 = ipaddress.IPv4Address('1.1.1.1')
+        ip3 = ipaddress.IPv4Address('1.1.1.2')
+        ip4 = ipaddress.IPv4Address('1.1.1.3')
+        ip5 = ipaddress.IPv4Address('1.1.1.4')
+        ip6 = ipaddress.IPv4Address('1.1.1.0')
+        # check that addreses are subsumed properly.
+        collapsed = ipaddress.collapse_addresses(
+            [ip1, ip2, ip3, ip4, ip5, ip6])
+        self.assertEqual(list(collapsed),
+                [ipaddress.IPv4Network('1.1.1.0/30'),
+                 ipaddress.IPv4Network('1.1.1.4/32')])
+
+        # test a mix of IP addresses and networks including some duplicates
+        ip1 = ipaddress.IPv4Address('1.1.1.0')
+        ip2 = ipaddress.IPv4Address('1.1.1.1')
+        ip3 = ipaddress.IPv4Address('1.1.1.2')
+        ip4 = ipaddress.IPv4Address('1.1.1.3')
+        #ip5 = ipaddress.IPv4Interface('1.1.1.4/30')
+        #ip6 = ipaddress.IPv4Interface('1.1.1.4/30')
+        # check that addreses are subsumed properly.
+        collapsed = ipaddress.collapse_addresses([ip1, ip2, ip3, ip4])
+        self.assertEqual(list(collapsed),
+                         [ipaddress.IPv4Network('1.1.1.0/30')])
+
+        # test only IP networks
+        ip1 = ipaddress.IPv4Network('1.1.0.0/24')
+        ip2 = ipaddress.IPv4Network('1.1.1.0/24')
+        ip3 = ipaddress.IPv4Network('1.1.2.0/24')
+        ip4 = ipaddress.IPv4Network('1.1.3.0/24')
+        ip5 = ipaddress.IPv4Network('1.1.4.0/24')
+        # stored in no particular order b/c we want CollapseAddr to call
+        # [].sort
+        ip6 = ipaddress.IPv4Network('1.1.0.0/22')
+        # check that addreses are subsumed properly.
+        collapsed = ipaddress.collapse_addresses([ip1, ip2, ip3, ip4, ip5,
+                                                     ip6])
+        self.assertEqual(list(collapsed),
+                         [ipaddress.IPv4Network('1.1.0.0/22'),
+                          ipaddress.IPv4Network('1.1.4.0/24')])
+
+        # test that two addresses are supernet'ed properly
+        collapsed = ipaddress.collapse_addresses([ip1, ip2])
+        self.assertEqual(list(collapsed),
+                         [ipaddress.IPv4Network('1.1.0.0/23')])
+
+        # test same IP networks
+        ip_same1 = ip_same2 = ipaddress.IPv4Network('1.1.1.1/32')
+        self.assertEqual(list(ipaddress.collapse_addresses(
+                    [ip_same1, ip_same2])),
+                         [ip_same1])
+
+        # test same IP addresses
+        ip_same1 = ip_same2 = ipaddress.IPv4Address('1.1.1.1')
+        self.assertEqual(list(ipaddress.collapse_addresses(
+                    [ip_same1, ip_same2])),
+                         [ipaddress.ip_network('1.1.1.1/32')])
+        ip1 = ipaddress.IPv6Network('2001::/100')
+        ip2 = ipaddress.IPv6Network('2001::/120')
+        ip3 = ipaddress.IPv6Network('2001::/96')
+        # test that ipv6 addresses are subsumed properly.
+        collapsed = ipaddress.collapse_addresses([ip1, ip2, ip3])
+        self.assertEqual(list(collapsed), [ip3])
+
+        # the toejam test
+        addr_tuples = [
+                (ipaddress.ip_address('1.1.1.1'),
+                 ipaddress.ip_address('::1')),
+                (ipaddress.IPv4Network('1.1.0.0/24'),
+                 ipaddress.IPv6Network('2001::/120')),
+                (ipaddress.IPv4Network('1.1.0.0/32'),
+                 ipaddress.IPv6Network('2001::/128')),
+        ]
+        for ip1, ip2 in addr_tuples:
+            self.assertRaises(TypeError, ipaddress.collapse_addresses,
+                              [ip1, ip2])
+
+    def testSummarizing(self):
+        #ip = ipaddress.ip_address
+        #ipnet = ipaddress.ip_network
+        summarize = ipaddress.summarize_address_range
+        ip1 = ipaddress.ip_address('1.1.1.0')
+        ip2 = ipaddress.ip_address('1.1.1.255')
+
+        # summarize works only for IPv4 & IPv6
+        class IPv7Address(ipaddress.IPv6Address):
+            @property
+            def version(self):
+                return 7
+        ip_invalid1 = IPv7Address('::1')
+        ip_invalid2 = IPv7Address('::1')
+        self.assertRaises(ValueError, list,
+                          summarize(ip_invalid1, ip_invalid2))
+        # test that a summary over ip4 & ip6 fails
+        self.assertRaises(TypeError, list,
+                          summarize(ip1, ipaddress.IPv6Address('::1')))
+        # test a /24 is summarized properly
+        self.assertEqual(list(summarize(ip1, ip2))[0],
+                         ipaddress.ip_network('1.1.1.0/24'))
+        # test an  IPv4 range that isn't on a network byte boundary
+        ip2 = ipaddress.ip_address('1.1.1.8')
+        self.assertEqual(list(summarize(ip1, ip2)),
+                         [ipaddress.ip_network('1.1.1.0/29'),
+                          ipaddress.ip_network('1.1.1.8')])
+        # all!
+        ip1 = ipaddress.IPv4Address(0)
+        ip2 = ipaddress.IPv4Address(ipaddress.IPv4Address._ALL_ONES)
+        self.assertEqual([ipaddress.IPv4Network('0.0.0.0/0')],
+                         list(summarize(ip1, ip2)))
+
+        ip1 = ipaddress.ip_address('1::')
+        ip2 = ipaddress.ip_address('1:ffff:ffff:ffff:ffff:ffff:ffff:ffff')
+        # test a IPv6 is sumamrized properly
+        self.assertEqual(list(summarize(ip1, ip2))[0],
+                         ipaddress.ip_network('1::/16'))
+        # test an IPv6 range that isn't on a network byte boundary
+        ip2 = ipaddress.ip_address('2::')
+        self.assertEqual(list(summarize(ip1, ip2)),
+                         [ipaddress.ip_network('1::/16'),
+                          ipaddress.ip_network('2::/128')])
+
+        # test exception raised when first is greater than last
+        self.assertRaises(ValueError, list,
+                          summarize(ipaddress.ip_address('1.1.1.0'),
+                                    ipaddress.ip_address('1.1.0.0')))
+        # test exception raised when first and last aren't IP addresses
+        self.assertRaises(TypeError, list,
+                          summarize(ipaddress.ip_network('1.1.1.0'),
+                                    ipaddress.ip_network('1.1.0.0')))
+        self.assertRaises(TypeError, list,
+                          summarize(ipaddress.ip_network('1.1.1.0'),
+                                    ipaddress.ip_network('1.1.0.0')))
+        # test exception raised when first and last are not same version
+        self.assertRaises(TypeError, list,
+                          summarize(ipaddress.ip_address('::'),
+                                    ipaddress.ip_network('1.1.0.0')))
+
+    def testAddressComparison(self):
+        self.assertTrue(ipaddress.ip_address('1.1.1.1') <=
+                        ipaddress.ip_address('1.1.1.1'))
+        self.assertTrue(ipaddress.ip_address('1.1.1.1') <=
+                        ipaddress.ip_address('1.1.1.2'))
+        self.assertTrue(ipaddress.ip_address('::1') <=
+                        ipaddress.ip_address('::1'))
+        self.assertTrue(ipaddress.ip_address('::1') <=
+                        ipaddress.ip_address('::2'))
+
+    def testInterfaceComparison(self):
+        self.assertTrue(ipaddress.ip_interface('1.1.1.1') <=
+                        ipaddress.ip_interface('1.1.1.1'))
+        self.assertTrue(ipaddress.ip_interface('1.1.1.1') <=
+                        ipaddress.ip_interface('1.1.1.2'))
+        self.assertTrue(ipaddress.ip_interface('::1') <=
+                        ipaddress.ip_interface('::1'))
+        self.assertTrue(ipaddress.ip_interface('::1') <=
+                        ipaddress.ip_interface('::2'))
+
+    def testNetworkComparison(self):
+        # ip1 and ip2 have the same network address
+        ip1 = ipaddress.IPv4Network('1.1.1.0/24')
+        ip2 = ipaddress.IPv4Network('1.1.1.0/32')
+        ip3 = ipaddress.IPv4Network('1.1.2.0/24')
+
+        self.assertTrue(ip1 < ip3)
+        self.assertTrue(ip3 > ip2)
+
+        self.assertEqual(ip1.compare_networks(ip1), 0)
+
+        # if addresses are the same, sort by netmask
+        self.assertEqual(ip1.compare_networks(ip2), -1)
+        self.assertEqual(ip2.compare_networks(ip1), 1)
+
+        self.assertEqual(ip1.compare_networks(ip3), -1)
+        self.assertEqual(ip3.compare_networks(ip1), 1)
+        self.assertTrue(ip1._get_networks_key() < ip3._get_networks_key())
+
+        ip1 = ipaddress.IPv6Network('2001:2000::/96')
+        ip2 = ipaddress.IPv6Network('2001:2001::/96')
+        ip3 = ipaddress.IPv6Network('2001:ffff:2000::/96')
+
+        self.assertTrue(ip1 < ip3)
+        self.assertTrue(ip3 > ip2)
+        self.assertEqual(ip1.compare_networks(ip3), -1)
+        self.assertTrue(ip1._get_networks_key() < ip3._get_networks_key())
+
+        # Test comparing different protocols.
+        # Should always raise a TypeError.
+        self.assertRaises(TypeError,
+                          self.ipv4_network.compare_networks,
+                          self.ipv6_network)
+        ipv6 = ipaddress.IPv6Interface('::/0')
+        ipv4 = ipaddress.IPv4Interface('0.0.0.0/0')
+        self.assertRaises(TypeError, ipv4.__lt__, ipv6)
+        self.assertRaises(TypeError, ipv4.__gt__, ipv6)
+        self.assertRaises(TypeError, ipv6.__lt__, ipv4)
+        self.assertRaises(TypeError, ipv6.__gt__, ipv4)
+
+        # Regression test for issue 19.
+        ip1 = ipaddress.ip_network('10.1.2.128/25')
+        self.assertFalse(ip1 < ip1)
+        self.assertFalse(ip1 > ip1)
+        ip2 = ipaddress.ip_network('10.1.3.0/24')
+        self.assertTrue(ip1 < ip2)
+        self.assertFalse(ip2 < ip1)
+        self.assertFalse(ip1 > ip2)
+        self.assertTrue(ip2 > ip1)
+        ip3 = ipaddress.ip_network('10.1.3.0/25')
+        self.assertTrue(ip2 < ip3)
+        self.assertFalse(ip3 < ip2)
+        self.assertFalse(ip2 > ip3)
+        self.assertTrue(ip3 > ip2)
+
+        # Regression test for issue 28.
+        ip1 = ipaddress.ip_network('10.10.10.0/31')
+        ip2 = ipaddress.ip_network('10.10.10.0')
+        ip3 = ipaddress.ip_network('10.10.10.2/31')
+        ip4 = ipaddress.ip_network('10.10.10.2')
+        sorted = [ip1, ip2, ip3, ip4]
+        unsorted = [ip2, ip4, ip1, ip3]
+        unsorted.sort()
+        self.assertEqual(sorted, unsorted)
+        unsorted = [ip4, ip1, ip3, ip2]
+        unsorted.sort()
+        self.assertEqual(sorted, unsorted)
+        self.assertRaises(TypeError, ip1.__lt__,
+                          ipaddress.ip_address('10.10.10.0'))
+        self.assertRaises(TypeError, ip2.__lt__,
+                          ipaddress.ip_address('10.10.10.0'))
+
+        # <=, >=
+        self.assertTrue(ipaddress.ip_network('1.1.1.1') <=
+                        ipaddress.ip_network('1.1.1.1'))
+        self.assertTrue(ipaddress.ip_network('1.1.1.1') <=
+                        ipaddress.ip_network('1.1.1.2'))
+        self.assertFalse(ipaddress.ip_network('1.1.1.2') <=
+                        ipaddress.ip_network('1.1.1.1'))
+        self.assertTrue(ipaddress.ip_network('::1') <=
+                        ipaddress.ip_network('::1'))
+        self.assertTrue(ipaddress.ip_network('::1') <=
+                        ipaddress.ip_network('::2'))
+        self.assertFalse(ipaddress.ip_network('::2') <=
+                         ipaddress.ip_network('::1'))
+
+    def testStrictNetworks(self):
+        self.assertRaises(ValueError, ipaddress.ip_network, '192.168.1.1/24')
+        self.assertRaises(ValueError, ipaddress.ip_network, '::1/120')
+
+    def testOverlaps(self):
+        other = ipaddress.IPv4Network('1.2.3.0/30')
+        other2 = ipaddress.IPv4Network('1.2.2.0/24')
+        other3 = ipaddress.IPv4Network('1.2.2.64/26')
+        self.assertTrue(self.ipv4_network.overlaps(other))
+        self.assertFalse(self.ipv4_network.overlaps(other2))
+        self.assertTrue(other2.overlaps(other3))
+
+    def testEmbeddedIpv4(self):
+        ipv4_string = '192.168.0.1'
+        ipv4 = ipaddress.IPv4Interface(ipv4_string)
+        v4compat_ipv6 = ipaddress.IPv6Interface('::%s' % ipv4_string)
+        self.assertEqual(int(v4compat_ipv6.ip), int(ipv4.ip))
+        v4mapped_ipv6 = ipaddress.IPv6Interface('::ffff:%s' % ipv4_string)
+        self.assertNotEqual(v4mapped_ipv6.ip, ipv4.ip)
+        self.assertRaises(ipaddress.AddressValueError, ipaddress.IPv6Interface,
+                          '2001:1.1.1.1:1.1.1.1')
+
+    # Issue 67: IPv6 with embedded IPv4 address not recognized.
+    def testIPv6AddressTooLarge(self):
+        # RFC4291 2.5.5.2
+        self.assertEqual(ipaddress.ip_address('::FFFF:192.0.2.1'),
+                          ipaddress.ip_address('::FFFF:c000:201'))
+        # RFC4291 2.2 (part 3) x::d.d.d.d
+        self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1'),
+                          ipaddress.ip_address('FFFF::c000:201'))
+
+    def testIPVersion(self):
+        self.assertEqual(self.ipv4_address.version, 4)
+        self.assertEqual(self.ipv6_address.version, 6)
+
+    def testMaxPrefixLength(self):
+        self.assertEqual(self.ipv4_interface.max_prefixlen, 32)
+        self.assertEqual(self.ipv6_interface.max_prefixlen, 128)
+
+    def testPacked(self):
+        self.assertEqual(self.ipv4_address.packed,
+                         b'\x01\x02\x03\x04')
+        self.assertEqual(ipaddress.IPv4Interface('255.254.253.252').packed,
+                         b'\xff\xfe\xfd\xfc')
+        self.assertEqual(self.ipv6_address.packed,
+                         b'\x20\x01\x06\x58\x02\x2a\xca\xfe'
+                         b'\x02\x00\x00\x00\x00\x00\x00\x01')
+        self.assertEqual(ipaddress.IPv6Interface('ffff:2:3:4:ffff::').packed,
+                         b'\xff\xff\x00\x02\x00\x03\x00\x04\xff\xff'
+                            + b'\x00' * 6)
+        self.assertEqual(ipaddress.IPv6Interface('::1:0:0:0:0').packed,
+                         b'\x00' * 6 + b'\x00\x01' + b'\x00' * 8)
+
+    def testIpStrFromPrefixlen(self):
+        ipv4 = ipaddress.IPv4Interface('1.2.3.4/24')
+        self.assertEqual(ipv4._ip_string_from_prefix(), '255.255.255.0')
+        self.assertEqual(ipv4._ip_string_from_prefix(28), '255.255.255.240')
+
+    def testIpType(self):
+        ipv4net = ipaddress.ip_network('1.2.3.4')
+        ipv4addr = ipaddress.ip_address('1.2.3.4')
+        ipv6net = ipaddress.ip_network('::1.2.3.4')
+        ipv6addr = ipaddress.ip_address('::1.2.3.4')
+        self.assertEqual(ipaddress.IPv4Network, type(ipv4net))
+        self.assertEqual(ipaddress.IPv4Address, type(ipv4addr))
+        self.assertEqual(ipaddress.IPv6Network, type(ipv6net))
+        self.assertEqual(ipaddress.IPv6Address, type(ipv6addr))
+
+    def testReservedIpv4(self):
+        # test networks
+        self.assertEqual(True, ipaddress.ip_interface(
+                '224.1.1.1/31').is_multicast)
+        self.assertEqual(False, ipaddress.ip_network('240.0.0.0').is_multicast)
+        self.assertEqual(True, ipaddress.ip_network('240.0.0.0').is_reserved)
+
+        self.assertEqual(True, ipaddress.ip_interface(
+                '192.168.1.1/17').is_private)
+        self.assertEqual(False, ipaddress.ip_network('192.169.0.0').is_private)
+        self.assertEqual(True, ipaddress.ip_network(
+                '10.255.255.255').is_private)
+        self.assertEqual(False, ipaddress.ip_network('11.0.0.0').is_private)
+        self.assertEqual(False, ipaddress.ip_network('11.0.0.0').is_reserved)
+        self.assertEqual(True, ipaddress.ip_network(
+                '172.31.255.255').is_private)
+        self.assertEqual(False, ipaddress.ip_network('172.32.0.0').is_private)
+        self.assertEqual(True,
+                         ipaddress.ip_network('169.254.1.0/24').is_link_local)
+
+        self.assertEqual(True,
+                          ipaddress.ip_interface(
+                              '169.254.100.200/24').is_link_local)
+        self.assertEqual(False,
+                          ipaddress.ip_interface(
+                              '169.255.100.200/24').is_link_local)
+
+        self.assertEqual(True,
+                          ipaddress.ip_network(
+                              '127.100.200.254/32').is_loopback)
+        self.assertEqual(True, ipaddress.ip_network(
+                '127.42.0.0/16').is_loopback)
+        self.assertEqual(False, ipaddress.ip_network('128.0.0.0').is_loopback)
+
+        # test addresses
+        self.assertEqual(True, ipaddress.ip_address('0.0.0.0').is_unspecified)
+        self.assertEqual(True, ipaddress.ip_address('224.1.1.1').is_multicast)
+        self.assertEqual(False, ipaddress.ip_address('240.0.0.0').is_multicast)
+        self.assertEqual(True, ipaddress.ip_address('240.0.0.1').is_reserved)
+        self.assertEqual(False,
+                         ipaddress.ip_address('239.255.255.255').is_reserved)
+
+        self.assertEqual(True, ipaddress.ip_address('192.168.1.1').is_private)
+        self.assertEqual(False, ipaddress.ip_address('192.169.0.0').is_private)
+        self.assertEqual(True, ipaddress.ip_address(
+                '10.255.255.255').is_private)
+        self.assertEqual(False, ipaddress.ip_address('11.0.0.0').is_private)
+        self.assertEqual(True, ipaddress.ip_address(
+                '172.31.255.255').is_private)
+        self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private)
+
+        self.assertEqual(True,
+                         ipaddress.ip_address('169.254.100.200').is_link_local)
+        self.assertEqual(False,
+                         ipaddress.ip_address('169.255.100.200').is_link_local)
+
+        self.assertEqual(True,
+                          ipaddress.ip_address('127.100.200.254').is_loopback)
+        self.assertEqual(True, ipaddress.ip_address('127.42.0.0').is_loopback)
+        self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback)
+        self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified)
+
+    def testReservedIpv6(self):
+
+        self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast)
+        self.assertEqual(True, ipaddress.ip_network(2**128 - 1).is_multicast)
+        self.assertEqual(True, ipaddress.ip_network('ff00::').is_multicast)
+        self.assertEqual(False, ipaddress.ip_network('fdff::').is_multicast)
+
+        self.assertEqual(True, ipaddress.ip_network('fecf::').is_site_local)
+        self.assertEqual(True, ipaddress.ip_network(
+                'feff:ffff:ffff:ffff::').is_site_local)
+        self.assertEqual(False, ipaddress.ip_network(
+                'fbf:ffff::').is_site_local)
+        self.assertEqual(False, ipaddress.ip_network('ff00::').is_site_local)
+
+        self.assertEqual(True, ipaddress.ip_network('fc00::').is_private)
+        self.assertEqual(True, ipaddress.ip_network(
+                'fc00:ffff:ffff:ffff::').is_private)
+        self.assertEqual(False, ipaddress.ip_network('fbff:ffff::').is_private)
+        self.assertEqual(False, ipaddress.ip_network('fe00::').is_private)
+
+        self.assertEqual(True, ipaddress.ip_network('fea0::').is_link_local)
+        self.assertEqual(True, ipaddress.ip_network(
+                'febf:ffff::').is_link_local)
+        self.assertEqual(False, ipaddress.ip_network(
+                'fe7f:ffff::').is_link_local)
+        self.assertEqual(False, ipaddress.ip_network('fec0::').is_link_local)
+
+        self.assertEqual(True, ipaddress.ip_interface('0:0::0:01').is_loopback)
+        self.assertEqual(False, ipaddress.ip_interface('::1/127').is_loopback)
+        self.assertEqual(False, ipaddress.ip_network('::').is_loopback)
+        self.assertEqual(False, ipaddress.ip_network('::2').is_loopback)
+
+        self.assertEqual(True, ipaddress.ip_network('0::0').is_unspecified)
+        self.assertEqual(False, ipaddress.ip_network('::1').is_unspecified)
+        self.assertEqual(False, ipaddress.ip_network('::/127').is_unspecified)
+
+        # test addresses
+        self.assertEqual(True, ipaddress.ip_address('ffff::').is_multicast)
+        self.assertEqual(True, ipaddress.ip_address(2**128 - 1).is_multicast)
+        self.assertEqual(True, ipaddress.ip_address('ff00::').is_multicast)
+        self.assertEqual(False, ipaddress.ip_address('fdff::').is_multicast)
+
+        self.assertEqual(True, ipaddress.ip_address('fecf::').is_site_local)
+        self.assertEqual(True, ipaddress.ip_address(
+                'feff:ffff:ffff:ffff::').is_site_local)
+        self.assertEqual(False, ipaddress.ip_address(
+                'fbf:ffff::').is_site_local)
+        self.assertEqual(False, ipaddress.ip_address('ff00::').is_site_local)
+
+        self.assertEqual(True, ipaddress.ip_address('fc00::').is_private)
+        self.assertEqual(True, ipaddress.ip_address(
+                'fc00:ffff:ffff:ffff::').is_private)
+        self.assertEqual(False, ipaddress.ip_address('fbff:ffff::').is_private)
+        self.assertEqual(False, ipaddress.ip_address('fe00::').is_private)
+
+        self.assertEqual(True, ipaddress.ip_address('fea0::').is_link_local)
+        self.assertEqual(True, ipaddress.ip_address(
+                'febf:ffff::').is_link_local)
+        self.assertEqual(False, ipaddress.ip_address(
+                'fe7f:ffff::').is_link_local)
+        self.assertEqual(False, ipaddress.ip_address('fec0::').is_link_local)
+
+        self.assertEqual(True, ipaddress.ip_address('0:0::0:01').is_loopback)
+        self.assertEqual(True, ipaddress.ip_address('::1').is_loopback)
+        self.assertEqual(False, ipaddress.ip_address('::2').is_loopback)
+
+        self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified)
+        self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified)
+
+        # some generic IETF reserved addresses
+        self.assertEqual(True, ipaddress.ip_address('100::').is_reserved)
+        self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved)
+
+    def testIpv4Mapped(self):
+        self.assertEqual(
+                ipaddress.ip_address('::ffff:192.168.1.1').ipv4_mapped,
+                ipaddress.ip_address('192.168.1.1'))
+        self.assertEqual(ipaddress.ip_address('::c0a8:101').ipv4_mapped, None)
+        self.assertEqual(ipaddress.ip_address('::ffff:c0a8:101').ipv4_mapped,
+                         ipaddress.ip_address('192.168.1.1'))
+
+    def testAddrExclude(self):
+        addr1 = ipaddress.ip_network('10.1.1.0/24')
+        addr2 = ipaddress.ip_network('10.1.1.0/26')
+        addr3 = ipaddress.ip_network('10.2.1.0/24')
+        addr4 = ipaddress.ip_address('10.1.1.0')
+        addr5 = ipaddress.ip_network('2001:db8::0/32')
+        self.assertEqual(sorted(list(addr1.address_exclude(addr2))),
+                         [ipaddress.ip_network('10.1.1.64/26'),
+                          ipaddress.ip_network('10.1.1.128/25')])
+        self.assertRaises(ValueError, list, addr1.address_exclude(addr3))
+        self.assertRaises(TypeError, list, addr1.address_exclude(addr4))
+        self.assertRaises(TypeError, list, addr1.address_exclude(addr5))
+        self.assertEqual(list(addr1.address_exclude(addr1)), [])
+
+    def testHash(self):
+        self.assertEqual(hash(ipaddress.ip_interface('10.1.1.0/24')),
+                         hash(ipaddress.ip_interface('10.1.1.0/24')))
+        self.assertEqual(hash(ipaddress.ip_network('10.1.1.0/24')),
+                         hash(ipaddress.ip_network('10.1.1.0/24')))
+        self.assertEqual(hash(ipaddress.ip_address('10.1.1.0')),
+                         hash(ipaddress.ip_address('10.1.1.0')))
+        # i70
+        self.assertEqual(hash(ipaddress.ip_address('1.2.3.4')),
+                         hash(ipaddress.ip_address(
+                    int(ipaddress.ip_address('1.2.3.4')._ip))))
+        ip1 = ipaddress.ip_address('10.1.1.0')
+        ip2 = ipaddress.ip_address('1::')
+        dummy = {}
+        dummy[self.ipv4_address] = None
+        dummy[self.ipv6_address] = None
+        dummy[ip1] = None
+        dummy[ip2] = None
+        self.assertTrue(self.ipv4_address in dummy)
+        self.assertTrue(ip2 in dummy)
+
+    def testIPBases(self):
+        net = self.ipv4_network
+        self.assertEqual('1.2.3.0/24', net.compressed)
+        self.assertEqual(
+                net._ip_int_from_prefix(24),
+                net._ip_int_from_prefix(None))
+        net = self.ipv6_network
+        self.assertRaises(ValueError, net._string_from_ip_int, 2**128 + 1)
+        self.assertEqual(
+                self.ipv6_address._string_from_ip_int(self.ipv6_address._ip),
+                self.ipv6_address._string_from_ip_int(None))
+
+    def testIPv6NetworkHelpers(self):
+        net = self.ipv6_network
+        self.assertEqual('2001:658:22a:cafe::/64', net.with_prefixlen)
+        self.assertEqual('2001:658:22a:cafe::/ffff:ffff:ffff:ffff::',
+                         net.with_netmask)
+        self.assertEqual('2001:658:22a:cafe::/::ffff:ffff:ffff:ffff',
+                         net.with_hostmask)
+        self.assertEqual('2001:658:22a:cafe::/64', str(net))
+
+    def testIPv4NetworkHelpers(self):
+        net = self.ipv4_network
+        self.assertEqual('1.2.3.0/24', net.with_prefixlen)
+        self.assertEqual('1.2.3.0/255.255.255.0', net.with_netmask)
+        self.assertEqual('1.2.3.0/0.0.0.255', net.with_hostmask)
+        self.assertEqual('1.2.3.0/24', str(net))
+
+    def testCopyConstructor(self):
+        addr1 = ipaddress.ip_network('10.1.1.0/24')
+        addr2 = ipaddress.ip_network(addr1)
+        addr3 = ipaddress.ip_interface('2001:658:22a:cafe:200::1/64')
+        addr4 = ipaddress.ip_interface(addr3)
+        addr5 = ipaddress.IPv4Address('1.1.1.1')
+        addr6 = ipaddress.IPv6Address('2001:658:22a:cafe:200::1')
+
+        self.assertEqual(addr1, addr2)
+        self.assertEqual(addr3, addr4)
+        self.assertEqual(addr5, ipaddress.IPv4Address(addr5))
+        self.assertEqual(addr6, ipaddress.IPv6Address(addr6))
+
+    def testCompressIPv6Address(self):
+        test_addresses = {
+            '1:2:3:4:5:6:7:8': '1:2:3:4:5:6:7:8/128',
+            '2001:0:0:4:0:0:0:8': '2001:0:0:4::8/128',
+            '2001:0:0:4:5:6:7:8': '2001::4:5:6:7:8/128',
+            '2001:0:3:4:5:6:7:8': '2001:0:3:4:5:6:7:8/128',
+            '2001:0:3:4:5:6:7:8': '2001:0:3:4:5:6:7:8/128',
+            '0:0:3:0:0:0:0:ffff': '0:0:3::ffff/128',
+            '0:0:0:4:0:0:0:ffff': '::4:0:0:0:ffff/128',
+            '0:0:0:0:5:0:0:ffff': '::5:0:0:ffff/128',
+            '1:0:0:4:0:0:7:8': '1::4:0:0:7:8/128',
+            '0:0:0:0:0:0:0:0': '::/128',
+            '0:0:0:0:0:0:0:0/0': '::/0',
+            '0:0:0:0:0:0:0:1': '::1/128',
+            '2001:0658:022a:cafe:0000:0000:0000:0000/66':
+            '2001:658:22a:cafe::/66',
+            '::1.2.3.4': '::102:304/128',
+            '1:2:3:4:5:ffff:1.2.3.4': '1:2:3:4:5:ffff:102:304/128',
+            '::7:6:5:4:3:2:1': '0:7:6:5:4:3:2:1/128',
+            '::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128',
+            '7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128',
+            '0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128',
+            }
+        for uncompressed, compressed in list(test_addresses.items()):
+            self.assertEqual(compressed, str(ipaddress.IPv6Interface(
+                uncompressed)))
+
+    def testExplodeShortHandIpStr(self):
+        addr1 = ipaddress.IPv6Interface('2001::1')
+        addr2 = ipaddress.IPv6Address('2001:0:5ef5:79fd:0:59d:a0e5:ba1')
+        addr3 = ipaddress.IPv6Network('2001::/96')
+        addr4 = ipaddress.IPv4Address('192.168.178.1')
+        self.assertEqual('2001:0000:0000:0000:0000:0000:0000:0001/128',
+                         addr1.exploded)
+        self.assertEqual('0000:0000:0000:0000:0000:0000:0000:0001/128',
+                         ipaddress.IPv6Interface('::1/128').exploded)
+        # issue 77
+        self.assertEqual('2001:0000:5ef5:79fd:0000:059d:a0e5:0ba1',
+                         addr2.exploded)
+        self.assertEqual('2001:0000:0000:0000:0000:0000:0000:0000/96',
+                         addr3.exploded)
+        self.assertEqual('192.168.178.1', addr4.exploded)
+
+    def testIntRepresentation(self):
+        self.assertEqual(16909060, int(self.ipv4_address))
+        self.assertEqual(42540616829182469433547762482097946625,
+                         int(self.ipv6_address))
+
+    def testForceVersion(self):
+        self.assertEqual(ipaddress.ip_network(1).version, 4)
+        self.assertEqual(ipaddress.IPv6Network(1).version, 6)
+
+    def testWithStar(self):
+        self.assertEqual(self.ipv4_interface.with_prefixlen, "1.2.3.4/24")
+        self.assertEqual(self.ipv4_interface.with_netmask,
+                         "1.2.3.4/255.255.255.0")
+        self.assertEqual(self.ipv4_interface.with_hostmask,
+                         "1.2.3.4/0.0.0.255")
+
+        self.assertEqual(self.ipv6_interface.with_prefixlen,
+                         '2001:658:22a:cafe:200::1/64')
+        self.assertEqual(self.ipv6_interface.with_netmask,
+                         '2001:658:22a:cafe:200::1/ffff:ffff:ffff:ffff::')
+        # this probably don't make much sense, but it's included for
+        # compatibility with ipv4
+        self.assertEqual(self.ipv6_interface.with_hostmask,
+                         '2001:658:22a:cafe:200::1/::ffff:ffff:ffff:ffff')
+
+    def testNetworkElementCaching(self):
+        # V4 - make sure we're empty
+        self.assertFalse('network_address' in self.ipv4_network._cache)
+        self.assertFalse('broadcast_address' in self.ipv4_network._cache)
+        self.assertFalse('hostmask' in self.ipv4_network._cache)
+
+        # V4 - populate and test
+        self.assertEqual(self.ipv4_network.network_address,
+                         ipaddress.IPv4Address('1.2.3.0'))
+        self.assertEqual(self.ipv4_network.broadcast_address,
+                         ipaddress.IPv4Address('1.2.3.255'))
+        self.assertEqual(self.ipv4_network.hostmask,
+                         ipaddress.IPv4Address('0.0.0.255'))
+
+        # V4 - check we're cached
+        self.assertTrue('broadcast_address' in self.ipv4_network._cache)
+        self.assertTrue('hostmask' in self.ipv4_network._cache)
+
+        # V6 - make sure we're empty
+        self.assertFalse('broadcast_address' in self.ipv6_network._cache)
+        self.assertFalse('hostmask' in self.ipv6_network._cache)
+
+        # V6 - populate and test
+        self.assertEqual(self.ipv6_network.network_address,
+                         ipaddress.IPv6Address('2001:658:22a:cafe::'))
+        self.assertEqual(self.ipv6_interface.network.network_address,
+                         ipaddress.IPv6Address('2001:658:22a:cafe::'))
+
+        self.assertEqual(
+            self.ipv6_network.broadcast_address,
+            ipaddress.IPv6Address('2001:658:22a:cafe:ffff:ffff:ffff:ffff'))
+        self.assertEqual(self.ipv6_network.hostmask,
+                         ipaddress.IPv6Address('::ffff:ffff:ffff:ffff'))
+        self.assertEqual(
+            self.ipv6_interface.network.broadcast_address,
+            ipaddress.IPv6Address('2001:658:22a:cafe:ffff:ffff:ffff:ffff'))
+        self.assertEqual(self.ipv6_interface.network.hostmask,
+                         ipaddress.IPv6Address('::ffff:ffff:ffff:ffff'))
+
+        # V6 - check we're cached
+        self.assertTrue('broadcast_address' in self.ipv6_network._cache)
+        self.assertTrue('hostmask' in self.ipv6_network._cache)
+        self.assertTrue(
+                'broadcast_address' in self.ipv6_interface.network._cache)
+        self.assertTrue('hostmask' in self.ipv6_interface.network._cache)
+
+    def testTeredo(self):
+        # stolen from wikipedia
+        server = ipaddress.IPv4Address('65.54.227.120')
+        client = ipaddress.IPv4Address('192.0.2.45')
+        teredo_addr = '2001:0000:4136:e378:8000:63bf:3fff:fdd2'
+        self.assertEqual((server, client),
+                         ipaddress.ip_address(teredo_addr).teredo)
+        bad_addr = '2000::4136:e378:8000:63bf:3fff:fdd2'
+        self.assertFalse(ipaddress.ip_address(bad_addr).teredo)
+        bad_addr = '2001:0001:4136:e378:8000:63bf:3fff:fdd2'
+        self.assertFalse(ipaddress.ip_address(bad_addr).teredo)
+
+        # i77
+        teredo_addr = ipaddress.IPv6Address('2001:0:5ef5:79fd:0:59d:a0e5:ba1')
+        self.assertEqual((ipaddress.IPv4Address('94.245.121.253'),
+                          ipaddress.IPv4Address('95.26.244.94')),
+                         teredo_addr.teredo)
+
+    def testsixtofour(self):
+        sixtofouraddr = ipaddress.ip_address('2002:ac1d:2d64::1')
+        bad_addr = ipaddress.ip_address('2000:ac1d:2d64::1')
+        self.assertEqual(ipaddress.IPv4Address('172.29.45.100'),
+                         sixtofouraddr.sixtofour)
+        self.assertFalse(bad_addr.sixtofour)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py
index dc2d07467a..7a6730e3a8 100644
--- a/Lib/test/test_isinstance.py
+++ b/Lib/test/test_isinstance.py
@@ -15,7 +15,7 @@ class TestIsInstanceExceptions(unittest.TestCase):
     # (leading to an "undetected error" in the debug build).  Set up is,
     # isinstance(inst, cls) where:
     #
-    # - cls isn't a a type, or a tuple
+    # - cls isn't a type, or a tuple
     # - cls has a __bases__ attribute
     # - inst has a __class__ attribute
     # - inst.__class__ as no __bases__ attribute
diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py
index 4d971835cb..43f8e1588b 100644
--- a/Lib/test/test_iter.py
+++ b/Lib/test/test_iter.py
@@ -2,6 +2,8 @@
 
 import unittest
 from test.support import run_unittest, TESTFN, unlink, cpython_only
+import pickle
+import collections.abc
 
 # Test result of triple loop (too big to inline)
 TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2),
@@ -28,6 +30,8 @@ class BasicIterClass:
             raise StopIteration
         self.i = res + 1
         return res
+    def __iter__(self):
+        return self
 
 class IteratingSequenceClass:
     def __init__(self, n):
@@ -49,7 +53,9 @@ class SequenceClass:
 class TestCase(unittest.TestCase):
 
     # Helper to check that an iterator returns a given sequence
-    def check_iterator(self, it, seq):
+    def check_iterator(self, it, seq, pickle=True):
+        if pickle:
+            self.check_pickle(it, seq)
         res = []
         while 1:
             try:
@@ -60,12 +66,33 @@ class TestCase(unittest.TestCase):
         self.assertEqual(res, seq)
 
     # Helper to check that a for loop generates a given sequence
-    def check_for_loop(self, expr, seq):
+    def check_for_loop(self, expr, seq, pickle=True):
+        if pickle:
+            self.check_pickle(iter(expr), seq)
         res = []
         for val in expr:
             res.append(val)
         self.assertEqual(res, seq)
 
+    # Helper to check picklability
+    def check_pickle(self, itorg, seq):
+        d = pickle.dumps(itorg)
+        it = pickle.loads(d)
+        # Cannot assert type equality because dict iterators unpickle as list
+        # iterators.
+        # self.assertEqual(type(itorg), type(it))
+        self.assertTrue(isinstance(it, collections.abc.Iterator))
+        self.assertEqual(list(it), seq)
+
+        it = pickle.loads(d)
+        try:
+            next(it)
+        except StopIteration:
+            return
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(list(it), seq[1:])
+
     # Test basic use of iter() function
     def test_iter_basic(self):
         self.check_iterator(iter(range(10)), list(range(10)))
@@ -138,7 +165,7 @@ class TestCase(unittest.TestCase):
                 if i > 100:
                     raise IndexError # Emergency stop
                 return i
-        self.check_iterator(iter(C(), 10), list(range(10)))
+        self.check_iterator(iter(C(), 10), list(range(10)), pickle=False)
 
     # Test two-argument iter() with function
     def test_iter_function(self):
@@ -146,7 +173,7 @@ class TestCase(unittest.TestCase):
             i = state[0]
             state[0] = i+1
             return i
-        self.check_iterator(iter(spam, 10), list(range(10)))
+        self.check_iterator(iter(spam, 10), list(range(10)), pickle=False)
 
     # Test two-argument iter() with function that raises StopIteration
     def test_iter_function_stop(self):
@@ -156,7 +183,7 @@ class TestCase(unittest.TestCase):
                 raise StopIteration
             state[0] = i+1
             return i
-        self.check_iterator(iter(spam, 20), list(range(10)))
+        self.check_iterator(iter(spam, 20), list(range(10)), pickle=False)
 
     # Test exception propagation through function iterator
     def test_exception_function(self):
@@ -198,7 +225,7 @@ class TestCase(unittest.TestCase):
                 if i == 10:
                     raise StopIteration
                 return SequenceClass.__getitem__(self, i)
-        self.check_for_loop(MySequenceClass(20), list(range(10)))
+        self.check_for_loop(MySequenceClass(20), list(range(10)), pickle=False)
 
     # Test a big range
     def test_iter_big_range(self):
@@ -237,8 +264,8 @@ class TestCase(unittest.TestCase):
             f.close()
         f = open(TESTFN, "r")
         try:
-            self.check_for_loop(f, ["0\n", "1\n", "2\n", "3\n", "4\n"])
-            self.check_for_loop(f, [])
+            self.check_for_loop(f, ["0\n", "1\n", "2\n", "3\n", "4\n"], pickle=False)
+            self.check_for_loop(f, [], pickle=False)
         finally:
             f.close()
             try:
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index dfa371e2b8..53926a9df5 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -37,6 +37,13 @@ def isOdd(x):
     'Test predicate'
     return x%2==1
 
+def tupleize(*args):
+    return args
+
+def irange(n):
+    for i in range(n):
+        yield i
+
 class StopNow:
     'Class emulating an empty iterable.'
     def __iter__(self):
@@ -55,8 +62,59 @@ def fact(n):
     'Factorial'
     return prod(range(1, n+1))
 
+# root level methods for pickling ability
+def testR(r):
+    return r[0]
+
+def testR2(r):
+    return r[2]
+
+def underten(x):
+    return x<10
+
 class TestBasicOps(unittest.TestCase):
 
+    def pickletest(self, it, stop=4, take=1, compare=None):
+        """Test that an iterator is the same after pickling, also when part-consumed"""
+        def expand(it, i=0):
+            # Recursively expand iterables, within sensible bounds
+            if i > 10:
+                raise RuntimeError("infinite recursion encountered")
+            if isinstance(it, str):
+                return it
+            try:
+                l = list(islice(it, stop))
+            except TypeError:
+                return it # can't expand it
+            return [expand(e, i+1) for e in l]
+
+        # Test the initial copy against the original
+        dump = pickle.dumps(it)
+        i2 = pickle.loads(dump)
+        self.assertEqual(type(it), type(i2))
+        a, b = expand(it), expand(i2)
+        self.assertEqual(a, b)
+        if compare:
+            c = expand(compare)
+            self.assertEqual(a, c)
+
+        # Take from the copy, and create another copy and compare them.
+        i3 = pickle.loads(dump)
+        took = 0
+        try:
+            for i in range(take):
+                next(i3)
+                took += 1
+        except StopIteration:
+            pass #in case there is less data than 'take'
+        dump = pickle.dumps(i3)
+        i4 = pickle.loads(dump)
+        a, b = expand(i3), expand(i4)
+        self.assertEqual(a, b)
+        if compare:
+            c = expand(compare[took:])
+            self.assertEqual(a, c);
+
     def test_accumulate(self):
         self.assertEqual(list(accumulate(range(10))),               # one positional arg
                           [0, 1, 3, 6, 10, 15, 21, 28, 36, 45])
@@ -69,11 +127,22 @@ class TestBasicOps(unittest.TestCase):
         self.assertEqual(list(accumulate('abc')), ['a', 'ab', 'abc'])   # works with non-numeric
         self.assertEqual(list(accumulate([])), [])                  # empty iterable
         self.assertEqual(list(accumulate([7])), [7])                # iterable of length one
-        self.assertRaises(TypeError, accumulate, range(10), 5)      # too many args
+        self.assertRaises(TypeError, accumulate, range(10), 5, 6)   # too many args
         self.assertRaises(TypeError, accumulate)                    # too few args
         self.assertRaises(TypeError, accumulate, x=range(10))       # unexpected kwd arg
         self.assertRaises(TypeError, list, accumulate([1, []]))     # args that don't add
 
+        s = [2, 8, 9, 5, 7, 0, 3, 4, 1, 6]
+        self.assertEqual(list(accumulate(s, min)),
+                         [2, 2, 2, 2, 2, 0, 0, 0, 0, 0])
+        self.assertEqual(list(accumulate(s, max)),
+                         [2, 8, 9, 9, 9, 9, 9, 9, 9, 9])
+        self.assertEqual(list(accumulate(s, operator.mul)),
+                         [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0])
+        with self.assertRaises(TypeError):
+            list(accumulate(s, chr))                                # unary-operation
+        self.pickletest(accumulate(range(10)))                      # test pickling
+
     def test_chain(self):
 
         def chain2(*iterables):
@@ -96,14 +165,43 @@ class TestBasicOps(unittest.TestCase):
         self.assertEqual(take(4, chain.from_iterable(['abc', 'def'])), list('abcd'))
         self.assertRaises(TypeError, list, chain.from_iterable([2, 3]))
 
+    def test_chain_reducible(self):
+        operators = [copy.deepcopy,
+                     lambda s: pickle.loads(pickle.dumps(s))]
+        for oper in operators:
+            it = chain('abc', 'def')
+            self.assertEqual(list(oper(it)), list('abcdef'))
+            self.assertEqual(next(it), 'a')
+            self.assertEqual(list(oper(it)), list('bcdef'))
+
+            self.assertEqual(list(oper(chain(''))), [])
+            self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd'))
+            self.assertRaises(TypeError, list, oper(chain(2, 3)))
+        self.pickletest(chain('abc', 'def'), compare=list('abcdef'))
+
     def test_combinations(self):
         self.assertRaises(TypeError, combinations, 'abc')       # missing r argument
         self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments
         self.assertRaises(TypeError, combinations, None)        # pool is not iterable
         self.assertRaises(ValueError, combinations, 'abc', -2)  # r is negative
-        self.assertEqual(list(combinations('abc', 32)), [])     # r > n
-        self.assertEqual(list(combinations(range(4), 3)),
-                                           [(0,1,2), (0,1,3), (0,2,3), (1,2,3)])
+
+        for op in (lambda a:a, lambda a:pickle.loads(pickle.dumps(a))):
+            self.assertEqual(list(op(combinations('abc', 32))), [])     # r > n
+
+            self.assertEqual(list(op(combinations('ABCD', 2))),
+                             [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')])
+            testIntermediate = combinations('ABCD', 2)
+            next(testIntermediate)
+            self.assertEqual(list(op(testIntermediate)),
+                             [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')])
+
+            self.assertEqual(list(op(combinations(range(4), 3))),
+                             [(0,1,2), (0,1,3), (0,2,3), (1,2,3)])
+            testIntermediate = combinations(range(4), 3)
+            next(testIntermediate)
+            self.assertEqual(list(op(testIntermediate)),
+                             [(0,1,3), (0,2,3), (1,2,3)])
+
 
         def combinations1(iterable, r):
             'Pure python version shown in the docs'
@@ -158,7 +256,11 @@ class TestBasicOps(unittest.TestCase):
                 self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version
                 self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version
 
+                self.pickletest(combinations(values, r))                 # test pickling
+
         # Test implementation detail:  tuple re-use
+    @support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_tuple_reuse(self):
         self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1)
 
@@ -168,8 +270,15 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(TypeError, cwr, 'abc', 2, 1) # too many arguments
         self.assertRaises(TypeError, cwr, None)        # pool is not iterable
         self.assertRaises(ValueError, cwr, 'abc', -2)  # r is negative
-        self.assertEqual(list(cwr('ABC', 2)),
-                         [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+
+        for op in (lambda a:a, lambda a:pickle.loads(pickle.dumps(a))):
+            self.assertEqual(list(op(cwr('ABC', 2))),
+                             [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+            testIntermediate = cwr('ABC', 2)
+            next(testIntermediate)
+            self.assertEqual(list(op(testIntermediate)),
+                             [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+
 
         def cwr1(iterable, r):
             'Pure python version shown in the docs'
@@ -228,7 +337,13 @@ class TestBasicOps(unittest.TestCase):
                 self.assertEqual(result, list(cwr1(values, r)))         # matches first pure python version
                 self.assertEqual(result, list(cwr2(values, r)))         # matches second pure python version
 
+                self.pickletest(cwr(values,r))                          # test pickling
+
         # Test implementation detail:  tuple re-use
+
+    @support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_with_replacement_tuple_reuse(self):
+        cwr = combinations_with_replacement
         self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1)
 
@@ -292,7 +407,10 @@ class TestBasicOps(unittest.TestCase):
                     self.assertEqual(result, list(permutations(values, None))) # test r as None
                     self.assertEqual(result, list(permutations(values)))       # test default r
 
-        # Test implementation detail:  tuple re-use
+                self.pickletest(permutations(values, r))                # test pickling
+
+    @support.impl_detail("tuple resuse is CPython specific")
+    def test_permutations_tuple_reuse(self):
         self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1)
 
@@ -345,6 +463,24 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(TypeError, compress, range(6))            # too few args
         self.assertRaises(TypeError, compress, range(6), None)      # too many args
 
+        # check copy, deepcopy, pickle
+        for op in (lambda a:copy.copy(a), lambda a:copy.deepcopy(a), lambda a:pickle.loads(pickle.dumps(a))):
+            for data, selectors, result1, result2 in [
+                ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'),
+                ('ABCDEF', [0,0,0,0,0,0], '', ''),
+                ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'),
+                ('ABCDEF', [1,0,1], 'AC', 'C'),
+                ('ABC', [0,1,1,1,1,1], 'BC', 'C'),
+                ]:
+
+                self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1))
+                self.assertEqual(list(op(compress(data, selectors))), list(result1))
+                testIntermediate = compress(data, selectors)
+                if result1:
+                    next(testIntermediate)
+                    self.assertEqual(list(op(testIntermediate)), list(result2))
+
+
     def test_count(self):
         self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)])
         self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)])
@@ -379,7 +515,7 @@ class TestBasicOps(unittest.TestCase):
             c = count(value)
             self.assertEqual(next(copy.copy(c)), value)
             self.assertEqual(next(copy.deepcopy(c)), value)
-            self.assertEqual(next(pickle.loads(pickle.dumps(c))), value)
+            self.pickletest(count(value))
 
         #check proper internal error handling for large "step' sizes
         count(1, maxsize+5); sys.exc_info()
@@ -426,6 +562,7 @@ class TestBasicOps(unittest.TestCase):
                 else:
                     r2 = ('count(%r, %r)' % (i, j)).replace('L', '')
                 self.assertEqual(r1, r2)
+                self.pickletest(count(i, j))
 
     def test_cycle(self):
         self.assertEqual(take(10, cycle('abc')), list('abcabcabca'))
@@ -434,6 +571,18 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(TypeError, cycle, 5)
         self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0])
 
+        # check copy, deepcopy, pickle
+        c = cycle('abc')
+        self.assertEqual(next(c), 'a')
+        #simple copy currently not supported, because __reduce__ returns
+        #an internal iterator
+        #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab'))
+        self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab'))
+        self.assertEqual(take(10, pickle.loads(pickle.dumps(c))), list('bcabcabcab'))
+        next(c)
+        self.assertEqual(take(10, pickle.loads(pickle.dumps(c))), list('cabcabcabc'))
+        self.pickletest(cycle('abc'))
+
     def test_groupby(self):
         # Check whether it accepts arguments correctly
         self.assertEqual([], list(groupby([])))
@@ -452,18 +601,37 @@ class TestBasicOps(unittest.TestCase):
                 dup.append(elem)
         self.assertEqual(s, dup)
 
+        # Check normal pickled
+        dup = []
+        for k, g in pickle.loads(pickle.dumps(groupby(s, testR))):
+            for elem in g:
+                self.assertEqual(k, elem[0])
+                dup.append(elem)
+        self.assertEqual(s, dup)
+
         # Check nested case
         dup = []
-        for k, g in groupby(s, lambda r:r[0]):
-            for ik, ig in groupby(g, lambda r:r[2]):
+        for k, g in groupby(s, testR):
+            for ik, ig in groupby(g, testR2):
+                for elem in ig:
+                    self.assertEqual(k, elem[0])
+                    self.assertEqual(ik, elem[2])
+                    dup.append(elem)
+        self.assertEqual(s, dup)
+
+        # Check nested and pickled
+        dup = []
+        for k, g in pickle.loads(pickle.dumps(groupby(s, testR))):
+            for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2))):
                 for elem in ig:
                     self.assertEqual(k, elem[0])
                     self.assertEqual(ik, elem[2])
                     dup.append(elem)
         self.assertEqual(s, dup)
 
+
         # Check case where inner iterator is not used
-        keys = [k for k, g in groupby(s, lambda r:r[0])]
+        keys = [k for k, g in groupby(s, testR)]
         expectedkeys = set([r[0] for r in s])
         self.assertEqual(set(keys), expectedkeys)
         self.assertEqual(len(keys), len(expectedkeys))
@@ -534,6 +702,20 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(TypeError, filter, isEven, 3)
         self.assertRaises(TypeError, next, filter(range(6), range(6)))
 
+        # check copy, deepcopy, pickle
+        ans = [0,2,4]
+
+        c = filter(isEven, range(6))
+        self.assertEqual(list(copy.copy(c)), ans)
+        c = filter(isEven, range(6))
+        self.assertEqual(list(copy.deepcopy(c)), ans)
+        c = filter(isEven, range(6))
+        self.assertEqual(list(pickle.loads(pickle.dumps(c))), ans)
+        next(c)
+        self.assertEqual(list(pickle.loads(pickle.dumps(c))), ans[1:])
+        c = filter(isEven, range(6))
+        self.pickletest(c)
+
     def test_filterfalse(self):
         self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5])
         self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0])
@@ -544,6 +726,7 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7)
         self.assertRaises(TypeError, filterfalse, isEven, 3)
         self.assertRaises(TypeError, next, filterfalse(range(6), range(6)))
+        self.pickletest(filterfalse(isEven, range(6)))
 
     def test_zip(self):
         # XXX This is rather silly now that builtin zip() calls zip()...
@@ -556,16 +739,35 @@ class TestBasicOps(unittest.TestCase):
         self.assertEqual(list(zip()), lzip())
         self.assertRaises(TypeError, zip, 3)
         self.assertRaises(TypeError, zip, range(3), 3)
-        # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in zip('abc', 'def')],
                          lzip('abc', 'def'))
         self.assertEqual([pair for pair in zip('abc', 'def')],
                          lzip('abc', 'def'))
+
+    @support.impl_detail("tuple reuse is specific to CPython")
+    def test_zip_tuple_reuse(self):
         ids = list(map(id, zip('abc', 'def')))
         self.assertEqual(min(ids), max(ids))
         ids = list(map(id, list(zip('abc', 'def'))))
         self.assertEqual(len(dict.fromkeys(ids)), len(ids))
 
+        # check copy, deepcopy, pickle
+        ans = [(x,y) for x, y in copy.copy(zip('abc',count()))]
+        self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+        ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))]
+        self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+        ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count())))]
+        self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+        testIntermediate = zip('abc',count())
+        next(testIntermediate)
+        ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate))]
+        self.assertEqual(ans, [('b', 1), ('c', 2)])
+
+        self.pickletest(zip('abc', count()))
+
     def test_ziplongest(self):
         for args in [
                 ['abc', range(6)],
@@ -603,16 +805,24 @@ class TestBasicOps(unittest.TestCase):
             else:
                 self.fail('Did not raise Type in:  ' + stmt)
 
-        # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in zip_longest('abc', 'def')],
                          list(zip('abc', 'def')))
         self.assertEqual([pair for pair in zip_longest('abc', 'def')],
                          list(zip('abc', 'def')))
+
+    @support.impl_detail("tuple reuse is specific to CPython")
+    def test_zip_longest_tuple_reuse(self):
         ids = list(map(id, zip_longest('abc', 'def')))
         self.assertEqual(min(ids), max(ids))
         ids = list(map(id, list(zip_longest('abc', 'def'))))
         self.assertEqual(len(dict.fromkeys(ids)), len(ids))
 
+    def test_zip_longest_pickling(self):
+        self.pickletest(zip_longest("abc", "def"))
+        self.pickletest(zip_longest("abc", "defgh"))
+        self.pickletest(zip_longest("abc", "defgh", fillvalue=1))
+        self.pickletest(zip_longest("", "defgh"))
+
     def test_bug_7244(self):
 
         class Repeater:
@@ -711,10 +921,25 @@ class TestBasicOps(unittest.TestCase):
             args = map(iter, args)
             self.assertEqual(len(list(product(*args))), expected_len)
 
-        # Test implementation detail:  tuple re-use
+    @support.impl_detail("tuple reuse is specific to CPython")
+    def test_product_tuple_reuse(self):
         self.assertEqual(len(set(map(id, product('abc', 'def')))), 1)
         self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1)
 
+    def test_product_pickling(self):
+        # check copy, deepcopy, pickle
+        for args, result in [
+            ([], [()]),                     # zero iterables
+            (['ab'], [('a',), ('b',)]),     # one iterable
+            ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]),     # two iterables
+            ([range(0), range(2), range(3)], []),           # first iterable with zero length
+            ([range(2), range(0), range(3)], []),           # middle iterable with zero length
+            ([range(2), range(3), range(0)], []),           # last iterable with zero length
+            ]:
+            self.assertEqual(list(copy.copy(product(*args))), result)
+            self.assertEqual(list(copy.deepcopy(product(*args))), result)
+            self.pickletest(product(*args))
+
     def test_repeat(self):
         self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a'])
         self.assertEqual(lzip(range(3),repeat('a')),
@@ -733,11 +958,16 @@ class TestBasicOps(unittest.TestCase):
         list(r)
         self.assertEqual(repr(r), 'repeat((1+0j), 0)')
 
+        # check copy, deepcopy, pickle
+        c = repeat(object='a', times=10)
+        self.assertEqual(next(c), 'a')
+        self.assertEqual(take(2, copy.copy(c)), list('a' * 2))
+        self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2))
+        self.pickletest(repeat(object='a', times=10))
+
     def test_map(self):
         self.assertEqual(list(map(operator.pow, range(3), range(1,7))),
                          [0**1, 1**2, 2**3])
-        def tupleize(*args):
-            return args
         self.assertEqual(list(map(tupleize, 'abc', range(5))),
                          [('a',0),('b',1),('c',2)])
         self.assertEqual(list(map(tupleize, 'abc', count())),
@@ -752,6 +982,18 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(ValueError, next, map(errfunc, [4], [5]))
         self.assertRaises(TypeError, next, map(onearg, [4], [5]))
 
+        # check copy, deepcopy, pickle
+        ans = [('a',0),('b',1),('c',2)]
+
+        c = map(tupleize, 'abc', count())
+        self.assertEqual(list(copy.copy(c)), ans)
+
+        c = map(tupleize, 'abc', count())
+        self.assertEqual(list(copy.deepcopy(c)), ans)
+
+        c = map(tupleize, 'abc', count())
+        self.pickletest(c)
+
     def test_starmap(self):
         self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))),
                          [0**1, 1**2, 2**3])
@@ -766,6 +1008,18 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)]))
         self.assertRaises(TypeError, next, starmap(onearg, [(4,5)]))
 
+        # check copy, deepcopy, pickle
+        ans = [0**1, 1**2, 2**3]
+
+        c = starmap(operator.pow, zip(range(3), range(1,7)))
+        self.assertEqual(list(copy.copy(c)), ans)
+
+        c = starmap(operator.pow, zip(range(3), range(1,7)))
+        self.assertEqual(list(copy.deepcopy(c)), ans)
+
+        c = starmap(operator.pow, zip(range(3), range(1,7)))
+        self.pickletest(c)
+
     def test_islice(self):
         for args in [          # islice(args) should agree with range(args)
                 (10, 20, 3),
@@ -798,17 +1052,18 @@ class TestBasicOps(unittest.TestCase):
         self.assertEqual(list(it), list(range(3, 10)))
 
         # Test invalid arguments
-        self.assertRaises(TypeError, islice, range(10))
-        self.assertRaises(TypeError, islice, range(10), 1, 2, 3, 4)
-        self.assertRaises(ValueError, islice, range(10), -5, 10, 1)
-        self.assertRaises(ValueError, islice, range(10), 1, -5, -1)
-        self.assertRaises(ValueError, islice, range(10), 1, 10, -1)
-        self.assertRaises(ValueError, islice, range(10), 1, 10, 0)
-        self.assertRaises(ValueError, islice, range(10), 'a')
-        self.assertRaises(ValueError, islice, range(10), 'a', 1)
-        self.assertRaises(ValueError, islice, range(10), 1, 'a')
-        self.assertRaises(ValueError, islice, range(10), 'a', 1, 1)
-        self.assertRaises(ValueError, islice, range(10), 1, 'a', 1)
+        ra = range(10)
+        self.assertRaises(TypeError, islice, ra)
+        self.assertRaises(TypeError, islice, ra, 1, 2, 3, 4)
+        self.assertRaises(ValueError, islice, ra, -5, 10, 1)
+        self.assertRaises(ValueError, islice, ra, 1, -5, -1)
+        self.assertRaises(ValueError, islice, ra, 1, 10, -1)
+        self.assertRaises(ValueError, islice, ra, 1, 10, 0)
+        self.assertRaises(ValueError, islice, ra, 'a')
+        self.assertRaises(ValueError, islice, ra, 'a', 1)
+        self.assertRaises(ValueError, islice, ra, 1, 'a')
+        self.assertRaises(ValueError, islice, ra, 'a', 1, 1)
+        self.assertRaises(ValueError, islice, ra, 1, 'a', 1)
         self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1)
 
         # Issue #10323:  Less islice in a predictable state
@@ -816,9 +1071,22 @@ class TestBasicOps(unittest.TestCase):
         self.assertEqual(list(islice(c, 1, 3, 50)), [1])
         self.assertEqual(next(c), 3)
 
+        # check copy, deepcopy, pickle
+        for args in [          # islice(args) should agree with range(args)
+                (10, 20, 3),
+                (10, 3, 20),
+                (10, 20),
+                (10, 3),
+                (20,)
+                ]:
+            self.assertEqual(list(copy.copy(islice(range(100), *args))),
+                             list(range(*args)))
+            self.assertEqual(list(copy.deepcopy(islice(range(100), *args))),
+                             list(range(*args)))
+            self.pickletest(islice(range(100), *args))
+
     def test_takewhile(self):
         data = [1, 3, 5, 20, 2, 4, 6, 8]
-        underten = lambda x: x<10
         self.assertEqual(list(takewhile(underten, data)), [1, 3, 5])
         self.assertEqual(list(takewhile(underten, [])), [])
         self.assertRaises(TypeError, takewhile)
@@ -830,9 +1098,14 @@ class TestBasicOps(unittest.TestCase):
         self.assertEqual(list(t), [1, 1, 1])
         self.assertRaises(StopIteration, next, t)
 
+        # check copy, deepcopy, pickle
+        self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5])
+        self.assertEqual(list(copy.deepcopy(takewhile(underten, data))),
+                        [1, 3, 5])
+        self.pickletest(takewhile(underten, data))
+
     def test_dropwhile(self):
         data = [1, 3, 5, 20, 2, 4, 6, 8]
-        underten = lambda x: x<10
         self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8])
         self.assertEqual(list(dropwhile(underten, [])), [])
         self.assertRaises(TypeError, dropwhile)
@@ -841,11 +1114,14 @@ class TestBasicOps(unittest.TestCase):
         self.assertRaises(TypeError, next, dropwhile(10, [(4,5)]))
         self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)]))
 
+        # check copy, deepcopy, pickle
+        self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8])
+        self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))),
+                        [20, 2, 4, 6, 8])
+        self.pickletest(dropwhile(underten, data))
+
     def test_tee(self):
         n = 200
-        def irange(n):
-            for i in range(n):
-                yield i
 
         a, b = tee([])        # test empty iterator
         self.assertEqual(list(a), [])
@@ -930,6 +1206,67 @@ class TestBasicOps(unittest.TestCase):
         del a
         self.assertRaises(ReferenceError, getattr, p, '__class__')
 
+        ans = list('abc')
+        long_ans = list(range(10000))
+
+        # check copy
+        a, b = tee('abc')
+        self.assertEqual(list(copy.copy(a)), ans)
+        self.assertEqual(list(copy.copy(b)), ans)
+        a, b = tee(list(range(10000)))
+        self.assertEqual(list(copy.copy(a)), long_ans)
+        self.assertEqual(list(copy.copy(b)), long_ans)
+
+        # check partially consumed copy
+        a, b = tee('abc')
+        take(2, a)
+        take(1, b)
+        self.assertEqual(list(copy.copy(a)), ans[2:])
+        self.assertEqual(list(copy.copy(b)), ans[1:])
+        self.assertEqual(list(a), ans[2:])
+        self.assertEqual(list(b), ans[1:])
+        a, b = tee(range(10000))
+        take(100, a)
+        take(60, b)
+        self.assertEqual(list(copy.copy(a)), long_ans[100:])
+        self.assertEqual(list(copy.copy(b)), long_ans[60:])
+        self.assertEqual(list(a), long_ans[100:])
+        self.assertEqual(list(b), long_ans[60:])
+
+        # check deepcopy
+        a, b = tee('abc')
+        self.assertEqual(list(copy.deepcopy(a)), ans)
+        self.assertEqual(list(copy.deepcopy(b)), ans)
+        self.assertEqual(list(a), ans)
+        self.assertEqual(list(b), ans)
+        a, b = tee(range(10000))
+        self.assertEqual(list(copy.deepcopy(a)), long_ans)
+        self.assertEqual(list(copy.deepcopy(b)), long_ans)
+        self.assertEqual(list(a), long_ans)
+        self.assertEqual(list(b), long_ans)
+
+        # check partially consumed deepcopy
+        a, b = tee('abc')
+        take(2, a)
+        take(1, b)
+        self.assertEqual(list(copy.deepcopy(a)), ans[2:])
+        self.assertEqual(list(copy.deepcopy(b)), ans[1:])
+        self.assertEqual(list(a), ans[2:])
+        self.assertEqual(list(b), ans[1:])
+        a, b = tee(range(10000))
+        take(100, a)
+        take(60, b)
+        self.assertEqual(list(copy.deepcopy(a)), long_ans[100:])
+        self.assertEqual(list(copy.deepcopy(b)), long_ans[60:])
+        self.assertEqual(list(a), long_ans[100:])
+        self.assertEqual(list(b), long_ans[60:])
+
+        # check pickle
+        self.pickletest(iter(tee('abc')))
+        a, b = tee('abc')
+        self.pickletest(a, compare=ans)
+        self.pickletest(b, compare=ans)
+
     # Issue 13454: Crash when deleting backward iterator from tee()
     def test_tee_del_backward(self):
         forward, backward = tee(repeat(None, 20000000))
@@ -961,9 +1298,21 @@ class TestBasicOps(unittest.TestCase):
 
 class TestExamples(unittest.TestCase):
 
-    def test_accumlate(self):
+    def test_accumulate(self):
         self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15])
 
+    def test_accumulate_reducible(self):
+        # check copy, deepcopy, pickle
+        data = [1, 2, 3, 4, 5]
+        accumulated = [1, 3, 6, 10, 15]
+        it = accumulate(data)
+
+        self.assertEqual(list(pickle.loads(pickle.dumps(it))), accumulated[:])
+        self.assertEqual(next(it), 1)
+        self.assertEqual(list(pickle.loads(pickle.dumps(it))), accumulated[1:])
+        self.assertEqual(list(copy.deepcopy(it)), accumulated[1:])
+        self.assertEqual(list(copy.copy(it)), accumulated[1:])
+
     def test_chain(self):
         self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF')
 
diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py
index 108ed18c59..0503a7fc6a 100644
--- a/Lib/test/test_keywordonlyarg.py
+++ b/Lib/test/test_keywordonlyarg.py
@@ -78,7 +78,7 @@ class KeywordOnlyArgTestCase(unittest.TestCase):
             pass
         with self.assertRaises(TypeError) as exc:
             f(1, 2, 3)
-        expected = "f() takes at most 2 positional arguments (3 given)"
+        expected = "f() takes from 1 to 2 positional arguments but 3 were given"
         self.assertEqual(str(exc.exception), expected)
 
     def testSyntaxErrorForFunctionCall(self):
diff --git a/Lib/test/test_lib2to3.py b/Lib/test/test_lib2to3.py
index 1afaf70ffa..df4c37b241 100644
--- a/Lib/test/test_lib2to3.py
+++ b/Lib/test/test_lib2to3.py
@@ -9,8 +9,8 @@ from test.support import run_unittest
 def suite():
     tests = unittest.TestSuite()
     loader = unittest.TestLoader()
-    for m in (test_fixers, test_pytree,test_util, test_refactor,
-              test_parser, test_main_):
+    for m in (test_fixers, test_pytree, test_util, test_refactor, test_parser,
+              test_main_):
         tests.addTests(loader.loadTestsFromModule(m))
     return tests
 
diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py
index 84b8afe3a4..5df27d30ad 100644
--- a/Lib/test/test_list.py
+++ b/Lib/test/test_list.py
@@ -1,5 +1,6 @@
 import sys
 from test import support, list_tests
+import pickle
 
 class ListTest(list_tests.CommonTest):
     type2test = list
@@ -69,6 +70,33 @@ class ListTest(list_tests.CommonTest):
         check(10)       # check our checking code
         check(1000000)
 
+    def test_iterator_pickle(self):
+        # Userlist iterators don't support pickling yet since
+        # they are based on generators.
+        data = self.type2test([4, 5, 6, 7])
+        it = itorg = iter(data)
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(type(itorg), type(it))
+        self.assertEqual(self.type2test(it), self.type2test(data))
+
+        it = pickle.loads(d)
+        next(it)
+        d = pickle.dumps(it)
+        self.assertEqual(self.type2test(it), self.type2test(data)[1:])
+
+    def test_reversed_pickle(self):
+        data = self.type2test([4, 5, 6, 7])
+        it = itorg = reversed(data)
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(type(itorg), type(it))
+        self.assertEqual(self.type2test(it), self.type2test(reversed(data)))
+
+        it = pickle.loads(d)
+        next(it)
+        d = pickle.dumps(it)
+        self.assertEqual(self.type2test(it), self.type2test(reversed(data))[1:])
 
     def test_no_comdat_folding(self):
         # Issue 8847: In the PGO build, the MSVC linker's COMDAT folding
diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py
index 8f7574b904..51a7bca4b6 100644
--- a/Lib/test/test_locale.py
+++ b/Lib/test/test_locale.py
@@ -11,7 +11,7 @@ def get_enUS_locale():
     if sys.platform == 'darwin':
         import os
         tlocs = ("en_US.UTF-8", "en_US.ISO8859-1", "en_US")
-        if int(os.uname()[2].split('.')[0]) < 10:
+        if int(os.uname().release.split('.')[0]) < 10:
             # The locale test work fine on OSX 10.6, I (ronaldoussoren)
             # haven't had time yet to verify if tests work on OSX 10.5
             # (10.4 is known to be bad)
@@ -401,6 +401,8 @@ class TestMiscellaneous(unittest.TestCase):
             # Unsupported locale on this system
             self.skipTest('test needs Turkish locale')
         loc = locale.getlocale(locale.LC_CTYPE)
+        if verbose:
+            print('got locale %a' % (loc,))
         locale.setlocale(locale.LC_CTYPE, loc)
         self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE))
 
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index 2a3c780d2c..cb908fb460 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -37,12 +37,11 @@ import random
 import re
 import select
 import socket
-from socketserver import ThreadingTCPServer, StreamRequestHandler
 import struct
 import sys
 import tempfile
-from test.support import captured_stdout, run_with_locale, run_unittest, patch
-from test.support import TestHandler, Matcher
+from test.support import (captured_stdout, run_with_locale, run_unittest,
+                          patch, requires_zlib, TestHandler, Matcher)
 import textwrap
 import time
 import unittest
@@ -50,8 +49,31 @@ import warnings
 import weakref
 try:
     import threading
+    # The following imports are needed only for tests which
+    # require threading
+    import asynchat
+    import asyncore
+    import errno
+    from http.server import HTTPServer, BaseHTTPRequestHandler
+    import smtpd
+    from urllib.parse import urlparse, parse_qs
+    from socketserver import (ThreadingUDPServer, DatagramRequestHandler,
+                              ThreadingTCPServer, StreamRequestHandler)
 except ImportError:
     threading = None
+try:
+    import win32evtlog
+except ImportError:
+    win32evtlog = None
+try:
+    import win32evtlogutil
+except ImportError:
+    win32evtlogutil = None
+    win32evtlog = None
+try:
+    import zlib
+except ImportError:
+    pass
 
 
 class BaseTest(unittest.TestCase):
@@ -79,9 +101,7 @@ class BaseTest(unittest.TestCase):
         finally:
             logging._releaseLock()
 
-        # Set two unused loggers: one non-ASCII and one Unicode.
-        # This is to test correct operation when sorting existing
-        # loggers in the configuration code. See issue 8201.
+        # Set two unused loggers
         self.logger1 = logging.getLogger("\xab\xd7\xbb")
         self.logger2 = logging.getLogger("\u013f\u00d6\u0047")
 
@@ -142,8 +162,7 @@ class BaseTest(unittest.TestCase):
         except AttributeError:
             # StringIO.StringIO lacks a reset() method.
             actual_lines = stream.getvalue().splitlines()
-        self.assertEqual(len(actual_lines), len(expected_values),
-                          '%s vs. %s' % (actual_lines, expected_values))
+        self.assertEqual(len(actual_lines), len(expected_values))
         for actual, expected in zip(actual_lines, expected_values):
             match = pat.search(actual)
             if not match:
@@ -181,17 +200,17 @@ class BuiltinLevelsTest(BaseTest):
 
         INF.log(logging.CRITICAL, m())
         INF.error(m())
-        INF.warn(m())
+        INF.warning(m())
         INF.info(m())
 
         DEB.log(logging.CRITICAL, m())
         DEB.error(m())
-        DEB.warn (m())
-        DEB.info (m())
+        DEB.warning(m())
+        DEB.info(m())
         DEB.debug(m())
 
         # These should not log.
-        ERR.warn(m())
+        ERR.warning(m())
         ERR.info(m())
         ERR.debug(m())
 
@@ -225,7 +244,7 @@ class BuiltinLevelsTest(BaseTest):
         INF_ERR.error(m())
 
         # These should not log.
-        INF_ERR.warn(m())
+        INF_ERR.warning(m())
         INF_ERR.info(m())
         INF_ERR.debug(m())
 
@@ -249,14 +268,14 @@ class BuiltinLevelsTest(BaseTest):
         # These should log.
         INF_UNDEF.log(logging.CRITICAL, m())
         INF_UNDEF.error(m())
-        INF_UNDEF.warn(m())
+        INF_UNDEF.warning(m())
         INF_UNDEF.info(m())
         INF_ERR_UNDEF.log(logging.CRITICAL, m())
         INF_ERR_UNDEF.error(m())
 
         # These should not log.
         INF_UNDEF.debug(m())
-        INF_ERR_UNDEF.warn(m())
+        INF_ERR_UNDEF.warning(m())
         INF_ERR_UNDEF.info(m())
         INF_ERR_UNDEF.debug(m())
 
@@ -295,8 +314,6 @@ class BuiltinLevelsTest(BaseTest):
             ('INF.BADPARENT', 'INFO', '4'),
         ])
 
-    def test_invalid_name(self):
-        self.assertRaises(TypeError, logging.getLogger, any)
 
 class BasicFilterTest(BaseTest):
 
@@ -355,6 +372,10 @@ class BasicFilterTest(BaseTest):
         finally:
             handler.removeFilter(filterfunc)
 
+    def test_empty_filter(self):
+        f = logging.Filter()
+        r = logging.makeLogRecord({'name': 'spam.eggs'})
+        self.assertTrue(f.filter(r))
 
 #
 #   First, we define our levels. There can be as many as you want - the only
@@ -498,6 +519,478 @@ class CustomLevelsAndFiltersTest(BaseTest):
             handler.removeFilter(garr)
 
 
+class HandlerTest(BaseTest):
+    def test_name(self):
+        h = logging.Handler()
+        h.name = 'generic'
+        self.assertEqual(h.name, 'generic')
+        h.name = 'anothergeneric'
+        self.assertEqual(h.name, 'anothergeneric')
+        self.assertRaises(NotImplementedError, h.emit, None)
+
+    def test_builtin_handlers(self):
+        # We can't actually *use* too many handlers in the tests,
+        # but we can try instantiating them with various options
+        if sys.platform in ('linux', 'darwin'):
+            for existing in (True, False):
+                fd, fn = tempfile.mkstemp()
+                os.close(fd)
+                if not existing:
+                    os.unlink(fn)
+                h = logging.handlers.WatchedFileHandler(fn, delay=True)
+                if existing:
+                    dev, ino = h.dev, h.ino
+                    self.assertEqual(dev, -1)
+                    self.assertEqual(ino, -1)
+                    r = logging.makeLogRecord({'msg': 'Test'})
+                    h.handle(r)
+                    # Now remove the file.
+                    os.unlink(fn)
+                    self.assertFalse(os.path.exists(fn))
+                    # The next call should recreate the file.
+                    h.handle(r)
+                    self.assertTrue(os.path.exists(fn))
+                else:
+                    self.assertEqual(h.dev, -1)
+                    self.assertEqual(h.ino, -1)
+                h.close()
+                if existing:
+                    os.unlink(fn)
+            if sys.platform == 'darwin':
+                sockname = '/var/run/syslog'
+            else:
+                sockname = '/dev/log'
+            try:
+                h = logging.handlers.SysLogHandler(sockname)
+                self.assertEqual(h.facility, h.LOG_USER)
+                self.assertTrue(h.unixsocket)
+                h.close()
+            except socket.error: # syslogd might not be available
+                pass
+        for method in ('GET', 'POST', 'PUT'):
+            if method == 'PUT':
+                self.assertRaises(ValueError, logging.handlers.HTTPHandler,
+                                  'localhost', '/log', method)
+            else:
+                h = logging.handlers.HTTPHandler('localhost', '/log', method)
+                h.close()
+        h = logging.handlers.BufferingHandler(0)
+        r = logging.makeLogRecord({})
+        self.assertTrue(h.shouldFlush(r))
+        h.close()
+        h = logging.handlers.BufferingHandler(1)
+        self.assertFalse(h.shouldFlush(r))
+        h.close()
+
+    @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.')
+    @unittest.skipUnless(threading, 'Threading required for this test.')
+    def test_race(self):
+        # Issue #14632 refers.
+        def remove_loop(fname, tries):
+            for _ in range(tries):
+                try:
+                    os.unlink(fname)
+                except OSError:
+                    pass
+                time.sleep(0.004 * random.randint(0, 4))
+
+        del_count = 500
+        log_count = 500
+
+        for delay in (False, True):
+            fd, fn = tempfile.mkstemp('.log', 'test_logging-3-')
+            os.close(fd)
+            remover = threading.Thread(target=remove_loop, args=(fn, del_count))
+            remover.daemon = True
+            remover.start()
+            h = logging.handlers.WatchedFileHandler(fn, delay=delay)
+            f = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s')
+            h.setFormatter(f)
+            try:
+                for _ in range(log_count):
+                    time.sleep(0.005)
+                    r = logging.makeLogRecord({'msg': 'testing' })
+                    h.handle(r)
+            finally:
+                remover.join()
+                h.close()
+                if os.path.exists(fn):
+                    os.unlink(fn)
+
+
+class BadStream(object):
+    def write(self, data):
+        raise RuntimeError('deliberate mistake')
+
+class TestStreamHandler(logging.StreamHandler):
+    def handleError(self, record):
+        self.error_record = record
+
+class StreamHandlerTest(BaseTest):
+    def test_error_handling(self):
+        h = TestStreamHandler(BadStream())
+        r = logging.makeLogRecord({})
+        old_raise = logging.raiseExceptions
+        old_stderr = sys.stderr
+        try:
+            h.handle(r)
+            self.assertIs(h.error_record, r)
+            h = logging.StreamHandler(BadStream())
+            sys.stderr = sio = io.StringIO()
+            h.handle(r)
+            self.assertIn('\nRuntimeError: deliberate mistake\n',
+                          sio.getvalue())
+            logging.raiseExceptions = False
+            sys.stderr = sio = io.StringIO()
+            h.handle(r)
+            self.assertEqual('', sio.getvalue())
+        finally:
+            logging.raiseExceptions = old_raise
+            sys.stderr = old_stderr
+
+# -- The following section could be moved into a server_helper.py module
+# -- if it proves to be of wider utility than just test_logging
+
+if threading:
+    class TestSMTPChannel(smtpd.SMTPChannel):
+        """
+        This derived class has had to be created because smtpd does not
+        support use of custom channel maps, although they are allowed by
+        asyncore's design. Issue #11959 has been raised to address this,
+        and if resolved satisfactorily, some of this code can be removed.
+        """
+        def __init__(self, server, conn, addr, sockmap):
+            asynchat.async_chat.__init__(self, conn, sockmap)
+            self.smtp_server = server
+            self.conn = conn
+            self.addr = addr
+            self.data_size_limit = None
+            self.received_lines = []
+            self.smtp_state = self.COMMAND
+            self.seen_greeting = ''
+            self.mailfrom = None
+            self.rcpttos = []
+            self.received_data = ''
+            self.fqdn = socket.getfqdn()
+            self.num_bytes = 0
+            try:
+                self.peer = conn.getpeername()
+            except socket.error as err:
+                # a race condition  may occur if the other end is closing
+                # before we can get the peername
+                self.close()
+                if err.args[0] != errno.ENOTCONN:
+                    raise
+                return
+            self.push('220 %s %s' % (self.fqdn, smtpd.__version__))
+            self.set_terminator(b'\r\n')
+            self.extended_smtp = False
+
+
+    class TestSMTPServer(smtpd.SMTPServer):
+        """
+        This class implements a test SMTP server.
+
+        :param addr: A (host, port) tuple which the server listens on.
+                     You can specify a port value of zero: the server's
+                     *port* attribute will hold the actual port number
+                     used, which can be used in client connections.
+        :param handler: A callable which will be called to process
+                        incoming messages. The handler will be passed
+                        the client address tuple, who the message is from,
+                        a list of recipients and the message data.
+        :param poll_interval: The interval, in seconds, used in the underlying
+                              :func:`select` or :func:`poll` call by
+                              :func:`asyncore.loop`.
+        :param sockmap: A dictionary which will be used to hold
+                        :class:`asyncore.dispatcher` instances used by
+                        :func:`asyncore.loop`. This avoids changing the
+                        :mod:`asyncore` module's global state.
+        """
+        channel_class = TestSMTPChannel
+
+        def __init__(self, addr, handler, poll_interval, sockmap):
+            self._localaddr = addr
+            self._remoteaddr = None
+            self.data_size_limit = None
+            self.sockmap = sockmap
+            asyncore.dispatcher.__init__(self, map=sockmap)
+            try:
+                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                sock.setblocking(0)
+                self.set_socket(sock, map=sockmap)
+                # try to re-use a server port if possible
+                self.set_reuse_addr()
+                self.bind(addr)
+                self.port = sock.getsockname()[1]
+                self.listen(5)
+            except:
+                self.close()
+                raise
+            self._handler = handler
+            self._thread = None
+            self.poll_interval = poll_interval
+
+        def handle_accepted(self, conn, addr):
+            """
+            Redefined only because the base class does not pass in a
+            map, forcing use of a global in :mod:`asyncore`.
+            """
+            channel = self.channel_class(self, conn, addr, self.sockmap)
+
+        def process_message(self, peer, mailfrom, rcpttos, data):
+            """
+            Delegates to the handler passed in to the server's constructor.
+
+            Typically, this will be a test case method.
+            :param peer: The client (host, port) tuple.
+            :param mailfrom: The address of the sender.
+            :param rcpttos: The addresses of the recipients.
+            :param data: The message.
+            """
+            self._handler(peer, mailfrom, rcpttos, data)
+
+        def start(self):
+            """
+            Start the server running on a separate daemon thread.
+            """
+            self._thread = t = threading.Thread(target=self.serve_forever,
+                                                args=(self.poll_interval,))
+            t.setDaemon(True)
+            t.start()
+
+        def serve_forever(self, poll_interval):
+            """
+            Run the :mod:`asyncore` loop until normal termination
+            conditions arise.
+            :param poll_interval: The interval, in seconds, used in the underlying
+                                  :func:`select` or :func:`poll` call by
+                                  :func:`asyncore.loop`.
+            """
+            try:
+                asyncore.loop(poll_interval, map=self.sockmap)
+            except select.error:
+                # On FreeBSD 8, closing the server repeatably
+                # raises this error. We swallow it if the
+                # server has been closed.
+                if self.connected or self.accepting:
+                    raise
+
+        def stop(self, timeout=None):
+            """
+            Stop the thread by closing the server instance.
+            Wait for the server thread to terminate.
+
+            :param timeout: How long to wait for the server thread
+                            to terminate.
+            """
+            self.close()
+            self._thread.join(timeout)
+            self._thread = None
+
+    class ControlMixin(object):
+        """
+        This mixin is used to start a server on a separate thread, and
+        shut it down programmatically. Request handling is simplified - instead
+        of needing to derive a suitable RequestHandler subclass, you just
+        provide a callable which will be passed each received request to be
+        processed.
+
+        :param handler: A handler callable which will be called with a
+                        single parameter - the request - in order to
+                        process the request. This handler is called on the
+                        server thread, effectively meaning that requests are
+                        processed serially. While not quite Web scale ;-),
+                        this should be fine for testing applications.
+        :param poll_interval: The polling interval in seconds.
+        """
+        def __init__(self, handler, poll_interval):
+            self._thread = None
+            self.poll_interval = poll_interval
+            self._handler = handler
+            self.ready = threading.Event()
+
+        def start(self):
+            """
+            Create a daemon thread to run the server, and start it.
+            """
+            self._thread = t = threading.Thread(target=self.serve_forever,
+                                                args=(self.poll_interval,))
+            t.setDaemon(True)
+            t.start()
+
+        def serve_forever(self, poll_interval):
+            """
+            Run the server. Set the ready flag before entering the
+            service loop.
+            """
+            self.ready.set()
+            super(ControlMixin, self).serve_forever(poll_interval)
+
+        def stop(self, timeout=None):
+            """
+            Tell the server thread to stop, and wait for it to do so.
+
+            :param timeout: How long to wait for the server thread
+                            to terminate.
+            """
+            self.shutdown()
+            if self._thread is not None:
+                self._thread.join(timeout)
+                self._thread = None
+            self.server_close()
+            self.ready.clear()
+
+    class TestHTTPServer(ControlMixin, HTTPServer):
+        """
+        An HTTP server which is controllable using :class:`ControlMixin`.
+
+        :param addr: A tuple with the IP address and port to listen on.
+        :param handler: A handler callable which will be called with a
+                        single parameter - the request - in order to
+                        process the request.
+        :param poll_interval: The polling interval in seconds.
+        :param log: Pass ``True`` to enable log messages.
+        """
+        def __init__(self, addr, handler, poll_interval=0.5,
+                     log=False, sslctx=None):
+            class DelegatingHTTPRequestHandler(BaseHTTPRequestHandler):
+                def __getattr__(self, name, default=None):
+                    if name.startswith('do_'):
+                        return self.process_request
+                    raise AttributeError(name)
+
+                def process_request(self):
+                    self.server._handler(self)
+
+                def log_message(self, format, *args):
+                    if log:
+                        super(DelegatingHTTPRequestHandler,
+                              self).log_message(format, *args)
+            HTTPServer.__init__(self, addr, DelegatingHTTPRequestHandler)
+            ControlMixin.__init__(self, handler, poll_interval)
+            self.sslctx = sslctx
+
+        def get_request(self):
+            try:
+                sock, addr = self.socket.accept()
+                if self.sslctx:
+                    sock = self.sslctx.wrap_socket(sock, server_side=True)
+            except socket.error as e:
+                # socket errors are silenced by the caller, print them here
+                sys.stderr.write("Got an error:\n%s\n" % e)
+                raise
+            return sock, addr
+
+    class TestTCPServer(ControlMixin, ThreadingTCPServer):
+        """
+        A TCP server which is controllable using :class:`ControlMixin`.
+
+        :param addr: A tuple with the IP address and port to listen on.
+        :param handler: A handler callable which will be called with a single
+                        parameter - the request - in order to process the request.
+        :param poll_interval: The polling interval in seconds.
+        :bind_and_activate: If True (the default), binds the server and starts it
+                            listening. If False, you need to call
+                            :meth:`server_bind` and :meth:`server_activate` at
+                            some later time before calling :meth:`start`, so that
+                            the server will set up the socket and listen on it.
+        """
+
+        allow_reuse_address = True
+
+        def __init__(self, addr, handler, poll_interval=0.5,
+                     bind_and_activate=True):
+            class DelegatingTCPRequestHandler(StreamRequestHandler):
+
+                def handle(self):
+                    self.server._handler(self)
+            ThreadingTCPServer.__init__(self, addr, DelegatingTCPRequestHandler,
+                                        bind_and_activate)
+            ControlMixin.__init__(self, handler, poll_interval)
+
+        def server_bind(self):
+            super(TestTCPServer, self).server_bind()
+            self.port = self.socket.getsockname()[1]
+
+    class TestUDPServer(ControlMixin, ThreadingUDPServer):
+        """
+        A UDP server which is controllable using :class:`ControlMixin`.
+
+        :param addr: A tuple with the IP address and port to listen on.
+        :param handler: A handler callable which will be called with a
+                        single parameter - the request - in order to
+                        process the request.
+        :param poll_interval: The polling interval for shutdown requests,
+                              in seconds.
+        :bind_and_activate: If True (the default), binds the server and
+                            starts it listening. If False, you need to
+                            call :meth:`server_bind` and
+                            :meth:`server_activate` at some later time
+                            before calling :meth:`start`, so that the server will
+                            set up the socket and listen on it.
+        """
+        def __init__(self, addr, handler, poll_interval=0.5,
+                     bind_and_activate=True):
+            class DelegatingUDPRequestHandler(DatagramRequestHandler):
+
+                def handle(self):
+                    self.server._handler(self)
+
+                def finish(self):
+                    data = self.wfile.getvalue()
+                    if data:
+                        try:
+                            super(DelegatingUDPRequestHandler, self).finish()
+                        except socket.error:
+                            if not self.server._closed:
+                                raise
+
+            ThreadingUDPServer.__init__(self, addr,
+                                        DelegatingUDPRequestHandler,
+                                        bind_and_activate)
+            ControlMixin.__init__(self, handler, poll_interval)
+            self._closed = False
+
+        def server_bind(self):
+            super(TestUDPServer, self).server_bind()
+            self.port = self.socket.getsockname()[1]
+
+        def server_close(self):
+            super(TestUDPServer, self).server_close()
+            self._closed = True
+
+# - end of server_helper section
+
+@unittest.skipUnless(threading, 'Threading required for this test.')
+class SMTPHandlerTest(BaseTest):
+    def test_basic(self):
+        sockmap = {}
+        server = TestSMTPServer(('localhost', 0), self.process_message, 0.001,
+                                sockmap)
+        server.start()
+        addr = ('localhost', server.port)
+        h = logging.handlers.SMTPHandler(addr, 'me', 'you', 'Log', timeout=5.0)
+        self.assertEqual(h.toaddrs, ['you'])
+        self.messages = []
+        r = logging.makeLogRecord({'msg': 'Hello'})
+        self.handled = threading.Event()
+        h.handle(r)
+        self.handled.wait(5.0)  # 14314: don't wait forever
+        server.stop()
+        self.assertTrue(self.handled.is_set())
+        self.assertEqual(len(self.messages), 1)
+        peer, mailfrom, rcpttos, data = self.messages[0]
+        self.assertEqual(mailfrom, 'me')
+        self.assertEqual(rcpttos, ['you'])
+        self.assertIn('\nSubject: Log\n', data)
+        self.assertTrue(data.endswith('\n\nHello'))
+        h.close()
+
+    def process_message(self, *args):
+        self.messages.append(args)
+        self.handled.set()
+
 class MemoryHandlerTest(BaseTest):
 
     """Tests for the MemoryHandler."""
@@ -525,7 +1018,7 @@ class MemoryHandlerTest(BaseTest):
         self.mem_logger.info(self.next_message())
         self.assert_log_lines([])
         # This will flush because the level is >= logging.WARNING
-        self.mem_logger.warn(self.next_message())
+        self.mem_logger.warning(self.next_message())
         lines = [
             ('DEBUG', '1'),
             ('INFO', '2'),
@@ -870,116 +1363,280 @@ class ConfigFileTest(BaseTest):
             # Original logger output is empty.
             self.assert_log_lines([])
 
-class LogRecordStreamHandler(StreamRequestHandler):
 
-    """Handler for a streaming logging request. It saves the log message in the
-    TCP server's 'log_output' attribute."""
+@unittest.skipUnless(threading, 'Threading required for this test.')
+class SocketHandlerTest(BaseTest):
+
+    """Test for SocketHandler objects."""
+
+    def setUp(self):
+        """Set up a TCP server to receive log messages, and a SocketHandler
+        pointing to that server's address and port."""
+        BaseTest.setUp(self)
+        addr = ('localhost', 0)
+        self.server = server = TestTCPServer(addr, self.handle_socket,
+                                                0.01)
+        server.start()
+        server.ready.wait()
+        self.sock_hdlr = logging.handlers.SocketHandler('localhost',
+                                                        server.port)
+        self.log_output = ''
+        self.root_logger.removeHandler(self.root_logger.handlers[0])
+        self.root_logger.addHandler(self.sock_hdlr)
+        self.handled = threading.Semaphore(0)
 
-    TCP_LOG_END = "!!!END!!!"
+    def tearDown(self):
+        """Shutdown the TCP server."""
+        try:
+            self.server.stop(2.0)
+            self.root_logger.removeHandler(self.sock_hdlr)
+            self.sock_hdlr.close()
+        finally:
+            BaseTest.tearDown(self)
 
-    def handle(self):
-        """Handle multiple requests - each expected to be of 4-byte length,
-        followed by the LogRecord in pickle format. Logs the record
-        according to whatever policy is configured locally."""
+    def handle_socket(self, request):
+        conn = request.connection
         while True:
-            chunk = self.connection.recv(4)
+            chunk = conn.recv(4)
             if len(chunk) < 4:
                 break
             slen = struct.unpack(">L", chunk)[0]
-            chunk = self.connection.recv(slen)
+            chunk = conn.recv(slen)
             while len(chunk) < slen:
-                chunk = chunk + self.connection.recv(slen - len(chunk))
-            obj = self.unpickle(chunk)
+                chunk = chunk + conn.recv(slen - len(chunk))
+            obj = pickle.loads(chunk)
             record = logging.makeLogRecord(obj)
-            self.handle_log_record(record)
+            self.log_output += record.msg + '\n'
+            self.handled.release()
+
+    def test_output(self):
+        # The log message sent to the SocketHandler is properly received.
+        logger = logging.getLogger("tcp")
+        logger.error("spam")
+        self.handled.acquire()
+        logger.debug("eggs")
+        self.handled.acquire()
+        self.assertEqual(self.log_output, "spam\neggs\n")
 
-    def unpickle(self, data):
-        return pickle.loads(data)
+    def test_noserver(self):
+        # Kill the server
+        self.server.stop(2.0)
+        #The logging call should try to connect, which should fail
+        try:
+            raise RuntimeError('Deliberate mistake')
+        except RuntimeError:
+            self.root_logger.exception('Never sent')
+        self.root_logger.error('Never sent, either')
+        now = time.time()
+        self.assertTrue(self.sock_hdlr.retryTime > now)
+        time.sleep(self.sock_hdlr.retryTime - now + 0.001)
+        self.root_logger.error('Nor this')
 
-    def handle_log_record(self, record):
-        # If the end-of-messages sentinel is seen, tell the server to
-        #  terminate.
-        if self.TCP_LOG_END in record.msg:
-            self.server.abort = 1
-            return
-        self.server.log_output += record.msg + "\n"
 
+@unittest.skipUnless(threading, 'Threading required for this test.')
+class DatagramHandlerTest(BaseTest):
 
-class LogRecordSocketReceiver(ThreadingTCPServer):
+    """Test for DatagramHandler."""
 
-    """A simple-minded TCP socket-based logging receiver suitable for test
-    purposes."""
+    def setUp(self):
+        """Set up a UDP server to receive log messages, and a DatagramHandler
+        pointing to that server's address and port."""
+        BaseTest.setUp(self)
+        addr = ('localhost', 0)
+        self.server = server = TestUDPServer(addr, self.handle_datagram, 0.01)
+        server.start()
+        server.ready.wait()
+        self.sock_hdlr = logging.handlers.DatagramHandler('localhost',
+                                                          server.port)
+        self.log_output = ''
+        self.root_logger.removeHandler(self.root_logger.handlers[0])
+        self.root_logger.addHandler(self.sock_hdlr)
+        self.handled = threading.Event()
 
-    allow_reuse_address = 1
-    log_output = ""
+    def tearDown(self):
+        """Shutdown the UDP server."""
+        try:
+            self.server.stop(2.0)
+            self.root_logger.removeHandler(self.sock_hdlr)
+            self.sock_hdlr.close()
+        finally:
+            BaseTest.tearDown(self)
 
-    def __init__(self, host='localhost',
-                             port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
-                     handler=LogRecordStreamHandler):
-        ThreadingTCPServer.__init__(self, (host, port), handler)
-        self.abort = False
-        self.timeout = 0.1
-        self.finished = threading.Event()
+    def handle_datagram(self, request):
+        slen = struct.pack('>L', 0) # length of prefix
+        packet = request.packet[len(slen):]
+        obj = pickle.loads(packet)
+        record = logging.makeLogRecord(obj)
+        self.log_output += record.msg + '\n'
+        self.handled.set()
 
-    def serve_until_stopped(self):
-        while not self.abort:
-            rd, wr, ex = select.select([self.socket.fileno()], [], [],
-                                       self.timeout)
-            if rd:
-                self.handle_request()
-        # Notify the main thread that we're about to exit
-        self.finished.set()
-        # close the listen socket
-        self.server_close()
+    def test_output(self):
+        # The log message sent to the DatagramHandler is properly received.
+        logger = logging.getLogger("udp")
+        logger.error("spam")
+        self.handled.wait()
+        self.handled.clear()
+        logger.error("eggs")
+        self.handled.wait()
+        self.assertEqual(self.log_output, "spam\neggs\n")
 
 
 @unittest.skipUnless(threading, 'Threading required for this test.')
-class SocketHandlerTest(BaseTest):
+class SysLogHandlerTest(BaseTest):
 
-    """Test for SocketHandler objects."""
+    """Test for SysLogHandler using UDP."""
 
     def setUp(self):
-        """Set up a TCP server to receive log messages, and a SocketHandler
+        """Set up a UDP server to receive log messages, and a SysLogHandler
         pointing to that server's address and port."""
         BaseTest.setUp(self)
-        self.tcpserver = LogRecordSocketReceiver(port=0)
-        self.port = self.tcpserver.socket.getsockname()[1]
-        self.threads = [
-                threading.Thread(target=self.tcpserver.serve_until_stopped)]
-        for thread in self.threads:
-            thread.start()
-
-        self.sock_hdlr = logging.handlers.SocketHandler('localhost', self.port)
-        self.sock_hdlr.setFormatter(self.root_formatter)
+        addr = ('localhost', 0)
+        self.server = server = TestUDPServer(addr, self.handle_datagram,
+                                                0.01)
+        server.start()
+        server.ready.wait()
+        self.sl_hdlr = logging.handlers.SysLogHandler(('localhost',
+                                                       server.port))
+        self.log_output = ''
         self.root_logger.removeHandler(self.root_logger.handlers[0])
-        self.root_logger.addHandler(self.sock_hdlr)
+        self.root_logger.addHandler(self.sl_hdlr)
+        self.handled = threading.Event()
 
     def tearDown(self):
-        """Shutdown the TCP server."""
+        """Shutdown the UDP server."""
         try:
-            self.tcpserver.abort = True
-            del self.tcpserver
-            self.root_logger.removeHandler(self.sock_hdlr)
-            self.sock_hdlr.close()
-            for thread in self.threads:
-                thread.join(2.0)
+            self.server.stop(2.0)
+            self.root_logger.removeHandler(self.sl_hdlr)
+            self.sl_hdlr.close()
         finally:
             BaseTest.tearDown(self)
 
-    def get_output(self):
-        """Get the log output as received by the TCP server."""
-        # Signal the TCP receiver and wait for it to terminate.
-        self.root_logger.critical(LogRecordStreamHandler.TCP_LOG_END)
-        self.tcpserver.finished.wait(2.0)
-        return self.tcpserver.log_output
+    def handle_datagram(self, request):
+        self.log_output = request.packet
+        self.handled.set()
 
     def test_output(self):
-        # The log message sent to the SocketHandler is properly received.
-        logger = logging.getLogger("tcp")
-        logger.error("spam")
-        logger.debug("eggs")
-        self.assertEqual(self.get_output(), "spam\neggs\n")
+        # The log message sent to the SysLogHandler is properly received.
+        logger = logging.getLogger("slh")
+        logger.error("sp\xe4m")
+        self.handled.wait()
+        self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m\x00')
+        self.handled.clear()
+        self.sl_hdlr.append_nul = False
+        logger.error("sp\xe4m")
+        self.handled.wait()
+        self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m')
+        self.handled.clear()
+        self.sl_hdlr.ident = "h\xe4m-"
+        logger.error("sp\xe4m")
+        self.handled.wait()
+        self.assertEqual(self.log_output, b'<11>h\xc3\xa4m-sp\xc3\xa4m')
+
+
+@unittest.skipUnless(threading, 'Threading required for this test.')
+class HTTPHandlerTest(BaseTest):
+    """Test for HTTPHandler."""
+
+    PEMFILE = """-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDGT4xS5r91rbLJQK2nUDenBhBG6qFk+bVOjuAGC/LSHlAoBnvG
+zQG3agOG+e7c5z2XT8m2ktORLqG3E4mYmbxgyhDrzP6ei2Anc+pszmnxPoK3Puh5
+aXV+XKt0bU0C1m2+ACmGGJ0t3P408art82nOxBw8ZHgIg9Dtp6xIUCyOqwIDAQAB
+AoGBAJFTnFboaKh5eUrIzjmNrKsG44jEyy+vWvHN/FgSC4l103HxhmWiuL5Lv3f7
+0tMp1tX7D6xvHwIG9VWvyKb/Cq9rJsDibmDVIOslnOWeQhG+XwJyitR0pq/KlJIB
+5LjORcBw795oKWOAi6RcOb1ON59tysEFYhAGQO9k6VL621gRAkEA/Gb+YXULLpbs
+piXN3q4zcHzeaVANo69tUZ6TjaQqMeTxE4tOYM0G0ZoSeHEdaP59AOZGKXXNGSQy
+2z/MddcYGQJBAMkjLSYIpOLJY11ja8OwwswFG2hEzHe0cS9bzo++R/jc1bHA5R0Y
+i6vA5iPi+wopPFvpytdBol7UuEBe5xZrxWMCQQCWxELRHiP2yWpEeLJ3gGDzoXMN
+PydWjhRju7Bx3AzkTtf+D6lawz1+eGTuEss5i0JKBkMEwvwnN2s1ce+EuF4JAkBb
+E96h1lAzkVW5OAfYOPY8RCPA90ZO/hoyg7PpSxR0ECuDrgERR8gXIeYUYfejBkEa
+rab4CfRoVJKKM28Yq/xZAkBvuq670JRCwOgfUTdww7WpdOQBYPkzQccsKNCslQW8
+/DyW6y06oQusSENUvynT6dr3LJxt/NgZPhZX2+k1eYDV
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICGzCCAYSgAwIBAgIJAIq84a2Q/OvlMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+BAMTCWxvY2FsaG9zdDAeFw0xMTA1MjExMDIzMzNaFw03NTAzMjEwMzU1MTdaMBQx
+EjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+xk+MUua/da2yyUCtp1A3pwYQRuqhZPm1To7gBgvy0h5QKAZ7xs0Bt2oDhvnu3Oc9
+l0/JtpLTkS6htxOJmJm8YMoQ68z+notgJ3PqbM5p8T6Ctz7oeWl1flyrdG1NAtZt
+vgAphhidLdz+NPGq7fNpzsQcPGR4CIPQ7aesSFAsjqsCAwEAAaN1MHMwHQYDVR0O
+BBYEFLWaUPO6N7efGiuoS9i3DVYcUwn0MEQGA1UdIwQ9MDuAFLWaUPO6N7efGiuo
+S9i3DVYcUwn0oRikFjAUMRIwEAYDVQQDEwlsb2NhbGhvc3SCCQCKvOGtkPzr5TAM
+BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAMK5whPjLNQK1Ivvk88oqJqq
+4f889OwikGP0eUhOBhbFlsZs+jq5YZC2UzHz+evzKBlgAP1u4lP/cB85CnjvWqM+
+1c/lywFHQ6HOdDeQ1L72tSYMrNOG4XNmLn0h7rx6GoTU7dcFRfseahBCq8mv0IDt
+IRbTpvlHWPjsSvHz0ZOH
+-----END CERTIFICATE-----"""
 
+    def setUp(self):
+        """Set up an HTTP server to receive log messages, and a HTTPHandler
+        pointing to that server's address and port."""
+        BaseTest.setUp(self)
+        self.handled = threading.Event()
+
+    def handle_request(self, request):
+        self.command = request.command
+        self.log_data = urlparse(request.path)
+        if self.command == 'POST':
+            try:
+                rlen = int(request.headers['Content-Length'])
+                self.post_data = request.rfile.read(rlen)
+            except:
+                self.post_data = None
+        request.send_response(200)
+        request.end_headers()
+        self.handled.set()
+
+    def test_output(self):
+        # The log message sent to the HTTPHandler is properly received.
+        logger = logging.getLogger("http")
+        root_logger = self.root_logger
+        root_logger.removeHandler(self.root_logger.handlers[0])
+        for secure in (False, True):
+            addr = ('localhost', 0)
+            if secure:
+                try:
+                    import ssl
+                    fd, fn = tempfile.mkstemp()
+                    os.close(fd)
+                    with open(fn, 'w') as f:
+                        f.write(self.PEMFILE)
+                    sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+                    sslctx.load_cert_chain(fn)
+                    os.unlink(fn)
+                except ImportError:
+                    sslctx = None
+            else:
+                sslctx = None
+            self.server = server = TestHTTPServer(addr, self.handle_request,
+                                                    0.01, sslctx=sslctx)
+            server.start()
+            server.ready.wait()
+            host = 'localhost:%d' % server.server_port
+            secure_client = secure and sslctx
+            self.h_hdlr = logging.handlers.HTTPHandler(host, '/frob',
+                                                       secure=secure_client)
+            self.log_data = None
+            root_logger.addHandler(self.h_hdlr)
+
+            for method in ('GET', 'POST'):
+                self.h_hdlr.method = method
+                self.handled.clear()
+                msg = "sp\xe4m"
+                logger.error(msg)
+                self.handled.wait()
+                self.assertEqual(self.log_data.path, '/frob')
+                self.assertEqual(self.command, method)
+                if method == 'GET':
+                    d = parse_qs(self.log_data.query)
+                else:
+                    d = parse_qs(self.post_data.decode('utf-8'))
+                self.assertEqual(d['name'], ['http'])
+                self.assertEqual(d['funcName'], ['test_output'])
+                self.assertEqual(d['msg'], [msg])
+
+            self.server.stop(2.0)
+            self.root_logger.removeHandler(self.h_hdlr)
+            self.h_hdlr.close()
 
 class MemoryTest(BaseTest):
 
@@ -1087,28 +1744,39 @@ class WarningsTest(BaseTest):
     def test_warnings(self):
         with warnings.catch_warnings():
             logging.captureWarnings(True)
-            try:
-                warnings.filterwarnings("always", category=UserWarning)
-                file = io.StringIO()
-                h = logging.StreamHandler(file)
-                logger = logging.getLogger("py.warnings")
-                logger.addHandler(h)
-                warnings.warn("I'm warning you...")
-                logger.removeHandler(h)
-                s = file.getvalue()
-                h.close()
-                self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0)
-
-                #See if an explicit file uses the original implementation
-                file = io.StringIO()
-                warnings.showwarning("Explicit", UserWarning, "dummy.py", 42,
-                                        file, "Dummy line")
-                s = file.getvalue()
-                file.close()
-                self.assertEqual(s,
-                        "dummy.py:42: UserWarning: Explicit\n  Dummy line\n")
-            finally:
-                logging.captureWarnings(False)
+            self.addCleanup(logging.captureWarnings, False)
+            warnings.filterwarnings("always", category=UserWarning)
+            stream = io.StringIO()
+            h = logging.StreamHandler(stream)
+            logger = logging.getLogger("py.warnings")
+            logger.addHandler(h)
+            warnings.warn("I'm warning you...")
+            logger.removeHandler(h)
+            s = stream.getvalue()
+            h.close()
+            self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0)
+
+            #See if an explicit file uses the original implementation
+            a_file = io.StringIO()
+            warnings.showwarning("Explicit", UserWarning, "dummy.py", 42,
+                                 a_file, "Dummy line")
+            s = a_file.getvalue()
+            a_file.close()
+            self.assertEqual(s,
+                "dummy.py:42: UserWarning: Explicit\n  Dummy line\n")
+
+    def test_warnings_no_handlers(self):
+        with warnings.catch_warnings():
+            logging.captureWarnings(True)
+            self.addCleanup(logging.captureWarnings, False)
+
+            # confirm our assumption: no loggers are set
+            logger = logging.getLogger("py.warnings")
+            self.assertEqual(logger.handlers, [])
+
+            warnings.showwarning("Explicit", UserWarning, "dummy.py", 42)
+            self.assertEqual(len(logger.handlers), 1)
+            self.assertIsInstance(logger.handlers[0], logging.NullHandler)
 
 
 def formatFunc(format, datefmt=None):
@@ -1961,6 +2629,7 @@ class ConfigDictTest(BaseTest):
             logging.config.stopListening()
             t.join(2.0)
 
+    @unittest.skipUnless(threading, 'Threading required for this test.')
     def test_listen_config_10_ok(self):
         with captured_stdout() as output:
             self.setup_via_listener(json.dumps(self.config10))
@@ -1980,6 +2649,7 @@ class ConfigDictTest(BaseTest):
                 ('ERROR', '4'),
             ], stream=output)
 
+    @unittest.skipUnless(threading, 'Threading required for this test.')
     def test_listen_config_1_ok(self):
         with captured_stdout() as output:
             self.setup_via_listener(textwrap.dedent(ConfigFileTest.config1))
@@ -1994,6 +2664,27 @@ class ConfigDictTest(BaseTest):
             # Original logger output is empty.
             self.assert_log_lines([])
 
+    def test_baseconfig(self):
+        d = {
+            'atuple': (1, 2, 3),
+            'alist': ['a', 'b', 'c'],
+            'adict': {'d': 'e', 'f': 3 },
+            'nest1': ('g', ('h', 'i'), 'j'),
+            'nest2': ['k', ['l', 'm'], 'n'],
+            'nest3': ['o', 'cfg://alist', 'p'],
+        }
+        bc = logging.config.BaseConfigurator(d)
+        self.assertEqual(bc.convert('cfg://atuple[1]'), 2)
+        self.assertEqual(bc.convert('cfg://alist[1]'), 'b')
+        self.assertEqual(bc.convert('cfg://nest1[1][0]'), 'h')
+        self.assertEqual(bc.convert('cfg://nest2[1][1]'), 'm')
+        self.assertEqual(bc.convert('cfg://adict.d'), 'e')
+        self.assertEqual(bc.convert('cfg://adict[f]'), 3)
+        v = bc.convert('cfg://nest3')
+        self.assertEqual(v.pop(1), ['a', 'b', 'c'])
+        self.assertRaises(KeyError, bc.convert, 'cfg://nosuch')
+        self.assertRaises(ValueError, bc.convert, 'cfg://!')
+        self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]')
 
 class ManagerTest(BaseTest):
     def test_manager_loggerclass(self):
@@ -2012,6 +2703,11 @@ class ManagerTest(BaseTest):
 
         self.assertEqual(logged, ['should appear in logged'])
 
+    def test_set_log_record_factory(self):
+        man = logging.Manager(None)
+        expected = object()
+        man.setLogRecordFactory(expected)
+        self.assertEqual(man.logRecordFactory, expected)
 
 class ChildLoggerTest(BaseTest):
     def test_child_loggers(self):
@@ -2113,6 +2809,18 @@ class QueueHandlerTest(BaseTest):
         self.assertTrue(handler.matches(levelno=logging.ERROR, message='2'))
         self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3'))
 
+ZERO = datetime.timedelta(0)
+
+class UTC(datetime.tzinfo):
+    def utcoffset(self, dt):
+        return ZERO
+
+    dst = utcoffset
+
+    def tzname(self, dt):
+        return 'UTC'
+
+utc = UTC()
 
 class FormatterTest(unittest.TestCase):
     def setUp(self):
@@ -2186,6 +2894,71 @@ class FormatterTest(unittest.TestCase):
         f = logging.Formatter('asctime', style='$')
         self.assertFalse(f.usesTime())
 
+    def test_invalid_style(self):
+        self.assertRaises(ValueError, logging.Formatter, None, None, 'x')
+
+    def test_time(self):
+        r = self.get_record()
+        dt = datetime.datetime(1993, 4, 21, 8, 3, 0, 0, utc)
+        # We use None to indicate we want the local timezone
+        # We're essentially converting a UTC time to local time
+        r.created = time.mktime(dt.astimezone(None).timetuple())
+        r.msecs = 123
+        f = logging.Formatter('%(asctime)s %(message)s')
+        f.converter = time.gmtime
+        self.assertEqual(f.formatTime(r), '1993-04-21 08:03:00,123')
+        self.assertEqual(f.formatTime(r, '%Y:%d'), '1993:21')
+        f.format(r)
+        self.assertEqual(r.asctime, '1993-04-21 08:03:00,123')
+
+class TestBufferingFormatter(logging.BufferingFormatter):
+    def formatHeader(self, records):
+        return '[(%d)' % len(records)
+
+    def formatFooter(self, records):
+        return '(%d)]' % len(records)
+
+class BufferingFormatterTest(unittest.TestCase):
+    def setUp(self):
+        self.records = [
+            logging.makeLogRecord({'msg': 'one'}),
+            logging.makeLogRecord({'msg': 'two'}),
+        ]
+
+    def test_default(self):
+        f = logging.BufferingFormatter()
+        self.assertEqual('', f.format([]))
+        self.assertEqual('onetwo', f.format(self.records))
+
+    def test_custom(self):
+        f = TestBufferingFormatter()
+        self.assertEqual('[(2)onetwo(2)]', f.format(self.records))
+        lf = logging.Formatter('<%(message)s>')
+        f = TestBufferingFormatter(lf)
+        self.assertEqual('[(2)<one><two>(2)]', f.format(self.records))
+
+class ExceptionTest(BaseTest):
+    def test_formatting(self):
+        r = self.root_logger
+        h = RecordingHandler()
+        r.addHandler(h)
+        try:
+            raise RuntimeError('deliberate mistake')
+        except:
+            logging.exception('failed', stack_info=True)
+        r.removeHandler(h)
+        h.close()
+        r = h.records[0]
+        self.assertTrue(r.exc_text.startswith('Traceback (most recent '
+                                              'call last):\n'))
+        self.assertTrue(r.exc_text.endswith('\nRuntimeError: '
+                                            'deliberate mistake'))
+        self.assertTrue(r.stack_info.startswith('Stack (most recent '
+                                              'call last):\n'))
+        self.assertTrue(r.stack_info.endswith('logging.exception(\'failed\', '
+                                            'stack_info=True)'))
+
+
 class LastResortTest(BaseTest):
     def test_last_resort(self):
         # Test the last resort handler
@@ -2196,6 +2969,8 @@ class LastResortTest(BaseTest):
         old_raise_exceptions = logging.raiseExceptions
         try:
             sys.stderr = sio = io.StringIO()
+            root.debug('This should not appear')
+            self.assertEqual(sio.getvalue(), '')
             root.warning('This is your final chance!')
             self.assertEqual(sio.getvalue(), 'This is your final chance!\n')
             #No handlers and no last resort, so 'No handlers' message
@@ -2220,152 +2995,236 @@ class LastResortTest(BaseTest):
             logging.raiseExceptions = old_raise_exceptions
 
 
-class BaseFileTest(BaseTest):
-    "Base class for handler tests that write log files"
+class FakeHandler:
+
+    def __init__(self, identifier, called):
+        for method in ('acquire', 'flush', 'close', 'release'):
+            setattr(self, method, self.record_call(identifier, method, called))
+
+    def record_call(self, identifier, method_name, called):
+        def inner():
+            called.append('{} - {}'.format(identifier, method_name))
+        return inner
+
+
+class RecordingHandler(logging.NullHandler):
+
+    def __init__(self, *args, **kwargs):
+        super(RecordingHandler, self).__init__(*args, **kwargs)
+        self.records = []
+
+    def handle(self, record):
+        """Keep track of all the emitted records."""
+        self.records.append(record)
+
+
+class ShutdownTest(BaseTest):
+
+    """Test suite for the shutdown method."""
 
     def setUp(self):
-        BaseTest.setUp(self)
-        fd, self.fn = tempfile.mkstemp(".log", "test_logging-2-")
-        os.close(fd)
-        self.rmfiles = []
+        super(ShutdownTest, self).setUp()
+        self.called = []
 
-    def tearDown(self):
-        for fn in self.rmfiles:
-            os.unlink(fn)
-        if os.path.exists(self.fn):
-            os.unlink(self.fn)
-        BaseTest.tearDown(self)
+        raise_exceptions = logging.raiseExceptions
+        self.addCleanup(setattr, logging, 'raiseExceptions', raise_exceptions)
 
-    def assertLogFile(self, filename):
-        "Assert a log file is there and register it for deletion"
-        self.assertTrue(os.path.exists(filename),
-                        msg="Log file %r does not exist")
-        self.rmfiles.append(filename)
+    def raise_error(self, error):
+        def inner():
+            raise error()
+        return inner
 
+    def test_no_failure(self):
+        # create some fake handlers
+        handler0 = FakeHandler(0, self.called)
+        handler1 = FakeHandler(1, self.called)
+        handler2 = FakeHandler(2, self.called)
 
-class RotatingFileHandlerTest(BaseFileTest):
-    def next_rec(self):
-        return logging.LogRecord('n', logging.DEBUG, 'p', 1,
-                                 self.next_message(), None, None, None)
+        # create live weakref to those handlers
+        handlers = map(logging.weakref.ref, [handler0, handler1, handler2])
 
-    def test_should_not_rollover(self):
-        # If maxbytes is zero rollover never occurs
-        rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=0)
-        self.assertFalse(rh.shouldRollover(None))
-        rh.close()
+        logging.shutdown(handlerList=list(handlers))
 
-    def test_should_rollover(self):
-        rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=1)
-        self.assertTrue(rh.shouldRollover(self.next_rec()))
-        rh.close()
+        expected = ['2 - acquire', '2 - flush', '2 - close', '2 - release',
+                    '1 - acquire', '1 - flush', '1 - close', '1 - release',
+                    '0 - acquire', '0 - flush', '0 - close', '0 - release']
+        self.assertEqual(expected, self.called)
 
-    def test_file_created(self):
-        # checks that the file is created and assumes it was created
-        # by us
-        rh = logging.handlers.RotatingFileHandler(self.fn)
-        rh.emit(self.next_rec())
-        self.assertLogFile(self.fn)
-        rh.close()
+    def _test_with_failure_in_method(self, method, error):
+        handler = FakeHandler(0, self.called)
+        setattr(handler, method, self.raise_error(error))
+        handlers = [logging.weakref.ref(handler)]
 
-    def test_rollover_filenames(self):
-        rh = logging.handlers.RotatingFileHandler(
-            self.fn, backupCount=2, maxBytes=1)
-        rh.emit(self.next_rec())
-        self.assertLogFile(self.fn)
-        rh.emit(self.next_rec())
-        self.assertLogFile(self.fn + ".1")
-        rh.emit(self.next_rec())
-        self.assertLogFile(self.fn + ".2")
-        self.assertFalse(os.path.exists(self.fn + ".3"))
-        rh.close()
+        logging.shutdown(handlerList=list(handlers))
 
-class TimedRotatingFileHandlerTest(BaseFileTest):
-    # test methods added below
-    pass
+        self.assertEqual('0 - release', self.called[-1])
 
-def secs(**kw):
-    return datetime.timedelta(**kw) // datetime.timedelta(seconds=1)
+    def test_with_ioerror_in_acquire(self):
+        self._test_with_failure_in_method('acquire', IOError)
 
-for when, exp in (('S', 1),
-                  ('M', 60),
-                  ('H', 60 * 60),
-                  ('D', 60 * 60 * 24),
-                  ('MIDNIGHT', 60 * 60 * 24),
-                  # current time (epoch start) is a Thursday, W0 means Monday
-                  ('W0', secs(days=4, hours=24)),
-                 ):
-    def test_compute_rollover(self, when=when, exp=exp):
-        rh = logging.handlers.TimedRotatingFileHandler(
-            self.fn, when=when, interval=1, backupCount=0, utc=True)
-        currentTime = 0.0
-        actual = rh.computeRollover(currentTime)
-        if exp != actual:
-            # Failures occur on some systems for MIDNIGHT and W0.
-            # Print detailed calculation for MIDNIGHT so we can try to see
-            # what's going on
-            if when == 'MIDNIGHT':
-                try:
-                    if rh.utc:
-                        t = time.gmtime(currentTime)
-                    else:
-                        t = time.localtime(currentTime)
-                    currentHour = t[3]
-                    currentMinute = t[4]
-                    currentSecond = t[5]
-                    # r is the number of seconds left between now and midnight
-                    r = logging.handlers._MIDNIGHT - ((currentHour * 60 +
-                                                       currentMinute) * 60 +
-                            currentSecond)
-                    result = currentTime + r
-                    print('t: %s (%s)' % (t, rh.utc), file=sys.stderr)
-                    print('currentHour: %s' % currentHour, file=sys.stderr)
-                    print('currentMinute: %s' % currentMinute, file=sys.stderr)
-                    print('currentSecond: %s' % currentSecond, file=sys.stderr)
-                    print('r: %s' % r, file=sys.stderr)
-                    print('result: %s' % result, file=sys.stderr)
-                except Exception:
-                    print('exception in diagnostic code: %s' % sys.exc_info()[1], file=sys.stderr)
-        self.assertEqual(exp, actual)
-        rh.close()
-    setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover)
+    def test_with_ioerror_in_flush(self):
+        self._test_with_failure_in_method('flush', IOError)
 
-class HandlerTest(BaseTest):
+    def test_with_ioerror_in_close(self):
+        self._test_with_failure_in_method('close', IOError)
 
-    @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.')
-    @unittest.skipUnless(threading, 'Threading required for this test.')
-    def test_race(self):
-        # Issue #14632 refers.
-        def remove_loop(fname, tries):
-            for _ in range(tries):
-                try:
-                    os.unlink(fname)
-                except OSError:
-                    pass
-                time.sleep(0.004 * random.randint(0, 4))
+    def test_with_valueerror_in_acquire(self):
+        self._test_with_failure_in_method('acquire', ValueError)
 
-        del_count = 500
-        log_count = 500
+    def test_with_valueerror_in_flush(self):
+        self._test_with_failure_in_method('flush', ValueError)
 
-        for delay in (False, True):
-            fd, fn = tempfile.mkstemp('.log', 'test_logging-3-')
-            os.close(fd)
-            remover = threading.Thread(target=remove_loop, args=(fn, del_count))
-            remover.daemon = True
-            remover.start()
-            h = logging.handlers.WatchedFileHandler(fn, delay=delay)
-            f = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s')
-            h.setFormatter(f)
-            try:
-                for _ in range(log_count):
-                    time.sleep(0.005)
-                    r = logging.makeLogRecord({'msg': 'testing' })
-                    h.handle(r)
-            finally:
-                remover.join()
-                h.close()
-                if os.path.exists(fn):
-                    os.unlink(fn)
+    def test_with_valueerror_in_close(self):
+        self._test_with_failure_in_method('close', ValueError)
+
+    def test_with_other_error_in_acquire_without_raise(self):
+        logging.raiseExceptions = False
+        self._test_with_failure_in_method('acquire', IndexError)
+
+    def test_with_other_error_in_flush_without_raise(self):
+        logging.raiseExceptions = False
+        self._test_with_failure_in_method('flush', IndexError)
+
+    def test_with_other_error_in_close_without_raise(self):
+        logging.raiseExceptions = False
+        self._test_with_failure_in_method('close', IndexError)
+
+    def test_with_other_error_in_acquire_with_raise(self):
+        logging.raiseExceptions = True
+        self.assertRaises(IndexError, self._test_with_failure_in_method,
+                          'acquire', IndexError)
 
+    def test_with_other_error_in_flush_with_raise(self):
+        logging.raiseExceptions = True
+        self.assertRaises(IndexError, self._test_with_failure_in_method,
+                          'flush', IndexError)
+
+    def test_with_other_error_in_close_with_raise(self):
+        logging.raiseExceptions = True
+        self.assertRaises(IndexError, self._test_with_failure_in_method,
+                          'close', IndexError)
+
+
+class ModuleLevelMiscTest(BaseTest):
+
+    """Test suite for some module level methods."""
+
+    def test_disable(self):
+        old_disable = logging.root.manager.disable
+        # confirm our assumptions are correct
+        self.assertEqual(old_disable, 0)
+        self.addCleanup(logging.disable, old_disable)
+
+        logging.disable(83)
+        self.assertEqual(logging.root.manager.disable, 83)
+
+    def _test_log(self, method, level=None):
+        called = []
+        patch(self, logging, 'basicConfig',
+              lambda *a, **kw: called.append((a, kw)))
+
+        recording = RecordingHandler()
+        logging.root.addHandler(recording)
+
+        log_method = getattr(logging, method)
+        if level is not None:
+            log_method(level, "test me: %r", recording)
+        else:
+            log_method("test me: %r", recording)
+
+        self.assertEqual(len(recording.records), 1)
+        record = recording.records[0]
+        self.assertEqual(record.getMessage(), "test me: %r" % recording)
+
+        expected_level = level if level is not None else getattr(logging, method.upper())
+        self.assertEqual(record.levelno, expected_level)
+
+        # basicConfig was not called!
+        self.assertEqual(called, [])
+
+    def test_log(self):
+        self._test_log('log', logging.ERROR)
+
+    def test_debug(self):
+        self._test_log('debug')
+
+    def test_info(self):
+        self._test_log('info')
+
+    def test_warning(self):
+        self._test_log('warning')
+
+    def test_error(self):
+        self._test_log('error')
+
+    def test_critical(self):
+        self._test_log('critical')
+
+    def test_set_logger_class(self):
+        self.assertRaises(TypeError, logging.setLoggerClass, object)
+
+        class MyLogger(logging.Logger):
+            pass
+
+        logging.setLoggerClass(MyLogger)
+        self.assertEqual(logging.getLoggerClass(), MyLogger)
+
+        logging.setLoggerClass(logging.Logger)
+        self.assertEqual(logging.getLoggerClass(), logging.Logger)
+
+class LogRecordTest(BaseTest):
+    def test_str_rep(self):
+        r = logging.makeLogRecord({})
+        s = str(r)
+        self.assertTrue(s.startswith('<LogRecord: '))
+        self.assertTrue(s.endswith('>'))
+
+    def test_dict_arg(self):
+        h = RecordingHandler()
+        r = logging.getLogger()
+        r.addHandler(h)
+        d = {'less' : 'more' }
+        logging.warning('less is %(less)s', d)
+        self.assertIs(h.records[0].args, d)
+        self.assertEqual(h.records[0].message, 'less is more')
+        r.removeHandler(h)
+        h.close()
+
+    def test_multiprocessing(self):
+        r = logging.makeLogRecord({})
+        self.assertEqual(r.processName, 'MainProcess')
+        try:
+            import multiprocessing as mp
+            r = logging.makeLogRecord({})
+            self.assertEqual(r.processName, mp.current_process().name)
+        except ImportError:
+            pass
+
+    def test_optional(self):
+        r = logging.makeLogRecord({})
+        NOT_NONE = self.assertIsNotNone
+        if threading:
+            NOT_NONE(r.thread)
+            NOT_NONE(r.threadName)
+        NOT_NONE(r.process)
+        NOT_NONE(r.processName)
+        log_threads = logging.logThreads
+        log_processes = logging.logProcesses
+        log_multiprocessing = logging.logMultiprocessing
+        try:
+            logging.logThreads = False
+            logging.logProcesses = False
+            logging.logMultiprocessing = False
+            r = logging.makeLogRecord({})
+            NONE = self.assertIsNone
+            NONE(r.thread)
+            NONE(r.threadName)
+            NONE(r.process)
+            NONE(r.processName)
+        finally:
+            logging.logThreads = log_threads
+            logging.logProcesses = log_processes
+            logging.logMultiprocessing = log_multiprocessing
 
 class BasicConfigTest(unittest.TestCase):
 
@@ -2471,6 +3330,17 @@ class BasicConfigTest(unittest.TestCase):
         logging.basicConfig(level=58)
         self.assertEqual(logging.root.level, 57)
 
+    def test_incompatible(self):
+        assertRaises = self.assertRaises
+        handlers = [logging.StreamHandler()]
+        stream = sys.stderr
+        assertRaises(ValueError, logging.basicConfig, filename='test.log',
+                                                     stream=stream)
+        assertRaises(ValueError, logging.basicConfig, filename='test.log',
+                                                     handlers=handlers)
+        assertRaises(ValueError, logging.basicConfig, stream=stream,
+                                                     handlers=handlers)
+
     def test_handlers(self):
         handlers = [
             logging.StreamHandler(),
@@ -2479,8 +3349,14 @@ class BasicConfigTest(unittest.TestCase):
         ]
         f = logging.Formatter()
         handlers[2].setFormatter(f)
-        self.assertRaises(ValueError, logging.basicConfig, level=logging.DEBUG,
-                          format='%(asctime)s %(message)s', handlers=handlers)
+        logging.basicConfig(handlers=handlers)
+        self.assertIs(handlers[0], logging.root.handlers[0])
+        self.assertIs(handlers[1], logging.root.handlers[1])
+        self.assertIs(handlers[2], logging.root.handlers[2])
+        self.assertIsNotNone(handlers[0].formatter)
+        self.assertIsNotNone(handlers[1].formatter)
+        self.assertIs(handlers[2].formatter, f)
+        self.assertIs(handlers[0].formatter, handlers[1].formatter)
 
     def _test_log(self, method, level=None):
         # logging.root has no handlers so basicConfig should be called
@@ -2523,20 +3399,445 @@ class BasicConfigTest(unittest.TestCase):
     def test_critical(self):
         self._test_log('critical')
 
+
+class LoggerAdapterTest(unittest.TestCase):
+
+    def setUp(self):
+        super(LoggerAdapterTest, self).setUp()
+        old_handler_list = logging._handlerList[:]
+
+        self.recording = RecordingHandler()
+        self.logger = logging.root
+        self.logger.addHandler(self.recording)
+        self.addCleanup(self.logger.removeHandler, self.recording)
+        self.addCleanup(self.recording.close)
+
+        def cleanup():
+            logging._handlerList[:] = old_handler_list
+
+        self.addCleanup(cleanup)
+        self.addCleanup(logging.shutdown)
+        self.adapter = logging.LoggerAdapter(logger=self.logger, extra=None)
+
+    def test_exception(self):
+        msg = 'testing exception: %r'
+        exc = None
+        try:
+            1 / 0
+        except ZeroDivisionError as e:
+            exc = e
+            self.adapter.exception(msg, self.recording)
+
+        self.assertEqual(len(self.recording.records), 1)
+        record = self.recording.records[0]
+        self.assertEqual(record.levelno, logging.ERROR)
+        self.assertEqual(record.msg, msg)
+        self.assertEqual(record.args, (self.recording,))
+        self.assertEqual(record.exc_info,
+                         (exc.__class__, exc, exc.__traceback__))
+
+    def test_critical(self):
+        msg = 'critical test! %r'
+        self.adapter.critical(msg, self.recording)
+
+        self.assertEqual(len(self.recording.records), 1)
+        record = self.recording.records[0]
+        self.assertEqual(record.levelno, logging.CRITICAL)
+        self.assertEqual(record.msg, msg)
+        self.assertEqual(record.args, (self.recording,))
+
+    def test_is_enabled_for(self):
+        old_disable = self.adapter.logger.manager.disable
+        self.adapter.logger.manager.disable = 33
+        self.addCleanup(setattr, self.adapter.logger.manager, 'disable',
+                        old_disable)
+        self.assertFalse(self.adapter.isEnabledFor(32))
+
+    def test_has_handlers(self):
+        self.assertTrue(self.adapter.hasHandlers())
+
+        for handler in self.logger.handlers:
+            self.logger.removeHandler(handler)
+
+        self.assertFalse(self.logger.hasHandlers())
+        self.assertFalse(self.adapter.hasHandlers())
+
+
+class LoggerTest(BaseTest):
+
+    def setUp(self):
+        super(LoggerTest, self).setUp()
+        self.recording = RecordingHandler()
+        self.logger = logging.Logger(name='blah')
+        self.logger.addHandler(self.recording)
+        self.addCleanup(self.logger.removeHandler, self.recording)
+        self.addCleanup(self.recording.close)
+        self.addCleanup(logging.shutdown)
+
+    def test_set_invalid_level(self):
+        self.assertRaises(TypeError, self.logger.setLevel, object())
+
+    def test_exception(self):
+        msg = 'testing exception: %r'
+        exc = None
+        try:
+            1 / 0
+        except ZeroDivisionError as e:
+            exc = e
+            self.logger.exception(msg, self.recording)
+
+        self.assertEqual(len(self.recording.records), 1)
+        record = self.recording.records[0]
+        self.assertEqual(record.levelno, logging.ERROR)
+        self.assertEqual(record.msg, msg)
+        self.assertEqual(record.args, (self.recording,))
+        self.assertEqual(record.exc_info,
+                         (exc.__class__, exc, exc.__traceback__))
+
+    def test_log_invalid_level_with_raise(self):
+        old_raise = logging.raiseExceptions
+        self.addCleanup(setattr, logging, 'raiseExecptions', old_raise)
+
+        logging.raiseExceptions = True
+        self.assertRaises(TypeError, self.logger.log, '10', 'test message')
+
+    def test_log_invalid_level_no_raise(self):
+        old_raise = logging.raiseExceptions
+        self.addCleanup(setattr, logging, 'raiseExecptions', old_raise)
+
+        logging.raiseExceptions = False
+        self.logger.log('10', 'test message')  # no exception happens
+
+    def test_find_caller_with_stack_info(self):
+        called = []
+        patch(self, logging.traceback, 'print_stack',
+              lambda f, file: called.append(file.getvalue()))
+
+        self.logger.findCaller(stack_info=True)
+
+        self.assertEqual(len(called), 1)
+        self.assertEqual('Stack (most recent call last):\n', called[0])
+
+    def test_make_record_with_extra_overwrite(self):
+        name = 'my record'
+        level = 13
+        fn = lno = msg = args = exc_info = func = sinfo = None
+        rv = logging._logRecordFactory(name, level, fn, lno, msg, args,
+                                       exc_info, func, sinfo)
+
+        for key in ('message', 'asctime') + tuple(rv.__dict__.keys()):
+            extra = {key: 'some value'}
+            self.assertRaises(KeyError, self.logger.makeRecord, name, level,
+                              fn, lno, msg, args, exc_info,
+                              extra=extra, sinfo=sinfo)
+
+    def test_make_record_with_extra_no_overwrite(self):
+        name = 'my record'
+        level = 13
+        fn = lno = msg = args = exc_info = func = sinfo = None
+        extra = {'valid_key': 'some value'}
+        result = self.logger.makeRecord(name, level, fn, lno, msg, args,
+                                        exc_info, extra=extra, sinfo=sinfo)
+        self.assertIn('valid_key', result.__dict__)
+
+    def test_has_handlers(self):
+        self.assertTrue(self.logger.hasHandlers())
+
+        for handler in self.logger.handlers:
+            self.logger.removeHandler(handler)
+        self.assertFalse(self.logger.hasHandlers())
+
+    def test_has_handlers_no_propagate(self):
+        child_logger = logging.getLogger('blah.child')
+        child_logger.propagate = False
+        self.assertFalse(child_logger.hasHandlers())
+
+    def test_is_enabled_for(self):
+        old_disable = self.logger.manager.disable
+        self.logger.manager.disable = 23
+        self.addCleanup(setattr, self.logger.manager, 'disable', old_disable)
+        self.assertFalse(self.logger.isEnabledFor(22))
+
+    def test_root_logger_aliases(self):
+        root = logging.getLogger()
+        self.assertIs(root, logging.root)
+        self.assertIs(root, logging.getLogger(None))
+        self.assertIs(root, logging.getLogger(''))
+        self.assertIs(root, logging.getLogger('foo').root)
+        self.assertIs(root, logging.getLogger('foo.bar').root)
+        self.assertIs(root, logging.getLogger('foo').parent)
+
+        self.assertIsNot(root, logging.getLogger('\0'))
+        self.assertIsNot(root, logging.getLogger('foo.bar').parent)
+
+    def test_invalid_names(self):
+        self.assertRaises(TypeError, logging.getLogger, any)
+        self.assertRaises(TypeError, logging.getLogger, b'foo')
+
+
+class BaseFileTest(BaseTest):
+    "Base class for handler tests that write log files"
+
+    def setUp(self):
+        BaseTest.setUp(self)
+        fd, self.fn = tempfile.mkstemp(".log", "test_logging-2-")
+        os.close(fd)
+        self.rmfiles = []
+
+    def tearDown(self):
+        for fn in self.rmfiles:
+            os.unlink(fn)
+        if os.path.exists(self.fn):
+            os.unlink(self.fn)
+        BaseTest.tearDown(self)
+
+    def assertLogFile(self, filename):
+        "Assert a log file is there and register it for deletion"
+        self.assertTrue(os.path.exists(filename),
+                        msg="Log file %r does not exist" % filename)
+        self.rmfiles.append(filename)
+
+
+class FileHandlerTest(BaseFileTest):
+    def test_delay(self):
+        os.unlink(self.fn)
+        fh = logging.FileHandler(self.fn, delay=True)
+        self.assertIsNone(fh.stream)
+        self.assertFalse(os.path.exists(self.fn))
+        fh.handle(logging.makeLogRecord({}))
+        self.assertIsNotNone(fh.stream)
+        self.assertTrue(os.path.exists(self.fn))
+        fh.close()
+
+class RotatingFileHandlerTest(BaseFileTest):
+    def next_rec(self):
+        return logging.LogRecord('n', logging.DEBUG, 'p', 1,
+                                 self.next_message(), None, None, None)
+
+    def test_should_not_rollover(self):
+        # If maxbytes is zero rollover never occurs
+        rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=0)
+        self.assertFalse(rh.shouldRollover(None))
+        rh.close()
+
+    def test_should_rollover(self):
+        rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=1)
+        self.assertTrue(rh.shouldRollover(self.next_rec()))
+        rh.close()
+
+    def test_file_created(self):
+        # checks that the file is created and assumes it was created
+        # by us
+        rh = logging.handlers.RotatingFileHandler(self.fn)
+        rh.emit(self.next_rec())
+        self.assertLogFile(self.fn)
+        rh.close()
+
+    def test_rollover_filenames(self):
+        def namer(name):
+            return name + ".test"
+        rh = logging.handlers.RotatingFileHandler(
+            self.fn, backupCount=2, maxBytes=1)
+        rh.namer = namer
+        rh.emit(self.next_rec())
+        self.assertLogFile(self.fn)
+        rh.emit(self.next_rec())
+        self.assertLogFile(namer(self.fn + ".1"))
+        rh.emit(self.next_rec())
+        self.assertLogFile(namer(self.fn + ".2"))
+        self.assertFalse(os.path.exists(namer(self.fn + ".3")))
+        rh.close()
+
+    @requires_zlib
+    def test_rotator(self):
+        def namer(name):
+            return name + ".gz"
+
+        def rotator(source, dest):
+            with open(source, "rb") as sf:
+                data = sf.read()
+                compressed = zlib.compress(data, 9)
+                with open(dest, "wb") as df:
+                    df.write(compressed)
+            os.remove(source)
+
+        rh = logging.handlers.RotatingFileHandler(
+            self.fn, backupCount=2, maxBytes=1)
+        rh.rotator = rotator
+        rh.namer = namer
+        m1 = self.next_rec()
+        rh.emit(m1)
+        self.assertLogFile(self.fn)
+        m2 = self.next_rec()
+        rh.emit(m2)
+        fn = namer(self.fn + ".1")
+        self.assertLogFile(fn)
+        newline = os.linesep
+        with open(fn, "rb") as f:
+            compressed = f.read()
+            data = zlib.decompress(compressed)
+            self.assertEqual(data.decode("ascii"), m1.msg + newline)
+        rh.emit(self.next_rec())
+        fn = namer(self.fn + ".2")
+        self.assertLogFile(fn)
+        with open(fn, "rb") as f:
+            compressed = f.read()
+            data = zlib.decompress(compressed)
+            self.assertEqual(data.decode("ascii"), m1.msg + newline)
+        rh.emit(self.next_rec())
+        fn = namer(self.fn + ".2")
+        with open(fn, "rb") as f:
+            compressed = f.read()
+            data = zlib.decompress(compressed)
+            self.assertEqual(data.decode("ascii"), m2.msg + newline)
+        self.assertFalse(os.path.exists(namer(self.fn + ".3")))
+        rh.close()
+
+class TimedRotatingFileHandlerTest(BaseFileTest):
+    # other test methods added below
+    def test_rollover(self):
+        fh = logging.handlers.TimedRotatingFileHandler(self.fn, 'S',
+                                                       backupCount=1)
+        fmt = logging.Formatter('%(asctime)s %(message)s')
+        fh.setFormatter(fmt)
+        r1 = logging.makeLogRecord({'msg': 'testing - initial'})
+        fh.emit(r1)
+        self.assertLogFile(self.fn)
+        time.sleep(1.1)    # a little over a second ...
+        r2 = logging.makeLogRecord({'msg': 'testing - after delay'})
+        fh.emit(r2)
+        fh.close()
+        # At this point, we should have a recent rotated file which we
+        # can test for the existence of. However, in practice, on some
+        # machines which run really slowly, we don't know how far back
+        # in time to go to look for the log file. So, we go back a fair
+        # bit, and stop as soon as we see a rotated file. In theory this
+        # could of course still fail, but the chances are lower.
+        found = False
+        now = datetime.datetime.now()
+        GO_BACK = 5 * 60 # seconds
+        for secs in range(GO_BACK):
+            prev = now - datetime.timedelta(seconds=secs)
+            fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S")
+            found = os.path.exists(fn)
+            if found:
+                self.rmfiles.append(fn)
+                break
+        msg = 'No rotated files found, went back %d seconds' % GO_BACK
+        if not found:
+            #print additional diagnostics
+            dn, fn = os.path.split(self.fn)
+            files = [f for f in os.listdir(dn) if f.startswith(fn)]
+            print('Test time: %s' % now.strftime("%Y-%m-%d %H-%M-%S"), file=sys.stderr)
+            print('The only matching files are: %s' % files, file=sys.stderr)
+            for f in files:
+                print('Contents of %s:' % f)
+                path = os.path.join(dn, f)
+                with open(path, 'r') as tf:
+                    print(tf.read())
+        self.assertTrue(found, msg=msg)
+
+    def test_invalid(self):
+        assertRaises = self.assertRaises
+        assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler,
+                     self.fn, 'X', delay=True)
+        assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler,
+                     self.fn, 'W', delay=True)
+        assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler,
+                     self.fn, 'W7', delay=True)
+
+def secs(**kw):
+    return datetime.timedelta(**kw) // datetime.timedelta(seconds=1)
+
+for when, exp in (('S', 1),
+                  ('M', 60),
+                  ('H', 60 * 60),
+                  ('D', 60 * 60 * 24),
+                  ('MIDNIGHT', 60 * 60 * 24),
+                  # current time (epoch start) is a Thursday, W0 means Monday
+                  ('W0', secs(days=4, hours=24)),
+                 ):
+    def test_compute_rollover(self, when=when, exp=exp):
+        rh = logging.handlers.TimedRotatingFileHandler(
+            self.fn, when=when, interval=1, backupCount=0, utc=True)
+        currentTime = 0.0
+        actual = rh.computeRollover(currentTime)
+        if exp != actual:
+            # Failures occur on some systems for MIDNIGHT and W0.
+            # Print detailed calculation for MIDNIGHT so we can try to see
+            # what's going on
+            if when == 'MIDNIGHT':
+                try:
+                    if rh.utc:
+                        t = time.gmtime(currentTime)
+                    else:
+                        t = time.localtime(currentTime)
+                    currentHour = t[3]
+                    currentMinute = t[4]
+                    currentSecond = t[5]
+                    # r is the number of seconds left between now and midnight
+                    r = logging.handlers._MIDNIGHT - ((currentHour * 60 +
+                                                       currentMinute) * 60 +
+                            currentSecond)
+                    result = currentTime + r
+                    print('t: %s (%s)' % (t, rh.utc), file=sys.stderr)
+                    print('currentHour: %s' % currentHour, file=sys.stderr)
+                    print('currentMinute: %s' % currentMinute, file=sys.stderr)
+                    print('currentSecond: %s' % currentSecond, file=sys.stderr)
+                    print('r: %s' % r, file=sys.stderr)
+                    print('result: %s' % result, file=sys.stderr)
+                except Exception:
+                    print('exception in diagnostic code: %s' % sys.exc_info()[1], file=sys.stderr)
+        self.assertEqual(exp, actual)
+        rh.close()
+    setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover)
+
+
+@unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil required for this test.')
+class NTEventLogHandlerTest(BaseTest):
+    def test_basic(self):
+        logtype = 'Application'
+        elh = win32evtlog.OpenEventLog(None, logtype)
+        num_recs = win32evtlog.GetNumberOfEventLogRecords(elh)
+        h = logging.handlers.NTEventLogHandler('test_logging')
+        r = logging.makeLogRecord({'msg': 'Test Log Message'})
+        h.handle(r)
+        h.close()
+        # Now see if the event is recorded
+        self.assertTrue(num_recs < win32evtlog.GetNumberOfEventLogRecords(elh))
+        flags = win32evtlog.EVENTLOG_BACKWARDS_READ | \
+                win32evtlog.EVENTLOG_SEQUENTIAL_READ
+        found = False
+        GO_BACK = 100
+        events = win32evtlog.ReadEventLog(elh, flags, GO_BACK)
+        for e in events:
+            if e.SourceName != 'test_logging':
+                continue
+            msg = win32evtlogutil.SafeFormatMessage(e, logtype)
+            if msg != 'Test Log Message\r\n':
+                continue
+            found = True
+            break
+        msg = 'Record not found in event log, went back %d records' % GO_BACK
+        self.assertTrue(found, msg=msg)
+
 # Set the locale to the platform-dependent default.  I have no idea
 # why the test does this, but in any case we save the current locale
 # first and restore it at the end.
 @run_with_locale('LC_ALL', '')
 def test_main():
     run_unittest(BuiltinLevelsTest, BasicFilterTest,
-                 CustomLevelsAndFiltersTest, MemoryHandlerTest,
-                 ConfigFileTest, SocketHandlerTest, MemoryTest,
-                 EncodingTest, WarningsTest, ConfigDictTest, ManagerTest,
-                 FormatterTest,
-                 LogRecordFactoryTest, ChildLoggerTest, QueueHandlerTest,
-                 RotatingFileHandlerTest,
-                 LastResortTest,
-                 TimedRotatingFileHandlerTest, HandlerTest, BasicConfigTest,
+                 CustomLevelsAndFiltersTest, HandlerTest, MemoryHandlerTest,
+                 ConfigFileTest, SocketHandlerTest, DatagramHandlerTest,
+                 MemoryTest, EncodingTest, WarningsTest, ConfigDictTest,
+                 ManagerTest, FormatterTest, BufferingFormatterTest,
+                 StreamHandlerTest, LogRecordFactoryTest, ChildLoggerTest,
+                 QueueHandlerTest, ShutdownTest, ModuleLevelMiscTest,
+                 BasicConfigTest, LoggerAdapterTest, LoggerTest,
+                 SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest,
+                 LastResortTest, LogRecordTest, ExceptionTest,
+                 SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest,
+                 TimedRotatingFileHandlerTest
                 )
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py
index 3ebc1f7fed..b417bea215 100644
--- a/Lib/test/test_long.py
+++ b/Lib/test/test_long.py
@@ -43,6 +43,53 @@ DBL_MIN_EXP = sys.float_info.min_exp
 DBL_MANT_DIG = sys.float_info.mant_dig
 DBL_MIN_OVERFLOW = 2**DBL_MAX_EXP - 2**(DBL_MAX_EXP - DBL_MANT_DIG - 1)
 
+
+# Pure Python version of correctly-rounded integer-to-float conversion.
+def int_to_float(n):
+    """
+    Correctly-rounded integer-to-float conversion.
+
+    """
+    # Constants, depending only on the floating-point format in use.
+    # We use an extra 2 bits of precision for rounding purposes.
+    PRECISION = sys.float_info.mant_dig + 2
+    SHIFT_MAX = sys.float_info.max_exp - PRECISION
+    Q_MAX = 1 << PRECISION
+    ROUND_HALF_TO_EVEN_CORRECTION = [0, -1, -2, 1, 0, -1, 2, 1]
+
+    # Reduce to the case where n is positive.
+    if n == 0:
+        return 0.0
+    elif n < 0:
+        return -int_to_float(-n)
+
+    # Convert n to a 'floating-point' number q * 2**shift, where q is an
+    # integer with 'PRECISION' significant bits.  When shifting n to create q,
+    # the least significant bit of q is treated as 'sticky'.  That is, the
+    # least significant bit of q is set if either the corresponding bit of n
+    # was already set, or any one of the bits of n lost in the shift was set.
+    shift = n.bit_length() - PRECISION
+    q = n << -shift if shift < 0 else (n >> shift) | bool(n & ~(-1 << shift))
+
+    # Round half to even (actually rounds to the nearest multiple of 4,
+    # rounding ties to a multiple of 8).
+    q += ROUND_HALF_TO_EVEN_CORRECTION[q & 7]
+
+    # Detect overflow.
+    if shift + (q == Q_MAX) > SHIFT_MAX:
+        raise OverflowError("integer too large to convert to float")
+
+    # Checks: q is exactly representable, and q**2**shift doesn't overflow.
+    assert q % 4 == 0 and q // 4 <= 2**(sys.float_info.mant_dig)
+    assert q * 2**shift <= sys.float_info.max
+
+    # Some circularity here, since float(q) is doing an int-to-float
+    # conversion.  But here q is of bounded size, and is exactly representable
+    # as a float.  In a low-level C-like language, this operation would be a
+    # simple cast (e.g., from unsigned long long to double).
+    return math.ldexp(float(q), shift)
+
+
 # pure Python version of correctly-rounded true division
 def truediv(a, b):
     """Correctly-rounded true division for integers."""
@@ -367,6 +414,23 @@ class LongTest(unittest.TestCase):
                 return 1729
         self.assertEqual(int(LongTrunc()), 1729)
 
+    def check_float_conversion(self, n):
+        # Check that int -> float conversion behaviour matches
+        # that of the pure Python version above.
+        try:
+            actual = float(n)
+        except OverflowError:
+            actual = 'overflow'
+
+        try:
+            expected = int_to_float(n)
+        except OverflowError:
+            expected = 'overflow'
+
+        msg = ("Error in conversion of integer {} to float.  "
+               "Got {}, expected {}.".format(n, actual, expected))
+        self.assertEqual(actual, expected, msg)
+
     @support.requires_IEEE_754
     def test_float_conversion(self):
 
@@ -421,6 +485,22 @@ class LongTest(unittest.TestCase):
             y = 2**p * 2**53
             self.assertEqual(int(float(x)), y)
 
+        # Compare builtin float conversion with pure Python int_to_float
+        # function above.
+        test_values = [
+            int_dbl_max-1, int_dbl_max, int_dbl_max+1,
+            halfway-1, halfway, halfway + 1,
+            top_power-1, top_power, top_power+1,
+            2*top_power-1, 2*top_power, top_power*top_power,
+        ]
+        test_values.extend(exact_values)
+        for p in range(-4, 8):
+            for x in range(-128, 128):
+                test_values.append(2**(p+53) + x)
+        for value in test_values:
+            self.check_float_conversion(value)
+            self.check_float_conversion(-value)
+
     def test_float_overflow(self):
         for x in -2.0, -1.0, 0.0, 1.0, 2.0:
             self.assertEqual(float(int(x)), x)
diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py
new file mode 100644
index 0000000000..20d85828e6
--- /dev/null
+++ b/Lib/test/test_lzma.py
@@ -0,0 +1,1531 @@
+from io import BytesIO, UnsupportedOperation
+import os
+import random
+import unittest
+
+from test.support import (
+    _4G, TESTFN, import_module, bigmemtest, run_unittest, unlink
+)
+
+lzma = import_module("lzma")
+from lzma import LZMACompressor, LZMADecompressor, LZMAError, LZMAFile
+
+
+class CompressorDecompressorTestCase(unittest.TestCase):
+
+    # Test error cases.
+
+    def test_simple_bad_args(self):
+        self.assertRaises(TypeError, LZMACompressor, [])
+        self.assertRaises(TypeError, LZMACompressor, format=3.45)
+        self.assertRaises(TypeError, LZMACompressor, check="")
+        self.assertRaises(TypeError, LZMACompressor, preset="asdf")
+        self.assertRaises(TypeError, LZMACompressor, filters=3)
+        # Can't specify FORMAT_AUTO when compressing.
+        self.assertRaises(ValueError, LZMACompressor, format=lzma.FORMAT_AUTO)
+        # Can't specify a preset and a custom filter chain at the same time.
+        with self.assertRaises(ValueError):
+            LZMACompressor(preset=7, filters=[{"id": lzma.FILTER_LZMA2}])
+
+        self.assertRaises(TypeError, LZMADecompressor, ())
+        self.assertRaises(TypeError, LZMADecompressor, memlimit=b"qw")
+        with self.assertRaises(TypeError):
+            LZMADecompressor(lzma.FORMAT_RAW, filters="zzz")
+        # Cannot specify a memory limit with FILTER_RAW.
+        with self.assertRaises(ValueError):
+            LZMADecompressor(lzma.FORMAT_RAW, memlimit=0x1000000)
+        # Can only specify a custom filter chain with FILTER_RAW.
+        self.assertRaises(ValueError, LZMADecompressor, filters=FILTERS_RAW_1)
+        with self.assertRaises(ValueError):
+            LZMADecompressor(format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1)
+        with self.assertRaises(ValueError):
+            LZMADecompressor(format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1)
+
+        lzc = LZMACompressor()
+        self.assertRaises(TypeError, lzc.compress)
+        self.assertRaises(TypeError, lzc.compress, b"foo", b"bar")
+        self.assertRaises(TypeError, lzc.flush, b"blah")
+        empty = lzc.flush()
+        self.assertRaises(ValueError, lzc.compress, b"quux")
+        self.assertRaises(ValueError, lzc.flush)
+
+        lzd = LZMADecompressor()
+        self.assertRaises(TypeError, lzd.decompress)
+        self.assertRaises(TypeError, lzd.decompress, b"foo", b"bar")
+        lzd.decompress(empty)
+        self.assertRaises(EOFError, lzd.decompress, b"quux")
+
+    def test_bad_filter_spec(self):
+        self.assertRaises(TypeError, LZMACompressor, filters=[b"wobsite"])
+        self.assertRaises(ValueError, LZMACompressor, filters=[{"xyzzy": 3}])
+        self.assertRaises(ValueError, LZMACompressor, filters=[{"id": 98765}])
+        with self.assertRaises(ValueError):
+            LZMACompressor(filters=[{"id": lzma.FILTER_LZMA2, "foo": 0}])
+        with self.assertRaises(ValueError):
+            LZMACompressor(filters=[{"id": lzma.FILTER_DELTA, "foo": 0}])
+        with self.assertRaises(ValueError):
+            LZMACompressor(filters=[{"id": lzma.FILTER_X86, "foo": 0}])
+
+    def test_decompressor_after_eof(self):
+        lzd = LZMADecompressor()
+        lzd.decompress(COMPRESSED_XZ)
+        self.assertRaises(EOFError, lzd.decompress, b"nyan")
+
+    def test_decompressor_memlimit(self):
+        lzd = LZMADecompressor(memlimit=1024)
+        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)
+
+        lzd = LZMADecompressor(lzma.FORMAT_XZ, memlimit=1024)
+        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)
+
+        lzd = LZMADecompressor(lzma.FORMAT_ALONE, memlimit=1024)
+        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE)
+
+    # Test LZMADecompressor on known-good input data.
+
+    def _test_decompressor(self, lzd, data, check, unused_data=b""):
+        self.assertFalse(lzd.eof)
+        out = lzd.decompress(data)
+        self.assertEqual(out, INPUT)
+        self.assertEqual(lzd.check, check)
+        self.assertTrue(lzd.eof)
+        self.assertEqual(lzd.unused_data, unused_data)
+
+    def test_decompressor_auto(self):
+        lzd = LZMADecompressor()
+        self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64)
+
+        lzd = LZMADecompressor()
+        self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE)
+
+    def test_decompressor_xz(self):
+        lzd = LZMADecompressor(lzma.FORMAT_XZ)
+        self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64)
+
+    def test_decompressor_alone(self):
+        lzd = LZMADecompressor(lzma.FORMAT_ALONE)
+        self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE)
+
+    def test_decompressor_raw_1(self):
+        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
+        self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE)
+
+    def test_decompressor_raw_2(self):
+        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2)
+        self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE)
+
+    def test_decompressor_raw_3(self):
+        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3)
+        self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE)
+
+    def test_decompressor_raw_4(self):
+        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
+        self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE)
+
+    def test_decompressor_chunks(self):
+        lzd = LZMADecompressor()
+        out = []
+        for i in range(0, len(COMPRESSED_XZ), 10):
+            self.assertFalse(lzd.eof)
+            out.append(lzd.decompress(COMPRESSED_XZ[i:i+10]))
+        out = b"".join(out)
+        self.assertEqual(out, INPUT)
+        self.assertEqual(lzd.check, lzma.CHECK_CRC64)
+        self.assertTrue(lzd.eof)
+        self.assertEqual(lzd.unused_data, b"")
+
+    def test_decompressor_unused_data(self):
+        lzd = LZMADecompressor()
+        extra = b"fooblibar"
+        self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64,
+                                unused_data=extra)
+
+    def test_decompressor_bad_input(self):
+        lzd = LZMADecompressor()
+        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1)
+
+        lzd = LZMADecompressor(lzma.FORMAT_XZ)
+        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE)
+
+        lzd = LZMADecompressor(lzma.FORMAT_ALONE)
+        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)
+
+        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
+        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)
+
+    # Test that LZMACompressor->LZMADecompressor preserves the input data.
+
+    def test_roundtrip_xz(self):
+        lzc = LZMACompressor()
+        cdata = lzc.compress(INPUT) + lzc.flush()
+        lzd = LZMADecompressor()
+        self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64)
+
+    def test_roundtrip_alone(self):
+        lzc = LZMACompressor(lzma.FORMAT_ALONE)
+        cdata = lzc.compress(INPUT) + lzc.flush()
+        lzd = LZMADecompressor()
+        self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
+
+    def test_roundtrip_raw(self):
+        lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
+        cdata = lzc.compress(INPUT) + lzc.flush()
+        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
+        self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
+
+    def test_roundtrip_chunks(self):
+        lzc = LZMACompressor()
+        cdata = []
+        for i in range(0, len(INPUT), 10):
+            cdata.append(lzc.compress(INPUT[i:i+10]))
+        cdata.append(lzc.flush())
+        cdata = b"".join(cdata)
+        lzd = LZMADecompressor()
+        self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64)
+
+    # LZMADecompressor intentionally does not handle concatenated streams.
+
+    def test_decompressor_multistream(self):
+        lzd = LZMADecompressor()
+        self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE,
+                                lzma.CHECK_CRC64, unused_data=COMPRESSED_ALONE)
+
+    # Test with inputs larger than 4GiB.
+
+    @bigmemtest(size=_4G + 100, memuse=2)
+    def test_compressor_bigmem(self, size):
+        lzc = LZMACompressor()
+        cdata = lzc.compress(b"x" * size) + lzc.flush()
+        ddata = lzma.decompress(cdata)
+        try:
+            self.assertEqual(len(ddata), size)
+            self.assertEqual(len(ddata.strip(b"x")), 0)
+        finally:
+            ddata = None
+
+    @bigmemtest(size=_4G + 100, memuse=3)
+    def test_decompressor_bigmem(self, size):
+        lzd = LZMADecompressor()
+        blocksize = 10 * 1024 * 1024
+        block = random.getrandbits(blocksize * 8).to_bytes(blocksize, "little")
+        try:
+            input = block * (size // blocksize + 1)
+            cdata = lzma.compress(input)
+            ddata = lzd.decompress(cdata)
+            self.assertEqual(ddata, input)
+        finally:
+            input = cdata = ddata = None
+
+
+class CompressDecompressFunctionTestCase(unittest.TestCase):
+
+    # Test error cases:
+
+    def test_bad_args(self):
+        self.assertRaises(TypeError, lzma.compress)
+        self.assertRaises(TypeError, lzma.compress, [])
+        self.assertRaises(TypeError, lzma.compress, b"", format="xz")
+        self.assertRaises(TypeError, lzma.compress, b"", check="none")
+        self.assertRaises(TypeError, lzma.compress, b"", preset="blah")
+        self.assertRaises(TypeError, lzma.compress, b"", filters=1024)
+        # Can't specify a preset and a custom filter chain at the same time.
+        with self.assertRaises(ValueError):
+            lzma.compress(b"", preset=3, filters=[{"id": lzma.FILTER_LZMA2}])
+
+        self.assertRaises(TypeError, lzma.decompress)
+        self.assertRaises(TypeError, lzma.decompress, [])
+        self.assertRaises(TypeError, lzma.decompress, b"", format="lzma")
+        self.assertRaises(TypeError, lzma.decompress, b"", memlimit=7.3e9)
+        with self.assertRaises(TypeError):
+            lzma.decompress(b"", format=lzma.FORMAT_RAW, filters={})
+        # Cannot specify a memory limit with FILTER_RAW.
+        with self.assertRaises(ValueError):
+            lzma.decompress(b"", format=lzma.FORMAT_RAW, memlimit=0x1000000)
+        # Can only specify a custom filter chain with FILTER_RAW.
+        with self.assertRaises(ValueError):
+            lzma.decompress(b"", filters=FILTERS_RAW_1)
+        with self.assertRaises(ValueError):
+            lzma.decompress(b"", format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1)
+        with self.assertRaises(ValueError):
+            lzma.decompress(
+                    b"", format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1)
+
+    def test_decompress_memlimit(self):
+        with self.assertRaises(LZMAError):
+            lzma.decompress(COMPRESSED_XZ, memlimit=1024)
+        with self.assertRaises(LZMAError):
+            lzma.decompress(
+                    COMPRESSED_XZ, format=lzma.FORMAT_XZ, memlimit=1024)
+        with self.assertRaises(LZMAError):
+            lzma.decompress(
+                    COMPRESSED_ALONE, format=lzma.FORMAT_ALONE, memlimit=1024)
+
+    # Test LZMADecompressor on known-good input data.
+
+    def test_decompress_good_input(self):
+        ddata = lzma.decompress(COMPRESSED_XZ)
+        self.assertEqual(ddata, INPUT)
+
+        ddata = lzma.decompress(COMPRESSED_ALONE)
+        self.assertEqual(ddata, INPUT)
+
+        ddata = lzma.decompress(COMPRESSED_XZ, lzma.FORMAT_XZ)
+        self.assertEqual(ddata, INPUT)
+
+        ddata = lzma.decompress(COMPRESSED_ALONE, lzma.FORMAT_ALONE)
+        self.assertEqual(ddata, INPUT)
+
+        ddata = lzma.decompress(
+                COMPRESSED_RAW_1, lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
+        self.assertEqual(ddata, INPUT)
+
+        ddata = lzma.decompress(
+                COMPRESSED_RAW_2, lzma.FORMAT_RAW, filters=FILTERS_RAW_2)
+        self.assertEqual(ddata, INPUT)
+
+        ddata = lzma.decompress(
+                COMPRESSED_RAW_3, lzma.FORMAT_RAW, filters=FILTERS_RAW_3)
+        self.assertEqual(ddata, INPUT)
+
+        ddata = lzma.decompress(
+                COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
+        self.assertEqual(ddata, INPUT)
+
+    def test_decompress_incomplete_input(self):
+        self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128])
+        self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128])
+        self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_1[:128],
+                          format=lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
+        self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_2[:128],
+                          format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2)
+        self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_3[:128],
+                          format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3)
+        self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128],
+                          format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
+
+    def test_decompress_bad_input(self):
+        with self.assertRaises(LZMAError):
+            lzma.decompress(COMPRESSED_RAW_1)
+        with self.assertRaises(LZMAError):
+            lzma.decompress(COMPRESSED_ALONE, format=lzma.FORMAT_XZ)
+        with self.assertRaises(LZMAError):
+            lzma.decompress(COMPRESSED_XZ, format=lzma.FORMAT_ALONE)
+        with self.assertRaises(LZMAError):
+            lzma.decompress(COMPRESSED_XZ, format=lzma.FORMAT_RAW,
+                            filters=FILTERS_RAW_1)
+
+    # Test that compress()->decompress() preserves the input data.
+
+    def test_roundtrip(self):
+        cdata = lzma.compress(INPUT)
+        ddata = lzma.decompress(cdata)
+        self.assertEqual(ddata, INPUT)
+
+        cdata = lzma.compress(INPUT, lzma.FORMAT_XZ)
+        ddata = lzma.decompress(cdata)
+        self.assertEqual(ddata, INPUT)
+
+        cdata = lzma.compress(INPUT, lzma.FORMAT_ALONE)
+        ddata = lzma.decompress(cdata)
+        self.assertEqual(ddata, INPUT)
+
+        cdata = lzma.compress(INPUT, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
+        ddata = lzma.decompress(cdata, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
+        self.assertEqual(ddata, INPUT)
+
+    # Unlike LZMADecompressor, decompress() *does* handle concatenated streams.
+
+    def test_decompress_multistream(self):
+        ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_ALONE)
+        self.assertEqual(ddata, INPUT * 2)
+
+
+class TempFile:
+    """Context manager - creates a file, and deletes it on __exit__."""
+
+    def __init__(self, filename, data=b""):
+        self.filename = filename
+        self.data = data
+
+    def __enter__(self):
+        with open(self.filename, "wb") as f:
+            f.write(self.data)
+
+    def __exit__(self, *args):
+        unlink(self.filename)
+
+
+class FileTestCase(unittest.TestCase):
+
+    def test_init(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            pass
+        with LZMAFile(BytesIO(), "w") as f:
+            pass
+        with LZMAFile(BytesIO(), "a") as f:
+            pass
+
+    def test_init_with_filename(self):
+        with TempFile(TESTFN, COMPRESSED_XZ):
+            with LZMAFile(TESTFN) as f:
+                pass
+            with LZMAFile(TESTFN, "w") as f:
+                pass
+            with LZMAFile(TESTFN, "a") as f:
+                pass
+
+    def test_init_mode(self):
+        with TempFile(TESTFN):
+            with LZMAFile(TESTFN, "r"):
+                pass
+            with LZMAFile(TESTFN, "rb"):
+                pass
+            with LZMAFile(TESTFN, "w"):
+                pass
+            with LZMAFile(TESTFN, "wb"):
+                pass
+            with LZMAFile(TESTFN, "a"):
+                pass
+            with LZMAFile(TESTFN, "ab"):
+                pass
+
+    def test_init_bad_mode(self):
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), (3, "x"))
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), "")
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), "x")
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), "rt")
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), "r+")
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), "wt")
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), "w+")
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), "rw")
+
+    def test_init_bad_check(self):
+        with self.assertRaises(TypeError):
+            LZMAFile(BytesIO(), "w", check=b"asd")
+        # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid.
+        with self.assertRaises(LZMAError):
+            LZMAFile(BytesIO(), "w", check=lzma.CHECK_UNKNOWN)
+        with self.assertRaises(LZMAError):
+            LZMAFile(BytesIO(), "w", check=lzma.CHECK_ID_MAX + 3)
+        # Cannot specify a check with mode="r".
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_NONE)
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_CRC32)
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_CRC64)
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_SHA256)
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_UNKNOWN)
+
+    def test_init_bad_preset(self):
+        with self.assertRaises(TypeError):
+            LZMAFile(BytesIO(), "w", preset=4.39)
+        with self.assertRaises(LZMAError):
+            LZMAFile(BytesIO(), "w", preset=10)
+        with self.assertRaises(LZMAError):
+            LZMAFile(BytesIO(), "w", preset=23)
+        with self.assertRaises(OverflowError):
+            LZMAFile(BytesIO(), "w", preset=-1)
+        with self.assertRaises(OverflowError):
+            LZMAFile(BytesIO(), "w", preset=-7)
+        with self.assertRaises(TypeError):
+            LZMAFile(BytesIO(), "w", preset="foo")
+        # Cannot specify a preset with mode="r".
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(COMPRESSED_XZ), preset=3)
+
+    def test_init_bad_filter_spec(self):
+        with self.assertRaises(TypeError):
+            LZMAFile(BytesIO(), "w", filters=[b"wobsite"])
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(), "w", filters=[{"xyzzy": 3}])
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(), "w", filters=[{"id": 98765}])
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(), "w",
+                     filters=[{"id": lzma.FILTER_LZMA2, "foo": 0}])
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(), "w",
+                     filters=[{"id": lzma.FILTER_DELTA, "foo": 0}])
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(), "w",
+                     filters=[{"id": lzma.FILTER_X86, "foo": 0}])
+
+    def test_init_with_preset_and_filters(self):
+        with self.assertRaises(ValueError):
+            LZMAFile(BytesIO(), "w", format=lzma.FORMAT_RAW,
+                     preset=6, filters=FILTERS_RAW_1)
+
+    def test_close(self):
+        with BytesIO(COMPRESSED_XZ) as src:
+            f = LZMAFile(src)
+            f.close()
+            # LZMAFile.close() should not close the underlying file object.
+            self.assertFalse(src.closed)
+            # Try closing an already-closed LZMAFile.
+            f.close()
+            self.assertFalse(src.closed)
+
+        # Test with a real file on disk, opened directly by LZMAFile.
+        with TempFile(TESTFN, COMPRESSED_XZ):
+            f = LZMAFile(TESTFN)
+            fp = f._fp
+            f.close()
+            # Here, LZMAFile.close() *should* close the underlying file object.
+            self.assertTrue(fp.closed)
+            # Try closing an already-closed LZMAFile.
+            f.close()
+
+    def test_closed(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        try:
+            self.assertFalse(f.closed)
+            f.read()
+            self.assertFalse(f.closed)
+        finally:
+            f.close()
+        self.assertTrue(f.closed)
+
+        f = LZMAFile(BytesIO(), "w")
+        try:
+            self.assertFalse(f.closed)
+        finally:
+            f.close()
+        self.assertTrue(f.closed)
+
+    def test_fileno(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        try:
+            self.assertRaises(UnsupportedOperation, f.fileno)
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.fileno)
+        with TempFile(TESTFN, COMPRESSED_XZ):
+            f = LZMAFile(TESTFN)
+            try:
+                self.assertEqual(f.fileno(), f._fp.fileno())
+                self.assertIsInstance(f.fileno(), int)
+            finally:
+                f.close()
+        self.assertRaises(ValueError, f.fileno)
+
+    def test_seekable(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        try:
+            self.assertTrue(f.seekable())
+            f.read()
+            self.assertTrue(f.seekable())
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.seekable)
+
+        f = LZMAFile(BytesIO(), "w")
+        try:
+            self.assertFalse(f.seekable())
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.seekable)
+
+        src = BytesIO(COMPRESSED_XZ)
+        src.seekable = lambda: False
+        f = LZMAFile(src)
+        try:
+            self.assertFalse(f.seekable())
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.seekable)
+
+    def test_readable(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        try:
+            self.assertTrue(f.readable())
+            f.read()
+            self.assertTrue(f.readable())
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.readable)
+
+        f = LZMAFile(BytesIO(), "w")
+        try:
+            self.assertFalse(f.readable())
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.readable)
+
+    def test_writable(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        try:
+            self.assertFalse(f.writable())
+            f.read()
+            self.assertFalse(f.writable())
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.writable)
+
+        f = LZMAFile(BytesIO(), "w")
+        try:
+            self.assertTrue(f.writable())
+        finally:
+            f.close()
+        self.assertRaises(ValueError, f.writable)
+
+    def test_read(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertEqual(f.read(), INPUT)
+            self.assertEqual(f.read(), b"")
+        with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f:
+            self.assertEqual(f.read(), INPUT)
+        with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f:
+            self.assertEqual(f.read(), INPUT)
+            self.assertEqual(f.read(), b"")
+        with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f:
+            self.assertEqual(f.read(), INPUT)
+            self.assertEqual(f.read(), b"")
+        with LZMAFile(BytesIO(COMPRESSED_RAW_1),
+                      format=lzma.FORMAT_RAW, filters=FILTERS_RAW_1) as f:
+            self.assertEqual(f.read(), INPUT)
+            self.assertEqual(f.read(), b"")
+        with LZMAFile(BytesIO(COMPRESSED_RAW_2),
+                      format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) as f:
+            self.assertEqual(f.read(), INPUT)
+            self.assertEqual(f.read(), b"")
+        with LZMAFile(BytesIO(COMPRESSED_RAW_3),
+                      format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) as f:
+            self.assertEqual(f.read(), INPUT)
+            self.assertEqual(f.read(), b"")
+        with LZMAFile(BytesIO(COMPRESSED_RAW_4),
+                      format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) as f:
+            self.assertEqual(f.read(), INPUT)
+            self.assertEqual(f.read(), b"")
+
+    def test_read_0(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertEqual(f.read(0), b"")
+        with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f:
+            self.assertEqual(f.read(0), b"")
+        with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f:
+            self.assertEqual(f.read(0), b"")
+        with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f:
+            self.assertEqual(f.read(0), b"")
+
+    def test_read_10(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            chunks = []
+            while True:
+                result = f.read(10)
+                if not result:
+                    break
+                self.assertLessEqual(len(result), 10)
+                chunks.append(result)
+            self.assertEqual(b"".join(chunks), INPUT)
+
+    def test_read_multistream(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f:
+            self.assertEqual(f.read(), INPUT * 5)
+        with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_ALONE)) as f:
+            self.assertEqual(f.read(), INPUT * 2)
+        with LZMAFile(BytesIO(COMPRESSED_RAW_3 * 4),
+                      format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) as f:
+            self.assertEqual(f.read(), INPUT * 4)
+
+    def test_read_multistream_buffer_size_aligned(self):
+        # Test the case where a stream boundary coincides with the end
+        # of the raw read buffer.
+        saved_buffer_size = lzma._BUFFER_SIZE
+        lzma._BUFFER_SIZE = len(COMPRESSED_XZ)
+        try:
+            with LZMAFile(BytesIO(COMPRESSED_XZ *  5)) as f:
+                self.assertEqual(f.read(), INPUT * 5)
+        finally:
+            lzma._BUFFER_SIZE = saved_buffer_size
+
+    def test_read_from_file(self):
+        with TempFile(TESTFN, COMPRESSED_XZ):
+            with LZMAFile(TESTFN) as f:
+                self.assertEqual(f.read(), INPUT)
+                self.assertEqual(f.read(), b"")
+
+    def test_read_from_file_with_bytes_filename(self):
+        try:
+            bytes_filename = TESTFN.encode("ascii")
+        except UnicodeEncodeError:
+            self.skipTest("Temporary file name needs to be ASCII")
+        with TempFile(TESTFN, COMPRESSED_XZ):
+            with LZMAFile(bytes_filename) as f:
+                self.assertEqual(f.read(), INPUT)
+                self.assertEqual(f.read(), b"")
+
+    def test_read_incomplete(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f:
+            self.assertRaises(EOFError, f.read)
+
+    def test_read_truncated(self):
+        # Drop stream footer: CRC (4 bytes), index size (4 bytes),
+        # flags (2 bytes) and magic number (2 bytes).
+        truncated = COMPRESSED_XZ[:-12]
+        with LZMAFile(BytesIO(truncated)) as f:
+            self.assertRaises(EOFError, f.read)
+        with LZMAFile(BytesIO(truncated)) as f:
+            self.assertEqual(f.read(len(INPUT)), INPUT)
+            self.assertRaises(EOFError, f.read, 1)
+        # Incomplete 12-byte header.
+        for i in range(12):
+            with LZMAFile(BytesIO(truncated[:i])) as f:
+                self.assertRaises(EOFError, f.read, 1)
+
+    def test_read_bad_args(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        f.close()
+        self.assertRaises(ValueError, f.read)
+        with LZMAFile(BytesIO(), "w") as f:
+            self.assertRaises(ValueError, f.read)
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertRaises(TypeError, f.read, None)
+
+    def test_read1(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            blocks = []
+            while True:
+                result = f.read1()
+                if not result:
+                    break
+                blocks.append(result)
+            self.assertEqual(b"".join(blocks), INPUT)
+            self.assertEqual(f.read1(), b"")
+
+    def test_read1_0(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertEqual(f.read1(0), b"")
+
+    def test_read1_10(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            blocks = []
+            while True:
+                result = f.read1(10)
+                if not result:
+                    break
+                blocks.append(result)
+            self.assertEqual(b"".join(blocks), INPUT)
+            self.assertEqual(f.read1(), b"")
+
+    def test_read1_multistream(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f:
+            blocks = []
+            while True:
+                result = f.read1()
+                if not result:
+                    break
+                blocks.append(result)
+            self.assertEqual(b"".join(blocks), INPUT * 5)
+            self.assertEqual(f.read1(), b"")
+
+    def test_read1_bad_args(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        f.close()
+        self.assertRaises(ValueError, f.read1)
+        with LZMAFile(BytesIO(), "w") as f:
+            self.assertRaises(ValueError, f.read1)
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertRaises(TypeError, f.read1, None)
+
+    def test_peek(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            result = f.peek()
+            self.assertGreater(len(result), 0)
+            self.assertTrue(INPUT.startswith(result))
+            self.assertEqual(f.read(), INPUT)
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            result = f.peek(10)
+            self.assertGreater(len(result), 0)
+            self.assertTrue(INPUT.startswith(result))
+            self.assertEqual(f.read(), INPUT)
+
+    def test_peek_bad_args(self):
+        with LZMAFile(BytesIO(), "w") as f:
+            self.assertRaises(ValueError, f.peek)
+
+    def test_iterator(self):
+        with BytesIO(INPUT) as f:
+            lines = f.readlines()
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertListEqual(list(iter(f)), lines)
+        with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f:
+            self.assertListEqual(list(iter(f)), lines)
+        with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f:
+            self.assertListEqual(list(iter(f)), lines)
+        with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f:
+            self.assertListEqual(list(iter(f)), lines)
+        with LZMAFile(BytesIO(COMPRESSED_RAW_2),
+                      format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) as f:
+            self.assertListEqual(list(iter(f)), lines)
+
+    def test_readline(self):
+        with BytesIO(INPUT) as f:
+            lines = f.readlines()
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            for line in lines:
+                self.assertEqual(f.readline(), line)
+
+    def test_readlines(self):
+        with BytesIO(INPUT) as f:
+            lines = f.readlines()
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertListEqual(f.readlines(), lines)
+
+    def test_write(self):
+        with BytesIO() as dst:
+            with LZMAFile(dst, "w") as f:
+                f.write(INPUT)
+            expected = lzma.compress(INPUT)
+            self.assertEqual(dst.getvalue(), expected)
+        with BytesIO() as dst:
+            with LZMAFile(dst, "w", format=lzma.FORMAT_XZ) as f:
+                f.write(INPUT)
+            expected = lzma.compress(INPUT, format=lzma.FORMAT_XZ)
+            self.assertEqual(dst.getvalue(), expected)
+        with BytesIO() as dst:
+            with LZMAFile(dst, "w", format=lzma.FORMAT_ALONE) as f:
+                f.write(INPUT)
+            expected = lzma.compress(INPUT, format=lzma.FORMAT_ALONE)
+            self.assertEqual(dst.getvalue(), expected)
+        with BytesIO() as dst:
+            with LZMAFile(dst, "w", format=lzma.FORMAT_RAW,
+                          filters=FILTERS_RAW_2) as f:
+                f.write(INPUT)
+            expected = lzma.compress(INPUT, format=lzma.FORMAT_RAW,
+                                     filters=FILTERS_RAW_2)
+            self.assertEqual(dst.getvalue(), expected)
+
+    def test_write_10(self):
+        with BytesIO() as dst:
+            with LZMAFile(dst, "w") as f:
+                for start in range(0, len(INPUT), 10):
+                    f.write(INPUT[start:start+10])
+            expected = lzma.compress(INPUT)
+            self.assertEqual(dst.getvalue(), expected)
+
+    def test_write_append(self):
+        part1 = INPUT[:1024]
+        part2 = INPUT[1024:1536]
+        part3 = INPUT[1536:]
+        expected = b"".join(lzma.compress(x) for x in (part1, part2, part3))
+        with BytesIO() as dst:
+            with LZMAFile(dst, "w") as f:
+                f.write(part1)
+            with LZMAFile(dst, "a") as f:
+                f.write(part2)
+            with LZMAFile(dst, "a") as f:
+                f.write(part3)
+            self.assertEqual(dst.getvalue(), expected)
+
+    def test_write_to_file(self):
+        try:
+            with LZMAFile(TESTFN, "w") as f:
+                f.write(INPUT)
+            expected = lzma.compress(INPUT)
+            with open(TESTFN, "rb") as f:
+                self.assertEqual(f.read(), expected)
+        finally:
+            unlink(TESTFN)
+
+    def test_write_to_file_with_bytes_filename(self):
+        try:
+            bytes_filename = TESTFN.encode("ascii")
+        except UnicodeEncodeError:
+            self.skipTest("Temporary file name needs to be ASCII")
+        try:
+            with LZMAFile(bytes_filename, "w") as f:
+                f.write(INPUT)
+            expected = lzma.compress(INPUT)
+            with open(TESTFN, "rb") as f:
+                self.assertEqual(f.read(), expected)
+        finally:
+            unlink(TESTFN)
+
+    def test_write_append_to_file(self):
+        part1 = INPUT[:1024]
+        part2 = INPUT[1024:1536]
+        part3 = INPUT[1536:]
+        expected = b"".join(lzma.compress(x) for x in (part1, part2, part3))
+        try:
+            with LZMAFile(TESTFN, "w") as f:
+                f.write(part1)
+            with LZMAFile(TESTFN, "a") as f:
+                f.write(part2)
+            with LZMAFile(TESTFN, "a") as f:
+                f.write(part3)
+            with open(TESTFN, "rb") as f:
+                self.assertEqual(f.read(), expected)
+        finally:
+            unlink(TESTFN)
+
+    def test_write_bad_args(self):
+        f = LZMAFile(BytesIO(), "w")
+        f.close()
+        self.assertRaises(ValueError, f.write, b"foo")
+        with LZMAFile(BytesIO(COMPRESSED_XZ), "r") as f:
+            self.assertRaises(ValueError, f.write, b"bar")
+        with LZMAFile(BytesIO(), "w") as f:
+            self.assertRaises(TypeError, f.write, None)
+            self.assertRaises(TypeError, f.write, "text")
+            self.assertRaises(TypeError, f.write, 789)
+
+    def test_writelines(self):
+        with BytesIO(INPUT) as f:
+            lines = f.readlines()
+        with BytesIO() as dst:
+            with LZMAFile(dst, "w") as f:
+                f.writelines(lines)
+            expected = lzma.compress(INPUT)
+            self.assertEqual(dst.getvalue(), expected)
+
+    def test_seek_forward(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            f.seek(555)
+            self.assertEqual(f.read(), INPUT[555:])
+
+    def test_seek_forward_across_streams(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ * 2)) as f:
+            f.seek(len(INPUT) + 123)
+            self.assertEqual(f.read(), INPUT[123:])
+
+    def test_seek_forward_relative_to_current(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            f.read(100)
+            f.seek(1236, 1)
+            self.assertEqual(f.read(), INPUT[1336:])
+
+    def test_seek_forward_relative_to_end(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            f.seek(-555, 2)
+            self.assertEqual(f.read(), INPUT[-555:])
+
+    def test_seek_backward(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            f.read(1001)
+            f.seek(211)
+            self.assertEqual(f.read(), INPUT[211:])
+
+    def test_seek_backward_across_streams(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ * 2)) as f:
+            f.read(len(INPUT) + 333)
+            f.seek(737)
+            self.assertEqual(f.read(), INPUT[737:] + INPUT)
+
+    def test_seek_backward_relative_to_end(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            f.seek(-150, 2)
+            self.assertEqual(f.read(), INPUT[-150:])
+
+    def test_seek_past_end(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            f.seek(len(INPUT) + 9001)
+            self.assertEqual(f.tell(), len(INPUT))
+            self.assertEqual(f.read(), b"")
+
+    def test_seek_past_start(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            f.seek(-88)
+            self.assertEqual(f.tell(), 0)
+            self.assertEqual(f.read(), INPUT)
+
+    def test_seek_bad_args(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        f.close()
+        self.assertRaises(ValueError, f.seek, 0)
+        with LZMAFile(BytesIO(), "w") as f:
+            self.assertRaises(ValueError, f.seek, 0)
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            self.assertRaises(ValueError, f.seek, 0, 3)
+            self.assertRaises(ValueError, f.seek, 9, ())
+            self.assertRaises(TypeError, f.seek, None)
+            self.assertRaises(TypeError, f.seek, b"derp")
+
+    def test_tell(self):
+        with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
+            pos = 0
+            while True:
+                self.assertEqual(f.tell(), pos)
+                result = f.read(183)
+                if not result:
+                    break
+                pos += len(result)
+            self.assertEqual(f.tell(), len(INPUT))
+        with LZMAFile(BytesIO(), "w") as f:
+            for pos in range(0, len(INPUT), 144):
+                self.assertEqual(f.tell(), pos)
+                f.write(INPUT[pos:pos+144])
+            self.assertEqual(f.tell(), len(INPUT))
+
+    def test_tell_bad_args(self):
+        f = LZMAFile(BytesIO(COMPRESSED_XZ))
+        f.close()
+        self.assertRaises(ValueError, f.tell)
+
+
+class OpenTestCase(unittest.TestCase):
+
+    def test_binary_modes(self):
+        with lzma.open(BytesIO(COMPRESSED_XZ), "rb") as f:
+            self.assertEqual(f.read(), INPUT)
+        with BytesIO() as bio:
+            with lzma.open(bio, "wb") as f:
+                f.write(INPUT)
+            file_data = lzma.decompress(bio.getvalue())
+            self.assertEqual(file_data, INPUT)
+            with lzma.open(bio, "ab") as f:
+                f.write(INPUT)
+            file_data = lzma.decompress(bio.getvalue())
+            self.assertEqual(file_data, INPUT * 2)
+
+    def test_text_modes(self):
+        uncompressed = INPUT.decode("ascii")
+        uncompressed_raw = uncompressed.replace("\n", os.linesep)
+        with lzma.open(BytesIO(COMPRESSED_XZ), "rt") as f:
+            self.assertEqual(f.read(), uncompressed)
+        with BytesIO() as bio:
+            with lzma.open(bio, "wt") as f:
+                f.write(uncompressed)
+            file_data = lzma.decompress(bio.getvalue()).decode("ascii")
+            self.assertEqual(file_data, uncompressed_raw)
+            with lzma.open(bio, "at") as f:
+                f.write(uncompressed)
+            file_data = lzma.decompress(bio.getvalue()).decode("ascii")
+            self.assertEqual(file_data, uncompressed_raw * 2)
+
+    def test_filename(self):
+        with TempFile(TESTFN):
+            with lzma.open(TESTFN, "wb") as f:
+                f.write(INPUT)
+            with open(TESTFN, "rb") as f:
+                file_data = lzma.decompress(f.read())
+                self.assertEqual(file_data, INPUT)
+            with lzma.open(TESTFN, "rb") as f:
+                self.assertEqual(f.read(), INPUT)
+            with lzma.open(TESTFN, "ab") as f:
+                f.write(INPUT)
+            with lzma.open(TESTFN, "rb") as f:
+                self.assertEqual(f.read(), INPUT * 2)
+
+    def test_bad_params(self):
+        # Test invalid parameter combinations.
+        with self.assertRaises(ValueError):
+            lzma.open(TESTFN, "")
+        with self.assertRaises(ValueError):
+            lzma.open(TESTFN, "x")
+        with self.assertRaises(ValueError):
+            lzma.open(TESTFN, "rbt")
+        with self.assertRaises(ValueError):
+            lzma.open(TESTFN, "rb", encoding="utf-8")
+        with self.assertRaises(ValueError):
+            lzma.open(TESTFN, "rb", errors="ignore")
+        with self.assertRaises(ValueError):
+            lzma.open(TESTFN, "rb", newline="\n")
+
+    def test_format_and_filters(self):
+        # Test non-default format and filter chain.
+        options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1}
+        with lzma.open(BytesIO(COMPRESSED_RAW_1), "rb", **options) as f:
+            self.assertEqual(f.read(), INPUT)
+        with BytesIO() as bio:
+            with lzma.open(bio, "wb", **options) as f:
+                f.write(INPUT)
+            file_data = lzma.decompress(bio.getvalue(), **options)
+            self.assertEqual(file_data, INPUT)
+
+    def test_encoding(self):
+        # Test non-default encoding.
+        uncompressed = INPUT.decode("ascii")
+        uncompressed_raw = uncompressed.replace("\n", os.linesep)
+        with BytesIO() as bio:
+            with lzma.open(bio, "wt", encoding="utf-16-le") as f:
+                f.write(uncompressed)
+            file_data = lzma.decompress(bio.getvalue()).decode("utf-16-le")
+            self.assertEqual(file_data, uncompressed_raw)
+            bio.seek(0)
+            with lzma.open(bio, "rt", encoding="utf-16-le") as f:
+                self.assertEqual(f.read(), uncompressed)
+
+    def test_encoding_error_handler(self):
+        # Test wih non-default encoding error handler.
+        with BytesIO(lzma.compress(b"foo\xffbar")) as bio:
+            with lzma.open(bio, "rt", encoding="ascii", errors="ignore") as f:
+                self.assertEqual(f.read(), "foobar")
+
+    def test_newline(self):
+        # Test with explicit newline (universal newline mode disabled).
+        text = INPUT.decode("ascii")
+        with BytesIO() as bio:
+            with lzma.open(bio, "wt", newline="\n") as f:
+                f.write(text)
+            bio.seek(0)
+            with lzma.open(bio, "rt", newline="\r") as f:
+                self.assertEqual(f.readlines(), [text])
+
+
+class MiscellaneousTestCase(unittest.TestCase):
+
+    def test_is_check_supported(self):
+        # CHECK_NONE and CHECK_CRC32 should always be supported,
+        # regardless of the options liblzma was compiled with.
+        self.assertTrue(lzma.is_check_supported(lzma.CHECK_NONE))
+        self.assertTrue(lzma.is_check_supported(lzma.CHECK_CRC32))
+
+        # The .xz format spec cannot store check IDs above this value.
+        self.assertFalse(lzma.is_check_supported(lzma.CHECK_ID_MAX + 1))
+
+        # This value should not be a valid check ID.
+        self.assertFalse(lzma.is_check_supported(lzma.CHECK_UNKNOWN))
+
+    def test__encode_filter_properties(self):
+        with self.assertRaises(TypeError):
+            lzma._encode_filter_properties(b"not a dict")
+        with self.assertRaises(ValueError):
+            lzma._encode_filter_properties({"id": 0x100})
+        with self.assertRaises(ValueError):
+            lzma._encode_filter_properties({"id": lzma.FILTER_LZMA2, "junk": 12})
+        with self.assertRaises(lzma.LZMAError):
+            lzma._encode_filter_properties({"id": lzma.FILTER_DELTA,
+                                           "dist": 9001})
+
+        # Test with parameters used by zipfile module.
+        props = lzma._encode_filter_properties({
+                "id": lzma.FILTER_LZMA1,
+                "pb": 2,
+                "lp": 0,
+                "lc": 3,
+                "dict_size": 8 << 20,
+            })
+        self.assertEqual(props, b"]\x00\x00\x80\x00")
+
+    def test__decode_filter_properties(self):
+        with self.assertRaises(TypeError):
+            lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes})
+        with self.assertRaises(lzma.LZMAError):
+            lzma._decode_filter_properties(lzma.FILTER_DELTA, b"too long")
+
+        # Test with parameters used by zipfile module.
+        filterspec = lzma._decode_filter_properties(
+                lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00")
+        self.assertEqual(filterspec["id"], lzma.FILTER_LZMA1)
+        self.assertEqual(filterspec["pb"], 2)
+        self.assertEqual(filterspec["lp"], 0)
+        self.assertEqual(filterspec["lc"], 3)
+        self.assertEqual(filterspec["dict_size"], 8 << 20)
+
+    def test_filter_properties_roundtrip(self):
+        spec1 = lzma._decode_filter_properties(
+                lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00")
+        reencoded = lzma._encode_filter_properties(spec1)
+        spec2 = lzma._decode_filter_properties(lzma.FILTER_LZMA1, reencoded)
+        self.assertEqual(spec1, spec2)
+
+
+# Test data:
+
+INPUT = b"""
+LAERTES
+
+       O, fear me not.
+       I stay too long: but here my father comes.
+
+       Enter POLONIUS
+
+       A double blessing is a double grace,
+       Occasion smiles upon a second leave.
+
+LORD POLONIUS
+
+       Yet here, Laertes! aboard, aboard, for shame!
+       The wind sits in the shoulder of your sail,
+       And you are stay'd for. There; my blessing with thee!
+       And these few precepts in thy memory
+       See thou character. Give thy thoughts no tongue,
+       Nor any unproportioned thought his act.
+       Be thou familiar, but by no means vulgar.
+       Those friends thou hast, and their adoption tried,
+       Grapple them to thy soul with hoops of steel;
+       But do not dull thy palm with entertainment
+       Of each new-hatch'd, unfledged comrade. Beware
+       Of entrance to a quarrel, but being in,
+       Bear't that the opposed may beware of thee.
+       Give every man thy ear, but few thy voice;
+       Take each man's censure, but reserve thy judgment.
+       Costly thy habit as thy purse can buy,
+       But not express'd in fancy; rich, not gaudy;
+       For the apparel oft proclaims the man,
+       And they in France of the best rank and station
+       Are of a most select and generous chief in that.
+       Neither a borrower nor a lender be;
+       For loan oft loses both itself and friend,
+       And borrowing dulls the edge of husbandry.
+       This above all: to thine ownself be true,
+       And it must follow, as the night the day,
+       Thou canst not then be false to any man.
+       Farewell: my blessing season this in thee!
+
+LAERTES
+
+       Most humbly do I take my leave, my lord.
+
+LORD POLONIUS
+
+       The time invites you; go; your servants tend.
+
+LAERTES
+
+       Farewell, Ophelia; and remember well
+       What I have said to you.
+
+OPHELIA
+
+       'Tis in my memory lock'd,
+       And you yourself shall keep the key of it.
+
+LAERTES
+
+       Farewell.
+"""
+
+COMPRESSED_XZ = (
+    b"\xfd7zXZ\x00\x00\x04\xe6\xd6\xb4F\x02\x00!\x01\x16\x00\x00\x00t/\xe5\xa3"
+    b"\xe0\x07\x80\x03\xdf]\x00\x05\x14\x07bX\x19\xcd\xddn\x98\x15\xe4\xb4\x9d"
+    b"o\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8\xe2\xfc\xe7\xd9\xfe6\xb8("
+    b"\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02\x17/\xa6=\xf0\xa2\xdf/M\x89"
+    b"\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ\x15\x80\x8c\xf8\x8do\xfa\x12"
+    b"\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t\xca6 BF$\xe5Q\xa4\x98\xee\xde"
+    b"l\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81\xe4N\xc8\x86\x153\xf5x2\xa2O"
+    b"\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z\xc4\xcdS\xb6t<\x16\xf2\x9cI#"
+    b"\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0\xaa\x96-Pe\xade:\x04\t\x1b\xf7"
+    b"\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b"
+    b"\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa"
+    b"\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8\x84b\xf6\xc3\xd4c-H\x93oJl\xd0iQ\xe4k"
+    b"\x84\x0b\xc1\xb7\xbc\xb1\x17\x88\xb1\xca?@\xf6\x07\xea\xe6x\xf1H12P\x0f"
+    b"\x8a\xc9\xeauw\xe3\xbe\xaai\xa9W\xd0\x80\xcd#cb5\x99\xd8]\xa9d\x0c\xbd"
+    b"\xa2\xdcWl\xedUG\xbf\x89yF\xf77\x81v\xbd5\x98\xbeh8\x18W\x08\xf0\x1b\x99"
+    b"5:\x1a?rD\x96\xa1\x04\x0f\xae\xba\x85\xeb\x9d5@\xf5\x83\xd37\x83\x8ac"
+    b"\x06\xd4\x97i\xcdt\x16S\x82k\xf6K\x01vy\x88\x91\x9b6T\xdae\r\xfd]:k\xbal"
+    b"\xa9\xbba\xc34\xf9r\xeb}r\xdb\xc7\xdb*\x8f\x03z\xdc8h\xcc\xc9\xd3\xbcl"
+    b"\xa5-\xcb\xeaK\xa2\xc5\x15\xc0\xe3\xc1\x86Z\xfb\xebL\xe13\xcf\x9c\xe3"
+    b"\x1d\xc9\xed\xc2\x06\xcc\xce!\x92\xe5\xfe\x9c^\xa59w \x9bP\xa3PK\x08d"
+    b"\xf9\xe2Z}\xa7\xbf\xed\xeb%$\x0c\x82\xb8/\xb0\x01\xa9&,\xf7qh{Q\x96)\xf2"
+    b"q\x96\xc3\x80\xb4\x12\xb0\xba\xe6o\xf4!\xb4[\xd4\x8aw\x10\xf7t\x0c\xb3"
+    b"\xd9\xd5\xc3`^\x81\x11??\\\xa4\x99\x85R\xd4\x8e\x83\xc9\x1eX\xbfa\xf1"
+    b"\xac\xb0\xea\xea\xd7\xd0\xab\x18\xe2\xf2\xed\xe1\xb7\xc9\x18\xcbS\xe4>"
+    b"\xc9\x95H\xe8\xcb\t\r%\xeb\xc7$.o\xf1\xf3R\x17\x1db\xbb\xd8U\xa5^\xccS"
+    b"\x16\x01\x87\xf3/\x93\xd1\xf0v\xc0r\xd7\xcc\xa2Gkz\xca\x80\x0e\xfd\xd0"
+    b"\x8b\xbb\xd2Ix\xb3\x1ey\xca-0\xe3z^\xd6\xd6\x8f_\xf1\x9dP\x9fi\xa7\xd1"
+    b"\xe8\x90\x84\xdc\xbf\xcdky\x8e\xdc\x81\x7f\xa3\xb2+\xbf\x04\xef\xd8\\"
+    b"\xc4\xdf\xe1\xb0\x01\xe9\x93\xe3Y\xf1\x1dY\xe8h\x81\xcf\xf1w\xcc\xb4\xef"
+    b" \x8b|\x04\xea\x83ej\xbe\x1f\xd4z\x9c`\xd3\x1a\x92A\x06\xe5\x8f\xa9\x13"
+    b"\t\x9e=\xfa\x1c\xe5_\x9f%v\x1bo\x11ZO\xd8\xf4\t\xddM\x16-\x04\xfc\x18<\""
+    b"CM\xddg~b\xf6\xef\x8e\x0c\xd0\xde|\xa0'\x8a\x0c\xd6x\xae!J\xa6F\x88\x15u"
+    b"\x008\x17\xbc7y\xb3\xd8u\xac_\x85\x8d\xe7\xc1@\x9c\xecqc\xa3#\xad\xf1"
+    b"\x935\xb5)_\r\xec3]\x0fo]5\xd0my\x07\x9b\xee\x81\xb5\x0f\xcfK+\x00\xc0"
+    b"\xe4b\x10\xe4\x0c\x1a \x9b\xe0\x97t\xf6\xa1\x9e\x850\xba\x0c\x9a\x8d\xc8"
+    b"\x8f\x07\xd7\xae\xc8\xf9+i\xdc\xb9k\xb0>f\x19\xb8\r\xa8\xf8\x1f$\xa5{p"
+    b"\xc6\x880\xce\xdb\xcf\xca_\x86\xac\x88h6\x8bZ%'\xd0\n\xbf\x0f\x9c\"\xba"
+    b"\xe5\x86\x9f\x0f7X=mNX[\xcc\x19FU\xc9\x860\xbc\x90a+* \xae_$\x03\x1e\xd3"
+    b"\xcd_\xa0\x9c\xde\xaf46q\xa5\xc9\x92\xd7\xca\xe3`\x9d\x85}\xb4\xff\xb3"
+    b"\x83\xfb\xb6\xca\xae`\x0bw\x7f\xfc\xd8\xacVe\x19\xc8\x17\x0bZ\xad\x88"
+    b"\xeb#\x97\x03\x13\xb1d\x0f{\x0c\x04w\x07\r\x97\xbd\xd6\xc1\xc3B:\x95\x08"
+    b"^\x10V\xaeaH\x02\xd9\xe3\n\\\x01X\xf6\x9c\x8a\x06u#%\xbe*\xa1\x18v\x85"
+    b"\xec!\t4\x00\x00\x00\x00Vj?uLU\xf3\xa6\x00\x01\xfb\x07\x81\x0f\x00\x00tw"
+    b"\x99P\xb1\xc4g\xfb\x02\x00\x00\x00\x00\x04YZ"
+)
+
+COMPRESSED_ALONE = (
+    b"]\x00\x00\x80\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x05\x14\x07bX\x19"
+    b"\xcd\xddn\x98\x15\xe4\xb4\x9do\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8"
+    b"\xe2\xfc\xe7\xd9\xfe6\xb8(\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02"
+    b"\x17/\xa6=\xf0\xa2\xdf/M\x89\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ"
+    b"\x15\x80\x8c\xf8\x8do\xfa\x12\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t"
+    b"\xca6 BF$\xe5Q\xa4\x98\xee\xdel\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81"
+    b"\xe4N\xc8\x86\x153\xf5x2\xa2O\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z"
+    b"\xc4\xcdS\xb6t<\x16\xf2\x9cI#\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0"
+    b"\xaa\x96-Pe\xade:\x04\t\x1b\xf7\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9"
+    b"\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7"
+    b"\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8"
+    b"\x84b\xf8\x1epl\xeajr\xd1=\t\x03\xdd\x13\x1b3!E\xf9vV\xdaF\xf3\xd7\xb4"
+    b"\x0c\xa9P~\xec\xdeE\xe37\xf6\x1d\xc6\xbb\xddc%\xb6\x0fI\x07\xf0;\xaf\xe7"
+    b"\xa0\x8b\xa7Z\x99(\xe9\xe2\xf0o\x18>`\xe1\xaa\xa8\xd9\xa1\xb2}\xe7\x8d"
+    b"\x834T\xb6\xef\xc1\xde\xe3\x98\xbcD\x03MA@\xd8\xed\xdc\xc8\x93\x03\x1a"
+    b"\x93\x0b\x7f\x94\x12\x0b\x02Sa\x18\xc9\xc5\x9bTJE}\xf6\xc8g\x17#ZV\x01"
+    b"\xc9\x9dc\x83\x0e>0\x16\x90S\xb8/\x03y_\x18\xfa(\xd7\x0br\xa2\xb0\xba?"
+    b"\x8c\xe6\x83@\x84\xdf\x02:\xc5z\x9e\xa6\x84\xc9\xf5BeyX\x83\x1a\xf1 :\t"
+    b"\xf7\x19\xfexD\\&G\xf3\x85Y\xa2J\xf9\x0bv{\x89\xf6\xe7)A\xaf\x04o\x00"
+    b"\x075\xd3\xe0\x7f\x97\x98F\x0f?v\x93\xedVtTf\xb5\x97\x83\xed\x19\xd7\x1a"
+    b"'k\xd7\xd9\xc5\\Y\xd1\xdc\x07\x15|w\xbc\xacd\x87\x08d\xec\xa7\xf6\x82"
+    b"\xfc\xb3\x93\xeb\xb9 \x8d\xbc ,\xb3X\xb0\xd2s\xd7\xd1\xffv\x05\xdf}\xa2"
+    b"\x96\xfb%\n\xdf\xa2\x7f\x08.\xa16\n\xe0\x19\x93\x7fh\n\x1c\x8c\x0f \x11"
+    b"\xc6Bl\x95\x19U}\xe4s\xb5\x10H\xea\x86pB\xe88\x95\xbe\x8cZ\xdb\xe4\x94A"
+    b"\x92\xb9;z\xaa\xa7{\x1c5!\xc0\xaf\xc1A\xf9\xda\xf0$\xb0\x02qg\xc8\xc7/|"
+    b"\xafr\x99^\x91\x88\xbf\x03\xd9=\xd7n\xda6{>8\n\xc7:\xa9'\xba.\x0b\xe2"
+    b"\xb5\x1d\x0e\n\x9a\x8e\x06\x8f:\xdd\x82'[\xc3\"wD$\xa7w\xecq\x8c,1\x93"
+    b"\xd0,\xae2w\x93\x12$Jd\x19mg\x02\x93\x9cA\x95\x9d&\xca8i\x9c\xb0;\xe7NQ"
+    b"\x1frh\x8beL;\xb0m\xee\x07Q\x9b\xc6\xd8\x03\xb5\xdeN\xd4\xfe\x98\xd0\xdc"
+    b"\x1a[\x04\xde\x1a\xf6\x91j\xf8EOli\x8eB^\x1d\x82\x07\xb2\xb5R]\xb7\xd7"
+    b"\xe9\xa6\xc3.\xfb\xf0-\xb4e\x9b\xde\x03\x88\xc6\xc1iN\x0e\x84wbQ\xdf~"
+    b"\xe9\xa4\x884\x96kM\xbc)T\xf3\x89\x97\x0f\x143\xe7)\xa0\xb3B\x00\xa8\xaf"
+    b"\x82^\xcb\xc7..\xdb\xc7\t\x9dH\xee5\xe9#\xe6NV\x94\xcb$Kk\xe3\x7f\r\xe3t"
+    b"\x12\xcf'\xefR\x8b\xf42\xcf-LH\xac\xe5\x1f0~?SO\xeb\xc1E\x1a\x1c]\xf2"
+    b"\xc4<\x11\x02\x10Z0a*?\xe4r\xff\xfb\xff\xf6\x14nG\xead^\xd6\xef8\xb6uEI"
+    b"\x99\nV\xe2\xb3\x95\x8e\x83\xf6i!\xb5&1F\xb1DP\xf4 SO3D!w\x99_G\x7f+\x90"
+    b".\xab\xbb]\x91>\xc9#h;\x0f5J\x91K\xf4^-[\x9e\x8a\\\x94\xca\xaf\xf6\x19"
+    b"\xd4\xa1\x9b\xc4\xb8p\xa1\xae\x15\xe9r\x84\xe0\xcar.l []\x8b\xaf+0\xf2g"
+    b"\x01aKY\xdfI\xcf,\n\xe8\xf0\xe7V\x80_#\xb2\xf2\xa9\x06\x8c>w\xe2W,\xf4"
+    b"\x8c\r\xf963\xf5J\xcc2\x05=kT\xeaUti\xe5_\xce\x1b\xfa\x8dl\x02h\xef\xa8"
+    b"\xfbf\x7f\xff\xf0\x19\xeax"
+)
+
+FILTERS_RAW_1 = [{"id": lzma.FILTER_LZMA2, "preset": 3}]
+COMPRESSED_RAW_1 = (
+    b"\xe0\x07\x80\x03\xfd]\x00\x05\x14\x07bX\x19\xcd\xddn\x96cyq\xa1\xdd\xee"
+    b"\xf8\xfam\xe3'\x88\xd3\xff\xe4\x9e \xceQ\x91\xa4\x14I\xf6\xb9\x9dVL8\x15"
+    b"_\x0e\x12\xc3\xeb\xbc\xa5\xcd\nW\x1d$=R;\x1d\xf8k8\t\xb1{\xd4\xc5+\x9d"
+    b"\x87c\xe5\xef\x98\xb4\xd7S3\xcd\xcc\xd2\xed\xa4\x0em\xe5\xf4\xdd\xd0b"
+    b"\xbe4*\xaa\x0b\xc5\x08\x10\x85+\x81.\x17\xaf9\xc9b\xeaZrA\xe20\x7fs\"r"
+    b"\xdaG\x81\xde\x90cu\xa5\xdb\xa9.A\x08l\xb0<\xf6\x03\xddOi\xd0\xc5\xb4"
+    b"\xec\xecg4t6\"\xa6\xb8o\xb5?\x18^}\xb6}\x03[:\xeb\x03\xa9\n[\x89l\x19g"
+    b"\x16\xc82\xed\x0b\xfb\x86n\xa2\x857@\x93\xcd6T\xc3u\xb0\t\xf9\x1b\x918"
+    b"\xfc[\x1b\x1e4\xb3\x14\x06PCV\xa8\"\xf5\x81x~\xe9\xb5N\x9cK\x9f\xc6\xc3%"
+    b"\xc8k:{6\xe7\xf7\xbd\x05\x02\xb4\xc4\xc3\xd3\xfd\xc3\xa8\\\xfc@\xb1F_"
+    b"\xc8\x90\xd9sU\x98\xad8\x05\x07\xde7J\x8bM\xd0\xb3;X\xec\x87\xef\xae\xb3"
+    b"eO,\xb1z,d\x11y\xeejlB\x02\x1d\xf28\x1f#\x896\xce\x0b\xf0\xf5\xa9PK\x0f"
+    b"\xb3\x13P\xd8\x88\xd2\xa1\x08\x04C?\xdb\x94_\x9a\"\xe9\xe3e\x1d\xde\x9b"
+    b"\xa1\xe8>H\x98\x10;\xc5\x03#\xb5\x9d4\x01\xe7\xc5\xba%v\xa49\x97A\xe0\""
+    b"\x8c\xc22\xe3i\xc1\x9d\xab3\xdf\xbe\xfdDm7\x1b\x9d\xab\xb5\x15o:J\x92"
+    b"\xdb\x816\x17\xc2O\x99\x1b\x0e\x8d\xf3\tQ\xed\x8e\x95S/\x16M\xb2S\x04"
+    b"\x0f\xc3J\xc6\xc7\xe4\xcb\xc5\xf4\xe7d\x14\xe4=^B\xfb\xd3E\xd3\x1e\xcd"
+    b"\x91\xa5\xd0G\x8f.\xf6\xf9\x0bb&\xd9\x9f\xc2\xfdj\xa2\x9e\xc4\\\x0e\x1dC"
+    b"v\xe8\xd2\x8a?^H\xec\xae\xeb>\xfe\xb8\xab\xd4IqY\x8c\xd4K7\x11\xf4D\xd0W"
+    b"\xa5\xbe\xeaO\xbf\xd0\x04\xfdl\x10\xae5\xd4U\x19\x06\xf9{\xaa\xe0\x81"
+    b"\x0f\xcf\xa3k{\x95\xbd\x19\xa2\xf8\xe4\xa3\x08O*\xf1\xf1B-\xc7(\x0eR\xfd"
+    b"@E\x9f\xd3\x1e:\xfdV\xb7\x04Y\x94\xeb]\x83\xc4\xa5\xd7\xc0gX\x98\xcf\x0f"
+    b"\xcd3\x00]n\x17\xec\xbd\xa3Y\x86\xc5\xf3u\xf6*\xbdT\xedA$A\xd9A\xe7\x98"
+    b"\xef\x14\x02\x9a\xfdiw\xec\xa0\x87\x11\xd9%\xc5\xeb\x8a=\xae\xc0\xc4\xc6"
+    b"D\x80\x8f\xa8\xd1\xbbq\xb2\xc0\xa0\xf5Cqp\xeeL\xe3\xe5\xdc \x84\"\xe9"
+    b"\x80t\x83\x05\xba\xf1\xc5~\x93\xc9\xf0\x01c\xceix\x9d\xed\xc5)l\x16)\xd1"
+    b"\x03@l\x04\x7f\x87\xa5yn\x1b\x01D\xaa:\xd2\x96\xb4\xb3?\xb0\xf9\xce\x07"
+    b"\xeb\x81\x00\xe4\xc3\xf5%_\xae\xd4\xf9\xeb\xe2\rh\xb2#\xd67Q\x16D\x82hn"
+    b"\xd1\xa3_?q\xf0\xe2\xac\xf317\x9e\xd0_\x83|\xf1\xca\xb7\x95S\xabW\x12"
+    b"\xff\xddt\xf69L\x01\xf2|\xdaW\xda\xees\x98L\x18\xb8_\xe8$\x82\xea\xd6"
+    b"\xd1F\xd4\x0b\xcdk\x01vf\x88h\xc3\xae\xb91\xc7Q\x9f\xa5G\xd9\xcc\x1f\xe3"
+    b"5\xb1\xdcy\x7fI\x8bcw\x8e\x10rIp\x02:\x19p_\xc8v\xcea\"\xc1\xd9\x91\x03"
+    b"\xbfe\xbe\xa6\xb3\xa8\x14\x18\xc3\xabH*m}\xc2\xc1\x9a}>l%\xce\x84\x99"
+    b"\xb3d\xaf\xd3\x82\x15\xdf\xc1\xfc5fOg\x9b\xfc\x8e^&\t@\xce\x9f\x06J\xb8"
+    b"\xb5\x86\x1d\xda{\x9f\xae\xb0\xff\x02\x81r\x92z\x8cM\xb7ho\xc9^\x9c\xb6"
+    b"\x9c\xae\xd1\xc9\xf4\xdfU7\xd6\\!\xea\x0b\x94k\xb9Ud~\x98\xe7\x86\x8az"
+    b"\x10;\xe3\x1d\xe5PG\xf8\xa4\x12\x05w\x98^\xc4\xb1\xbb\xfb\xcf\xe0\x7f"
+    b"\x033Sf\x0c \xb1\xf6@\x94\xe5\xa3\xb2\xa7\x10\x9a\xc0\x14\xc3s\xb5xRD"
+    b"\xf4`W\xd9\xe5\xd3\xcf\x91\rTZ-X\xbe\xbf\xb5\xe2\xee|\x1a\xbf\xfb\x08"
+    b"\x91\xe1\xfc\x9a\x18\xa3\x8b\xd6^\x89\xf5[\xef\x87\xd1\x06\x1c7\xd6\xa2"
+    b"\t\tQ5/@S\xc05\xd2VhAK\x03VC\r\x9b\x93\xd6M\xf1xO\xaaO\xed\xb9<\x0c\xdae"
+    b"*\xd0\x07Hk6\x9fG+\xa1)\xcd\x9cl\x87\xdb\xe1\xe7\xefK}\x875\xab\xa0\x19u"
+    b"\xf6*F\xb32\x00\x00\x00"
+)
+
+FILTERS_RAW_2 = [{"id": lzma.FILTER_DELTA, "dist": 2},
+                 {"id": lzma.FILTER_LZMA2,
+                  "preset": lzma.PRESET_DEFAULT | lzma.PRESET_EXTREME}]
+COMPRESSED_RAW_2 = (
+    b"\xe0\x07\x80\x05\x91]\x00\x05\x14\x06-\xd4\xa8d?\xef\xbe\xafH\xee\x042"
+    b"\xcb.\xb5g\x8f\xfb\x14\xab\xa5\x9f\x025z\xa4\xdd\xd8\t[}W\xf8\x0c\x1dmH"
+    b"\xfa\x05\xfcg\xba\xe5\x01Q\x0b\x83R\xb6A\x885\xc0\xba\xee\n\x1cv~\xde:o"
+    b"\x06:J\xa7\x11Cc\xea\xf7\xe5*o\xf7\x83\\l\xbdE\x19\x1f\r\xa8\x10\xb42"
+    b"\x0caU{\xd7\xb8w\xdc\xbe\x1b\xfc8\xb4\xcc\xd38\\\xf6\x13\xf6\xe7\x98\xfa"
+    b"\xc7[\x17_9\x86%\xa8\xf8\xaa\xb8\x8dfs#\x1e=\xed<\x92\x10\\t\xff\x86\xfb"
+    b"=\x9e7\x18\x1dft\\\xb5\x01\x95Q\xc5\x19\xb38\xe0\xd4\xaa\x07\xc3\x7f\xd8"
+    b"\xa2\x00>-\xd3\x8e\xa1#\xfa\x83ArAm\xdbJ~\x93\xa3B\x82\xe0\xc7\xcc(\x08`"
+    b"WK\xad\x1b\x94kaj\x04 \xde\xfc\xe1\xed\xb0\x82\x91\xefS\x84%\x86\xfbi"
+    b"\x99X\xf1B\xe7\x90;E\xfde\x98\xda\xca\xd6T\xb4bg\xa4\n\x9aj\xd1\x83\x9e]"
+    b"\"\x7fM\xb5\x0fr\xd2\\\xa5j~P\x10GH\xbfN*Z\x10.\x81\tpE\x8a\x08\xbe1\xbd"
+    b"\xcd\xa9\xe1\x8d\x1f\x04\xf9\x0eH\xb9\xae\xd6\xc3\xc1\xa5\xa9\x95P\xdc~"
+    b"\xff\x01\x930\xa9\x04\xf6\x03\xfe\xb5JK\xc3]\xdd9\xb1\xd3\xd7F\xf5\xd1"
+    b"\x1e\xa0\x1c_\xed[\x0c\xae\xd4\x8b\x946\xeb\xbf\xbb\xe3$kS{\xb5\x80,f:Sj"
+    b"\x0f\x08z\x1c\xf5\xe8\xe6\xae\x98\xb0Q~r\x0f\xb0\x05?\xb6\x90\x19\x02&"
+    b"\xcb\x80\t\xc4\xea\x9c|x\xce\x10\x9c\xc5|\xcbdhh+\x0c'\xc5\x81\xc33\xb5"
+    b"\x14q\xd6\xc5\xe3`Z#\xdc\x8a\xab\xdd\xea\x08\xc2I\xe7\x02l{\xec\x196\x06"
+    b"\x91\x8d\xdc\xd5\xb3x\xe1hz%\xd1\xf8\xa5\xdd\x98!\x8c\x1c\xc1\x17RUa\xbb"
+    b"\x95\x0f\xe4X\xea1\x0c\xf1=R\xbe\xc60\xe3\xa4\x9a\x90bd\x97$]B\x01\xdd"
+    b"\x1f\xe3h2c\x1e\xa0L`4\xc6x\xa3Z\x8a\r\x14]T^\xd8\x89\x1b\x92\r;\xedY"
+    b"\x0c\xef\x8d9z\xf3o\xb6)f\xa9]$n\rp\x93\xd0\x10\xa4\x08\xb8\xb2\x8b\xb6"
+    b"\x8f\x80\xae;\xdcQ\xf1\xfa\x9a\x06\x8e\xa5\x0e\x8cK\x9c @\xaa:UcX\n!\xc6"
+    b"\x02\x12\xcb\x1b\"=\x16.\x1f\x176\xf2g=\xe1Wn\xe9\xe1\xd4\xf1O\xad\x15"
+    b"\x86\xe9\xa3T\xaf\xa9\xd7D\xb5\xd1W3pnt\x11\xc7VOj\xb7M\xc4i\xa1\xf1$3"
+    b"\xbb\xdc\x8af\xb0\xc5Y\r\xd1\xfb\xf2\xe7K\xe6\xc5hwO\xfe\x8c2^&\x07\xd5"
+    b"\x1fV\x19\xfd\r\x14\xd2i=yZ\xe6o\xaf\xc6\xb6\x92\x9d\xc4\r\xb3\xafw\xac%"
+    b"\xcfc\x1a\xf1`]\xf2\x1a\x9e\x808\xedm\xedQ\xb2\xfe\xe4h`[q\xae\xe0\x0f"
+    b"\xba0g\xb6\"N\xc3\xfb\xcfR\x11\xc5\x18)(\xc40\\\xa3\x02\xd9G!\xce\x1b"
+    b"\xc1\x96x\xb5\xc8z\x1f\x01\xb4\xaf\xde\xc2\xcd\x07\xe7H\xb3y\xa8M\n\\A\t"
+    b"ar\xddM\x8b\x9a\xea\x84\x9b!\xf1\x8d\xb1\xf1~\x1e\r\xa5H\xba\xf1\x84o"
+    b"\xda\x87\x01h\xe9\xa2\xbe\xbeqN\x9d\x84\x0b!WG\xda\xa1\xa5A\xb7\xc7`j"
+    b"\x15\xf2\xe9\xdd?\x015B\xd2~E\x06\x11\xe0\x91!\x05^\x80\xdd\xa8y\x15}"
+    b"\xa1)\xb1)\x81\x18\xf4\xf4\xf8\xc0\xefD\xe3\xdb2f\x1e\x12\xabu\xc9\x97"
+    b"\xcd\x1e\xa7\x0c\x02x4_6\x03\xc4$t\xf39\x94\x1d=\xcb\xbfv\\\xf5\xa3\x1d"
+    b"\x9d8jk\x95\x13)ff\xf9n\xc4\xa9\xe3\x01\xb8\xda\xfb\xab\xdfM\x99\xfb\x05"
+    b"\xe0\xe9\xb0I\xf4E\xab\xe2\x15\xa3\x035\xe7\xdeT\xee\x82p\xb4\x88\xd3"
+    b"\x893\x9c/\xc0\xd6\x8fou;\xf6\x95PR\xa9\xb2\xc1\xefFj\xe2\xa7$\xf7h\xf1"
+    b"\xdfK(\xc9c\xba7\xe8\xe3)\xdd\xb2,\x83\xfb\x84\x18.y\x18Qi\x88\xf8`h-"
+    b"\xef\xd5\xed\x8c\t\xd8\xc3^\x0f\x00\xb7\xd0[!\xafM\x9b\xd7.\x07\xd8\xfb"
+    b"\xd9\xe2-S+\xaa8,\xa0\x03\x1b \xea\xa8\x00\xc3\xab~\xd0$e\xa5\x7f\xf7"
+    b"\x95P]\x12\x19i\xd9\x7fo\x0c\xd8g^\rE\xa5\x80\x18\xc5\x01\x80\xaek`\xff~"
+    b"\xb6y\xe7+\xe5\x11^D\xa7\x85\x18\"!\xd6\xd2\xa7\xf4\x1eT\xdb\x02\xe15"
+    b"\x02Y\xbc\x174Z\xe7\x9cH\x1c\xbf\x0f\xc6\xe9f]\xcf\x8cx\xbc\xe5\x15\x94"
+    b"\xfc3\xbc\xa7TUH\xf1\x84\x1b\xf7\xa9y\xc07\x84\xf8X\xd8\xef\xfc \x1c\xd8"
+    b"( /\xf2\xb7\xec\xc1\\\x8c\xf6\x95\xa1\x03J\x83vP8\xe1\xe3\xbb~\xc24kA"
+    b"\x98y\xa1\xf2P\xe9\x9d\xc9J\xf8N\x99\xb4\xceaO\xde\x16\x1e\xc2\x19\xa7"
+    b"\x03\xd2\xe0\x8f:\x15\xf3\x84\x9e\xee\xe6e\xb8\x02q\xc7AC\x1emw\xfd\t"
+    b"\x9a\x1eu\xc1\xa9\xcaCwUP\x00\xa5\xf78L4w!\x91L2 \x87\xd0\xf2\x06\x81j"
+    b"\x80;\x03V\x06\x87\x92\xcb\x90lv@E\x8d\x8d\xa5\xa6\xe7Z[\xdf\xd6E\x03`>"
+    b"\x8f\xde\xa1bZ\x84\xd0\xa9`\x05\x0e{\x80;\xe3\xbef\x8d\x1d\xebk1.\xe3"
+    b"\xe9N\x15\xf7\xd4(\xfa\xbb\x15\xbdu\xf7\x7f\x86\xae!\x03L\x1d\xb5\xc1"
+    b"\xb9\x11\xdb\xd0\x93\xe4\x02\xe1\xd2\xcbBjc_\xe8}d\xdb\xc3\xa0Y\xbe\xc9/"
+    b"\x95\x01\xa3,\xe6bl@\x01\xdbp\xc2\xce\x14\x168\xc2q\xe3uH\x89X\xa4\xa9"
+    b"\x19\x1d\xc1}\x7fOX\x19\x9f\xdd\xbe\x85\x83\xff\x96\x1ee\x82O`CF=K\xeb$I"
+    b"\x17_\xefX\x8bJ'v\xde\x1f+\xd9.v\xf8Tv\x17\xf2\x9f5\x19\xe1\xb9\x91\xa8S"
+    b"\x86\xbd\x1a\"(\xa5x\x8dC\x03X\x81\x91\xa8\x11\xc4pS\x13\xbc\xf2'J\xae!"
+    b"\xef\xef\x84G\t\x8d\xc4\x10\x132\x00oS\x9e\xe0\xe4d\x8f\xb8y\xac\xa6\x9f"
+    b",\xb8f\x87\r\xdf\x9eE\x0f\xe1\xd0\\L\x00\xb2\xe1h\x84\xef}\x98\xa8\x11"
+    b"\xccW#\\\x83\x7fo\xbbz\x8f\x00"
+)
+
+FILTERS_RAW_3 = [{"id": lzma.FILTER_IA64, "start_offset": 0x100},
+                 {"id": lzma.FILTER_LZMA2}]
+COMPRESSED_RAW_3 = (
+    b"\xe0\x07\x80\x03\xdf]\x00\x05\x14\x07bX\x19\xcd\xddn\x98\x15\xe4\xb4\x9d"
+    b"o\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8\xe2\xfc\xe7\xd9\xfe6\xb8("
+    b"\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02\x17/\xa6=\xf0\xa2\xdf/M\x89"
+    b"\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ\x15\x80\x8c\xf8\x8do\xfa\x12"
+    b"\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t\xca6 BF$\xe5Q\xa4\x98\xee\xde"
+    b"l\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81\xe4N\xc8\x86\x153\xf5x2\xa2O"
+    b"\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z\xc4\xcdS\xb6t<\x16\xf2\x9cI#"
+    b"\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0\xaa\x96-Pe\xade:\x04\t\x1b\xf7"
+    b"\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b"
+    b"\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa"
+    b"\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8\x84b\xf6\xc3\xd4c-H\x93oJl\xd0iQ\xe4k"
+    b"\x84\x0b\xc1\xb7\xbc\xb1\x17\x88\xb1\xca?@\xf6\x07\xea\xe6x\xf1H12P\x0f"
+    b"\x8a\xc9\xeauw\xe3\xbe\xaai\xa9W\xd0\x80\xcd#cb5\x99\xd8]\xa9d\x0c\xbd"
+    b"\xa2\xdcWl\xedUG\xbf\x89yF\xf77\x81v\xbd5\x98\xbeh8\x18W\x08\xf0\x1b\x99"
+    b"5:\x1a?rD\x96\xa1\x04\x0f\xae\xba\x85\xeb\x9d5@\xf5\x83\xd37\x83\x8ac"
+    b"\x06\xd4\x97i\xcdt\x16S\x82k\xf6K\x01vy\x88\x91\x9b6T\xdae\r\xfd]:k\xbal"
+    b"\xa9\xbba\xc34\xf9r\xeb}r\xdb\xc7\xdb*\x8f\x03z\xdc8h\xcc\xc9\xd3\xbcl"
+    b"\xa5-\xcb\xeaK\xa2\xc5\x15\xc0\xe3\xc1\x86Z\xfb\xebL\xe13\xcf\x9c\xe3"
+    b"\x1d\xc9\xed\xc2\x06\xcc\xce!\x92\xe5\xfe\x9c^\xa59w \x9bP\xa3PK\x08d"
+    b"\xf9\xe2Z}\xa7\xbf\xed\xeb%$\x0c\x82\xb8/\xb0\x01\xa9&,\xf7qh{Q\x96)\xf2"
+    b"q\x96\xc3\x80\xb4\x12\xb0\xba\xe6o\xf4!\xb4[\xd4\x8aw\x10\xf7t\x0c\xb3"
+    b"\xd9\xd5\xc3`^\x81\x11??\\\xa4\x99\x85R\xd4\x8e\x83\xc9\x1eX\xbfa\xf1"
+    b"\xac\xb0\xea\xea\xd7\xd0\xab\x18\xe2\xf2\xed\xe1\xb7\xc9\x18\xcbS\xe4>"
+    b"\xc9\x95H\xe8\xcb\t\r%\xeb\xc7$.o\xf1\xf3R\x17\x1db\xbb\xd8U\xa5^\xccS"
+    b"\x16\x01\x87\xf3/\x93\xd1\xf0v\xc0r\xd7\xcc\xa2Gkz\xca\x80\x0e\xfd\xd0"
+    b"\x8b\xbb\xd2Ix\xb3\x1ey\xca-0\xe3z^\xd6\xd6\x8f_\xf1\x9dP\x9fi\xa7\xd1"
+    b"\xe8\x90\x84\xdc\xbf\xcdky\x8e\xdc\x81\x7f\xa3\xb2+\xbf\x04\xef\xd8\\"
+    b"\xc4\xdf\xe1\xb0\x01\xe9\x93\xe3Y\xf1\x1dY\xe8h\x81\xcf\xf1w\xcc\xb4\xef"
+    b" \x8b|\x04\xea\x83ej\xbe\x1f\xd4z\x9c`\xd3\x1a\x92A\x06\xe5\x8f\xa9\x13"
+    b"\t\x9e=\xfa\x1c\xe5_\x9f%v\x1bo\x11ZO\xd8\xf4\t\xddM\x16-\x04\xfc\x18<\""
+    b"CM\xddg~b\xf6\xef\x8e\x0c\xd0\xde|\xa0'\x8a\x0c\xd6x\xae!J\xa6F\x88\x15u"
+    b"\x008\x17\xbc7y\xb3\xd8u\xac_\x85\x8d\xe7\xc1@\x9c\xecqc\xa3#\xad\xf1"
+    b"\x935\xb5)_\r\xec3]\x0fo]5\xd0my\x07\x9b\xee\x81\xb5\x0f\xcfK+\x00\xc0"
+    b"\xe4b\x10\xe4\x0c\x1a \x9b\xe0\x97t\xf6\xa1\x9e\x850\xba\x0c\x9a\x8d\xc8"
+    b"\x8f\x07\xd7\xae\xc8\xf9+i\xdc\xb9k\xb0>f\x19\xb8\r\xa8\xf8\x1f$\xa5{p"
+    b"\xc6\x880\xce\xdb\xcf\xca_\x86\xac\x88h6\x8bZ%'\xd0\n\xbf\x0f\x9c\"\xba"
+    b"\xe5\x86\x9f\x0f7X=mNX[\xcc\x19FU\xc9\x860\xbc\x90a+* \xae_$\x03\x1e\xd3"
+    b"\xcd_\xa0\x9c\xde\xaf46q\xa5\xc9\x92\xd7\xca\xe3`\x9d\x85}\xb4\xff\xb3"
+    b"\x83\xfb\xb6\xca\xae`\x0bw\x7f\xfc\xd8\xacVe\x19\xc8\x17\x0bZ\xad\x88"
+    b"\xeb#\x97\x03\x13\xb1d\x0f{\x0c\x04w\x07\r\x97\xbd\xd6\xc1\xc3B:\x95\x08"
+    b"^\x10V\xaeaH\x02\xd9\xe3\n\\\x01X\xf6\x9c\x8a\x06u#%\xbe*\xa1\x18v\x85"
+    b"\xec!\t4\x00\x00\x00"
+)
+
+FILTERS_RAW_4 = [{"id": lzma.FILTER_DELTA, "dist": 4},
+                 {"id": lzma.FILTER_X86, "start_offset": 0x40},
+                 {"id": lzma.FILTER_LZMA2, "preset": 4, "lc": 2}]
+COMPRESSED_RAW_4 = (
+    b"\xe0\x07\x80\x06\x0e\\\x00\x05\x14\x07bW\xaah\xdd\x10\xdc'\xd6\x90,\xc6v"
+    b"Jq \x14l\xb7\x83xB\x0b\x97f=&fx\xba\n>Tn\xbf\x8f\xfb\x1dF\xca\xc3v_\xca?"
+    b"\xfbV<\x92#\xd4w\xa6\x8a\xeb\xf6\x03\xc0\x01\x94\xd8\x9e\x13\x12\x98\xd1"
+    b"*\xfa]c\xe8\x1e~\xaf\xb5]Eg\xfb\x9e\x01\"8\xb2\x90\x06=~\xe9\x91W\xcd"
+    b"\xecD\x12\xc7\xfa\xe1\x91\x06\xc7\x99\xb9\xe3\x901\x87\x19u\x0f\x869\xff"
+    b"\xc1\xb0hw|\xb0\xdcl\xcck\xb16o7\x85\xee{Y_b\xbf\xbc$\xf3=\x8d\x8bw\xe5Z"
+    b"\x08@\xc4kmE\xad\xfb\xf6*\xd8\xad\xa1\xfb\xc5{\xdej,)\x1emB\x1f<\xaeca"
+    b"\x80(\xee\x07 \xdf\xe9\xf8\xeb\x0e-\x97\x86\x90c\xf9\xea'B\xf7`\xd7\xb0"
+    b"\x92\xbd\xa0\x82]\xbd\x0e\x0eB\x19\xdc\x96\xc6\x19\xd86D\xf0\xd5\x831"
+    b"\x03\xb7\x1c\xf7&5\x1a\x8f PZ&j\xf8\x98\x1bo\xcc\x86\x9bS\xd3\xa5\xcdu"
+    b"\xf9$\xcc\x97o\xe5V~\xfb\x97\xb5\x0b\x17\x9c\xfdxW\x10\xfep4\x80\xdaHDY"
+    b"\xfa)\xfet\xb5\"\xd4\xd3F\x81\xf4\x13\x1f\xec\xdf\xa5\x13\xfc\"\x91x\xb7"
+    b"\x99\xce\xc8\x92\n\xeb[\x10l*Y\xd8\xb1@\x06\xc8o\x8d7r\xebu\xfd5\x0e\x7f"
+    b"\xf1$U{\t}\x1fQ\xcfxN\x9d\x9fXX\xe9`\x83\xc1\x06\xf4\x87v-f\x11\xdb/\\"
+    b"\x06\xff\xd7)B\xf3g\x06\x88#2\x1eB244\x7f4q\t\xc893?mPX\x95\xa6a\xfb)d"
+    b"\x9b\xfc\x98\x9aj\x04\xae\x9b\x9d\x19w\xba\xf92\xfaA\x11\\\x17\x97C3\xa4"
+    b"\xbc!\x88\xcdo[\xec:\x030\x91.\x85\xe0@\\4\x16\x12\x9d\xcaJv\x97\xb04"
+    b"\xack\xcbkf\xa3ss\xfc\x16^\x8ce\x85a\xa5=&\xecr\xb3p\xd1E\xd5\x80y\xc7"
+    b"\xda\xf6\xfek\xbcT\xbfH\xee\x15o\xc5\x8c\x830\xec\x1d\x01\xae\x0c-e\\"
+    b"\x91\x90\x94\xb2\xf8\x88\x91\xe8\x0b\xae\xa7>\x98\xf6\x9ck\xd2\xc6\x08"
+    b"\xe6\xab\t\x98\xf2!\xa0\x8c^\xacqA\x99<\x1cEG\x97\xc8\xf1\xb6\xb9\x82"
+    b"\x8d\xf7\x08s\x98a\xff\xe3\xcc\x92\x0e\xd2\xb6U\xd7\xd9\x86\x7fa\xe5\x1c"
+    b"\x8dTG@\t\x1e\x0e7*\xfc\xde\xbc]6N\xf7\xf1\x84\x9e\x9f\xcf\xe9\x1e\xb5'"
+    b"\xf4<\xdf\x99sq\xd0\x9d\xbd\x99\x0b\xb4%p4\xbf{\xbb\x8a\xd2\x0b\xbc=M"
+    b"\x94H:\xf5\xa8\xd6\xa4\xc90\xc2D\xb9\xd3\xa8\xb0S\x87 `\xa2\xeb\xf3W\xce"
+    b" 7\xf9N#\r\xe6\xbe\t\x9d\xe7\x811\xf9\x10\xc1\xc2\x14\xf6\xfc\xcba\xb7"
+    b"\xb1\x7f\x95l\xe4\tjA\xec:\x10\xe5\xfe\xc2\\=D\xe2\x0c\x0b3]\xf7\xc1\xf7"
+    b"\xbceZ\xb1A\xea\x16\xe5\xfddgFQ\xed\xaf\x04\xa3\xd3\xf8\xa2q\x19B\xd4r"
+    b"\xc5\x0c\x9a\x14\x94\xea\x91\xc4o\xe4\xbb\xb4\x99\xf4@\xd1\xe6\x0c\xe3"
+    b"\xc6d\xa0Q\n\xf2/\xd8\xb8S5\x8a\x18:\xb5g\xac\x95D\xce\x17\x07\xd4z\xda"
+    b"\x90\xe65\x07\x19H!\t\xfdu\x16\x8e\x0eR\x19\xf4\x8cl\x0c\xf9Q\xf1\x80"
+    b"\xe3\xbf\xd7O\xf8\x8c\x18\x0b\x9c\xf1\x1fb\xe1\tR\xb2\xf1\xe1A\xea \xcf-"
+    b"IGE\xf1\x14\x98$\x83\x15\xc9\xd8j\xbf\x19\x0f\xd5\xd1\xaa\xb3\xf3\xa5I2s"
+    b"\x8d\x145\xca\xd5\xd93\x9c\xb8D0\xe6\xaa%\xd0\xc0P}JO^h\x8e\x08\xadlV."
+    b"\x18\x88\x13\x05o\xb0\x07\xeaw\xe0\xb6\xa4\xd5*\xe4r\xef\x07G+\xc1\xbei["
+    b"w\xe8\xab@_\xef\x15y\xe5\x12\xc9W\x1b.\xad\x85-\xc2\xf7\xe3mU6g\x8eSA"
+    b"\x01(\xd3\xdb\x16\x13=\xde\x92\xf9,D\xb8\x8a\xb2\xb4\xc9\xc3\xefnE\xe8\\"
+    b"\xa6\xe2Y\xd2\xcf\xcb\x8c\xb6\xd5\xe9\x1d\x1e\x9a\x8b~\xe2\xa6\rE\x84uV"
+    b"\xed\xc6\x99\xddm<\x10[\x0fu\x1f\xc1\x1d1\n\xcfw\xb2%!\xf0[\xce\x87\x83B"
+    b"\x08\xaa,\x08%d\xcef\x94\"\xd9g.\xc83\xcbXY+4\xec\x85qA\n\x1d=9\xf0*\xb1"
+    b"\x1f/\xf3s\xd61b\x7f@\xfb\x9d\xe3FQ\\\xbd\x82\x1e\x00\xf3\xce\xd3\xe1"
+    b"\xca,E\xfd7[\xab\xb6\xb7\xac!mA}\xbd\x9d3R5\x9cF\xabH\xeb\x92)cc\x13\xd0"
+    b"\xbd\xee\xe9n{\x1dIJB\xa5\xeb\x11\xe8`w&`\x8b}@Oxe\t\x8a\x07\x02\x95\xf2"
+    b"\xed\xda|\xb1e\xbe\xaa\xbbg\x19@\xe1Y\x878\x84\x0f\x8c\xe3\xc98\xf2\x9e"
+    b"\xd5N\xb5J\xef\xab!\xe2\x8dq\xe1\xe5q\xc5\xee\x11W\xb7\xe4k*\x027\xa0"
+    b"\xa3J\xf4\xd8m\xd0q\x94\xcb\x07\n:\xb6`.\xe4\x9c\x15+\xc0)\xde\x80X\xd4"
+    b"\xcfQm\x01\xc2cP\x1cA\x85'\xc9\xac\x8b\xe6\xb2)\xe6\x84t\x1c\x92\xe4Z"
+    b"\x1cR\xb0\x9e\x96\xd1\xfb\x1c\xa6\x8b\xcb`\x10\x12]\xf2gR\x9bFT\xe0\xc8H"
+    b"S\xfb\xac<\x04\xc7\xc1\xe8\xedP\xf4\x16\xdb\xc0\xd7e\xc2\x17J^\x1f\xab"
+    b"\xff[\x08\x19\xb4\xf5\xfb\x19\xb4\x04\xe5c~']\xcb\xc2A\xec\x90\xd0\xed"
+    b"\x06,\xc5K{\x86\x03\xb1\xcdMx\xdeQ\x8c3\xf9\x8a\xea=\x89\xaba\xd2\xc89a"
+    b"\xd72\xf0\xc3\x19\x8a\xdfs\xd4\xfd\xbb\x81b\xeaE\"\xd8\xf4d\x0cD\xf7IJ!"
+    b"\xe5d\xbbG\xe9\xcam\xaa\x0f_r\x95\x91NBq\xcaP\xce\xa7\xa9\xb5\x10\x94eP!"
+    b"|\x856\xcd\xbfIir\xb8e\x9bjP\x97q\xabwS7\x1a\x0ehM\xe7\xca\x86?\xdeP}y~"
+    b"\x0f\x95I\xfc\x13\xe1<Q\x1b\x868\x1d\x11\xdf\x94\xf4\x82>r\xa9k\x88\xcb"
+    b"\xfd\xc3v\xe2\xb9\x8a\x02\x8eq\x92I\xf8\xf6\xf1\x03s\x9b\xb8\xe3\"\xe3"
+    b"\xa9\xa5>D\xb8\x96;\xe7\x92\xd133\xe8\xdd'e\xc9.\xdc;\x17\x1f\xf5H\x13q"
+    b"\xa4W\x0c\xdb~\x98\x01\xeb\xdf\xe32\x13\x0f\xddx\n6\xa0\t\x10\xb6\xbb"
+    b"\xb0\xc3\x18\xb6;\x9fj[\xd9\xd5\xc9\x06\x8a\x87\xcd\xe5\xee\xfc\x9c-%@"
+    b"\xee\xe0\xeb\xd2\xe3\xe8\xfb\xc0\x122\\\xc7\xaf\xc2\xa1Oth\xb3\x8f\x82"
+    b"\xb3\x18\xa8\x07\xd5\xee_\xbe\xe0\x1cA\x1e_\r\x9a\xb0\x17W&\xa2D\x91\x94"
+    b"\x1a\xb2\xef\xf2\xdc\x85;X\xb0,\xeb>-7S\xe5\xca\x07)\x1fp\x7f\xcaQBL\xca"
+    b"\xf3\xb9d\xfc\xb5su\xb0\xc8\x95\x90\xeb*)\xa0v\xe4\x9a{FW\xf4l\xde\xcdj"
+    b"\x00"
+)
+
+
+def test_main():
+    run_unittest(
+        CompressorDecompressorTestCase,
+        CompressDecompressFunctionTestCase,
+        FileTestCase,
+        OpenTestCase,
+        MiscellaneousTestCase,
+    )
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_macpath.py b/Lib/test/test_macpath.py
index d732e14a3a..ae4e5640fa 100644
--- a/Lib/test/test_macpath.py
+++ b/Lib/test/test_macpath.py
@@ -115,13 +115,9 @@ class MacPathTestCase(unittest.TestCase):
         self.assertEqual(normpath(b"a:b:"), b"a:b")
 
 
-class MacCommonTest(test_genericpath.CommonTest):
+class MacCommonTest(test_genericpath.CommonTest, unittest.TestCase):
     pathmodule = macpath
 
 
-def test_main():
-    support.run_unittest(MacPathTestCase, MacCommonTest)
-
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py
index 7a842929d5..39e8643414 100644
--- a/Lib/test/test_mailbox.py
+++ b/Lib/test/test_mailbox.py
@@ -22,6 +22,10 @@ except ImportError:
 
 class TestBase:
 
+    all_mailbox_types = (mailbox.Message, mailbox.MaildirMessage,
+                         mailbox.mboxMessage, mailbox.MHMessage,
+                         mailbox.BabylMessage, mailbox.MMDFMessage)
+
     def _check_sample(self, msg):
         # Inspect a mailbox.Message representation of the sample message
         self.assertIsInstance(msg, email.message.Message)
@@ -91,14 +95,14 @@ class TestMailbox(TestBase):
             """)
 
     def test_add_invalid_8bit_bytes_header(self):
-        key = self._box.add(self._nonascii_msg.encode('latin1'))
+        key = self._box.add(self._nonascii_msg.encode('latin-1'))
         self.assertEqual(len(self._box), 1)
         self.assertEqual(self._box.get_bytes(key),
-            self._nonascii_msg.encode('latin1'))
+            self._nonascii_msg.encode('latin-1'))
 
     def test_invalid_nonascii_header_as_string(self):
         subj = self._nonascii_msg.splitlines()[1]
-        key = self._box.add(subj.encode('latin1'))
+        key = self._box.add(subj.encode('latin-1'))
         self.assertEqual(self._box.get_string(key),
             'Subject: =?unknown-8bit?b?RmFsaW5hcHThciBo4Xpob3pzeuFsbO104XNz'
             'YWwuIE3hciByZW5kZWx06Ww/?=\n\n')
@@ -930,8 +934,7 @@ class TestMaildir(TestMailbox, unittest.TestCase):
         # the mtime and should cause a re-read. Note that "sleep
         # emulation" is still in effect, as skewfactor is -3.
         filename = os.path.join(self._path, 'cur', 'stray-file')
-        f = open(filename, 'w')
-        f.close()
+        support.create_empty_file(filename)
         os.unlink(filename)
         self._box._refresh()
         self.assertTrue(refreshed())
@@ -1397,6 +1400,14 @@ class TestMessage(TestBase, unittest.TestCase):
         # Initialize with invalid argument
         self.assertRaises(TypeError, lambda: self._factory(object()))
 
+    def test_all_eMM_attribues_exist(self):
+        # Issue 12537
+        eMM = email.message_from_string(_sample_message)
+        msg = self._factory(_sample_message)
+        for attr in eMM.__dict__:
+            self.assertTrue(attr in msg.__dict__,
+                '{} attribute does not exist'.format(attr))
+
     def test_become_message(self):
         # Take on the state of another message
         eMM = email.message_from_string(_sample_message)
@@ -1408,9 +1419,7 @@ class TestMessage(TestBase, unittest.TestCase):
         # Copy self's format-specific data to other message formats.
         # This test is superficial; better ones are in TestMessageConversion.
         msg = self._factory()
-        for class_ in (mailbox.Message, mailbox.MaildirMessage,
-                       mailbox.mboxMessage, mailbox.MHMessage,
-                       mailbox.BabylMessage, mailbox.MMDFMessage):
+        for class_ in self.all_mailbox_types:
             other_msg = class_()
             msg._explain_to(other_msg)
         other_msg = email.message.Message()
@@ -1642,37 +1651,44 @@ class TestMessageConversion(TestBase, unittest.TestCase):
 
     def test_plain_to_x(self):
         # Convert Message to all formats
-        for class_ in (mailbox.Message, mailbox.MaildirMessage,
-                       mailbox.mboxMessage, mailbox.MHMessage,
-                       mailbox.BabylMessage, mailbox.MMDFMessage):
+        for class_ in self.all_mailbox_types:
             msg_plain = mailbox.Message(_sample_message)
             msg = class_(msg_plain)
             self._check_sample(msg)
 
     def test_x_to_plain(self):
         # Convert all formats to Message
-        for class_ in (mailbox.Message, mailbox.MaildirMessage,
-                       mailbox.mboxMessage, mailbox.MHMessage,
-                       mailbox.BabylMessage, mailbox.MMDFMessage):
+        for class_ in self.all_mailbox_types:
             msg = class_(_sample_message)
             msg_plain = mailbox.Message(msg)
             self._check_sample(msg_plain)
 
     def test_x_from_bytes(self):
         # Convert all formats to Message
-        for class_ in (mailbox.Message, mailbox.MaildirMessage,
-                       mailbox.mboxMessage, mailbox.MHMessage,
-                       mailbox.BabylMessage, mailbox.MMDFMessage):
+        for class_ in self.all_mailbox_types:
             msg = class_(_bytes_sample_message)
             self._check_sample(msg)
 
     def test_x_to_invalid(self):
         # Convert all formats to an invalid format
-        for class_ in (mailbox.Message, mailbox.MaildirMessage,
-                       mailbox.mboxMessage, mailbox.MHMessage,
-                       mailbox.BabylMessage, mailbox.MMDFMessage):
+        for class_ in self.all_mailbox_types:
             self.assertRaises(TypeError, lambda: class_(False))
 
+    def test_type_specific_attributes_removed_on_conversion(self):
+        reference = {class_: class_(_sample_message).__dict__
+                        for class_ in self.all_mailbox_types}
+        for class1 in self.all_mailbox_types:
+            for class2 in self.all_mailbox_types:
+                if class1 is class2:
+                    continue
+                source = class1(_sample_message)
+                target = class2(source)
+                type_specific = [a for a in reference[class1]
+                                   if a not in reference[class2]]
+                for attr in type_specific:
+                    self.assertNotIn(attr, target.__dict__,
+                        "while converting {} to {}".format(class1, class2))
+
     def test_maildir_to_maildir(self):
         # Convert MaildirMessage to MaildirMessage
         msg_maildir = mailbox.MaildirMessage(_sample_message)
diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py
new file mode 100644
index 0000000000..a4cd09c706
--- /dev/null
+++ b/Lib/test/test_mailcap.py
@@ -0,0 +1,221 @@
+import mailcap
+import os
+import shutil
+import test.support
+import unittest
+
+# Location of mailcap file
+MAILCAPFILE = test.support.findfile("mailcap.txt")
+
+# Dict to act as mock mailcap entry for this test
+# The keys and values should match the contents of MAILCAPFILE
+MAILCAPDICT = {
+    'application/x-movie':
+        [{'compose': 'moviemaker %s',
+          'x11-bitmap': '"/usr/lib/Zmail/bitmaps/movie.xbm"',
+          'description': '"Movie"',
+          'view': 'movieplayer %s'}],
+    'application/*':
+        [{'copiousoutput': '',
+          'view': 'echo "This is \\"%t\\" but        is 50 \\% Greek to me" \\; cat %s'}],
+    'audio/basic':
+        [{'edit': 'audiocompose %s',
+          'compose': 'audiocompose %s',
+          'description': '"An audio fragment"',
+          'view': 'showaudio %s'}],
+    'video/mpeg':
+        [{'view': 'mpeg_play %s'}],
+    'application/postscript':
+        [{'needsterminal': '', 'view': 'ps-to-terminal %s'},
+         {'compose': 'idraw %s', 'view': 'ps-to-terminal %s'}],
+    'application/x-dvi':
+        [{'view': 'xdvi %s'}],
+    'message/external-body':
+        [{'composetyped': 'extcompose %s',
+          'description': '"A reference to data stored in an external location"',
+          'needsterminal': '',
+          'view': 'showexternal %s %{access-type} %{name} %{site}     %{directory} %{mode} %{server}'}],
+    'text/richtext':
+        [{'test': 'test "`echo     %{charset} | tr \'[A-Z]\' \'[a-z]\'`"  = iso-8859-8',
+          'copiousoutput': '',
+          'view': 'shownonascii iso-8859-8 -e richtext -p %s'}],
+    'image/x-xwindowdump':
+        [{'view': 'display %s'}],
+    'audio/*':
+        [{'view': '/usr/local/bin/showaudio %t'}],
+    'video/*':
+        [{'view': 'animate %s'}],
+    'application/frame':
+        [{'print': '"cat %s | lp"', 'view': 'showframe %s'}],
+    'image/rgb':
+        [{'view': 'display %s'}]
+}
+
+
+class HelperFunctionTest(unittest.TestCase):
+
+    def test_listmailcapfiles(self):
+        # The return value for listmailcapfiles() will vary by system.
+        # So verify that listmailcapfiles() returns a list of strings that is of
+        # non-zero length.
+        mcfiles = mailcap.listmailcapfiles()
+        self.assertIsInstance(mcfiles, list)
+        for m in mcfiles:
+            self.assertIsInstance(m, str)
+        with test.support.EnvironmentVarGuard() as env:
+            # According to RFC 1524, if MAILCAPS env variable exists, use that
+            # and only that.
+            if "MAILCAPS" in env:
+                env_mailcaps = env["MAILCAPS"].split(os.pathsep)
+            else:
+                env_mailcaps = ["/testdir1/.mailcap", "/testdir2/mailcap"]
+                env["MAILCAPS"] = os.pathsep.join(env_mailcaps)
+                mcfiles = mailcap.listmailcapfiles()
+        self.assertEqual(env_mailcaps, mcfiles)
+
+    def test_readmailcapfile(self):
+        # Test readmailcapfile() using test file. It should match MAILCAPDICT.
+        with open(MAILCAPFILE, 'r') as mcf:
+            d = mailcap.readmailcapfile(mcf)
+        self.assertDictEqual(d, MAILCAPDICT)
+
+    def test_lookup(self):
+        # Test without key
+        expected = [{'view': 'mpeg_play %s'}, {'view': 'animate %s'}]
+        actual = mailcap.lookup(MAILCAPDICT, 'video/mpeg')
+        self.assertListEqual(expected, actual)
+
+        # Test with key
+        key = 'compose'
+        expected = [{'edit': 'audiocompose %s',
+                     'compose': 'audiocompose %s',
+                     'description': '"An audio fragment"',
+                     'view': 'showaudio %s'}]
+        actual = mailcap.lookup(MAILCAPDICT, 'audio/basic', key)
+        self.assertListEqual(expected, actual)
+
+    def test_subst(self):
+        plist = ['id=1', 'number=2', 'total=3']
+        # test case: ([field, MIMEtype, filename, plist=[]], <expected string>)
+        test_cases = [
+            (["", "audio/*", "foo.txt"], ""),
+            (["echo foo", "audio/*", "foo.txt"], "echo foo"),
+            (["echo %s", "audio/*", "foo.txt"], "echo foo.txt"),
+            (["echo %t", "audio/*", "foo.txt"], "echo audio/*"),
+            (["echo \%t", "audio/*", "foo.txt"], "echo %t"),
+            (["echo foo", "audio/*", "foo.txt", plist], "echo foo"),
+            (["echo %{total}", "audio/*", "foo.txt", plist], "echo 3")
+        ]
+        for tc in test_cases:
+            self.assertEqual(mailcap.subst(*tc[0]), tc[1])
+
+
+class GetcapsTest(unittest.TestCase):
+
+    def test_mock_getcaps(self):
+        # Test mailcap.getcaps() using mock mailcap file in this dir.
+        # Temporarily override any existing system mailcap file by pointing the
+        # MAILCAPS environment variable to our mock file.
+        with test.support.EnvironmentVarGuard() as env:
+            env["MAILCAPS"] = MAILCAPFILE
+            caps = mailcap.getcaps()
+            self.assertDictEqual(caps, MAILCAPDICT)
+
+    def test_system_mailcap(self):
+        # Test mailcap.getcaps() with mailcap file(s) on system, if any.
+        caps = mailcap.getcaps()
+        self.assertIsInstance(caps, dict)
+        mailcapfiles = mailcap.listmailcapfiles()
+        existingmcfiles = [mcf for mcf in mailcapfiles if os.path.exists(mcf)]
+        if existingmcfiles:
+            # At least 1 mailcap file exists, so test that.
+            for (k, v) in caps.items():
+                self.assertIsInstance(k, str)
+                self.assertIsInstance(v, list)
+                for e in v:
+                    self.assertIsInstance(e, dict)
+        else:
+            # No mailcap files on system. getcaps() should return empty dict.
+            self.assertEqual({}, caps)
+
+
+class FindmatchTest(unittest.TestCase):
+
+    def test_findmatch(self):
+
+        # default findmatch arguments
+        c = MAILCAPDICT
+        fname = "foo.txt"
+        plist = ["access-type=default", "name=john", "site=python.org",
+                 "directory=/tmp", "mode=foo", "server=bar"]
+        audio_basic_entry = {
+            'edit': 'audiocompose %s',
+            'compose': 'audiocompose %s',
+            'description': '"An audio fragment"',
+            'view': 'showaudio %s'
+        }
+        audio_entry = {"view": "/usr/local/bin/showaudio %t"}
+        video_entry = {'view': 'animate %s'}
+        message_entry = {
+            'composetyped': 'extcompose %s',
+            'description': '"A reference to data stored in an external location"', 'needsterminal': '',
+            'view': 'showexternal %s %{access-type} %{name} %{site}     %{directory} %{mode} %{server}'
+        }
+
+        # test case: (findmatch args, findmatch keyword args, expected output)
+        #   positional args: caps, MIMEtype
+        #   keyword args: key="view", filename="/dev/null", plist=[]
+        #   output: (command line, mailcap entry)
+        cases = [
+            ([{}, "video/mpeg"], {}, (None, None)),
+            ([c, "foo/bar"], {}, (None, None)),
+            ([c, "video/mpeg"], {}, ('mpeg_play /dev/null', {'view': 'mpeg_play %s'})),
+            ([c, "audio/basic", "edit"], {}, ("audiocompose /dev/null", audio_basic_entry)),
+            ([c, "audio/basic", "compose"], {}, ("audiocompose /dev/null", audio_basic_entry)),
+            ([c, "audio/basic", "description"], {}, ('"An audio fragment"', audio_basic_entry)),
+            ([c, "audio/basic", "foobar"], {}, (None, None)),
+            ([c, "video/*"], {"filename": fname}, ("animate %s" % fname, video_entry)),
+            ([c, "audio/basic", "compose"],
+             {"filename": fname},
+             ("audiocompose %s" % fname, audio_basic_entry)),
+            ([c, "audio/basic"],
+             {"key": "description", "filename": fname},
+             ('"An audio fragment"', audio_basic_entry)),
+            ([c, "audio/*"],
+             {"filename": fname},
+             ("/usr/local/bin/showaudio audio/*", audio_entry)),
+            ([c, "message/external-body"],
+             {"plist": plist},
+             ("showexternal /dev/null default john python.org     /tmp foo bar", message_entry))
+        ]
+        self._run_cases(cases)
+
+    @unittest.skipUnless(os.name == "posix", "Requires 'test' command on system")
+    def test_test(self):
+        # findmatch() will automatically check any "test" conditions and skip
+        # the entry if the check fails.
+        caps = {"test/pass": [{"test": "test 1 -eq 1"}],
+                "test/fail": [{"test": "test 1 -eq 0"}]}
+        # test case: (findmatch args, findmatch keyword args, expected output)
+        #   positional args: caps, MIMEtype, key ("test")
+        #   keyword args: N/A
+        #   output: (command line, mailcap entry)
+        cases = [
+            # findmatch will return the mailcap entry for test/pass because it evaluates to true
+            ([caps, "test/pass", "test"], {}, ("test 1 -eq 1", {"test": "test 1 -eq 1"})),
+            # findmatch will return None because test/fail evaluates to false
+            ([caps, "test/fail", "test"], {}, (None, None))
+        ]
+        self._run_cases(cases)
+
+    def _run_cases(self, cases):
+        for c in cases:
+            self.assertEqual(mailcap.findmatch(*c[0], **c[1]), c[2])
+
+
+def test_main():
+    test.support.run_unittest(HelperFunctionTest, GetcapsTest, FindmatchTest)
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
index 02ef62a983..025b53ccbe 100644
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -6,6 +6,7 @@ import marshal
 import sys
 import unittest
 import os
+import types
 
 class HelperMixin:
     def helper(self, sample, *extra):
@@ -114,6 +115,22 @@ class CodeTestCase(unittest.TestCase):
         codes = (ExceptionTestCase.test_exceptions.__code__,) * count
         marshal.loads(marshal.dumps(codes))
 
+    def test_different_filenames(self):
+        co1 = compile("x", "f1", "exec")
+        co2 = compile("y", "f2", "exec")
+        co1, co2 = marshal.loads(marshal.dumps((co1, co2)))
+        self.assertEqual(co1.co_filename, "f1")
+        self.assertEqual(co2.co_filename, "f2")
+
+    @support.cpython_only
+    def test_same_filename_used(self):
+        s = """def f(): pass\ndef g(): pass"""
+        co = compile(s, "myfile", "exec")
+        co = marshal.loads(marshal.dumps(co))
+        for obj in co.co_consts:
+            if isinstance(obj, types.CodeType):
+                self.assertIs(co.co_filename, obj.co_filename)
+
 class ContainerTestCase(unittest.TestCase, HelperMixin):
     d = {'astring': 'foo@bar.baz.spam',
          'afloat': 7283.43,
@@ -263,7 +280,6 @@ class BugsTestCase(unittest.TestCase):
         self.assertRaises(TypeError, marshal.loads, unicode_string)
 
 LARGE_SIZE = 2**31
-character_size = 4 if sys.maxunicode > 0xFFFF else 2
 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
 
 class NullWriter:
@@ -279,7 +295,7 @@ class LargeValuesTestCase(unittest.TestCase):
     def test_bytes(self, size):
         self.check_unmarshallable(b'x' * size)
 
-    @support.bigmemtest(size=LARGE_SIZE, memuse=character_size, dry_run=False)
+    @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
     def test_str(self, size):
         self.check_unmarshallable('x' * size)
 
diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py
index dddc889375..715003af05 100644
--- a/Lib/test/test_math.py
+++ b/Lib/test/test_math.py
@@ -2,11 +2,12 @@
 # XXXX Should not do tests around zero only
 
 from test.support import run_unittest, verbose, requires_IEEE_754
+from test import support
 import unittest
 import math
 import os
+import platform
 import sys
-import random
 import struct
 import sysconfig
 
@@ -457,12 +458,12 @@ class MathTests(unittest.TestCase):
 
     def testFmod(self):
         self.assertRaises(TypeError, math.fmod)
-        self.ftest('fmod(10,1)', math.fmod(10,1), 0)
-        self.ftest('fmod(10,0.5)', math.fmod(10,0.5), 0)
-        self.ftest('fmod(10,1.5)', math.fmod(10,1.5), 1)
-        self.ftest('fmod(-10,1)', math.fmod(-10,1), 0)
-        self.ftest('fmod(-10,0.5)', math.fmod(-10,0.5), 0)
-        self.ftest('fmod(-10,1.5)', math.fmod(-10,1.5), -1)
+        self.ftest('fmod(10, 1)', math.fmod(10, 1), 0.0)
+        self.ftest('fmod(10, 0.5)', math.fmod(10, 0.5), 0.0)
+        self.ftest('fmod(10, 1.5)', math.fmod(10, 1.5), 1.0)
+        self.ftest('fmod(-10, 1)', math.fmod(-10, 1), -0.0)
+        self.ftest('fmod(-10, 0.5)', math.fmod(-10, 0.5), -0.0)
+        self.ftest('fmod(-10, 1.5)', math.fmod(-10, 1.5), -1.0)
         self.assertTrue(math.isnan(math.fmod(NAN, 1.)))
         self.assertTrue(math.isnan(math.fmod(1., NAN)))
         self.assertTrue(math.isnan(math.fmod(NAN, NAN)))
@@ -650,6 +651,33 @@ class MathTests(unittest.TestCase):
         n= 2**90
         self.assertAlmostEqual(math.log1p(n), math.log1p(float(n)))
 
+    @requires_IEEE_754
+    def testLog2(self):
+        self.assertRaises(TypeError, math.log2)
+
+        # Check some integer values
+        self.assertEqual(math.log2(1), 0.0)
+        self.assertEqual(math.log2(2), 1.0)
+        self.assertEqual(math.log2(4), 2.0)
+
+        # Large integer values
+        self.assertEqual(math.log2(2**1023), 1023.0)
+        self.assertEqual(math.log2(2**1024), 1024.0)
+        self.assertEqual(math.log2(2**2000), 2000.0)
+
+        self.assertRaises(ValueError, math.log2, -1.5)
+        self.assertRaises(ValueError, math.log2, NINF)
+        self.assertTrue(math.isnan(math.log2(NAN)))
+
+    @requires_IEEE_754
+    # log2() is not accurate enough on Mac OS X Tiger (10.4)
+    @support.requires_mac_ver(10, 5)
+    def testLog2Exact(self):
+        # Check that we get exact equality for log2 of powers of 2.
+        actual = [math.log2(math.ldexp(1.0, n)) for n in range(-1074, 1024)]
+        expected = [float(n) for n in range(-1074, 1024)]
+        self.assertEqual(actual, expected)
+
     def testLog10(self):
         self.assertRaises(TypeError, math.log10)
         self.ftest('log10(0.1)', math.log10(0.1), -1)
@@ -1010,7 +1038,6 @@ class MathTests(unittest.TestCase):
 
     @requires_IEEE_754
     def test_mtestfile(self):
-        ALLOWED_ERROR = 20  # permitted error, in ulps
         fail_fmt = "{}:{}({!r}): expected {!r}, got {!r}"
 
         failures = []
diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py
index 7c0c84f0d9..e5db94566f 100644
--- a/Lib/test/test_memoryio.py
+++ b/Lib/test/test_memoryio.py
@@ -658,7 +658,7 @@ class CBytesIOTest(PyBytesIOTest):
 
     @support.cpython_only
     def test_sizeof(self):
-        basesize = support.calcobjsize('P2PP2PP')
+        basesize = support.calcobjsize('P2nN2Pn')
         check = self.check_sizeof
         self.assertEqual(object.__sizeof__(io.BytesIO()), basesize)
         check(io.BytesIO(), basesize )
diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py
index 0bfddd97ed..ee6b15ac14 100644
--- a/Lib/test/test_memoryview.py
+++ b/Lib/test/test_memoryview.py
@@ -1,6 +1,7 @@
 """Unit tests for the memoryview
 
-XXX We need more tests! Some tests are in test_bytes
+   Some tests are in test_bytes. Many tests that require _testbuffer.ndarray
+   are in test_buffer.
 """
 
 import unittest
@@ -24,15 +25,14 @@ class AbstractMemoryTests:
         return filter(None, [self.ro_type, self.rw_type])
 
     def check_getitem_with_type(self, tp):
-        item = self.getitem_type
         b = tp(self._source)
         oldrefcount = sys.getrefcount(b)
         m = self._view(b)
-        self.assertEqual(m[0], item(b"a"))
-        self.assertIsInstance(m[0], bytes)
-        self.assertEqual(m[5], item(b"f"))
-        self.assertEqual(m[-1], item(b"f"))
-        self.assertEqual(m[-6], item(b"a"))
+        self.assertEqual(m[0], ord(b"a"))
+        self.assertIsInstance(m[0], int)
+        self.assertEqual(m[5], ord(b"f"))
+        self.assertEqual(m[-1], ord(b"f"))
+        self.assertEqual(m[-6], ord(b"a"))
         # Bounds checking
         self.assertRaises(IndexError, lambda: m[6])
         self.assertRaises(IndexError, lambda: m[-7])
@@ -76,7 +76,9 @@ class AbstractMemoryTests:
         b = self.rw_type(self._source)
         oldrefcount = sys.getrefcount(b)
         m = self._view(b)
-        m[0] = tp(b"0")
+        m[0] = ord(b'1')
+        self._check_contents(tp, b, b"1bcdef")
+        m[0:1] = tp(b"0")
         self._check_contents(tp, b, b"0bcdef")
         m[1:3] = tp(b"12")
         self._check_contents(tp, b, b"012def")
@@ -102,10 +104,17 @@ class AbstractMemoryTests:
         # Wrong index/slice types
         self.assertRaises(TypeError, setitem, 0.0, b"a")
         self.assertRaises(TypeError, setitem, (0,), b"a")
+        self.assertRaises(TypeError, setitem, (slice(0,1,1), 0), b"a")
+        self.assertRaises(TypeError, setitem, (0, slice(0,1,1)), b"a")
+        self.assertRaises(TypeError, setitem, (0,), b"a")
         self.assertRaises(TypeError, setitem, "a", b"a")
+        # Not implemented: multidimensional slices
+        slices = (slice(0,1,1), slice(0,1,2))
+        self.assertRaises(NotImplementedError, setitem, slices, b"a")
         # Trying to resize the memory object
-        self.assertRaises(ValueError, setitem, 0, b"")
-        self.assertRaises(ValueError, setitem, 0, b"ab")
+        exc = ValueError if m.format == 'c' else TypeError
+        self.assertRaises(exc, setitem, 0, b"")
+        self.assertRaises(exc, setitem, 0, b"ab")
         self.assertRaises(ValueError, setitem, slice(1,1), b"a")
         self.assertRaises(ValueError, setitem, slice(0,2), b"a")
 
@@ -175,7 +184,7 @@ class AbstractMemoryTests:
         self.assertEqual(m.shape, (6,))
         self.assertEqual(len(m), 6)
         self.assertEqual(m.strides, (self.itemsize,))
-        self.assertEqual(m.suboffsets, None)
+        self.assertEqual(m.suboffsets, ())
         return m
 
     def test_attributes_readonly(self):
@@ -209,12 +218,16 @@ class AbstractMemoryTests:
                 # If tp is a factory rather than a plain type, skip
                 continue
 
+            class MyView():
+                def __init__(self, base):
+                    self.m = memoryview(base)
             class MySource(tp):
                 pass
             class MyObject:
                 pass
 
-            # Create a reference cycle through a memoryview object
+            # Create a reference cycle through a memoryview object.
+            # This exercises mbuf_clear().
             b = MySource(tp(b'abc'))
             m = self._view(b)
             o = MyObject()
@@ -226,6 +239,17 @@ class AbstractMemoryTests:
             gc.collect()
             self.assertTrue(wr() is None, wr())
 
+            # This exercises memory_clear().
+            m = MyView(tp(b'abc'))
+            o = MyObject()
+            m.x = m
+            m.o = o
+            wr = weakref.ref(o)
+            m = o = None
+            # The cycle must be broken
+            gc.collect()
+            self.assertTrue(wr() is None, wr())
+
     def _check_released(self, m, tp):
         check = self.assertRaisesRegex(ValueError, "released")
         with check: bytes(m)
@@ -283,6 +307,51 @@ class AbstractMemoryTests:
         i = io.BytesIO(b'ZZZZ')
         self.assertRaises(TypeError, i.readinto, m)
 
+    def test_getbuf_fail(self):
+        self.assertRaises(TypeError, self._view, {})
+
+    def test_hash(self):
+        # Memoryviews of readonly (hashable) types are hashable, and they
+        # hash as hash(obj.tobytes()).
+        tp = self.ro_type
+        if tp is None:
+            self.skipTest("no read-only type to test")
+        b = tp(self._source)
+        m = self._view(b)
+        self.assertEqual(hash(m), hash(b"abcdef"))
+        # Releasing the memoryview keeps the stored hash value (as with weakrefs)
+        m.release()
+        self.assertEqual(hash(m), hash(b"abcdef"))
+        # Hashing a memoryview for the first time after it is released
+        # results in an error (as with weakrefs).
+        m = self._view(b)
+        m.release()
+        self.assertRaises(ValueError, hash, m)
+
+    def test_hash_writable(self):
+        # Memoryviews of writable types are unhashable
+        tp = self.rw_type
+        if tp is None:
+            self.skipTest("no writable type to test")
+        b = tp(self._source)
+        m = self._view(b)
+        self.assertRaises(ValueError, hash, m)
+
+    def test_weakref(self):
+        # Check memoryviews are weakrefable
+        for tp in self._types:
+            b = tp(self._source)
+            m = self._view(b)
+            L = []
+            def callback(wr, b=b):
+                L.append(b)
+            wr = weakref.ref(m, callback)
+            self.assertIs(wr(), m)
+            del m
+            test.support.gc_collect()
+            self.assertIs(wr(), None)
+            self.assertIs(L[0], b)
+
 # Variations on source objects for the buffer: bytes-like objects, then arrays
 # with itemsize > 1.
 # NOTE: support for multi-dimensional objects is unimplemented.
diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py
index 219ab99840..e6fe20a107 100644
--- a/Lib/test/test_metaclass.py
+++ b/Lib/test/test_metaclass.py
@@ -159,6 +159,7 @@ Use a __prepare__ method that returns an instrumented dict.
     ...     bar = 123
     ...
     d['__module__'] = 'test.test_metaclass'
+    d['__qualname__'] = 'C'
     d['foo'] = 4
     d['foo'] = 42
     d['bar'] = 123
@@ -177,12 +178,12 @@ Use a metaclass that doesn't derive from type.
     ...     b = 24
     ...
     meta: C ()
-    ns: [('__module__', 'test.test_metaclass'), ('a', 42), ('b', 24)]
+    ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)]
     kw: []
     >>> type(C) is dict
     True
     >>> print(sorted(C.items()))
-    [('__module__', 'test.test_metaclass'), ('a', 42), ('b', 24)]
+    [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)]
     >>>
 
 And again, with a __prepare__ attribute.
@@ -199,11 +200,12 @@ And again, with a __prepare__ attribute.
     ...
     prepare: C () [('other', 'booh')]
     d['__module__'] = 'test.test_metaclass'
+    d['__qualname__'] = 'C'
     d['a'] = 1
     d['a'] = 2
     d['b'] = 3
     meta: C ()
-    ns: [('__module__', 'test.test_metaclass'), ('a', 2), ('b', 3)]
+    ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)]
     kw: [('other', 'booh')]
     >>>
 
@@ -246,7 +248,13 @@ Test failures in looking up the __prepare__ method work.
 
 """
 
-__test__ = {'doctests' : doctests}
+import sys
+
+# Trace function introduces __locals__ which causes various tests to fail.
+if hasattr(sys, 'gettrace') and sys.gettrace():
+    __test__ = {}
+else:
+    __test__ = {'doctests' : doctests}
 
 def test_main(verbose=False):
     from test import support
diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py
index 80812c8cfa..5867b2dcba 100644
--- a/Lib/test/test_minidom.py
+++ b/Lib/test/test_minidom.py
@@ -4,9 +4,7 @@ import pickle
 from test.support import verbose, run_unittest, findfile
 import unittest
 
-import xml.dom
 import xml.dom.minidom
-import xml.parsers.expat
 
 from xml.dom.minidom import parse, Node, Document, parseString
 from xml.dom.minidom import getDOMImplementation
@@ -14,7 +12,6 @@ from xml.dom.minidom import getDOMImplementation
 
 tstfile = findfile("test.xml", subdir="xmltestdata")
 
-
 # The tests of DocumentType importing use these helpers to construct
 # the documents to work with, since not all DOM builders actually
 # create the DocumentType nodes.
@@ -50,7 +47,7 @@ class MinidomTest(unittest.TestCase):
 
     def checkWholeText(self, node, s):
         t = node.wholeText
-        self.confirm(t == s, "looking for %s, found %s" % (repr(s), repr(t)))
+        self.confirm(t == s, "looking for %r, found %r" % (s, t))
 
     def testParseFromFile(self):
         with open(tstfile) as file:
@@ -282,6 +279,7 @@ class MinidomTest(unittest.TestCase):
 
         child.setAttribute("def", "ghi")
         self.confirm(len(child.attributes) == 1)
+        self.assertRaises(xml.dom.NotFoundErr, child.removeAttribute, "foo")
         child.removeAttribute("def")
         self.confirm(len(child.attributes) == 0)
         dom.unlink()
@@ -293,6 +291,8 @@ class MinidomTest(unittest.TestCase):
         child.setAttributeNS("http://www.w3.org", "xmlns:python",
                                                 "http://www.python.org")
         child.setAttributeNS("http://www.python.org", "python:abcattr", "foo")
+        self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNS,
+            "foo", "http://www.python.org")
         self.confirm(len(child.attributes) == 2)
         child.removeAttributeNS("http://www.python.org", "abcattr")
         self.confirm(len(child.attributes) == 1)
@@ -304,11 +304,23 @@ class MinidomTest(unittest.TestCase):
         child.setAttribute("spam", "jam")
         self.confirm(len(child.attributes) == 1)
         node = child.getAttributeNode("spam")
+        self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNode,
+            None)
         child.removeAttributeNode(node)
         self.confirm(len(child.attributes) == 0
                 and child.getAttributeNode("spam") is None)
+        dom2 = Document()
+        child2 = dom2.appendChild(dom.createElement("foo"))
+        self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNode,
+            node)
         dom.unlink()
 
+    def testHasAttribute(self):
+        dom = Document()
+        child = dom.appendChild(dom.createElement("foo"))
+        child.setAttribute("spam", "jam")
+        self.confirm(child.hasAttribute("spam"))
+
     def testChangeAttr(self):
         dom = parseString("<abc/>")
         el = dom.documentElement
@@ -592,7 +604,13 @@ class MinidomTest(unittest.TestCase):
 
     def testFirstChild(self): pass
 
-    def testHasChildNodes(self): pass
+    def testHasChildNodes(self):
+        dom = parseString("<doc><foo/></doc>")
+        doc = dom.documentElement
+        self.assertTrue(dom.hasChildNodes())
+        dom2 = parseString("<doc/>")
+        doc2 = dom2.documentElement
+        self.assertFalse(doc2.hasChildNodes())
 
     def _testCloneElementCopiesAttributes(self, e1, e2, test):
         attrs1 = e1.attributes
@@ -1041,41 +1059,6 @@ class MinidomTest(unittest.TestCase):
                 "test NodeList.item()")
         doc.unlink()
 
-    def testSAX2DOM(self):
-        from xml.dom import pulldom
-
-        sax2dom = pulldom.SAX2DOM()
-        sax2dom.startDocument()
-        sax2dom.startElement("doc", {})
-        sax2dom.characters("text")
-        sax2dom.startElement("subelm", {})
-        sax2dom.characters("text")
-        sax2dom.endElement("subelm")
-        sax2dom.characters("text")
-        sax2dom.endElement("doc")
-        sax2dom.endDocument()
-
-        doc = sax2dom.document
-        root = doc.documentElement
-        (text1, elm1, text2) = root.childNodes
-        text3 = elm1.childNodes[0]
-
-        self.confirm(text1.previousSibling is None and
-                text1.nextSibling is elm1 and
-                elm1.previousSibling is text1 and
-                elm1.nextSibling is text2 and
-                text2.previousSibling is elm1 and
-                text2.nextSibling is None and
-                text3.previousSibling is None and
-                text3.nextSibling is None, "testSAX2DOM - siblings")
-
-        self.confirm(root.parentNode is doc and
-                text1.parentNode is root and
-                elm1.parentNode is root and
-                text2.parentNode is root and
-                text3.parentNode is elm1, "testSAX2DOM - parents")
-        doc.unlink()
-
     def testEncodings(self):
         doc = parseString('<foo>&#x20ac;</foo>')
         self.assertEqual(doc.toxml(),
@@ -1084,6 +1067,11 @@ class MinidomTest(unittest.TestCase):
             b'<?xml version="1.0" encoding="utf-8"?><foo>\xe2\x82\xac</foo>')
         self.assertEqual(doc.toxml('iso-8859-15'),
             b'<?xml version="1.0" encoding="iso-8859-15"?><foo>\xa4</foo>')
+        self.assertEqual(doc.toxml('us-ascii'),
+            b'<?xml version="1.0" encoding="us-ascii"?><foo>&#8364;</foo>')
+        self.assertEqual(doc.toxml('utf-16'),
+            '<?xml version="1.0" encoding="utf-16"?>'
+            '<foo>\u20ac</foo>'.encode('utf-16'))
 
         # Verify that character decoding errors raise exceptions instead
         # of crashing
@@ -1522,12 +1510,21 @@ class MinidomTest(unittest.TestCase):
         doc.appendChild(doc.createComment("foo--bar"))
         self.assertRaises(ValueError, doc.toxml)
 
+
     def testEmptyXMLNSValue(self):
         doc = parseString("<element xmlns=''>\n"
                           "<foo/>\n</element>")
         doc2 = parseString(doc.toxml())
         self.confirm(doc2.namespaceURI == xml.dom.EMPTY_NAMESPACE)
 
+    def testDocRemoveChild(self):
+        doc = parse(tstfile)
+        title_tag = doc.documentElement.getElementsByTagName("TITLE")[0]
+        self.assertRaises( xml.dom.NotFoundErr, doc.removeChild, title_tag)
+        num_children_before = len(doc.childNodes)
+        doc.removeChild(doc.childNodes[0])
+        num_children_after = len(doc.childNodes)
+        self.assertTrue(num_children_after == num_children_before - 1)
 
 def test_main():
     run_unittest(MinidomTest)
diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py
index 4407c5cfc9..d066368726 100644
--- a/Lib/test/test_mmap.py
+++ b/Lib/test/test_mmap.py
@@ -417,6 +417,35 @@ class MmapTests(unittest.TestCase):
             m[x] = b
             self.assertEqual(m[x], b)
 
+    def test_read_all(self):
+        m = mmap.mmap(-1, 16)
+        self.addCleanup(m.close)
+
+        # With no parameters, or None or a negative argument, reads all
+        m.write(bytes(range(16)))
+        m.seek(0)
+        self.assertEqual(m.read(), bytes(range(16)))
+        m.seek(8)
+        self.assertEqual(m.read(), bytes(range(8, 16)))
+        m.seek(16)
+        self.assertEqual(m.read(), b'')
+        m.seek(3)
+        self.assertEqual(m.read(None), bytes(range(3, 16)))
+        m.seek(4)
+        self.assertEqual(m.read(-1), bytes(range(4, 16)))
+        m.seek(5)
+        self.assertEqual(m.read(-2), bytes(range(5, 16)))
+        m.seek(9)
+        self.assertEqual(m.read(-42), bytes(range(9, 16)))
+
+    def test_read_invalid_arg(self):
+        m = mmap.mmap(-1, 16)
+        self.addCleanup(m.close)
+
+        self.assertRaises(TypeError, m.read, 'foo')
+        self.assertRaises(TypeError, m.read, 5.5)
+        self.assertRaises(TypeError, m.read, [1, 2, 3])
+
     def test_extended_getslice(self):
         # Test extended slicing by comparing with list slicing.
         s = bytes(reversed(range(256)))
@@ -543,8 +572,7 @@ class MmapTests(unittest.TestCase):
         f.close()
 
     def test_error(self):
-        self.assertTrue(issubclass(mmap.error, EnvironmentError))
-        self.assertIn("mmap.error", str(mmap.error))
+        self.assertIs(mmap.error, OSError)
 
     def test_io_methods(self):
         data = b"0123456789"
diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py
index 15836cac93..e5a2525d18 100644
--- a/Lib/test/test_module.py
+++ b/Lib/test/test_module.py
@@ -5,6 +5,15 @@ from test.support import run_unittest, gc_collect
 import sys
 ModuleType = type(sys)
 
+class FullLoader:
+    @classmethod
+    def module_repr(cls, m):
+        return "<module '{}' (crafted)>".format(m.__name__)
+
+class BareLoader:
+    pass
+
+
 class ModuleTests(unittest.TestCase):
     def test_uninitialized(self):
         # An uninitialized module has no __dict__ or __name__,
@@ -70,16 +79,100 @@ class ModuleTests(unittest.TestCase):
         m = ModuleType("foo")
         m.destroyed = destroyed
         s = """class A:
-    def __del__(self, destroyed=destroyed):
-        destroyed.append(1)
-a = A()"""
+    def __init__(self, l):
+        self.l = l
+    def __del__(self):
+        self.l.append(1)
+a = A(destroyed)"""
         exec(s, m.__dict__)
         del m
         gc_collect()
         self.assertEqual(destroyed, [1])
 
+    def test_module_repr_minimal(self):
+        # reprs when modules have no __file__, __name__, or __loader__
+        m = ModuleType('foo')
+        del m.__name__
+        self.assertEqual(repr(m), "<module '?'>")
+
+    def test_module_repr_with_name(self):
+        m = ModuleType('foo')
+        self.assertEqual(repr(m), "<module 'foo'>")
+
+    def test_module_repr_with_name_and_filename(self):
+        m = ModuleType('foo')
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module 'foo' from '/tmp/foo.py'>")
+
+    def test_module_repr_with_filename_only(self):
+        m = ModuleType('foo')
+        del m.__name__
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module '?' from '/tmp/foo.py'>")
+
+    def test_module_repr_with_bare_loader_but_no_name(self):
+        m = ModuleType('foo')
+        del m.__name__
+        # Yes, a class not an instance.
+        m.__loader__ = BareLoader
+        self.assertEqual(
+            repr(m), "<module '?' (<class 'test.test_module.BareLoader'>)>")
+
+    def test_module_repr_with_full_loader_but_no_name(self):
+        # m.__loader__.module_repr() will fail because the module has no
+        # m.__name__.  This exception will get suppressed and instead the
+        # loader's repr will be used.
+        m = ModuleType('foo')
+        del m.__name__
+        # Yes, a class not an instance.
+        m.__loader__ = FullLoader
+        self.assertEqual(
+            repr(m), "<module '?' (<class 'test.test_module.FullLoader'>)>")
+
+    def test_module_repr_with_bare_loader(self):
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = BareLoader
+        self.assertEqual(
+            repr(m), "<module 'foo' (<class 'test.test_module.BareLoader'>)>")
+
+    def test_module_repr_with_full_loader(self):
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = FullLoader
+        self.assertEqual(
+            repr(m), "<module 'foo' (crafted)>")
+
+    def test_module_repr_with_bare_loader_and_filename(self):
+        # Because the loader has no module_repr(), use the file name.
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = BareLoader
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module 'foo' from '/tmp/foo.py'>")
+
+    def test_module_repr_with_full_loader_and_filename(self):
+        # Even though the module has an __file__, use __loader__.module_repr()
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = FullLoader
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module 'foo' (crafted)>")
+
+    def test_module_repr_builtin(self):
+        self.assertEqual(repr(sys), "<module 'sys' (built-in)>")
+
+    def test_module_repr_source(self):
+        r = repr(unittest)
+        self.assertEqual(r[:25], "<module 'unittest' from '")
+        self.assertEqual(r[-13:], "__init__.py'>")
+
+    # frozen and namespace module reprs are tested in importlib.
+
+
 def test_main():
     run_unittest(ModuleTests)
 
+
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py
index a1842178df..53ea232174 100644
--- a/Lib/test/test_modulefinder.py
+++ b/Lib/test/test_modulefinder.py
@@ -1,7 +1,7 @@
-import __future__
 import os
+import errno
+import shutil
 import unittest
-import distutils.dir_util
 import tempfile
 
 from test import support
@@ -9,7 +9,7 @@ from test import support
 import modulefinder
 
 TEST_DIR = tempfile.mkdtemp()
-TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)]
+TEST_PATH = [TEST_DIR, os.path.dirname(tempfile.__file__)]
 
 # Each test description is a list of 5 items:
 #
@@ -196,12 +196,29 @@ a/module.py
                                 from . import bar
 """]
 
+relative_import_test_4 = [
+    "a.module",
+    ["a", "a.module"],
+    [],
+    [],
+    """\
+a/__init__.py
+                                def foo(): pass
+a/module.py
+                                from . import *
+"""]
+
+
 def open_file(path):
-    ##print "#", os.path.abspath(path)
     dirname = os.path.dirname(path)
-    distutils.dir_util.mkpath(dirname)
+    try:
+        os.makedirs(dirname)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
     return open(path, "w")
 
+
 def create_package(source):
     ofi = None
     try:
@@ -216,6 +233,7 @@ def create_package(source):
         if ofi:
             ofi.close()
 
+
 class ModuleFinderTest(unittest.TestCase):
     def _do_test(self, info, report=False):
         import_this, modules, missing, maybe_missing, source = info
@@ -234,19 +252,17 @@ class ModuleFinderTest(unittest.TestCase):
 ##                    import traceback; traceback.print_exc()
 ##                sys.path = opath
 ##                return
-            modules = set(modules)
-            found = set(mf.modules.keys())
-            more = list(found - modules)
-            less = list(modules - found)
+            modules = sorted(set(modules))
+            found = sorted(mf.modules)
             # check if we found what we expected, not more, not less
-            self.assertEqual((more, less), ([], []))
+            self.assertEqual(found, modules)
 
             # check for missing and maybe missing modules
             bad, maybe = mf.any_missing_maybe()
             self.assertEqual(bad, missing)
             self.assertEqual(maybe, maybe_missing)
         finally:
-            distutils.dir_util.remove_tree(TEST_DIR)
+            shutil.rmtree(TEST_DIR)
 
     def test_package(self):
         self._do_test(package_test)
@@ -254,25 +270,26 @@ class ModuleFinderTest(unittest.TestCase):
     def test_maybe(self):
         self._do_test(maybe_test)
 
-    if getattr(__future__, "absolute_import", None):
+    def test_maybe_new(self):
+        self._do_test(maybe_test_new)
+
+    def test_absolute_imports(self):
+        self._do_test(absolute_import_test)
 
-        def test_maybe_new(self):
-            self._do_test(maybe_test_new)
+    def test_relative_imports(self):
+        self._do_test(relative_import_test)
 
-        def test_absolute_imports(self):
-            self._do_test(absolute_import_test)
+    def test_relative_imports_2(self):
+        self._do_test(relative_import_test_2)
 
-        def test_relative_imports(self):
-            self._do_test(relative_import_test)
+    def test_relative_imports_3(self):
+        self._do_test(relative_import_test_3)
 
-        def test_relative_imports_2(self):
-            self._do_test(relative_import_test_2)
+    def test_relative_imports_4(self):
+        self._do_test(relative_import_test_4)
 
-        def test_relative_imports_3(self):
-            self._do_test(relative_import_test_3)
 
 def test_main():
-    distutils.log.set_threshold(distutils.log.WARN)
     support.run_unittest(ModuleFinderTest)
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index 86c68dcd9a..feb7bd595a 100644
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -247,20 +247,16 @@ class Test_ISO2022(unittest.TestCase):
             self.assertFalse(any(x > 0x80 for x in e))
 
     def test_bug1572832(self):
-        if sys.maxunicode >= 0x10000:
-            myunichr = chr
-        else:
-            myunichr = lambda x: chr(0xD7C0+(x>>10)) + chr(0xDC00+(x&0x3FF))
-
         for x in range(0x10000, 0x110000):
             # Any ISO 2022 codec will cause the segfault
-            myunichr(x).encode('iso_2022_jp', 'ignore')
+            chr(x).encode('iso_2022_jp', 'ignore')
 
 class TestStateful(unittest.TestCase):
     text = '\u4E16\u4E16'
     encoding = 'iso-2022-jp'
     expected = b'\x1b$B@$@$'
-    expected_reset = b'\x1b$B@$@$\x1b(B'
+    reset = b'\x1b(B'
+    expected_reset = expected + reset
 
     def test_encode(self):
         self.assertEqual(self.text.encode(self.encoding), self.expected_reset)
@@ -271,6 +267,8 @@ class TestStateful(unittest.TestCase):
             encoder.encode(char)
             for char in self.text)
         self.assertEqual(output, self.expected)
+        self.assertEqual(encoder.encode('', final=True), self.reset)
+        self.assertEqual(encoder.encode('', final=True), b'')
 
     def test_incrementalencoder_final(self):
         encoder = codecs.getincrementalencoder(self.encoding)()
@@ -279,12 +277,14 @@ class TestStateful(unittest.TestCase):
             encoder.encode(char, index == last_index)
             for index, char in enumerate(self.text))
         self.assertEqual(output, self.expected_reset)
+        self.assertEqual(encoder.encode('', final=True), b'')
 
 class TestHZStateful(TestStateful):
     text = '\u804a\u804a'
     encoding = 'hz'
     expected = b'~{ADAD'
-    expected_reset = b'~{ADAD~}'
+    reset = b'~}'
+    expected_reset = expected + reset
 
 def test_main():
     support.run_unittest(__name__)
diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py
index 05fba7fb68..3fb07f6a54 100644
--- a/Lib/test/test_multiprocessing.py
+++ b/Lib/test/test_multiprocessing.py
@@ -8,6 +8,7 @@ import unittest
 import queue as pyqueue
 import time
 import io
+import itertools
 import sys
 import os
 import gc
@@ -17,6 +18,7 @@ import array
 import socket
 import random
 import logging
+import struct
 import test.support
 import test.script_helper
 
@@ -84,6 +86,13 @@ HAVE_GETVALUE = not getattr(_multiprocessing,
 
 WIN32 = (sys.platform == "win32")
 
+from multiprocessing.connection import wait
+
+def wait_for_handle(handle, timeout):
+    if timeout is not None and timeout < 0.0:
+        timeout = None
+    return wait([handle], timeout)
+
 try:
     MAXFD = os.sysconf("SC_OPEN_MAX")
 except:
@@ -197,6 +206,18 @@ class _TestProcess(BaseTestCase):
         self.assertEqual(current.ident, os.getpid())
         self.assertEqual(current.exitcode, None)
 
+    def test_daemon_argument(self):
+        if self.TYPE == "threads":
+            return
+
+        # By default uses the current process's daemon flag.
+        proc0 = self.Process(target=self._test)
+        self.assertEqual(proc0.daemon, self.current_process().daemon)
+        proc1 = self.Process(target=self._test, daemon=True)
+        self.assertTrue(proc1.daemon)
+        proc2 = self.Process(target=self._test, daemon=False)
+        self.assertFalse(proc2.daemon)
+
     @classmethod
     def _test(cls, q, *args, **kwds):
         current = cls.current_process()
@@ -262,9 +283,18 @@ class _TestProcess(BaseTestCase):
         self.assertIn(p, self.active_children())
         self.assertEqual(p.exitcode, None)
 
+        join = TimingWrapper(p.join)
+
+        self.assertEqual(join(0), None)
+        self.assertTimingAlmostEqual(join.elapsed, 0.0)
+        self.assertEqual(p.is_alive(), True)
+
+        self.assertEqual(join(-1), None)
+        self.assertTimingAlmostEqual(join.elapsed, 0.0)
+        self.assertEqual(p.is_alive(), True)
+
         p.terminate()
 
-        join = TimingWrapper(p.join)
         self.assertEqual(join(), None)
         self.assertTimingAlmostEqual(join.elapsed, 0.0)
 
@@ -329,6 +359,26 @@ class _TestProcess(BaseTestCase):
             ]
         self.assertEqual(result, expected)
 
+    @classmethod
+    def _test_sentinel(cls, event):
+        event.wait(10.0)
+
+    def test_sentinel(self):
+        if self.TYPE == "threads":
+            return
+        event = self.Event()
+        p = self.Process(target=self._test_sentinel, args=(event,))
+        with self.assertRaises(ValueError):
+            p.sentinel
+        p.start()
+        self.addCleanup(p.join)
+        sentinel = p.sentinel
+        self.assertIsInstance(sentinel, int)
+        self.assertFalse(wait_for_handle(sentinel, timeout=0.0))
+        event.set()
+        p.join()
+        self.assertTrue(wait_for_handle(sentinel, timeout=DELTA))
+
 #
 #
 #
@@ -868,6 +918,104 @@ class _TestCondition(BaseTestCase):
         self.assertEqual(res, False)
         self.assertTimingAlmostEqual(wait.elapsed, TIMEOUT1)
 
+    @classmethod
+    def _test_waitfor_f(cls, cond, state):
+        with cond:
+            state.value = 0
+            cond.notify()
+            result = cond.wait_for(lambda : state.value==4)
+            if not result or state.value != 4:
+                sys.exit(1)
+
+    @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
+    def test_waitfor(self):
+        # based on test in test/lock_tests.py
+        cond = self.Condition()
+        state = self.Value('i', -1)
+
+        p = self.Process(target=self._test_waitfor_f, args=(cond, state))
+        p.daemon = True
+        p.start()
+
+        with cond:
+            result = cond.wait_for(lambda : state.value==0)
+            self.assertTrue(result)
+            self.assertEqual(state.value, 0)
+
+        for i in range(4):
+            time.sleep(0.01)
+            with cond:
+                state.value += 1
+                cond.notify()
+
+        p.join(5)
+        self.assertFalse(p.is_alive())
+        self.assertEqual(p.exitcode, 0)
+
+    @classmethod
+    def _test_waitfor_timeout_f(cls, cond, state, success, sem):
+        sem.release()
+        with cond:
+            expected = 0.1
+            dt = time.time()
+            result = cond.wait_for(lambda : state.value==4, timeout=expected)
+            dt = time.time() - dt
+            # borrow logic in assertTimeout() from test/lock_tests.py
+            if not result and expected * 0.6 < dt < expected * 10.0:
+                success.value = True
+
+    @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
+    def test_waitfor_timeout(self):
+        # based on test in test/lock_tests.py
+        cond = self.Condition()
+        state = self.Value('i', 0)
+        success = self.Value('i', False)
+        sem = self.Semaphore(0)
+
+        p = self.Process(target=self._test_waitfor_timeout_f,
+                         args=(cond, state, success, sem))
+        p.daemon = True
+        p.start()
+        self.assertTrue(sem.acquire(timeout=10))
+
+        # Only increment 3 times, so state == 4 is never reached.
+        for i in range(3):
+            time.sleep(0.01)
+            with cond:
+                state.value += 1
+                cond.notify()
+
+        p.join(5)
+        self.assertTrue(success.value)
+
+    @classmethod
+    def _test_wait_result(cls, c, pid):
+        with c:
+            c.notify()
+        time.sleep(1)
+        if pid is not None:
+            os.kill(pid, signal.SIGINT)
+
+    def test_wait_result(self):
+        if isinstance(self, ProcessesMixin) and sys.platform != 'win32':
+            pid = os.getpid()
+        else:
+            pid = None
+
+        c = self.Condition()
+        with c:
+            self.assertFalse(c.wait(0))
+            self.assertFalse(c.wait(0.1))
+
+            p = self.Process(target=self._test_wait_result, args=(c, pid))
+            p.start()
+
+            self.assertTrue(c.wait(10))
+            if pid is not None:
+                self.assertRaises(KeyboardInterrupt, c.wait, 10)
+
+            p.join()
+
 
 class _TestEvent(BaseTestCase):
 
@@ -911,6 +1059,340 @@ class _TestEvent(BaseTestCase):
         self.assertEqual(wait(), True)
 
 #
+# Tests for Barrier - adapted from tests in test/lock_tests.py
+#
+
+# Many of the tests for threading.Barrier use a list as an atomic
+# counter: a value is appended to increment the counter, and the
+# length of the list gives the value.  We use the class DummyList
+# for the same purpose.
+
+class _DummyList(object):
+
+    def __init__(self):
+        wrapper = multiprocessing.heap.BufferWrapper(struct.calcsize('i'))
+        lock = multiprocessing.Lock()
+        self.__setstate__((wrapper, lock))
+        self._lengthbuf[0] = 0
+
+    def __setstate__(self, state):
+        (self._wrapper, self._lock) = state
+        self._lengthbuf = self._wrapper.create_memoryview().cast('i')
+
+    def __getstate__(self):
+        return (self._wrapper, self._lock)
+
+    def append(self, _):
+        with self._lock:
+            self._lengthbuf[0] += 1
+
+    def __len__(self):
+        with self._lock:
+            return self._lengthbuf[0]
+
+def _wait():
+    # A crude wait/yield function not relying on synchronization primitives.
+    time.sleep(0.01)
+
+
+class Bunch(object):
+    """
+    A bunch of threads.
+    """
+    def __init__(self, namespace, f, args, n, wait_before_exit=False):
+        """
+        Construct a bunch of `n` threads running the same function `f`.
+        If `wait_before_exit` is True, the threads won't terminate until
+        do_finish() is called.
+        """
+        self.f = f
+        self.args = args
+        self.n = n
+        self.started = namespace.DummyList()
+        self.finished = namespace.DummyList()
+        self._can_exit = namespace.Event()
+        if not wait_before_exit:
+            self._can_exit.set()
+        for i in range(n):
+            p = namespace.Process(target=self.task)
+            p.daemon = True
+            p.start()
+
+    def task(self):
+        pid = os.getpid()
+        self.started.append(pid)
+        try:
+            self.f(*self.args)
+        finally:
+            self.finished.append(pid)
+            self._can_exit.wait(30)
+            assert self._can_exit.is_set()
+
+    def wait_for_started(self):
+        while len(self.started) < self.n:
+            _wait()
+
+    def wait_for_finished(self):
+        while len(self.finished) < self.n:
+            _wait()
+
+    def do_finish(self):
+        self._can_exit.set()
+
+
+class AppendTrue(object):
+    def __init__(self, obj):
+        self.obj = obj
+    def __call__(self):
+        self.obj.append(True)
+
+
+class _TestBarrier(BaseTestCase):
+    """
+    Tests for Barrier objects.
+    """
+    N = 5
+    defaultTimeout = 30.0  # XXX Slow Windows buildbots need generous timeout
+
+    def setUp(self):
+        self.barrier = self.Barrier(self.N, timeout=self.defaultTimeout)
+
+    def tearDown(self):
+        self.barrier.abort()
+        self.barrier = None
+
+    def DummyList(self):
+        if self.TYPE == 'threads':
+            return []
+        elif self.TYPE == 'manager':
+            return self.manager.list()
+        else:
+            return _DummyList()
+
+    def run_threads(self, f, args):
+        b = Bunch(self, f, args, self.N-1)
+        f(*args)
+        b.wait_for_finished()
+
+    @classmethod
+    def multipass(cls, barrier, results, n):
+        m = barrier.parties
+        assert m == cls.N
+        for i in range(n):
+            results[0].append(True)
+            assert len(results[1]) == i * m
+            barrier.wait()
+            results[1].append(True)
+            assert len(results[0]) == (i + 1) * m
+            barrier.wait()
+        try:
+            assert barrier.n_waiting == 0
+        except NotImplementedError:
+            pass
+        assert not barrier.broken
+
+    def test_barrier(self, passes=1):
+        """
+        Test that a barrier is passed in lockstep
+        """
+        results = [self.DummyList(), self.DummyList()]
+        self.run_threads(self.multipass, (self.barrier, results, passes))
+
+    def test_barrier_10(self):
+        """
+        Test that a barrier works for 10 consecutive runs
+        """
+        return self.test_barrier(10)
+
+    @classmethod
+    def _test_wait_return_f(cls, barrier, queue):
+        res = barrier.wait()
+        queue.put(res)
+
+    def test_wait_return(self):
+        """
+        test the return value from barrier.wait
+        """
+        queue = self.Queue()
+        self.run_threads(self._test_wait_return_f, (self.barrier, queue))
+        results = [queue.get() for i in range(self.N)]
+        self.assertEqual(results.count(0), 1)
+
+    @classmethod
+    def _test_action_f(cls, barrier, results):
+        barrier.wait()
+        if len(results) != 1:
+            raise RuntimeError
+
+    def test_action(self):
+        """
+        Test the 'action' callback
+        """
+        results = self.DummyList()
+        barrier = self.Barrier(self.N, action=AppendTrue(results))
+        self.run_threads(self._test_action_f, (barrier, results))
+        self.assertEqual(len(results), 1)
+
+    @classmethod
+    def _test_abort_f(cls, barrier, results1, results2):
+        try:
+            i = barrier.wait()
+            if i == cls.N//2:
+                raise RuntimeError
+            barrier.wait()
+            results1.append(True)
+        except threading.BrokenBarrierError:
+            results2.append(True)
+        except RuntimeError:
+            barrier.abort()
+
+    def test_abort(self):
+        """
+        Test that an abort will put the barrier in a broken state
+        """
+        results1 = self.DummyList()
+        results2 = self.DummyList()
+        self.run_threads(self._test_abort_f,
+                         (self.barrier, results1, results2))
+        self.assertEqual(len(results1), 0)
+        self.assertEqual(len(results2), self.N-1)
+        self.assertTrue(self.barrier.broken)
+
+    @classmethod
+    def _test_reset_f(cls, barrier, results1, results2, results3):
+        i = barrier.wait()
+        if i == cls.N//2:
+            # Wait until the other threads are all in the barrier.
+            while barrier.n_waiting < cls.N-1:
+                time.sleep(0.001)
+            barrier.reset()
+        else:
+            try:
+                barrier.wait()
+                results1.append(True)
+            except threading.BrokenBarrierError:
+                results2.append(True)
+        # Now, pass the barrier again
+        barrier.wait()
+        results3.append(True)
+
+    def test_reset(self):
+        """
+        Test that a 'reset' on a barrier frees the waiting threads
+        """
+        results1 = self.DummyList()
+        results2 = self.DummyList()
+        results3 = self.DummyList()
+        self.run_threads(self._test_reset_f,
+                         (self.barrier, results1, results2, results3))
+        self.assertEqual(len(results1), 0)
+        self.assertEqual(len(results2), self.N-1)
+        self.assertEqual(len(results3), self.N)
+
+    @classmethod
+    def _test_abort_and_reset_f(cls, barrier, barrier2,
+                                results1, results2, results3):
+        try:
+            i = barrier.wait()
+            if i == cls.N//2:
+                raise RuntimeError
+            barrier.wait()
+            results1.append(True)
+        except threading.BrokenBarrierError:
+            results2.append(True)
+        except RuntimeError:
+            barrier.abort()
+        # Synchronize and reset the barrier.  Must synchronize first so
+        # that everyone has left it when we reset, and after so that no
+        # one enters it before the reset.
+        if barrier2.wait() == cls.N//2:
+            barrier.reset()
+        barrier2.wait()
+        barrier.wait()
+        results3.append(True)
+
+    def test_abort_and_reset(self):
+        """
+        Test that a barrier can be reset after being broken.
+        """
+        results1 = self.DummyList()
+        results2 = self.DummyList()
+        results3 = self.DummyList()
+        barrier2 = self.Barrier(self.N)
+
+        self.run_threads(self._test_abort_and_reset_f,
+                         (self.barrier, barrier2, results1, results2, results3))
+        self.assertEqual(len(results1), 0)
+        self.assertEqual(len(results2), self.N-1)
+        self.assertEqual(len(results3), self.N)
+
+    @classmethod
+    def _test_timeout_f(cls, barrier, results):
+        i = barrier.wait()
+        if i == cls.N//2:
+            # One thread is late!
+            time.sleep(1.0)
+        try:
+            barrier.wait(0.5)
+        except threading.BrokenBarrierError:
+            results.append(True)
+
+    def test_timeout(self):
+        """
+        Test wait(timeout)
+        """
+        results = self.DummyList()
+        self.run_threads(self._test_timeout_f, (self.barrier, results))
+        self.assertEqual(len(results), self.barrier.parties)
+
+    @classmethod
+    def _test_default_timeout_f(cls, barrier, results):
+        i = barrier.wait(cls.defaultTimeout)
+        if i == cls.N//2:
+            # One thread is later than the default timeout
+            time.sleep(1.0)
+        try:
+            barrier.wait()
+        except threading.BrokenBarrierError:
+            results.append(True)
+
+    def test_default_timeout(self):
+        """
+        Test the barrier's default timeout
+        """
+        barrier = self.Barrier(self.N, timeout=0.5)
+        results = self.DummyList()
+        self.run_threads(self._test_default_timeout_f, (barrier, results))
+        self.assertEqual(len(results), barrier.parties)
+
+    def test_single_thread(self):
+        b = self.Barrier(1)
+        b.wait()
+        b.wait()
+
+    @classmethod
+    def _test_thousand_f(cls, barrier, passes, conn, lock):
+        for i in range(passes):
+            barrier.wait()
+            with lock:
+                conn.send(i)
+
+    def test_thousand(self):
+        if self.TYPE == 'manager':
+            return
+        passes = 1000
+        lock = self.Lock()
+        conn, child_conn = self.Pipe(False)
+        for j in range(self.N):
+            p = self.Process(target=self._test_thousand_f,
+                           args=(self.barrier, passes, child_conn, lock))
+            p.start()
+
+        for i in range(passes):
+            for j in range(self.N):
+                self.assertEqual(conn.recv(), i)
+
+#
 #
 #
 
@@ -1130,6 +1612,9 @@ def sqr(x, wait=0.0):
     time.sleep(wait)
     return x*x
 
+def mul(x, y):
+    return x*y
+
 class _TestPool(BaseTestCase):
 
     def test_apply(self):
@@ -1143,6 +1628,37 @@ class _TestPool(BaseTestCase):
         self.assertEqual(pmap(sqr, list(range(100)), chunksize=20),
                          list(map(sqr, list(range(100)))))
 
+    def test_starmap(self):
+        psmap = self.pool.starmap
+        tuples = list(zip(range(10), range(9,-1, -1)))
+        self.assertEqual(psmap(mul, tuples),
+                         list(itertools.starmap(mul, tuples)))
+        tuples = list(zip(range(100), range(99,-1, -1)))
+        self.assertEqual(psmap(mul, tuples, chunksize=20),
+                         list(itertools.starmap(mul, tuples)))
+
+    def test_starmap_async(self):
+        tuples = list(zip(range(100), range(99,-1, -1)))
+        self.assertEqual(self.pool.starmap_async(mul, tuples).get(),
+                         list(itertools.starmap(mul, tuples)))
+
+    def test_map_async(self):
+        self.assertEqual(self.pool.map_async(sqr, list(range(10))).get(),
+                         list(map(sqr, list(range(10)))))
+
+    def test_map_async_callbacks(self):
+        call_args = self.manager.list() if self.TYPE == 'manager' else []
+        self.pool.map_async(int, ['1'],
+                            callback=call_args.append,
+                            error_callback=call_args.append).wait()
+        self.assertEqual(1, len(call_args))
+        self.assertEqual([1], call_args[0])
+        self.pool.map_async(int, ['a'],
+                            callback=call_args.append,
+                            error_callback=call_args.append).wait()
+        self.assertEqual(2, len(call_args))
+        self.assertIsInstance(call_args[1], ValueError)
+
     def test_map_chunksize(self):
         try:
             self.pool.map_async(sqr, [], chunksize=1).get(timeout=TIMEOUT1)
@@ -1221,6 +1737,16 @@ class _TestPool(BaseTestCase):
         p.close()
         p.join()
 
+    def test_context(self):
+        if self.TYPE == 'processes':
+            L = list(range(10))
+            expected = [sqr(i) for i in L]
+            with multiprocessing.Pool(2) as p:
+                r = p.map_async(sqr, L)
+                self.assertEqual(r.get(), expected)
+            print(p._state)
+            self.assertRaises(ValueError, p.map_async, sqr, L)
+
 def raising():
     raise KeyError("key")
 
@@ -1322,6 +1848,11 @@ class _TestZZZNumberOfObjects(BaseTestCase):
     # run after all the other tests for the manager.  It tests that
     # there have been no "reference leaks" for the manager's shared
     # objects.  Note the comment in _TestPool.test_terminate().
+
+    # If some other test using ManagerMixin.manager fails, then the
+    # raised exception may keep alive a frame which holds a reference
+    # to a managed object.  This will cause test_number_of_objects to
+    # also fail.
     ALLOWED_TYPES = ('manager',)
 
     def test_number_of_objects(self):
@@ -1376,7 +1907,27 @@ class _TestMyManager(BaseTestCase):
     def test_mymanager(self):
         manager = MyManager()
         manager.start()
+        self.common(manager)
+        manager.shutdown()
+
+        # If the manager process exited cleanly then the exitcode
+        # will be zero.  Otherwise (after a short timeout)
+        # terminate() is used, resulting in an exitcode of -SIGTERM.
+        self.assertEqual(manager._process.exitcode, 0)
+
+    def test_mymanager_context(self):
+        with MyManager() as manager:
+            self.common(manager)
+        self.assertEqual(manager._process.exitcode, 0)
+
+    def test_mymanager_context_prestarted(self):
+        manager = MyManager()
+        manager.start()
+        with manager:
+            self.common(manager)
+        self.assertEqual(manager._process.exitcode, 0)
 
+    def common(self, manager):
         foo = manager.Foo()
         bar = manager.Bar()
         baz = manager.baz()
@@ -1399,7 +1950,6 @@ class _TestMyManager(BaseTestCase):
 
         self.assertEqual(list(baz), [i*i for i in range(10)])
 
-        manager.shutdown()
 
 #
 # Test of connecting to a remote server and using xmlrpclib for serialization
@@ -1570,6 +2120,9 @@ class _TestConnection(BaseTestCase):
         self.assertEqual(poll(), False)
         self.assertTimingAlmostEqual(poll.elapsed, 0)
 
+        self.assertEqual(poll(-1), False)
+        self.assertTimingAlmostEqual(poll.elapsed, 0)
+
         self.assertEqual(poll(TIMEOUT1), False)
         self.assertTimingAlmostEqual(poll.elapsed, TIMEOUT1)
 
@@ -1756,6 +2309,43 @@ class _TestConnection(BaseTestCase):
         self.assertRaises(RuntimeError, reduction.recv_handle, conn)
         p.join()
 
+    def test_context(self):
+        a, b = self.Pipe()
+
+        with a, b:
+            a.send(1729)
+            self.assertEqual(b.recv(), 1729)
+            if self.TYPE == 'processes':
+                self.assertFalse(a.closed)
+                self.assertFalse(b.closed)
+
+        if self.TYPE == 'processes':
+            self.assertTrue(a.closed)
+            self.assertTrue(b.closed)
+            self.assertRaises(IOError, a.recv)
+            self.assertRaises(IOError, b.recv)
+
+class _TestListener(BaseTestCase):
+
+    ALLOWED_TYPES = ('processes',)
+
+    def test_multiple_bind(self):
+        for family in self.connection.families:
+            l = self.connection.Listener(family=family)
+            self.addCleanup(l.close)
+            self.assertRaises(OSError, self.connection.Listener,
+                              l.address, family)
+
+    def test_context(self):
+        with self.connection.Listener() as l:
+            with self.connection.Client(l.address) as c:
+                with l.accept() as d:
+                    c.send(1729)
+                    self.assertEqual(d.recv(), 1729)
+
+        if self.TYPE == 'processes':
+            self.assertRaises(IOError, l.accept)
+
 class _TestListenerClient(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', 'threads')
@@ -1793,52 +2383,146 @@ class _TestListenerClient(BaseTestCase):
         p.join()
         l.close()
 
+    def test_issue16955(self):
+        for fam in self.connection.families:
+            l = self.connection.Listener(family=fam)
+            c = self.connection.Client(l.address)
+            a = l.accept()
+            a.send_bytes(b"hello")
+            self.assertTrue(c.poll(1))
+            a.close()
+            c.close()
+            l.close()
+
+class _TestPoll(unittest.TestCase):
+
+    ALLOWED_TYPES = ('processes', 'threads')
+
+    def test_empty_string(self):
+        a, b = self.Pipe()
+        self.assertEqual(a.poll(), False)
+        b.send_bytes(b'')
+        self.assertEqual(a.poll(), True)
+        self.assertEqual(a.poll(), True)
+
+    @classmethod
+    def _child_strings(cls, conn, strings):
+        for s in strings:
+            time.sleep(0.1)
+            conn.send_bytes(s)
+        conn.close()
+
+    def test_strings(self):
+        strings = (b'hello', b'', b'a', b'b', b'', b'bye', b'', b'lop')
+        a, b = self.Pipe()
+        p = self.Process(target=self._child_strings, args=(b, strings))
+        p.start()
+
+        for s in strings:
+            for i in range(200):
+                if a.poll(0.01):
+                    break
+            x = a.recv_bytes()
+            self.assertEqual(s, x)
+
+        p.join()
+
+    @classmethod
+    def _child_boundaries(cls, r):
+        # Polling may "pull" a message in to the child process, but we
+        # don't want it to pull only part of a message, as that would
+        # corrupt the pipe for any other processes which might later
+        # read from it.
+        r.poll(5)
+
+    def test_boundaries(self):
+        r, w = self.Pipe(False)
+        p = self.Process(target=self._child_boundaries, args=(r,))
+        p.start()
+        time.sleep(2)
+        L = [b"first", b"second"]
+        for obj in L:
+            w.send_bytes(obj)
+        w.close()
+        p.join()
+        self.assertIn(r.recv_bytes(), L)
+
+    @classmethod
+    def _child_dont_merge(cls, b):
+        b.send_bytes(b'a')
+        b.send_bytes(b'b')
+        b.send_bytes(b'cd')
+
+    def test_dont_merge(self):
+        a, b = self.Pipe()
+        self.assertEqual(a.poll(0.0), False)
+        self.assertEqual(a.poll(0.1), False)
+
+        p = self.Process(target=self._child_dont_merge, args=(b,))
+        p.start()
+
+        self.assertEqual(a.recv_bytes(), b'a')
+        self.assertEqual(a.poll(1.0), True)
+        self.assertEqual(a.poll(1.0), True)
+        self.assertEqual(a.recv_bytes(), b'b')
+        self.assertEqual(a.poll(1.0), True)
+        self.assertEqual(a.poll(1.0), True)
+        self.assertEqual(a.poll(0.0), True)
+        self.assertEqual(a.recv_bytes(), b'cd')
+
+        p.join()
+
 #
 # Test of sending connection and socket objects between processes
 #
-"""
+
+@unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction")
 class _TestPicklingConnections(BaseTestCase):
 
     ALLOWED_TYPES = ('processes',)
 
-    def _listener(self, conn, families):
+    @classmethod
+    def tearDownClass(cls):
+        from multiprocessing.reduction import resource_sharer
+        resource_sharer.stop(timeout=5)
+
+    @classmethod
+    def _listener(cls, conn, families):
         for fam in families:
-            l = self.connection.Listener(family=fam)
+            l = cls.connection.Listener(family=fam)
             conn.send(l.address)
             new_conn = l.accept()
             conn.send(new_conn)
+            new_conn.close()
+            l.close()
 
-        if self.TYPE == 'processes':
-            l = socket.socket()
-            l.bind(('localhost', 0))
-            conn.send(l.getsockname())
-            l.listen(1)
-            new_conn, addr = l.accept()
-            conn.send(new_conn)
+        l = socket.socket()
+        l.bind(('localhost', 0))
+        l.listen(1)
+        conn.send(l.getsockname())
+        new_conn, addr = l.accept()
+        conn.send(new_conn)
+        new_conn.close()
+        l.close()
 
         conn.recv()
 
-    def _remote(self, conn):
+    @classmethod
+    def _remote(cls, conn):
         for (address, msg) in iter(conn.recv, None):
-            client = self.connection.Client(address)
+            client = cls.connection.Client(address)
             client.send(msg.upper())
             client.close()
 
-        if self.TYPE == 'processes':
-            address, msg = conn.recv()
-            client = socket.socket()
-            client.connect(address)
-            client.sendall(msg.upper())
-            client.close()
+        address, msg = conn.recv()
+        client = socket.socket()
+        client.connect(address)
+        client.sendall(msg.upper())
+        client.close()
 
         conn.close()
 
     def test_pickling(self):
-        try:
-            multiprocessing.allow_connection_pickling()
-        except ImportError:
-            return
-
         families = self.connection.families
 
         lconn, lconn0 = self.Pipe()
@@ -1862,16 +2546,19 @@ class _TestPicklingConnections(BaseTestCase):
 
         rconn.send(None)
 
-        if self.TYPE == 'processes':
-            msg = latin('This connection uses a normal socket')
-            address = lconn.recv()
-            rconn.send((address, msg))
-            if hasattr(socket, 'fromfd'):
-                new_conn = lconn.recv()
-                self.assertEqual(new_conn.recv(100), msg.upper())
-            else:
-                # XXX On Windows with Py2.6 need to backport fromfd()
-                discard = lconn.recv_bytes()
+        msg = latin('This connection uses a normal socket')
+        address = lconn.recv()
+        rconn.send((address, msg))
+        new_conn = lconn.recv()
+        buf = []
+        while True:
+            s = new_conn.recv(100)
+            if not s:
+                break
+            buf.append(s)
+        buf = b''.join(buf)
+        self.assertEqual(buf, msg.upper())
+        new_conn.close()
 
         lconn.send(None)
 
@@ -1880,7 +2567,46 @@ class _TestPicklingConnections(BaseTestCase):
 
         lp.join()
         rp.join()
-"""
+
+    @classmethod
+    def child_access(cls, conn):
+        w = conn.recv()
+        w.send('all is well')
+        w.close()
+
+        r = conn.recv()
+        msg = r.recv()
+        conn.send(msg*2)
+
+        conn.close()
+
+    def test_access(self):
+        # On Windows, if we do not specify a destination pid when
+        # using DupHandle then we need to be careful to use the
+        # correct access flags for DuplicateHandle(), or else
+        # DupHandle.detach() will raise PermissionError.  For example,
+        # for a read only pipe handle we should use
+        # access=FILE_GENERIC_READ.  (Unfortunately
+        # DUPLICATE_SAME_ACCESS does not work.)
+        conn, child_conn = self.Pipe()
+        p = self.Process(target=self.child_access, args=(child_conn,))
+        p.daemon = True
+        p.start()
+        child_conn.close()
+
+        r, w = self.Pipe(duplex=False)
+        conn.send(w)
+        w.close()
+        self.assertEqual(r.recv(), 'all is well')
+        r.close()
+
+        r, w = self.Pipe(duplex=False)
+        conn.send(r)
+        r.close()
+        w.send('foobar')
+        w.close()
+        self.assertEqual(conn.recv(), 'foobar'*2)
+
 #
 #
 #
@@ -2207,9 +2933,15 @@ class TestInvalidHandle(unittest.TestCase):
 
     @unittest.skipIf(WIN32, "skipped on Windows")
     def test_invalid_handles(self):
-        conn = _multiprocessing.Connection(44977608)
-        self.assertRaises(IOError, conn.poll)
-        self.assertRaises(IOError, _multiprocessing.Connection, -1)
+        conn = multiprocessing.connection.Connection(44977608)
+        try:
+            self.assertRaises((ValueError, IOError), conn.poll)
+        finally:
+            # Hack private attribute _handle to avoid printing an error
+            # in conn.__del__
+            conn._handle = None
+        self.assertRaises((ValueError, IOError),
+                          multiprocessing.connection.Connection, -1)
 
 #
 # Functions used to create test cases from the base ones in this module
@@ -2228,10 +2960,12 @@ def create_test_cases(Mixin, type):
     result = {}
     glob = globals()
     Type = type.capitalize()
+    ALL_TYPES = {'processes', 'threads', 'manager'}
 
     for name in list(glob.keys()):
         if name.startswith('_Test'):
             base = glob[name]
+            assert set(base.ALLOWED_TYPES) <= ALL_TYPES, set(base.ALLOWED_TYPES)
             if type in base.ALLOWED_TYPES:
                 newname = 'With' + Type + name[1:]
                 class Temp(base, unittest.TestCase, Mixin):
@@ -2250,7 +2984,7 @@ class ProcessesMixin(object):
     Process = multiprocessing.Process
     locals().update(get_attributes(multiprocessing, (
         'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore',
-        'Condition', 'Event', 'Value', 'Array', 'RawValue',
+        'Condition', 'Event', 'Barrier', 'Value', 'Array', 'RawValue',
         'RawArray', 'current_process', 'active_children', 'Pipe',
         'connection', 'JoinableQueue', 'Pool'
         )))
@@ -2265,7 +2999,7 @@ class ManagerMixin(object):
     manager = object.__new__(multiprocessing.managers.SyncManager)
     locals().update(get_attributes(manager, (
         'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore',
-       'Condition', 'Event', 'Value', 'Array', 'list', 'dict',
+        'Condition', 'Event', 'Barrier', 'Value', 'Array', 'list', 'dict',
         'Namespace', 'JoinableQueue', 'Pool'
         )))
 
@@ -2278,7 +3012,7 @@ class ThreadsMixin(object):
     Process = multiprocessing.dummy.Process
     locals().update(get_attributes(multiprocessing.dummy, (
         'Queue', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore',
-        'Condition', 'Event', 'Value', 'Array', 'current_process',
+        'Condition', 'Event', 'Barrier', 'Value', 'Array', 'current_process',
         'active_children', 'Pipe', 'connection', 'dict', 'list',
         'Namespace', 'JoinableQueue', 'Pool'
         )))
@@ -2330,6 +3064,7 @@ class TestInitializers(unittest.TestCase):
 
     def tearDown(self):
         self.mgr.shutdown()
+        self.mgr.join()
 
     def test_manager_initializer(self):
         m = multiprocessing.managers.SyncManager()
@@ -2337,6 +3072,7 @@ class TestInitializers(unittest.TestCase):
         m.start(initializer, (self.ns,))
         self.assertEqual(self.ns.test, 1)
         m.shutdown()
+        m.join()
 
     def test_pool_initializer(self):
         self.assertRaises(TypeError, multiprocessing.Pool, initializer=1)
@@ -2369,6 +3105,8 @@ def _afunc(x):
 def pool_in_process():
     pool = multiprocessing.Pool(processes=4)
     x = pool.map(_afunc, [1, 2, 3, 4, 5, 6, 7])
+    pool.close()
+    pool.join()
 
 class _file_like(object):
     def __init__(self, delegate):
@@ -2413,6 +3151,181 @@ class TestStdinBadfiledescriptor(unittest.TestCase):
         assert sio.getvalue() == 'foo'
 
 
+class TestWait(unittest.TestCase):
+
+    @classmethod
+    def _child_test_wait(cls, w, slow):
+        for i in range(10):
+            if slow:
+                time.sleep(random.random()*0.1)
+            w.send((i, os.getpid()))
+        w.close()
+
+    def test_wait(self, slow=False):
+        from multiprocessing.connection import wait
+        readers = []
+        procs = []
+        messages = []
+
+        for i in range(4):
+            r, w = multiprocessing.Pipe(duplex=False)
+            p = multiprocessing.Process(target=self._child_test_wait, args=(w, slow))
+            p.daemon = True
+            p.start()
+            w.close()
+            readers.append(r)
+            procs.append(p)
+            self.addCleanup(p.join)
+
+        while readers:
+            for r in wait(readers):
+                try:
+                    msg = r.recv()
+                except EOFError:
+                    readers.remove(r)
+                    r.close()
+                else:
+                    messages.append(msg)
+
+        messages.sort()
+        expected = sorted((i, p.pid) for i in range(10) for p in procs)
+        self.assertEqual(messages, expected)
+
+    @classmethod
+    def _child_test_wait_socket(cls, address, slow):
+        s = socket.socket()
+        s.connect(address)
+        for i in range(10):
+            if slow:
+                time.sleep(random.random()*0.1)
+            s.sendall(('%s\n' % i).encode('ascii'))
+        s.close()
+
+    def test_wait_socket(self, slow=False):
+        from multiprocessing.connection import wait
+        l = socket.socket()
+        l.bind(('', 0))
+        l.listen(4)
+        addr = ('localhost', l.getsockname()[1])
+        readers = []
+        procs = []
+        dic = {}
+
+        for i in range(4):
+            p = multiprocessing.Process(target=self._child_test_wait_socket,
+                                        args=(addr, slow))
+            p.daemon = True
+            p.start()
+            procs.append(p)
+            self.addCleanup(p.join)
+
+        for i in range(4):
+            r, _ = l.accept()
+            readers.append(r)
+            dic[r] = []
+        l.close()
+
+        while readers:
+            for r in wait(readers):
+                msg = r.recv(32)
+                if not msg:
+                    readers.remove(r)
+                    r.close()
+                else:
+                    dic[r].append(msg)
+
+        expected = ''.join('%s\n' % i for i in range(10)).encode('ascii')
+        for v in dic.values():
+            self.assertEqual(b''.join(v), expected)
+
+    def test_wait_slow(self):
+        self.test_wait(True)
+
+    def test_wait_socket_slow(self):
+        self.test_wait_socket(True)
+
+    def test_wait_timeout(self):
+        from multiprocessing.connection import wait
+
+        expected = 5
+        a, b = multiprocessing.Pipe()
+
+        start = time.time()
+        res = wait([a, b], expected)
+        delta = time.time() - start
+
+        self.assertEqual(res, [])
+        self.assertLess(delta, expected * 2)
+        self.assertGreater(delta, expected * 0.5)
+
+        b.send(None)
+
+        start = time.time()
+        res = wait([a, b], 20)
+        delta = time.time() - start
+
+        self.assertEqual(res, [a])
+        self.assertLess(delta, 0.4)
+
+    @classmethod
+    def signal_and_sleep(cls, sem, period):
+        sem.release()
+        time.sleep(period)
+
+    def test_wait_integer(self):
+        from multiprocessing.connection import wait
+
+        expected = 3
+        sorted_ = lambda l: sorted(l, key=lambda x: id(x))
+        sem = multiprocessing.Semaphore(0)
+        a, b = multiprocessing.Pipe()
+        p = multiprocessing.Process(target=self.signal_and_sleep,
+                                    args=(sem, expected))
+
+        p.start()
+        self.assertIsInstance(p.sentinel, int)
+        self.assertTrue(sem.acquire(timeout=20))
+
+        start = time.time()
+        res = wait([a, p.sentinel, b], expected + 20)
+        delta = time.time() - start
+
+        self.assertEqual(res, [p.sentinel])
+        self.assertLess(delta, expected + 2)
+        self.assertGreater(delta, expected - 2)
+
+        a.send(None)
+
+        start = time.time()
+        res = wait([a, p.sentinel, b], 20)
+        delta = time.time() - start
+
+        self.assertEqual(sorted_(res), sorted_([p.sentinel, b]))
+        self.assertLess(delta, 0.4)
+
+        b.send(None)
+
+        start = time.time()
+        res = wait([a, p.sentinel, b], 20)
+        delta = time.time() - start
+
+        self.assertEqual(sorted_(res), sorted_([a, p.sentinel, b]))
+        self.assertLess(delta, 0.4)
+
+        p.terminate()
+        p.join()
+
+    def test_neg_timeout(self):
+        from multiprocessing.connection import wait
+        a, b = multiprocessing.Pipe()
+        t = time.time()
+        res = wait([a], timeout=-1)
+        t = time.time() - t
+        self.assertEqual(res, [])
+        self.assertLess(t, 1)
+        a.close()
+        b.close()
+
 #
 # Issue 14151: Test invalid family on invalid environment
 #
@@ -2430,6 +3343,38 @@ class TestInvalidFamily(unittest.TestCase):
             multiprocessing.connection.Listener('/var/test.pipe')
 
 #
+# Issue 12098: check sys.flags of child matches that for parent
+#
+
+class TestFlags(unittest.TestCase):
+    @classmethod
+    def run_in_grandchild(cls, conn):
+        conn.send(tuple(sys.flags))
+
+    @classmethod
+    def run_in_child(cls):
+        import json
+        r, w = multiprocessing.Pipe(duplex=False)
+        p = multiprocessing.Process(target=cls.run_in_grandchild, args=(w,))
+        p.start()
+        grandchild_flags = r.recv()
+        p.join()
+        r.close()
+        w.close()
+        flags = (tuple(sys.flags), grandchild_flags)
+        print(json.dumps(flags))
+
+    def test_flags(self):
+        import json, subprocess
+        # start child process using unusual flags
+        prog = ('from test.test_multiprocessing import TestFlags; ' +
+                'TestFlags.run_in_child()')
+        data = subprocess.check_output(
+            [sys.executable, '-E', '-S', '-O', '-c', prog])
+        child_flags, grandchild_flags = json.loads(data.decode('ascii'))
+        self.assertEqual(child_flags, grandchild_flags)
+
+#
 # Test interaction with socket timeouts - see Issue #6056
 #
 
@@ -2484,8 +3429,8 @@ class TestNoForkBomb(unittest.TestCase):
 #
 
 testcases_other = [OtherTest, TestInvalidHandle, TestInitializers,
-                   TestStdinBadfiledescriptor, TestInvalidFamily,
-                   TestTimeouts, TestNoForkBomb]
+                   TestStdinBadfiledescriptor, TestWait, TestInvalidFamily,
+                   TestFlags, TestTimeouts, TestNoForkBomb]
 
 #
 #
@@ -2522,14 +3467,18 @@ def test_main(run=None):
 
     loadTestsFromTestCase = unittest.defaultTestLoader.loadTestsFromTestCase
     suite = unittest.TestSuite(loadTestsFromTestCase(tc) for tc in testcases)
-    run(suite)
-
-    ThreadsMixin.pool.terminate()
-    ProcessesMixin.pool.terminate()
-    ManagerMixin.pool.terminate()
-    ManagerMixin.manager.shutdown()
-
-    del ProcessesMixin.pool, ThreadsMixin.pool, ManagerMixin.pool
+    try:
+        run(suite)
+    finally:
+        ThreadsMixin.pool.terminate()
+        ProcessesMixin.pool.terminate()
+        ManagerMixin.pool.terminate()
+        ManagerMixin.pool.join()
+        ManagerMixin.manager.shutdown()
+        ManagerMixin.manager.join()
+        ThreadsMixin.pool.join()
+        ProcessesMixin.pool.join()
+        del ProcessesMixin.pool, ThreadsMixin.pool, ManagerMixin.pool
 
 def main():
     test_main(unittest.TextTestRunner(verbosity=2).run)
diff --git a/Lib/test/test_mutants.py b/Lib/test/test_mutants.py
deleted file mode 100644
index b43fa47626..0000000000
--- a/Lib/test/test_mutants.py
+++ /dev/null
@@ -1,291 +0,0 @@
-from test.support import verbose, TESTFN
-import random
-import os
-
-# From SF bug #422121:  Insecurities in dict comparison.
-
-# Safety of code doing comparisons has been an historical Python weak spot.
-# The problem is that comparison of structures written in C *naturally*
-# wants to hold on to things like the size of the container, or "the
-# biggest" containee so far, across a traversal of the container; but
-# code to do containee comparisons can call back into Python and mutate
-# the container in arbitrary ways while the C loop is in midstream.  If the
-# C code isn't extremely paranoid about digging things out of memory on
-# each trip, and artificially boosting refcounts for the duration, anything
-# from infinite loops to OS crashes can result (yes, I use Windows <wink>).
-#
-# The other problem is that code designed to provoke a weakness is usually
-# white-box code, and so catches only the particular vulnerabilities the
-# author knew to protect against.  For example, Python's list.sort() code
-# went thru many iterations as one "new" vulnerability after another was
-# discovered.
-#
-# So the dict comparison test here uses a black-box approach instead,
-# generating dicts of various sizes at random, and performing random
-# mutations on them at random times.  This proved very effective,
-# triggering at least six distinct failure modes the first 20 times I
-# ran it.  Indeed, at the start, the driver never got beyond 6 iterations
-# before the test died.
-
-# The dicts are global to make it easy to mutate tham from within functions.
-dict1 = {}
-dict2 = {}
-
-# The current set of keys in dict1 and dict2.  These are materialized as
-# lists to make it easy to pick a dict key at random.
-dict1keys = []
-dict2keys = []
-
-# Global flag telling maybe_mutate() whether to *consider* mutating.
-mutate = 0
-
-# If global mutate is true, consider mutating a dict.  May or may not
-# mutate a dict even if mutate is true.  If it does decide to mutate a
-# dict, it picks one of {dict1, dict2} at random, and deletes a random
-# entry from it; or, more rarely, adds a random element.
-
-def maybe_mutate():
-    global mutate
-    if not mutate:
-        return
-    if random.random() < 0.5:
-        return
-
-    if random.random() < 0.5:
-        target, keys = dict1, dict1keys
-    else:
-        target, keys = dict2, dict2keys
-
-    if random.random() < 0.2:
-        # Insert a new key.
-        mutate = 0   # disable mutation until key inserted
-        while 1:
-            newkey = Horrid(random.randrange(100))
-            if newkey not in target:
-                break
-        target[newkey] = Horrid(random.randrange(100))
-        keys.append(newkey)
-        mutate = 1
-
-    elif keys:
-        # Delete a key at random.
-        mutate = 0   # disable mutation until key deleted
-        i = random.randrange(len(keys))
-        key = keys[i]
-        del target[key]
-        del keys[i]
-        mutate = 1
-
-# A horrid class that triggers random mutations of dict1 and dict2 when
-# instances are compared.
-
-class Horrid:
-    def __init__(self, i):
-        # Comparison outcomes are determined by the value of i.
-        self.i = i
-
-        # An artificial hashcode is selected at random so that we don't
-        # have any systematic relationship between comparison outcomes
-        # (based on self.i and other.i) and relative position within the
-        # hash vector (based on hashcode).
-        # XXX This is no longer effective.
-        ##self.hashcode = random.randrange(1000000000)
-
-    def __hash__(self):
-        return 42
-        return self.hashcode
-
-    def __eq__(self, other):
-        maybe_mutate()   # The point of the test.
-        return self.i == other.i
-
-    def __ne__(self, other):
-        raise RuntimeError("I didn't expect some kind of Spanish inquisition!")
-
-    __lt__ = __le__ = __gt__ = __ge__ = __ne__
-
-    def __repr__(self):
-        return "Horrid(%d)" % self.i
-
-# Fill dict d with numentries (Horrid(i), Horrid(j)) key-value pairs,
-# where i and j are selected at random from the candidates list.
-# Return d.keys() after filling.
-
-def fill_dict(d, candidates, numentries):
-    d.clear()
-    for i in range(numentries):
-        d[Horrid(random.choice(candidates))] = \
-            Horrid(random.choice(candidates))
-    return list(d.keys())
-
-# Test one pair of randomly generated dicts, each with n entries.
-# Note that dict comparison is trivial if they don't have the same number
-# of entires (then the "shorter" dict is instantly considered to be the
-# smaller one, without even looking at the entries).
-
-def test_one(n):
-    global mutate, dict1, dict2, dict1keys, dict2keys
-
-    # Fill the dicts without mutating them.
-    mutate = 0
-    dict1keys = fill_dict(dict1, range(n), n)
-    dict2keys = fill_dict(dict2, range(n), n)
-
-    # Enable mutation, then compare the dicts so long as they have the
-    # same size.
-    mutate = 1
-    if verbose:
-        print("trying w/ lengths", len(dict1), len(dict2), end=' ')
-    while dict1 and len(dict1) == len(dict2):
-        if verbose:
-            print(".", end=' ')
-        c = dict1 == dict2
-    if verbose:
-        print()
-
-# Run test_one n times.  At the start (before the bugs were fixed), 20
-# consecutive runs of this test each blew up on or before the sixth time
-# test_one was run.  So n doesn't have to be large to get an interesting
-# test.
-# OTOH, calling with large n is also interesting, to ensure that the fixed
-# code doesn't hold on to refcounts *too* long (in which case memory would
-# leak).
-
-def test(n):
-    for i in range(n):
-        test_one(random.randrange(1, 100))
-
-# See last comment block for clues about good values for n.
-test(100)
-
-##########################################################################
-# Another segfault bug, distilled by Michael Hudson from a c.l.py post.
-
-class Child:
-    def __init__(self, parent):
-        self.__dict__['parent'] = parent
-    def __getattr__(self, attr):
-        self.parent.a = 1
-        self.parent.b = 1
-        self.parent.c = 1
-        self.parent.d = 1
-        self.parent.e = 1
-        self.parent.f = 1
-        self.parent.g = 1
-        self.parent.h = 1
-        self.parent.i = 1
-        return getattr(self.parent, attr)
-
-class Parent:
-    def __init__(self):
-        self.a = Child(self)
-
-# Hard to say what this will print!  May vary from time to time.  But
-# we're specifically trying to test the tp_print slot here, and this is
-# the clearest way to do it.  We print the result to a temp file so that
-# the expected-output file doesn't need to change.
-
-f = open(TESTFN, "w")
-print(Parent().__dict__, file=f)
-f.close()
-os.unlink(TESTFN)
-
-##########################################################################
-# And another core-dumper from Michael Hudson.
-
-dict = {}
-
-# Force dict to malloc its table.
-for i in range(1, 10):
-    dict[i] = i
-
-f = open(TESTFN, "w")
-
-class Machiavelli:
-    def __repr__(self):
-        dict.clear()
-
-        # Michael sez:  "doesn't crash without this.  don't know why."
-        # Tim sez:  "luck of the draw; crashes with or without for me."
-        print(file=f)
-
-        return repr("machiavelli")
-
-    def __hash__(self):
-        return 0
-
-dict[Machiavelli()] = Machiavelli()
-
-print(str(dict), file=f)
-f.close()
-os.unlink(TESTFN)
-del f, dict
-
-
-##########################################################################
-# And another core-dumper from Michael Hudson.
-
-dict = {}
-
-# let's force dict to malloc its table
-for i in range(1, 10):
-    dict[i] = i
-
-class Machiavelli2:
-    def __eq__(self, other):
-        dict.clear()
-        return 1
-
-    def __hash__(self):
-        return 0
-
-dict[Machiavelli2()] = Machiavelli2()
-
-try:
-    dict[Machiavelli2()]
-except KeyError:
-    pass
-
-del dict
-
-##########################################################################
-# And another core-dumper from Michael Hudson.
-
-dict = {}
-
-# let's force dict to malloc its table
-for i in range(1, 10):
-    dict[i] = i
-
-class Machiavelli3:
-    def __init__(self, id):
-        self.id = id
-
-    def __eq__(self, other):
-        if self.id == other.id:
-            dict.clear()
-            return 1
-        else:
-            return 0
-
-    def __repr__(self):
-        return "%s(%s)"%(self.__class__.__name__, self.id)
-
-    def __hash__(self):
-        return 0
-
-dict[Machiavelli3(1)] = Machiavelli3(0)
-dict[Machiavelli3(2)] = Machiavelli3(0)
-
-f = open(TESTFN, "w")
-try:
-    try:
-        print(dict[Machiavelli3(2)], file=f)
-    except KeyError:
-        pass
-finally:
-    f.close()
-    os.unlink(TESTFN)
-
-del dict
-del dict1, dict2, dict1keys, dict2keys
diff --git a/Lib/test/test_namespace_pkgs.py b/Lib/test/test_namespace_pkgs.py
new file mode 100644
index 0000000000..7067b12e8f
--- /dev/null
+++ b/Lib/test/test_namespace_pkgs.py
@@ -0,0 +1,294 @@
+import sys
+import contextlib
+import unittest
+import os
+
+from test.test_importlib import util
+from test.support import run_unittest
+
+# needed tests:
+#
+# need to test when nested, so that the top-level path isn't sys.path
+# need to test dynamic path detection, both at top-level and nested
+# with dynamic path, check when a loader is returned on path reload (that is,
+#  trying to switch from a namespace package to a regular package)
+
+
+@contextlib.contextmanager
+def sys_modules_context():
+    """
+    Make sure sys.modules is the same object and has the same content
+    when exiting the context as when entering.
+
+    Similar to importlib.test.util.uncache, but doesn't require explicit
+    names.
+    """
+    sys_modules_saved = sys.modules
+    sys_modules_copy = sys.modules.copy()
+    try:
+        yield
+    finally:
+        sys.modules = sys_modules_saved
+        sys.modules.clear()
+        sys.modules.update(sys_modules_copy)
+
+
+@contextlib.contextmanager
+def namespace_tree_context(**kwargs):
+    """
+    Save import state and sys.modules cache and restore it on exit.
+    Typical usage:
+
+    >>> with namespace_tree_context(path=['/tmp/xxyy/portion1',
+    ...         '/tmp/xxyy/portion2']):
+    ...     pass
+    """
+    # use default meta_path and path_hooks unless specified otherwise
+    kwargs.setdefault('meta_path', sys.meta_path)
+    kwargs.setdefault('path_hooks', sys.path_hooks)
+    import_context = util.import_state(**kwargs)
+    with import_context, sys_modules_context():
+        yield
+
+class NamespacePackageTest(unittest.TestCase):
+    """
+    Subclasses should define self.root and self.paths (under that root)
+    to be added to sys.path.
+    """
+    root = os.path.join(os.path.dirname(__file__), 'namespace_pkgs')
+
+    def setUp(self):
+        self.resolved_paths = [
+            os.path.join(self.root, path) for path in self.paths
+        ]
+        self.ctx = namespace_tree_context(path=self.resolved_paths)
+        self.ctx.__enter__()
+
+    def tearDown(self):
+        # TODO: will we ever want to pass exc_info to __exit__?
+        self.ctx.__exit__(None, None, None)
+
+class SingleNamespacePackage(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+    def test_module_repr(self):
+        import foo.one
+        self.assertEqual(repr(foo), "<module 'foo' (namespace)>")
+
+
+class DynamicPatheNamespacePackage(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_dynamic_path(self):
+        # Make sure only 'foo.one' can be imported
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+        with self.assertRaises(ImportError):
+            import foo.two
+
+        # Now modify sys.path
+        sys.path.append(os.path.join(self.root, 'portion2'))
+
+        # And make sure foo.two is now importable
+        import foo.two
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class CombinedNamespacePackages(NamespacePackageTest):
+    paths = ['both_portions']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'both_portions foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SeparatedNamespacePackages(NamespacePackageTest):
+    paths = ['portion1', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class SeparatedOverlappingNamespacePackages(NamespacePackageTest):
+    paths = ['portion1', 'both_portions']
+
+    def test_first_path_wins(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+    def test_first_path_wins_again(self):
+        sys.path.reverse()
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'both_portions foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+    def test_first_path_wins_importing_second_first(self):
+        import foo.two
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SingleZipNamespacePackage(NamespacePackageTest):
+    paths = ['top_level_portion1.zip']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+
+class SeparatedZipNamespacePackages(NamespacePackageTest):
+    paths = ['top_level_portion1.zip', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+        self.assertIn('top_level_portion1.zip', foo.one.__file__)
+        self.assertNotIn('.zip', foo.two.__file__)
+
+
+class SingleNestedZipNamespacePackage(NamespacePackageTest):
+    paths = ['nested_portion1.zip/nested_portion1']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+
+class SeparatedNestedZipNamespacePackages(NamespacePackageTest):
+    paths = ['nested_portion1.zip/nested_portion1', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+        fn = os.path.join('nested_portion1.zip', 'nested_portion1')
+        self.assertIn(fn, foo.one.__file__)
+        self.assertNotIn('.zip', foo.two.__file__)
+
+
+class LegacySupport(NamespacePackageTest):
+    paths = ['not_a_namespace_pkg', 'portion1', 'portion2', 'both_portions']
+
+    def test_non_namespace_package_takes_precedence(self):
+        import foo.one
+        with self.assertRaises(ImportError):
+            import foo.two
+        self.assertIn('__init__', foo.__file__)
+        self.assertNotIn('namespace', str(foo.__loader__).lower())
+
+
+class DynamicPathCalculation(NamespacePackageTest):
+    paths = ['project1', 'project2']
+
+    def test_project3_fails(self):
+        import parent.child.one
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+        import parent.child.two
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+
+        self.assertEqual(parent.child.one.attr, 'parent child one')
+        self.assertEqual(parent.child.two.attr, 'parent child two')
+
+        with self.assertRaises(ImportError):
+            import parent.child.three
+
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+
+    def test_project3_succeeds(self):
+        import parent.child.one
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+        import parent.child.two
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+
+        self.assertEqual(parent.child.one.attr, 'parent child one')
+        self.assertEqual(parent.child.two.attr, 'parent child two')
+
+        with self.assertRaises(ImportError):
+            import parent.child.three
+
+        # now add project3
+        sys.path.append(os.path.join(self.root, 'project3'))
+        import parent.child.three
+
+        # the paths dynamically get longer, to include the new directories
+        self.assertEqual(len(parent.__path__), 3)
+        self.assertEqual(len(parent.child.__path__), 3)
+
+        self.assertEqual(parent.child.three.attr, 'parent child three')
+
+
+class ZipWithMissingDirectory(NamespacePackageTest):
+    paths = ['missing_directory.zip']
+
+    @unittest.expectedFailure
+    def test_missing_directory(self):
+        # This will fail because missing_directory.zip contains:
+        #   Length      Date    Time    Name
+        # ---------  ---------- -----   ----
+        #        29  2012-05-03 18:13   foo/one.py
+        #         0  2012-05-03 20:57   bar/
+        #        38  2012-05-03 20:57   bar/two.py
+        # ---------                     -------
+        #        67                     3 files
+
+        # Because there is no 'foo/', the zipimporter currently doesn't
+        #  know that foo is a namespace package
+
+        import foo.one
+
+    def test_present_directory(self):
+        # This succeeds because there is a "bar/" in the zip file
+        import bar.two
+        self.assertEqual(bar.two.attr, 'missing_directory foo two')
+
+
+class ModuleAndNamespacePackageInSameDir(NamespacePackageTest):
+    paths = ['module_and_namespace_package']
+
+    def test_module_before_namespace_package(self):
+        # Make sure we find the module in preference to the
+        #  namespace package.
+        import a_test
+        self.assertEqual(a_test.attr, 'in module')
+
+
+def test_main():
+    run_unittest(*NamespacePackageTest.__subclasses__())
+
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py
index 7335b23b2d..19fb10a9c7 100644
--- a/Lib/test/test_nntplib.py
+++ b/Lib/test/test_nntplib.py
@@ -1,4 +1,5 @@
 import io
+import socket
 import datetime
 import textwrap
 import unittest
@@ -257,6 +258,26 @@ class NetworkedNNTPTestsMixin:
             # value
             setattr(cls, name, wrap_meth(meth))
 
+    def test_with_statement(self):
+        def is_connected():
+            if not hasattr(server, 'file'):
+                return False
+            try:
+                server.help()
+            except (socket.error, EOFError):
+                return False
+            return True
+
+        with self.NNTP_CLASS(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) as server:
+            self.assertTrue(is_connected())
+            self.assertTrue(server.help())
+        self.assertFalse(is_connected())
+
+        with self.NNTP_CLASS(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) as server:
+            server.quit()
+        self.assertFalse(is_connected())
+
+
 NetworkedNNTPTestsMixin.wrap_methods()
 
 
@@ -894,7 +915,7 @@ class NNTPv1v2TestsMixin:
 
     def _check_article_body(self, lines):
         self.assertEqual(len(lines), 4)
-        self.assertEqual(lines[-1].decode('utf8'), "-- Signed by André.")
+        self.assertEqual(lines[-1].decode('utf-8'), "-- Signed by André.")
         self.assertEqual(lines[-2], b"")
         self.assertEqual(lines[-3], b".Here is a dot-starting line.")
         self.assertEqual(lines[-4], b"This is just a test article.")
@@ -1133,12 +1154,12 @@ class NNTPv1v2TestsMixin:
         self.assertEqual(resp, success_resp)
         # With an iterable of terminated lines
         def iterlines(b):
-            return iter(b.splitlines(True))
+            return iter(b.splitlines(keepends=True))
         resp = self._check_post_ihave_sub(func, *args, file_factory=iterlines)
         self.assertEqual(resp, success_resp)
         # With an iterable of non-terminated lines
         def iterlines(b):
-            return iter(b.splitlines(False))
+            return iter(b.splitlines(keepends=False))
         resp = self._check_post_ihave_sub(func, *args, file_factory=iterlines)
         self.assertEqual(resp, success_resp)
 
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 6464950982..f8098767ce 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -1,10 +1,11 @@
 import ntpath
 import os
 import sys
+import unittest
+import warnings
 from test.support import TestFailed
 from test import support, test_genericpath
 from tempfile import TemporaryFile
-import unittest
 
 
 def tester(fn, wantResult):
@@ -21,7 +22,9 @@ def tester(fn, wantResult):
     fn = fn.replace('["', '[b"')
     fn = fn.replace(", '", ", b'")
     fn = fn.replace(', "', ', b"')
-    gotResult = eval(fn)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", DeprecationWarning)
+        gotResult = eval(fn)
     if isinstance(wantResult, str):
         wantResult = wantResult.encode('ascii')
     elif isinstance(wantResult, tuple):
@@ -254,14 +257,10 @@ class TestNtpath(unittest.TestCase):
                     ntpath.sameopenfile(-1, -1)
 
 
-class NtCommonTest(test_genericpath.CommonTest):
+class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
     pathmodule = ntpath
     attributes = ['relpath', 'splitunc']
 
 
-def test_main():
-    support.run_unittest(TestNtpath, NtCommonTest)
-
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/Lib/test/test_numeric_tower.py b/Lib/test/test_numeric_tower.py
index bef3d4c35d..3423d4e526 100644
--- a/Lib/test/test_numeric_tower.py
+++ b/Lib/test/test_numeric_tower.py
@@ -150,7 +150,7 @@ class ComparisonTest(unittest.TestCase):
         # int, float, Fraction, Decimal
         test_values = [
             float('-inf'),
-            D('-1e999999999'),
+            D('-1e425000000'),
             -1e308,
             F(-22, 7),
             -3.14,
diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py
index 7b95612024..78de278f76 100644
--- a/Lib/test/test_optparse.py
+++ b/Lib/test/test_optparse.py
@@ -318,6 +318,22 @@ class TestOptionChecks(BaseTest):
             ["-b"], {'action': 'store',
                      'callback_kwargs': 'foo'})
 
+    def test_no_single_dash(self):
+        self.assertOptionError(
+            "invalid long option string '-debug': "
+            "must start with --, followed by non-dash",
+            ["-debug"])
+
+        self.assertOptionError(
+            "option -d: invalid long option string '-debug': must start with"
+            " --, followed by non-dash",
+            ["-d", "-debug"])
+
+        self.assertOptionError(
+            "invalid long option string '-debug': "
+            "must start with --, followed by non-dash",
+            ["-debug", "--debug"])
+
 class TestOptionParser(BaseTest):
     def setUp(self):
         self.parser = OptionParser()
@@ -631,7 +647,7 @@ class TestStandard(BaseTest):
                                                option_list=options)
 
     def test_required_value(self):
-        self.assertParseFail(["-a"], "-a option requires an argument")
+        self.assertParseFail(["-a"], "-a option requires 1 argument")
 
     def test_invalid_integer(self):
         self.assertParseFail(["-b", "5x"],
@@ -1023,7 +1039,7 @@ class TestExtendAddTypes(BaseTest):
         TYPE_CHECKER["file"] = check_file
 
     def test_filetype_ok(self):
-        open(support.TESTFN, "w").close()
+        support.create_empty_file(support.TESTFN)
         self.assertParseOK(["--file", support.TESTFN, "-afoo"],
                            {'file': support.TESTFN, 'a': 'foo'},
                            [])
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 720e78b317..184c9ae31f 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -14,20 +14,43 @@ import shutil
 from test import support
 import contextlib
 import mmap
+import platform
+import re
 import uuid
+import asyncore
+import asynchat
+import socket
+import itertools
 import stat
+import locale
+import codecs
+try:
+    import threading
+except ImportError:
+    threading = None
 from test.script_helper import assert_python_ok
 
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    os.stat_float_times(True)
+st = os.stat(__file__)
+stat_supports_subsecond = (
+    # check if float and int timestamps are different
+    (st.st_atime != st[7])
+    or (st.st_mtime != st[8])
+    or (st.st_ctime != st[9]))
+
 # Detect whether we're on a Linux system that uses the (now outdated
 # and unmaintained) linuxthreads threading library.  There's an issue
 # when combining linuxthreads with a failed execv call: see
 # http://bugs.python.org/issue4970.
-if (hasattr(os, "confstr_names") and
-    "CS_GNU_LIBPTHREAD_VERSION" in os.confstr_names):
-    libpthread = os.confstr("CS_GNU_LIBPTHREAD_VERSION")
-    USING_LINUXTHREADS= libpthread.startswith("linuxthreads")
+if hasattr(sys, 'thread_info') and sys.thread_info.version:
+    USING_LINUXTHREADS = sys.thread_info.version.startswith("linuxthreads")
 else:
-    USING_LINUXTHREADS= False
+    USING_LINUXTHREADS = False
+
+# Issue #14110: Some tests fail on FreeBSD if the user is in the wheel group.
+HAVE_WHEEL_GROUP = sys.platform.startswith('freebsd') and os.getgid() == 0
 
 # Tests creating TESTFN
 class FileTests(unittest.TestCase):
@@ -124,6 +147,18 @@ class FileTests(unittest.TestCase):
         self.fdopen_helper('r')
         self.fdopen_helper('r', 100)
 
+    def test_replace(self):
+        TESTFN2 = support.TESTFN + ".2"
+        with open(support.TESTFN, 'w') as f:
+            f.write("1")
+        with open(TESTFN2, 'w') as f:
+            f.write("2")
+        self.addCleanup(os.unlink, TESTFN2)
+        os.replace(support.TESTFN, TESTFN2)
+        self.assertRaises(FileNotFoundError, os.stat, support.TESTFN)
+        with open(TESTFN2, 'r') as f:
+            self.assertEqual(f.read(), "1")
+
 
 # Test attributes on return values from os.*stat* family.
 class StatAttributeTests(unittest.TestCase):
@@ -142,7 +177,6 @@ class StatAttributeTests(unittest.TestCase):
         if not hasattr(os, "stat"):
             return
 
-        import stat
         result = os.stat(fname)
 
         # Make sure direct access works
@@ -162,6 +196,13 @@ class StatAttributeTests(unittest.TestCase):
                                   result[getattr(stat, name)])
                 self.assertIn(attr, members)
 
+        # Make sure that the st_?time and st_?time_ns fields roughly agree
+        # (they should always agree up to around tens-of-microseconds)
+        for name in 'st_atime st_mtime st_ctime'.split():
+            floaty = int(getattr(result, name) * 100000)
+            nanosecondy = getattr(result, name + "_ns") // 10000
+            self.assertAlmostEqual(floaty, nanosecondy, delta=2)
+
         try:
             result[200]
             self.fail("No exception raised")
@@ -208,7 +249,9 @@ class StatAttributeTests(unittest.TestCase):
             fname = self.fname.encode(sys.getfilesystemencoding())
         except UnicodeEncodeError:
             self.skipTest("cannot encode %a for the filesystem" % self.fname)
-        self.check_stat_attributes(fname)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.check_stat_attributes(fname)
 
     def test_statvfs_attributes(self):
         if not hasattr(os, "statvfs"):
@@ -265,6 +308,143 @@ class StatAttributeTests(unittest.TestCase):
         st2 = os.stat(support.TESTFN)
         self.assertEqual(st2.st_mtime, int(st.st_mtime-delta))
 
+    def _test_utime(self, filename, attr, utime, delta):
+        # Issue #13327 removed the requirement to pass None as the
+        # second argument. Check that the previous methods of passing
+        # a time tuple or None work in addition to no argument.
+        st0 = os.stat(filename)
+        # Doesn't set anything new, but sets the time tuple way
+        utime(filename, (attr(st0, "st_atime"), attr(st0, "st_mtime")))
+        # Setting the time to the time you just read, then reading again,
+        # should always return exactly the same times.
+        st1 = os.stat(filename)
+        self.assertEqual(attr(st0, "st_mtime"), attr(st1, "st_mtime"))
+        self.assertEqual(attr(st0, "st_atime"), attr(st1, "st_atime"))
+        # Set to the current time in the old explicit way.
+        os.utime(filename, None)
+        st2 = os.stat(support.TESTFN)
+        # Set to the current time in the new way
+        os.utime(filename)
+        st3 = os.stat(filename)
+        self.assertAlmostEqual(attr(st2, "st_mtime"), attr(st3, "st_mtime"), delta=delta)
+
+    def test_utime(self):
+        def utime(file, times):
+            return os.utime(file, times)
+        self._test_utime(self.fname, getattr, utime, 10)
+        self._test_utime(support.TESTFN, getattr, utime, 10)
+
+
+    def _test_utime_ns(self, set_times_ns, test_dir=True):
+        def getattr_ns(o, attr):
+            return getattr(o, attr + "_ns")
+        ten_s = 10 * 1000 * 1000 * 1000
+        self._test_utime(self.fname, getattr_ns, set_times_ns, ten_s)
+        if test_dir:
+            self._test_utime(support.TESTFN, getattr_ns, set_times_ns, ten_s)
+
+    def test_utime_ns(self):
+        def utime_ns(file, times):
+            return os.utime(file, ns=times)
+        self._test_utime_ns(utime_ns)
+
+    requires_utime_dir_fd = unittest.skipUnless(
+                                os.utime in os.supports_dir_fd,
+                                "dir_fd support for utime required for this test.")
+    requires_utime_fd = unittest.skipUnless(
+                                os.utime in os.supports_fd,
+                                "fd support for utime required for this test.")
+    requires_utime_nofollow_symlinks = unittest.skipUnless(
+                                os.utime in os.supports_follow_symlinks,
+                                "follow_symlinks support for utime required for this test.")
+
+    @requires_utime_nofollow_symlinks
+    def test_lutimes_ns(self):
+        def lutimes_ns(file, times):
+            return os.utime(file, ns=times, follow_symlinks=False)
+        self._test_utime_ns(lutimes_ns)
+
+    @requires_utime_fd
+    def test_futimes_ns(self):
+        def futimes_ns(file, times):
+            with open(file, "wb") as f:
+                os.utime(f.fileno(), ns=times)
+        self._test_utime_ns(futimes_ns, test_dir=False)
+
+    def _utime_invalid_arguments(self, name, arg):
+        with self.assertRaises(ValueError):
+            getattr(os, name)(arg, (5, 5), ns=(5, 5))
+
+    def test_utime_invalid_arguments(self):
+        self._utime_invalid_arguments('utime', self.fname)
+
+
+    @unittest.skipUnless(stat_supports_subsecond,
+                         "os.stat() doesn't has a subsecond resolution")
+    def _test_utime_subsecond(self, set_time_func):
+        asec, amsec = 1, 901
+        atime = asec + amsec * 1e-3
+        msec, mmsec = 2, 901
+        mtime = msec + mmsec * 1e-3
+        filename = self.fname
+        os.utime(filename, (0, 0))
+        set_time_func(filename, atime, mtime)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            os.stat_float_times(True)
+        st = os.stat(filename)
+        self.assertAlmostEqual(st.st_atime, atime, places=3)
+        self.assertAlmostEqual(st.st_mtime, mtime, places=3)
+
+    def test_utime_subsecond(self):
+        def set_time(filename, atime, mtime):
+            os.utime(filename, (atime, mtime))
+        self._test_utime_subsecond(set_time)
+
+    @requires_utime_fd
+    def test_futimes_subsecond(self):
+        def set_time(filename, atime, mtime):
+            with open(filename, "wb") as f:
+                os.utime(f.fileno(), times=(atime, mtime))
+        self._test_utime_subsecond(set_time)
+
+    @requires_utime_fd
+    def test_futimens_subsecond(self):
+        def set_time(filename, atime, mtime):
+            with open(filename, "wb") as f:
+                os.utime(f.fileno(), times=(atime, mtime))
+        self._test_utime_subsecond(set_time)
+
+    @requires_utime_dir_fd
+    def test_futimesat_subsecond(self):
+        def set_time(filename, atime, mtime):
+            dirname = os.path.dirname(filename)
+            dirfd = os.open(dirname, os.O_RDONLY)
+            try:
+                os.utime(os.path.basename(filename), dir_fd=dirfd,
+                             times=(atime, mtime))
+            finally:
+                os.close(dirfd)
+        self._test_utime_subsecond(set_time)
+
+    @requires_utime_nofollow_symlinks
+    def test_lutimes_subsecond(self):
+        def set_time(filename, atime, mtime):
+            os.utime(filename, (atime, mtime), follow_symlinks=False)
+        self._test_utime_subsecond(set_time)
+
+    @requires_utime_dir_fd
+    def test_utimensat_subsecond(self):
+        def set_time(filename, atime, mtime):
+            dirname = os.path.dirname(filename)
+            dirfd = os.open(dirname, os.O_RDONLY)
+            try:
+                os.utime(os.path.basename(filename), dir_fd=dirfd,
+                             times=(atime, mtime))
+            finally:
+                os.close(dirfd)
+        self._test_utime_subsecond(set_time)
+
     # Restrict test to Win32, since there is no guarantee other
     # systems support centiseconds
     if sys.platform == 'win32':
@@ -296,6 +476,19 @@ class StatAttributeTests(unittest.TestCase):
                     return
                 self.fail("Could not stat pagefile.sys")
 
+        @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
+        def test_15261(self):
+            # Verify that stat'ing a closed fd does not cause crash
+            r, w = os.pipe()
+            try:
+                os.stat(r)          # should not raise error
+            finally:
+                os.close(r)
+                os.close(w)
+            with self.assertRaises(OSError) as ctx:
+                os.stat(r)
+            self.assertEqual(ctx.exception.errno, errno.EBADF)
+
 from test import mapping_tests
 
 class EnvironTests(mapping_tests.BasicTestMappingProtocol):
@@ -324,23 +517,23 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
         return os.environ
 
     # Bug 1110478
+    @unittest.skipUnless(os.path.exists('/bin/sh'), 'requires /bin/sh')
     def test_update2(self):
         os.environ.clear()
-        if os.path.exists("/bin/sh"):
-            os.environ.update(HELLO="World")
-            with os.popen("/bin/sh -c 'echo $HELLO'") as popen:
-                value = popen.read().strip()
-                self.assertEqual(value, "World")
+        os.environ.update(HELLO="World")
+        with os.popen("/bin/sh -c 'echo $HELLO'") as popen:
+            value = popen.read().strip()
+            self.assertEqual(value, "World")
 
+    @unittest.skipUnless(os.path.exists('/bin/sh'), 'requires /bin/sh')
     def test_os_popen_iter(self):
-        if os.path.exists("/bin/sh"):
-            with os.popen(
-                "/bin/sh -c 'echo \"line1\nline2\nline3\"'") as popen:
-                it = iter(popen)
-                self.assertEqual(next(it), "line1\n")
-                self.assertEqual(next(it), "line2\n")
-                self.assertEqual(next(it), "line3\n")
-                self.assertRaises(StopIteration, next, it)
+        with os.popen(
+            "/bin/sh -c 'echo \"line1\nline2\nline3\"'") as popen:
+            it = iter(popen)
+            self.assertEqual(next(it), "line1\n")
+            self.assertEqual(next(it), "line2\n")
+            self.assertEqual(next(it), "line3\n")
+            self.assertRaises(StopIteration, next, it)
 
     # Verify environ keys and values from the OS are of the
     # correct str type.
@@ -427,8 +620,8 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
 
     # On FreeBSD < 7 and OS X < 10.6, unsetenv() doesn't return a value (issue
     # #13415).
-    @unittest.skipIf(sys.platform.startswith(('freebsd', 'darwin')),
-                     "due to known OS bug: see issue #13415")
+    @support.requires_freebsd_version(7)
+    @support.requires_mac_ver(10, 6)
     def test_unset_error(self):
         if sys.platform == "win32":
             # an environment variable is limited to 32,767 characters
@@ -442,7 +635,7 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
 class WalkTests(unittest.TestCase):
     """Tests for os.walk()."""
 
-    def test_traversal(self):
+    def setUp(self):
         import os
         from os.path import join
 
@@ -456,6 +649,7 @@ class WalkTests(unittest.TestCase):
         #         SUB2/             a file kid and a dirsymlink kid
         #           tmp3
         #           link/           a symlink to TESTFN.2
+        #           broken_link
         #       TEST2/
         #         tmp4              a lone file
         walk_path = join(support.TESTFN, "TEST1")
@@ -468,6 +662,8 @@ class WalkTests(unittest.TestCase):
         link_path = join(sub2_path, "link")
         t2_path = join(support.TESTFN, "TEST2")
         tmp4_path = join(support.TESTFN, "TEST2", "tmp4")
+        link_path = join(sub2_path, "link")
+        broken_link_path = join(sub2_path, "broken_link")
 
         # Create stuff.
         os.makedirs(sub11_path)
@@ -484,7 +680,8 @@ class WalkTests(unittest.TestCase):
             else:
                 symlink_to_dir = os.symlink
             symlink_to_dir(os.path.abspath(t2_path), link_path)
-            sub2_tree = (sub2_path, ["link"], ["tmp3"])
+            symlink_to_dir('broken', broken_link_path)
+            sub2_tree = (sub2_path, ["link"], ["broken_link", "tmp3"])
         else:
             sub2_tree = (sub2_path, [], ["tmp3"])
 
@@ -496,6 +693,7 @@ class WalkTests(unittest.TestCase):
         #     flipped:  TESTFN, SUB2, SUB1, SUB11
         flipped = all[0][1][0] != "SUB1"
         all[0][1].sort()
+        all[3 - 2 * flipped][-1].sort()
         self.assertEqual(all[0], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
         self.assertEqual(all[1 + flipped], (sub1_path, ["SUB11"], ["tmp2"]))
         self.assertEqual(all[2 + flipped], (sub11_path, [], []))
@@ -511,6 +709,7 @@ class WalkTests(unittest.TestCase):
                 dirs.remove('SUB1')
         self.assertEqual(len(all), 2)
         self.assertEqual(all[0], (walk_path, ["SUB2"], ["tmp1"]))
+        all[1][-1].sort()
         self.assertEqual(all[1], sub2_tree)
 
         # Walk bottom-up.
@@ -521,6 +720,7 @@ class WalkTests(unittest.TestCase):
         #     flipped:  SUB2, SUB11, SUB1, TESTFN
         flipped = all[3][1][0] != "SUB1"
         all[3][1].sort()
+        all[2 - 2 * flipped][-1].sort()
         self.assertEqual(all[3], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
         self.assertEqual(all[flipped], (sub11_path, [], []))
         self.assertEqual(all[flipped + 1], (sub1_path, ["SUB11"], ["tmp2"]))
@@ -552,6 +752,82 @@ class WalkTests(unittest.TestCase):
                     os.remove(dirname)
         os.rmdir(support.TESTFN)
 
+
+@unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
+class FwalkTests(WalkTests):
+    """Tests for os.fwalk()."""
+
+    def _compare_to_walk(self, walk_kwargs, fwalk_kwargs):
+        """
+        compare with walk() results.
+        """
+        walk_kwargs = walk_kwargs.copy()
+        fwalk_kwargs = fwalk_kwargs.copy()
+        for topdown, follow_symlinks in itertools.product((True, False), repeat=2):
+            walk_kwargs.update(topdown=topdown, followlinks=follow_symlinks)
+            fwalk_kwargs.update(topdown=topdown, follow_symlinks=follow_symlinks)
+
+            expected = {}
+            for root, dirs, files in os.walk(**walk_kwargs):
+                expected[root] = (set(dirs), set(files))
+
+            for root, dirs, files, rootfd in os.fwalk(**fwalk_kwargs):
+                self.assertIn(root, expected)
+                self.assertEqual(expected[root], (set(dirs), set(files)))
+
+    def test_compare_to_walk(self):
+        kwargs = {'top': support.TESTFN}
+        self._compare_to_walk(kwargs, kwargs)
+
+    def test_dir_fd(self):
+        try:
+            fd = os.open(".", os.O_RDONLY)
+            walk_kwargs = {'top': support.TESTFN}
+            fwalk_kwargs = walk_kwargs.copy()
+            fwalk_kwargs['dir_fd'] = fd
+            self._compare_to_walk(walk_kwargs, fwalk_kwargs)
+        finally:
+            os.close(fd)
+
+    def test_yields_correct_dir_fd(self):
+        # check returned file descriptors
+        for topdown, follow_symlinks in itertools.product((True, False), repeat=2):
+            args = support.TESTFN, topdown, None
+            for root, dirs, files, rootfd in os.fwalk(*args, follow_symlinks=follow_symlinks):
+                # check that the FD is valid
+                os.fstat(rootfd)
+                # redundant check
+                os.stat(rootfd)
+                # check that listdir() returns consistent information
+                self.assertEqual(set(os.listdir(rootfd)), set(dirs) | set(files))
+
+    def test_fd_leak(self):
+        # Since we're opening a lot of FDs, we must be careful to avoid leaks:
+        # we both check that calling fwalk() a large number of times doesn't
+        # yield EMFILE, and that the minimum allocated FD hasn't changed.
+        minfd = os.dup(1)
+        os.close(minfd)
+        for i in range(256):
+            for x in os.fwalk(support.TESTFN):
+                pass
+        newfd = os.dup(1)
+        self.addCleanup(os.close, newfd)
+        self.assertEqual(newfd, minfd)
+
+    def tearDown(self):
+        # cleanup
+        for root, dirs, files, rootfd in os.fwalk(support.TESTFN, topdown=False):
+            for name in files:
+                os.unlink(name, dir_fd=rootfd)
+            for name in dirs:
+                st = os.stat(name, dir_fd=rootfd, follow_symlinks=False)
+                if stat.S_ISDIR(st.st_mode):
+                    os.rmdir(name, dir_fd=rootfd)
+                else:
+                    os.unlink(name, dir_fd=rootfd)
+        os.rmdir(support.TESTFN)
+
+
 class MakedirTests(unittest.TestCase):
     def setUp(self):
         os.mkdir(support.TESTFN)
@@ -575,14 +851,12 @@ class MakedirTests(unittest.TestCase):
         path = os.path.join(support.TESTFN, 'dir1')
         mode = 0o777
         old_mask = os.umask(0o022)
-        try:
-            os.makedirs(path, mode)
-            self.assertRaises(OSError, os.makedirs, path, mode)
-            self.assertRaises(OSError, os.makedirs, path, mode, exist_ok=False)
-            self.assertRaises(OSError, os.makedirs, path, 0o776, exist_ok=True)
-            os.makedirs(path, mode=mode, exist_ok=True)
-        finally:
-            os.umask(old_mask)
+        os.makedirs(path, mode)
+        self.assertRaises(OSError, os.makedirs, path, mode)
+        self.assertRaises(OSError, os.makedirs, path, mode, exist_ok=False)
+        self.assertRaises(OSError, os.makedirs, path, 0o776, exist_ok=True)
+        os.makedirs(path, mode=mode, exist_ok=True)
+        os.umask(old_mask)
 
     def test_exist_ok_s_isgid_directory(self):
         path = os.path.join(support.TESTFN, 'dir1')
@@ -594,7 +868,7 @@ class MakedirTests(unittest.TestCase):
                     os.lstat(support.TESTFN).st_mode)
             try:
                 os.chmod(support.TESTFN, existing_testfn_mode | S_ISGID)
-            except OSError:
+            except PermissionError:
                 raise unittest.SkipTest('Cannot set S_ISGID for dir.')
             if (os.lstat(support.TESTFN).st_mode & S_ISGID != S_ISGID):
                 raise unittest.SkipTest('No support for S_ISGID dir mode.')
@@ -894,10 +1168,12 @@ class TestInvalidFD(unittest.TestCase):
 
     def test_fpathconf(self):
         if hasattr(os, "fpathconf"):
+            self.check(os.pathconf, "PC_NAME_MAX")
             self.check(os.fpathconf, "PC_NAME_MAX")
 
     def test_ftruncate(self):
         if hasattr(os, "ftruncate"):
+            self.check(os.truncate, 0)
             self.check(os.ftruncate, 0)
 
     def test_lseek(self):
@@ -931,7 +1207,9 @@ class LinkTests(unittest.TestCase):
         with open(file1, "w") as f1:
             f1.write("test")
 
-        os.link(file1, file2)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            os.link(file1, file2)
         with open(file1, "r") as f1, open(file2, "r") as f2:
             self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno()))
 
@@ -965,7 +1243,7 @@ if sys.platform != 'win32':
 
         if hasattr(os, 'setgid'):
             def test_setgid(self):
-                if os.getuid() != 0:
+                if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
                     self.assertRaises(os.error, os.setgid, 0)
                 self.assertRaises(OverflowError, os.setgid, 1<<32)
 
@@ -977,7 +1255,7 @@ if sys.platform != 'win32':
 
         if hasattr(os, 'setegid'):
             def test_setegid(self):
-                if os.getuid() != 0:
+                if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
                     self.assertRaises(os.error, os.setegid, 0)
                 self.assertRaises(OverflowError, os.setegid, 1<<32)
 
@@ -997,7 +1275,7 @@ if sys.platform != 'win32':
 
         if hasattr(os, 'setregid'):
             def test_setregid(self):
-                if os.getuid() != 0:
+                if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
                     self.assertRaises(os.error, os.setregid, 0, 0)
                 self.assertRaises(OverflowError, os.setregid, 1<<32, 0)
                 self.assertRaises(OverflowError, os.setregid, 0, 1<<32)
@@ -1038,8 +1316,7 @@ if sys.platform != 'win32':
             os.mkdir(self.dir)
             try:
                 for fn in bytesfn:
-                    f = open(os.path.join(self.bdir, fn), "w")
-                    f.close()
+                    support.create_empty_file(os.path.join(self.bdir, fn))
                     fn = os.fsdecode(fn)
                     if fn in self.unicodefn:
                         raise ValueError("duplicate filename")
@@ -1055,6 +1332,13 @@ if sys.platform != 'win32':
             expected = self.unicodefn
             found = set(os.listdir(self.dir))
             self.assertEqual(found, expected)
+            # test listdir without arguments
+            current_directory = os.getcwd()
+            try:
+                os.chdir(os.sep)
+                self.assertEqual(set(os.listdir()), set(os.listdir(os.sep)))
+            finally:
+                os.chdir(current_directory)
 
         def test_open(self):
             for fn in self.unicodefn:
@@ -1267,8 +1551,10 @@ class Win32SymlinkTests(unittest.TestCase):
         self.assertNotEqual(os.lstat(link), os.stat(link))
 
         bytes_link = os.fsencode(link)
-        self.assertEqual(os.stat(bytes_link), os.stat(target))
-        self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link))
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertEqual(os.stat(bytes_link), os.stat(target))
+            self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link))
 
     def test_12084(self):
         level1 = os.path.abspath(support.TESTFN)
@@ -1327,6 +1613,22 @@ class FSEncodingTests(unittest.TestCase):
             self.assertEqual(os.fsdecode(bytesfn), fn)
 
 
+
+class DeviceEncodingTests(unittest.TestCase):
+
+    def test_bad_fd(self):
+        # Return None when an fd doesn't actually exist.
+        self.assertIsNone(os.device_encoding(123456))
+
+    @unittest.skipUnless(os.isatty(0) and (sys.platform.startswith('win') or
+            (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET'))),
+            'test requires a tty and either Windows or nl_langinfo(CODESET)')
+    def test_device_encoding(self):
+        encoding = os.device_encoding(0)
+        self.assertIsNotNone(encoding)
+        self.assertTrue(codecs.lookup(encoding))
+
+
 class PidTests(unittest.TestCase):
     @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid")
     def test_getppid(self):
@@ -1348,12 +1650,471 @@ class LoginTests(unittest.TestCase):
         self.assertNotEqual(len(user_name), 0)
 
 
+@unittest.skipUnless(hasattr(os, 'getpriority') and hasattr(os, 'setpriority'),
+                     "needs os.getpriority and os.setpriority")
+class ProgramPriorityTests(unittest.TestCase):
+    """Tests for os.getpriority() and os.setpriority()."""
+
+    def test_set_get_priority(self):
+
+        base = os.getpriority(os.PRIO_PROCESS, os.getpid())
+        os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1)
+        try:
+            new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid())
+            if base >= 19 and new_prio <= 19:
+                raise unittest.SkipTest(
+      "unable to reliably test setpriority at current nice level of %s" % base)
+            else:
+                self.assertEqual(new_prio, base + 1)
+        finally:
+            try:
+                os.setpriority(os.PRIO_PROCESS, os.getpid(), base)
+            except OSError as err:
+                if err.errno != errno.EACCES:
+                    raise
+
+
+if threading is not None:
+    class SendfileTestServer(asyncore.dispatcher, threading.Thread):
+
+        class Handler(asynchat.async_chat):
+
+            def __init__(self, conn):
+                asynchat.async_chat.__init__(self, conn)
+                self.in_buffer = []
+                self.closed = False
+                self.push(b"220 ready\r\n")
+
+            def handle_read(self):
+                data = self.recv(4096)
+                self.in_buffer.append(data)
+
+            def get_data(self):
+                return b''.join(self.in_buffer)
+
+            def handle_close(self):
+                self.close()
+                self.closed = True
+
+            def handle_error(self):
+                raise
+
+        def __init__(self, address):
+            threading.Thread.__init__(self)
+            asyncore.dispatcher.__init__(self)
+            self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.bind(address)
+            self.listen(5)
+            self.host, self.port = self.socket.getsockname()[:2]
+            self.handler_instance = None
+            self._active = False
+            self._active_lock = threading.Lock()
+
+        # --- public API
+
+        @property
+        def running(self):
+            return self._active
+
+        def start(self):
+            assert not self.running
+            self.__flag = threading.Event()
+            threading.Thread.start(self)
+            self.__flag.wait()
+
+        def stop(self):
+            assert self.running
+            self._active = False
+            self.join()
+
+        def wait(self):
+            # wait for handler connection to be closed, then stop the server
+            while not getattr(self.handler_instance, "closed", False):
+                time.sleep(0.001)
+            self.stop()
+
+        # --- internals
+
+        def run(self):
+            self._active = True
+            self.__flag.set()
+            while self._active and asyncore.socket_map:
+                self._active_lock.acquire()
+                asyncore.loop(timeout=0.001, count=1)
+                self._active_lock.release()
+            asyncore.close_all()
+
+        def handle_accept(self):
+            conn, addr = self.accept()
+            self.handler_instance = self.Handler(conn)
+
+        def handle_connect(self):
+            self.close()
+        handle_read = handle_connect
+
+        def writable(self):
+            return 0
+
+        def handle_error(self):
+            raise
+
+
+@unittest.skipUnless(threading is not None, "test needs threading module")
+@unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()")
+class TestSendfile(unittest.TestCase):
+
+    DATA = b"12345abcde" * 16 * 1024  # 160 KB
+    SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \
+                               not sys.platform.startswith("solaris") and \
+                               not sys.platform.startswith("sunos")
+
+    @classmethod
+    def setUpClass(cls):
+        with open(support.TESTFN, "wb") as f:
+            f.write(cls.DATA)
+
+    @classmethod
+    def tearDownClass(cls):
+        support.unlink(support.TESTFN)
+
+    def setUp(self):
+        self.server = SendfileTestServer((support.HOST, 0))
+        self.server.start()
+        self.client = socket.socket()
+        self.client.connect((self.server.host, self.server.port))
+        self.client.settimeout(1)
+        # synchronize by waiting for "220 ready" response
+        self.client.recv(1024)
+        self.sockno = self.client.fileno()
+        self.file = open(support.TESTFN, 'rb')
+        self.fileno = self.file.fileno()
+
+    def tearDown(self):
+        self.file.close()
+        self.client.close()
+        if self.server.running:
+            self.server.stop()
+
+    def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
+        """A higher level wrapper representing how an application is
+        supposed to use sendfile().
+        """
+        while 1:
+            try:
+                if self.SUPPORT_HEADERS_TRAILERS:
+                    return os.sendfile(sock, file, offset, nbytes, headers,
+                                       trailers)
+                else:
+                    return os.sendfile(sock, file, offset, nbytes)
+            except OSError as err:
+                if err.errno == errno.ECONNRESET:
+                    # disconnected
+                    raise
+                elif err.errno in (errno.EAGAIN, errno.EBUSY):
+                    # we have to retry send data
+                    continue
+                else:
+                    raise
+
+    def test_send_whole_file(self):
+        # normal send
+        total_sent = 0
+        offset = 0
+        nbytes = 4096
+        while total_sent < len(self.DATA):
+            sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes)
+            if sent == 0:
+                break
+            offset += sent
+            total_sent += sent
+            self.assertTrue(sent <= nbytes)
+            self.assertEqual(offset, total_sent)
+
+        self.assertEqual(total_sent, len(self.DATA))
+        self.client.shutdown(socket.SHUT_RDWR)
+        self.client.close()
+        self.server.wait()
+        data = self.server.handler_instance.get_data()
+        self.assertEqual(len(data), len(self.DATA))
+        self.assertEqual(data, self.DATA)
+
+    def test_send_at_certain_offset(self):
+        # start sending a file at a certain offset
+        total_sent = 0
+        offset = len(self.DATA) // 2
+        must_send = len(self.DATA) - offset
+        nbytes = 4096
+        while total_sent < must_send:
+            sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes)
+            if sent == 0:
+                break
+            offset += sent
+            total_sent += sent
+            self.assertTrue(sent <= nbytes)
+
+        self.client.shutdown(socket.SHUT_RDWR)
+        self.client.close()
+        self.server.wait()
+        data = self.server.handler_instance.get_data()
+        expected = self.DATA[len(self.DATA) // 2:]
+        self.assertEqual(total_sent, len(expected))
+        self.assertEqual(len(data), len(expected))
+        self.assertEqual(data, expected)
+
+    def test_offset_overflow(self):
+        # specify an offset > file size
+        offset = len(self.DATA) + 4096
+        try:
+            sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
+        except OSError as e:
+            # Solaris can raise EINVAL if offset >= file length, ignore.
+            if e.errno != errno.EINVAL:
+                raise
+        else:
+            self.assertEqual(sent, 0)
+        self.client.shutdown(socket.SHUT_RDWR)
+        self.client.close()
+        self.server.wait()
+        data = self.server.handler_instance.get_data()
+        self.assertEqual(data, b'')
+
+    def test_invalid_offset(self):
+        with self.assertRaises(OSError) as cm:
+            os.sendfile(self.sockno, self.fileno, -1, 4096)
+        self.assertEqual(cm.exception.errno, errno.EINVAL)
+
+    # --- headers / trailers tests
+
+    if SUPPORT_HEADERS_TRAILERS:
+
+        def test_headers(self):
+            total_sent = 0
+            sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
+                               headers=[b"x" * 512])
+            total_sent += sent
+            offset = 4096
+            nbytes = 4096
+            while 1:
+                sent = self.sendfile_wrapper(self.sockno, self.fileno,
+                                                     offset, nbytes)
+                if sent == 0:
+                    break
+                total_sent += sent
+                offset += sent
+
+            expected_data = b"x" * 512 + self.DATA
+            self.assertEqual(total_sent, len(expected_data))
+            self.client.close()
+            self.server.wait()
+            data = self.server.handler_instance.get_data()
+            self.assertEqual(hash(data), hash(expected_data))
+
+        def test_trailers(self):
+            TESTFN2 = support.TESTFN + "2"
+            with open(TESTFN2, 'wb') as f:
+                f.write(b"abcde")
+            with open(TESTFN2, 'rb')as f:
+                self.addCleanup(os.remove, TESTFN2)
+                os.sendfile(self.sockno, f.fileno(), 0, 4096,
+                            trailers=[b"12345"])
+                self.client.close()
+                self.server.wait()
+                data = self.server.handler_instance.get_data()
+                self.assertEqual(data, b"abcde12345")
+
+        if hasattr(os, "SF_NODISKIO"):
+            def test_flags(self):
+                try:
+                    os.sendfile(self.sockno, self.fileno, 0, 4096,
+                                flags=os.SF_NODISKIO)
+                except OSError as err:
+                    if err.errno not in (errno.EBUSY, errno.EAGAIN):
+                        raise
+
+
+def supports_extended_attributes():
+    if not hasattr(os, "setxattr"):
+        return False
+    try:
+        with open(support.TESTFN, "wb") as fp:
+            try:
+                os.setxattr(fp.fileno(), b"user.test", b"")
+            except OSError:
+                return False
+    finally:
+        support.unlink(support.TESTFN)
+    # Kernels < 2.6.39 don't respect setxattr flags.
+    kernel_version = platform.release()
+    m = re.match("2.6.(\d{1,2})", kernel_version)
+    return m is None or int(m.group(1)) >= 39
+
+
+@unittest.skipUnless(supports_extended_attributes(),
+                     "no non-broken extended attribute support")
+class ExtendedAttributeTests(unittest.TestCase):
+
+    def tearDown(self):
+        support.unlink(support.TESTFN)
+
+    def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwargs):
+        fn = support.TESTFN
+        open(fn, "wb").close()
+        with self.assertRaises(OSError) as cm:
+            getxattr(fn, s("user.test"), **kwargs)
+        self.assertEqual(cm.exception.errno, errno.ENODATA)
+        init_xattr = listxattr(fn)
+        self.assertIsInstance(init_xattr, list)
+        setxattr(fn, s("user.test"), b"", **kwargs)
+        xattr = set(init_xattr)
+        xattr.add("user.test")
+        self.assertEqual(set(listxattr(fn)), xattr)
+        self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"")
+        setxattr(fn, s("user.test"), b"hello", os.XATTR_REPLACE, **kwargs)
+        self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"hello")
+        with self.assertRaises(OSError) as cm:
+            setxattr(fn, s("user.test"), b"bye", os.XATTR_CREATE, **kwargs)
+        self.assertEqual(cm.exception.errno, errno.EEXIST)
+        with self.assertRaises(OSError) as cm:
+            setxattr(fn, s("user.test2"), b"bye", os.XATTR_REPLACE, **kwargs)
+        self.assertEqual(cm.exception.errno, errno.ENODATA)
+        setxattr(fn, s("user.test2"), b"foo", os.XATTR_CREATE, **kwargs)
+        xattr.add("user.test2")
+        self.assertEqual(set(listxattr(fn)), xattr)
+        removexattr(fn, s("user.test"), **kwargs)
+        with self.assertRaises(OSError) as cm:
+            getxattr(fn, s("user.test"), **kwargs)
+        self.assertEqual(cm.exception.errno, errno.ENODATA)
+        xattr.remove("user.test")
+        self.assertEqual(set(listxattr(fn)), xattr)
+        self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo")
+        setxattr(fn, s("user.test"), b"a"*1024, **kwargs)
+        self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*1024)
+        removexattr(fn, s("user.test"), **kwargs)
+        many = sorted("user.test{}".format(i) for i in range(100))
+        for thing in many:
+            setxattr(fn, thing, b"x", **kwargs)
+        self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many))
+
+    def _check_xattrs(self, *args, **kwargs):
+        def make_bytes(s):
+            return bytes(s, "ascii")
+        self._check_xattrs_str(str, *args, **kwargs)
+        support.unlink(support.TESTFN)
+        self._check_xattrs_str(make_bytes, *args, **kwargs)
+
+    def test_simple(self):
+        self._check_xattrs(os.getxattr, os.setxattr, os.removexattr,
+                           os.listxattr)
+
+    def test_lpath(self):
+        self._check_xattrs(os.getxattr, os.setxattr, os.removexattr,
+                           os.listxattr, follow_symlinks=False)
+
+    def test_fds(self):
+        def getxattr(path, *args):
+            with open(path, "rb") as fp:
+                return os.getxattr(fp.fileno(), *args)
+        def setxattr(path, *args):
+            with open(path, "wb") as fp:
+                os.setxattr(fp.fileno(), *args)
+        def removexattr(path, *args):
+            with open(path, "wb") as fp:
+                os.removexattr(fp.fileno(), *args)
+        def listxattr(path, *args):
+            with open(path, "rb") as fp:
+                return os.listxattr(fp.fileno(), *args)
+        self._check_xattrs(getxattr, setxattr, removexattr, listxattr)
+
+
+@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+class Win32DeprecatedBytesAPI(unittest.TestCase):
+    def test_deprecated(self):
+        import nt
+        filename = os.fsencode(support.TESTFN)
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", DeprecationWarning)
+            for func, *args in (
+                (nt._getfullpathname, filename),
+                (nt._isdir, filename),
+                (os.access, filename, os.R_OK),
+                (os.chdir, filename),
+                (os.chmod, filename, 0o777),
+                (os.getcwdb,),
+                (os.link, filename, filename),
+                (os.listdir, filename),
+                (os.lstat, filename),
+                (os.mkdir, filename),
+                (os.open, filename, os.O_RDONLY),
+                (os.rename, filename, filename),
+                (os.rmdir, filename),
+                (os.startfile, filename),
+                (os.stat, filename),
+                (os.unlink, filename),
+                (os.utime, filename),
+            ):
+                self.assertRaises(DeprecationWarning, func, *args)
+
+    @support.skip_unless_symlink
+    def test_symlink(self):
+        filename = os.fsencode(support.TESTFN)
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", DeprecationWarning)
+            self.assertRaises(DeprecationWarning,
+                              os.symlink, filename, filename)
+
+
+@unittest.skipUnless(hasattr(os, 'get_terminal_size'), "requires os.get_terminal_size")
+class TermsizeTests(unittest.TestCase):
+    def test_does_not_crash(self):
+        """Check if get_terminal_size() returns a meaningful value.
+
+        There's no easy portable way to actually check the size of the
+        terminal, so let's check if it returns something sensible instead.
+        """
+        try:
+            size = os.get_terminal_size()
+        except OSError as e:
+            if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY):
+                # Under win32 a generic OSError can be thrown if the
+                # handle cannot be retrieved
+                self.skipTest("failed to query terminal size")
+            raise
+
+        self.assertGreaterEqual(size.columns, 0)
+        self.assertGreaterEqual(size.lines, 0)
+
+    def test_stty_match(self):
+        """Check if stty returns the same results
+
+        stty actually tests stdin, so get_terminal_size is invoked on
+        stdin explicitly. If stty succeeded, then get_terminal_size()
+        should work too.
+        """
+        try:
+            size = subprocess.check_output(['stty', 'size']).decode().split()
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            self.skipTest("stty invocation failed")
+        expected = (int(size[1]), int(size[0])) # reversed order
+
+        try:
+            actual = os.get_terminal_size(sys.__stdin__.fileno())
+        except OSError as e:
+            if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY):
+                # Under win32 a generic OSError can be thrown if the
+                # handle cannot be retrieved
+                self.skipTest("failed to query terminal size")
+            raise
+        self.assertEqual(expected, actual)
+
+
+@support.reap_threads
 def test_main():
     support.run_unittest(
         FileTests,
         StatAttributeTests,
         EnvironTests,
         WalkTests,
+        FwalkTests,
         MakedirTests,
         DevNullTests,
         URandomTests,
@@ -1365,9 +2126,15 @@ def test_main():
         Win32KillTests,
         Win32SymlinkTests,
         FSEncodingTests,
+        DeviceEncodingTests,
         PidTests,
         LoginTests,
         LinkTests,
+        TestSendfile,
+        ProgramPriorityTests,
+        ExtendedAttributeTests,
+        Win32DeprecatedBytesAPI,
+        TermsizeTests,
         RemoveDirsTests,
     )
 
diff --git a/Lib/test/test_ossaudiodev.py b/Lib/test/test_ossaudiodev.py
index 9cb89d69a6..3908a0506e 100644
--- a/Lib/test/test_ossaudiodev.py
+++ b/Lib/test/test_ossaudiodev.py
@@ -170,6 +170,22 @@ class OSSAudioDevTests(unittest.TestCase):
             pass
         self.assertTrue(dsp.closed)
 
+    def test_on_closed(self):
+        dsp = ossaudiodev.open('w')
+        dsp.close()
+        self.assertRaises(ValueError, dsp.fileno)
+        self.assertRaises(ValueError, dsp.read, 1)
+        self.assertRaises(ValueError, dsp.write, b'x')
+        self.assertRaises(ValueError, dsp.writeall, b'x')
+        self.assertRaises(ValueError, dsp.bufsize)
+        self.assertRaises(ValueError, dsp.obufcount)
+        self.assertRaises(ValueError, dsp.obufcount)
+        self.assertRaises(ValueError, dsp.obuffree)
+        self.assertRaises(ValueError, dsp.getptr)
+
+        mixer = ossaudiodev.openmixer()
+        mixer.close()
+        self.assertRaises(ValueError, mixer.fileno)
 
 def test_main():
     try:
diff --git a/Lib/test/test_osx_env.py b/Lib/test/test_osx_env.py
index 8b3df3783b..24ec2b4403 100644
--- a/Lib/test/test_osx_env.py
+++ b/Lib/test/test_osx_env.py
@@ -5,6 +5,7 @@ Test suite for OS X interpreter environment variables.
 from test.support import EnvironmentVarGuard, run_unittest
 import subprocess
 import sys
+import sysconfig
 import unittest
 
 class OSXEnvironmentVariableTestCase(unittest.TestCase):
@@ -27,8 +28,6 @@ class OSXEnvironmentVariableTestCase(unittest.TestCase):
         self._check_sys('PYTHONEXECUTABLE', '==', 'sys.executable')
 
 def test_main():
-    from distutils import sysconfig
-
     if sys.platform == 'darwin' and sysconfig.get_config_var('WITH_NEXT_FRAMEWORK'):
         run_unittest(OSXEnvironmentVariableTestCase)
 
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
index 70eb9c0cc2..c93a2ca0be 100644
--- a/Lib/test/test_parser.py
+++ b/Lib/test/test_parser.py
@@ -51,6 +51,10 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
         self.check_suite("def f(): (yield 1)*2")
         self.check_suite("def f(): return; yield 1")
         self.check_suite("def f(): yield 1; return")
+        self.check_suite("def f(): yield from 1")
+        self.check_suite("def f(): x = yield from 1")
+        self.check_suite("def f(): f((yield from 1))")
+        self.check_suite("def f(): yield 1; return 1")
         self.check_suite("def f():\n"
                          "    for x in range(30):\n"
                          "        yield x\n")
@@ -717,6 +721,12 @@ class STObjectTestCase(unittest.TestCase):
 
     # XXX tests for pickling and unpickling of ST objects should go here
 
+class OtherParserCase(unittest.TestCase):
+
+    def test_two_args_to_expr(self):
+        # See bug #12264
+        with self.assertRaises(TypeError):
+            parser.expr("a", "b")
 
 def test_main():
     support.run_unittest(
@@ -725,6 +735,7 @@ def test_main():
         CompileTestCase,
         ParserStackLimitTestCase,
         STObjectTestCase,
+        OtherParserCase,
     )
 
 
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 8355e7d955..4fb07a3cc0 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -21,9 +21,12 @@ class PdbTestInput(object):
     def __enter__(self):
         self.real_stdin = sys.stdin
         sys.stdin = _FakeInput(self.input)
+        self.orig_trace = sys.gettrace() if hasattr(sys, 'gettrace') else None
 
     def __exit__(self, *exc):
         sys.stdin = self.real_stdin
+        if self.orig_trace:
+            sys.settrace(self.orig_trace)
 
 
 def test_pdb_displayhook():
diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py
index e268ae288d..1cacdea569 100644
--- a/Lib/test/test_peepholer.py
+++ b/Lib/test/test_peepholer.py
@@ -3,13 +3,16 @@ import re
 import sys
 from io import StringIO
 import unittest
+from math import copysign
 
 def disassemble(func):
     f = StringIO()
     tmp = sys.stdout
     sys.stdout = f
-    dis.dis(func)
-    sys.stdout = tmp
+    try:
+        dis.dis(func)
+    finally:
+        sys.stdout = tmp
     result = f.getvalue()
     f.close()
     return result
@@ -17,6 +20,7 @@ def disassemble(func):
 def dis_single(line):
     return disassemble(compile(line, '', 'single'))
 
+
 class TestTranforms(unittest.TestCase):
 
     def test_unot(self):
@@ -99,6 +103,12 @@ class TestTranforms(unittest.TestCase):
             self.assertIn(elem, asm)
             self.assertNotIn('BUILD_TUPLE', asm)
 
+        # Long tuples should be folded too.
+        asm = dis_single(repr(tuple(range(10000))))
+        # One LOAD_CONST for the tuple, one for the None return value
+        self.assertEqual(asm.count('LOAD_CONST'), 2)
+        self.assertNotIn('BUILD_TUPLE', asm)
+
         # Bug 1053819:  Tuple of constants misidentified when presented with:
         # . . . opcode_with_arg 100   unary_opcode   BUILD_TUPLE 1  . . .
         # The following would segfault upon compilation
@@ -196,27 +206,28 @@ class TestTranforms(unittest.TestCase):
         self.assertIn('(1000)', asm)
 
     def test_binary_subscr_on_unicode(self):
-        # unicode strings don't get optimized
+        # valid code get optimized
         asm = dis_single('"foo"[0]')
-        self.assertNotIn("('f')", asm)
-        self.assertIn('BINARY_SUBSCR', asm)
+        self.assertIn("('f')", asm)
+        self.assertNotIn('BINARY_SUBSCR', asm)
         asm = dis_single('"\u0061\uffff"[1]')
-        self.assertNotIn("('\\uffff')", asm)
-        self.assertIn('BINARY_SUBSCR', asm)
+        self.assertIn("('\\uffff')", asm)
+        self.assertNotIn('BINARY_SUBSCR', asm)
+        asm = dis_single('"\U00012345abcdef"[3]')
+        self.assertIn("('c')", asm)
+        self.assertNotIn('BINARY_SUBSCR', asm)
 
+        # invalid code doesn't get optimized
         # out of range
         asm = dis_single('"fuu"[10]')
         self.assertIn('BINARY_SUBSCR', asm)
-        # non-BMP char (see #5057)
-        asm = dis_single('"\U00012345"[0]')
-        self.assertIn('BINARY_SUBSCR', asm)
-        asm = dis_single('"\U00012345abcdef"[3]')
-        self.assertIn('BINARY_SUBSCR', asm)
-
 
     def test_folding_of_unaryops_on_constants(self):
         for line, elem in (
             ('-0.5', '(-0.5)'),                     # unary negative
+            ('-0.0', '(-0.0)'),                     # -0.0
+            ('-(1.0-1.0)','(-0.0)'),                # -0.0 after folding
+            ('-0', '(0)'),                          # -0
             ('~-2', '(1)'),                         # unary invert
             ('+1', '(1)'),                          # unary positive
         ):
@@ -224,6 +235,13 @@ class TestTranforms(unittest.TestCase):
             self.assertIn(elem, asm, asm)
             self.assertNotIn('UNARY_', asm)
 
+        # Check that -0.0 works after marshaling
+        def negzero():
+            return -(1.0-1.0)
+
+        self.assertNotIn('UNARY_', disassemble(negzero))
+        self.assertTrue(copysign(1.0, negzero()) < 0)
+
         # Verify that unfoldables are skipped
         for line, elem in (
             ('-"abc"', "('abc')"),                  # unary negative
@@ -286,6 +304,25 @@ class TestTranforms(unittest.TestCase):
         asm = disassemble(f)
         self.assertNotIn('BINARY_ADD', asm)
 
+    def test_constant_folding(self):
+        # Issue #11244: aggressive constant folding.
+        exprs = [
+            "3 * -5",
+            "-3 * 5",
+            "2 * (3 * 4)",
+            "(2 * 3) * 4",
+            "(-1, 2, 3)",
+            "(1, -2, 3)",
+            "(1, 2, -3)",
+            "(1, 2, -3) * 6",
+            "lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}",
+        ]
+        for e in exprs:
+            asm = dis_single(e)
+            self.assertNotIn('UNARY_', asm, e)
+            self.assertNotIn('BINARY_', asm, e)
+            self.assertNotIn('BUILD_', asm, e)
+
 class TestBuglets(unittest.TestCase):
 
     def test_bug_11510(self):
diff --git a/Lib/test/test_pep277.py b/Lib/test/test_pep277.py
index 6d891e541e..4b16cbb383 100644
--- a/Lib/test/test_pep277.py
+++ b/Lib/test/test_pep277.py
@@ -1,6 +1,9 @@
 # Test the Unicode versions of normal file functions
 # open, os.open, os.stat. os.listdir, os.rename, os.remove, os.mkdir, os.chdir, os.rmdir
-import sys, os, unittest
+import os
+import sys
+import unittest
+import warnings
 from unicodedata import normalize
 from test import support
 
@@ -38,8 +41,8 @@ if sys.platform != 'darwin':
         '17_\u2001\u2001\u2001A',
         '18_\u2003\u2003\u2003A',  # == NFC('\u2001\u2001\u2001A')
         '19_\u0020\u0020\u0020A',  # '\u0020' == ' ' == NFKC('\u2000') ==
-                                #  NFKC('\u2001') == NFKC('\u2003')
-])
+                                   #  NFKC('\u2001') == NFKC('\u2003')
+    ])
 
 
 # Is it Unicode-friendly?
@@ -71,7 +74,7 @@ class UnicodeFileTests(unittest.TestCase):
     def setUp(self):
         try:
             os.mkdir(support.TESTFN)
-        except OSError:
+        except FileExistsError:
             pass
         files = set()
         for name in self.files:
@@ -90,15 +93,17 @@ class UnicodeFileTests(unittest.TestCase):
             return normalize(self.normal_form, s)
         return s
 
-    def _apply_failure(self, fn, filename, expected_exception,
-                       check_fn_in_exception = True):
+    def _apply_failure(self, fn, filename,
+                       expected_exception=FileNotFoundError,
+                       check_filename=True):
         with self.assertRaises(expected_exception) as c:
             fn(filename)
         exc_filename = c.exception.filename
-        # the "filename" exception attribute may be encoded
-        if isinstance(exc_filename, bytes):
-            filename = filename.encode(sys.getfilesystemencoding())
-        if check_fn_in_exception:
+        # listdir may append a wildcard to the filename
+        if fn is os.listdir and sys.platform == 'win32':
+            exc_filename, _, wildcard = exc_filename.rpartition(os.sep)
+            self.assertEqual(wildcard, '*.*')
+        if check_filename:
             self.assertEqual(exc_filename, filename, "Function '%s(%a) failed "
                              "with bad filename in the exception: %a" %
                              (fn.__name__, filename, exc_filename))
@@ -107,13 +112,18 @@ class UnicodeFileTests(unittest.TestCase):
         # Pass non-existing Unicode filenames all over the place.
         for name in self.files:
             name = "not_" + name
-            self._apply_failure(open, name, IOError)
-            self._apply_failure(os.stat, name, OSError)
-            self._apply_failure(os.chdir, name, OSError)
-            self._apply_failure(os.rmdir, name, OSError)
-            self._apply_failure(os.remove, name, OSError)
-            # listdir may append a wildcard to the filename, so dont check
-            self._apply_failure(os.listdir, name, OSError, False)
+            self._apply_failure(open, name)
+            self._apply_failure(os.stat, name)
+            self._apply_failure(os.chdir, name)
+            self._apply_failure(os.rmdir, name)
+            self._apply_failure(os.remove, name)
+            self._apply_failure(os.listdir, name)
+
+    if sys.platform == 'win32':
+        # Windows is lunatic. Issue #13366.
+        _listdir_failure = NotADirectoryError, FileNotFoundError
+    else:
+        _listdir_failure = NotADirectoryError
 
     def test_open(self):
         for name in self.files:
@@ -121,12 +131,13 @@ class UnicodeFileTests(unittest.TestCase):
             f.write((name+'\n').encode("utf-8"))
             f.close()
             os.stat(name)
+            self._apply_failure(os.listdir, name, self._listdir_failure)
 
     # Skip the test on darwin, because darwin does normalize the filename to
     # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC,
     # NFKD in Python is useless, because darwin will normalize it later and so
     # open(), os.stat(), etc. don't raise any exception.
-    @unittest.skipIf(sys.platform == 'darwin', 'irrevelant test on Mac OS X')
+    @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
     def test_normalize(self):
         files = set(self.files)
         others = set()
@@ -134,21 +145,22 @@ class UnicodeFileTests(unittest.TestCase):
             others |= set(normalize(nf, file) for file in files)
         others -= files
         for name in others:
-            self._apply_failure(open, name, IOError)
-            self._apply_failure(os.stat, name, OSError)
-            self._apply_failure(os.chdir, name, OSError)
-            self._apply_failure(os.rmdir, name, OSError)
-            self._apply_failure(os.remove, name, OSError)
-            # listdir may append a wildcard to the filename, so dont check
-            self._apply_failure(os.listdir, name, OSError, False)
+            self._apply_failure(open, name)
+            self._apply_failure(os.stat, name)
+            self._apply_failure(os.chdir, name)
+            self._apply_failure(os.rmdir, name)
+            self._apply_failure(os.remove, name)
+            self._apply_failure(os.listdir, name)
 
     # Skip the test on darwin, because darwin uses a normalization different
     # than Python NFD normalization: filenames are different even if we use
     # Python NFD normalization.
-    @unittest.skipIf(sys.platform == 'darwin', 'irrevelant test on Mac OS X')
+    @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
     def test_listdir(self):
         sf0 = set(self.files)
-        f1 = os.listdir(support.TESTFN.encode(sys.getfilesystemencoding()))
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            f1 = os.listdir(support.TESTFN.encode(sys.getfilesystemencoding()))
         f2 = os.listdir(support.TESTFN)
         sf2 = set(os.path.join(support.TESTFN, f) for f in f2)
         self.assertEqual(sf0, sf2, "%a != %a" % (sf0, sf2))
diff --git a/Lib/test/test_pep292.py b/Lib/test/test_pep292.py
index 119c7ea8a1..6da8d2e3e1 100644
--- a/Lib/test/test_pep292.py
+++ b/Lib/test/test_pep292.py
@@ -42,19 +42,6 @@ class TestTemplate(unittest.TestCase):
         s = Template('$who likes $$')
         eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $')
 
-    def test_invalid(self):
-        class MyPattern(Template):
-            pattern = r"""
-            (?:
-            (?P<invalid>)            |
-            (?P<escaped>%(delim)s)   |
-            @(?P<named>%(id)s)       |
-            @{(?P<braced>%(id)s)}    
-            )
-            """
-        s = MyPattern('$')
-        self.assertRaises(ValueError, s.substitute, dict()) 
-
     def test_percents(self):
         eq = self.assertEqual
         s = Template('%(foo)s $foo ${foo}')
@@ -172,6 +159,26 @@ class TestTemplate(unittest.TestCase):
         val = t.safe_substitute({'location': 'Cleveland'})
         self.assertEqual(val, 'PyCon in Cleveland')
 
+    def test_invalid_with_no_lines(self):
+        # The error formatting for invalid templates
+        # has a special case for no data that the default
+        # pattern can't trigger (always has at least '$')
+        # So we craft a pattern that is always invalid
+        # with no leading data.
+        class MyTemplate(Template):
+            pattern = r"""
+              (?P<invalid>) |
+              unreachable(
+                (?P<named>)   |
+                (?P<braced>)  |
+                (?P<escaped>)
+              )
+            """
+        s = MyTemplate('')
+        with self.assertRaises(ValueError) as err:
+            s.substitute({})
+        self.assertIn('line 1, col 1', str(err.exception))
+
     def test_unicode_values(self):
         s = Template('$who likes $what')
         d = dict(who='t\xffm', what='f\xfe\fed')
diff --git a/Lib/test/test_pep3131.py b/Lib/test/test_pep3131.py
index df0f64d86a..2e6b90a35c 100644
--- a/Lib/test/test_pep3131.py
+++ b/Lib/test/test_pep3131.py
@@ -17,12 +17,7 @@ class PEP3131Test(unittest.TestCase):
 
     def test_non_bmp_normalized(self):
         ������� = 1
-        # On wide builds, this is normalized, but on narrow ones it is not. See
-        # #12746.
-        try:
-            self.assertIn("�������", dir())
-        except AssertionError:
-            raise unittest.case._ExpectedFailure(sys.exc_info())
+        self.assertIn("Unicode", dir())
 
     def test_invalid(self):
         try:
diff --git a/Lib/test/test_pep3151.py b/Lib/test/test_pep3151.py
new file mode 100644
index 0000000000..2792c10f98
--- /dev/null
+++ b/Lib/test/test_pep3151.py
@@ -0,0 +1,211 @@
+import builtins
+import os
+import select
+import socket
+import sys
+import unittest
+import errno
+from errno import EEXIST
+
+from test import support
+
+class SubOSError(OSError):
+    pass
+
+class SubOSErrorWithInit(OSError):
+    def __init__(self, message, bar):
+        self.bar = bar
+        super().__init__(message)
+
+class SubOSErrorWithNew(OSError):
+    def __new__(cls, message, baz):
+        self = super().__new__(cls, message)
+        self.baz = baz
+        return self
+
+class SubOSErrorCombinedInitFirst(SubOSErrorWithInit, SubOSErrorWithNew):
+    pass
+
+class SubOSErrorCombinedNewFirst(SubOSErrorWithNew, SubOSErrorWithInit):
+    pass
+
+class SubOSErrorWithStandaloneInit(OSError):
+    def __init__(self):
+        pass
+
+
+class HierarchyTest(unittest.TestCase):
+
+    def test_builtin_errors(self):
+        self.assertEqual(OSError.__name__, 'OSError')
+        self.assertIs(IOError, OSError)
+        self.assertIs(EnvironmentError, OSError)
+
+    def test_socket_errors(self):
+        self.assertIs(socket.error, IOError)
+        self.assertIs(socket.gaierror.__base__, OSError)
+        self.assertIs(socket.herror.__base__, OSError)
+        self.assertIs(socket.timeout.__base__, OSError)
+
+    def test_select_error(self):
+        self.assertIs(select.error, OSError)
+
+    # mmap.error is tested in test_mmap
+
+    _pep_map = """
+        +-- BlockingIOError        EAGAIN, EALREADY, EWOULDBLOCK, EINPROGRESS
+        +-- ChildProcessError                                          ECHILD
+        +-- ConnectionError
+            +-- BrokenPipeError                              EPIPE, ESHUTDOWN
+            +-- ConnectionAbortedError                           ECONNABORTED
+            +-- ConnectionRefusedError                           ECONNREFUSED
+            +-- ConnectionResetError                               ECONNRESET
+        +-- FileExistsError                                            EEXIST
+        +-- FileNotFoundError                                          ENOENT
+        +-- InterruptedError                                            EINTR
+        +-- IsADirectoryError                                          EISDIR
+        +-- NotADirectoryError                                        ENOTDIR
+        +-- PermissionError                                     EACCES, EPERM
+        +-- ProcessLookupError                                          ESRCH
+        +-- TimeoutError                                            ETIMEDOUT
+    """
+    def _make_map(s):
+        _map = {}
+        for line in s.splitlines():
+            line = line.strip('+- ')
+            if not line:
+                continue
+            excname, _, errnames = line.partition(' ')
+            for errname in filter(None, errnames.strip().split(', ')):
+                _map[getattr(errno, errname)] = getattr(builtins, excname)
+        return _map
+    _map = _make_map(_pep_map)
+
+    def test_errno_mapping(self):
+        # The OSError constructor maps errnos to subclasses
+        # A sample test for the basic functionality
+        e = OSError(EEXIST, "Bad file descriptor")
+        self.assertIs(type(e), FileExistsError)
+        # Exhaustive testing
+        for errcode, exc in self._map.items():
+            e = OSError(errcode, "Some message")
+            self.assertIs(type(e), exc)
+        othercodes = set(errno.errorcode) - set(self._map)
+        for errcode in othercodes:
+            e = OSError(errcode, "Some message")
+            self.assertIs(type(e), OSError)
+
+    def test_try_except(self):
+        filename = "some_hopefully_non_existing_file"
+
+        # This checks that try .. except checks the concrete exception
+        # (FileNotFoundError) and not the base type specified when
+        # PyErr_SetFromErrnoWithFilenameObject was called.
+        # (it is therefore deliberate that it doesn't use assertRaises)
+        try:
+            open(filename)
+        except FileNotFoundError:
+            pass
+        else:
+            self.fail("should have raised a FileNotFoundError")
+
+        # Another test for PyErr_SetExcFromWindowsErrWithFilenameObject()
+        self.assertFalse(os.path.exists(filename))
+        try:
+            os.unlink(filename)
+        except FileNotFoundError:
+            pass
+        else:
+            self.fail("should have raised a FileNotFoundError")
+
+
+class AttributesTest(unittest.TestCase):
+
+    def test_windows_error(self):
+        if os.name == "nt":
+            self.assertIn('winerror', dir(OSError))
+        else:
+            self.assertNotIn('winerror', dir(OSError))
+
+    def test_posix_error(self):
+        e = OSError(EEXIST, "File already exists", "foo.txt")
+        self.assertEqual(e.errno, EEXIST)
+        self.assertEqual(e.args[0], EEXIST)
+        self.assertEqual(e.strerror, "File already exists")
+        self.assertEqual(e.filename, "foo.txt")
+        if os.name == "nt":
+            self.assertEqual(e.winerror, None)
+
+    @unittest.skipUnless(os.name == "nt", "Windows-specific test")
+    def test_errno_translation(self):
+        # ERROR_ALREADY_EXISTS (183) -> EEXIST
+        e = OSError(0, "File already exists", "foo.txt", 183)
+        self.assertEqual(e.winerror, 183)
+        self.assertEqual(e.errno, EEXIST)
+        self.assertEqual(e.args[0], EEXIST)
+        self.assertEqual(e.strerror, "File already exists")
+        self.assertEqual(e.filename, "foo.txt")
+
+    def test_blockingioerror(self):
+        args = ("a", "b", "c", "d", "e")
+        for n in range(6):
+            e = BlockingIOError(*args[:n])
+            with self.assertRaises(AttributeError):
+                e.characters_written
+        e = BlockingIOError("a", "b", 3)
+        self.assertEqual(e.characters_written, 3)
+        e.characters_written = 5
+        self.assertEqual(e.characters_written, 5)
+
+    # XXX VMSError not tested
+
+
+class ExplicitSubclassingTest(unittest.TestCase):
+
+    def test_errno_mapping(self):
+        # When constructing an OSError subclass, errno mapping isn't done
+        e = SubOSError(EEXIST, "Bad file descriptor")
+        self.assertIs(type(e), SubOSError)
+
+    def test_init_overriden(self):
+        e = SubOSErrorWithInit("some message", "baz")
+        self.assertEqual(e.bar, "baz")
+        self.assertEqual(e.args, ("some message",))
+
+    def test_init_kwdargs(self):
+        e = SubOSErrorWithInit("some message", bar="baz")
+        self.assertEqual(e.bar, "baz")
+        self.assertEqual(e.args, ("some message",))
+
+    def test_new_overriden(self):
+        e = SubOSErrorWithNew("some message", "baz")
+        self.assertEqual(e.baz, "baz")
+        self.assertEqual(e.args, ("some message",))
+
+    def test_new_kwdargs(self):
+        e = SubOSErrorWithNew("some message", baz="baz")
+        self.assertEqual(e.baz, "baz")
+        self.assertEqual(e.args, ("some message",))
+
+    def test_init_new_overriden(self):
+        e = SubOSErrorCombinedInitFirst("some message", "baz")
+        self.assertEqual(e.bar, "baz")
+        self.assertEqual(e.baz, "baz")
+        self.assertEqual(e.args, ("some message",))
+        e = SubOSErrorCombinedNewFirst("some message", "baz")
+        self.assertEqual(e.bar, "baz")
+        self.assertEqual(e.baz, "baz")
+        self.assertEqual(e.args, ("some message",))
+
+    def test_init_standalone(self):
+        # __init__ doesn't propagate to OSError.__init__ (see issue #15229)
+        e = SubOSErrorWithStandaloneInit()
+        self.assertEqual(e.args, ())
+        self.assertEqual(str(e), '')
+
+
+def test_main():
+    support.run_unittest(__name__)
+
+if __name__=="__main__":
+    test_main()
diff --git a/Lib/test/test_pep380.py b/Lib/test/test_pep380.py
new file mode 100644
index 0000000000..f3eed4353b
--- /dev/null
+++ b/Lib/test/test_pep380.py
@@ -0,0 +1,965 @@
+# -*- coding: utf-8 -*-
+
+"""
+Test suite for PEP 380 implementation
+
+adapted from original tests written by Greg Ewing
+see <http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Python3.1.2-rev5.zip>
+"""
+
+import unittest
+import io
+import sys
+import inspect
+import parser
+
+from test.support import captured_stderr
+
+class TestPEP380Operation(unittest.TestCase):
+    """
+    Test semantics.
+    """
+
+    def test_delegation_of_initial_next_to_subgenerator(self):
+        """
+        Test delegation of initial next() call to subgenerator
+        """
+        trace = []
+        def g1():
+            trace.append("Starting g1")
+            yield from g2()
+            trace.append("Finishing g1")
+        def g2():
+            trace.append("Starting g2")
+            yield 42
+            trace.append("Finishing g2")
+        for x in g1():
+            trace.append("Yielded %s" % (x,))
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Starting g2",
+            "Yielded 42",
+            "Finishing g2",
+            "Finishing g1",
+        ])
+
+    def test_raising_exception_in_initial_next_call(self):
+        """
+        Test raising exception in initial next() call
+        """
+        trace = []
+        def g1():
+            try:
+                trace.append("Starting g1")
+                yield from g2()
+            finally:
+                trace.append("Finishing g1")
+        def g2():
+            try:
+                trace.append("Starting g2")
+                raise ValueError("spanish inquisition occurred")
+            finally:
+                trace.append("Finishing g2")
+        try:
+            for x in g1():
+                trace.append("Yielded %s" % (x,))
+        except ValueError as e:
+            self.assertEqual(e.args[0], "spanish inquisition occurred")
+        else:
+            self.fail("subgenerator failed to raise ValueError")
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Starting g2",
+            "Finishing g2",
+            "Finishing g1",
+        ])
+
+    def test_delegation_of_next_call_to_subgenerator(self):
+        """
+        Test delegation of next() call to subgenerator
+        """
+        trace = []
+        def g1():
+            trace.append("Starting g1")
+            yield "g1 ham"
+            yield from g2()
+            yield "g1 eggs"
+            trace.append("Finishing g1")
+        def g2():
+            trace.append("Starting g2")
+            yield "g2 spam"
+            yield "g2 more spam"
+            trace.append("Finishing g2")
+        for x in g1():
+            trace.append("Yielded %s" % (x,))
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Yielded g1 ham",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Yielded g2 more spam",
+            "Finishing g2",
+            "Yielded g1 eggs",
+            "Finishing g1",
+        ])
+
+    def test_raising_exception_in_delegated_next_call(self):
+        """
+        Test raising exception in delegated next() call
+        """
+        trace = []
+        def g1():
+            try:
+                trace.append("Starting g1")
+                yield "g1 ham"
+                yield from g2()
+                yield "g1 eggs"
+            finally:
+                trace.append("Finishing g1")
+        def g2():
+            try:
+                trace.append("Starting g2")
+                yield "g2 spam"
+                raise ValueError("hovercraft is full of eels")
+                yield "g2 more spam"
+            finally:
+                trace.append("Finishing g2")
+        try:
+            for x in g1():
+                trace.append("Yielded %s" % (x,))
+        except ValueError as e:
+            self.assertEqual(e.args[0], "hovercraft is full of eels")
+        else:
+            self.fail("subgenerator failed to raise ValueError")
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Yielded g1 ham",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Finishing g2",
+            "Finishing g1",
+        ])
+
+    def test_delegation_of_send(self):
+        """
+        Test delegation of send()
+        """
+        trace = []
+        def g1():
+            trace.append("Starting g1")
+            x = yield "g1 ham"
+            trace.append("g1 received %s" % (x,))
+            yield from g2()
+            x = yield "g1 eggs"
+            trace.append("g1 received %s" % (x,))
+            trace.append("Finishing g1")
+        def g2():
+            trace.append("Starting g2")
+            x = yield "g2 spam"
+            trace.append("g2 received %s" % (x,))
+            x = yield "g2 more spam"
+            trace.append("g2 received %s" % (x,))
+            trace.append("Finishing g2")
+        g = g1()
+        y = next(g)
+        x = 1
+        try:
+            while 1:
+                y = g.send(x)
+                trace.append("Yielded %s" % (y,))
+                x += 1
+        except StopIteration:
+            pass
+        self.assertEqual(trace,[
+            "Starting g1",
+            "g1 received 1",
+            "Starting g2",
+            "Yielded g2 spam",
+            "g2 received 2",
+            "Yielded g2 more spam",
+            "g2 received 3",
+            "Finishing g2",
+            "Yielded g1 eggs",
+            "g1 received 4",
+            "Finishing g1",
+        ])
+
+    def test_handling_exception_while_delegating_send(self):
+        """
+        Test handling exception while delegating 'send'
+        """
+        trace = []
+        def g1():
+            trace.append("Starting g1")
+            x = yield "g1 ham"
+            trace.append("g1 received %s" % (x,))
+            yield from g2()
+            x = yield "g1 eggs"
+            trace.append("g1 received %s" % (x,))
+            trace.append("Finishing g1")
+        def g2():
+            trace.append("Starting g2")
+            x = yield "g2 spam"
+            trace.append("g2 received %s" % (x,))
+            raise ValueError("hovercraft is full of eels")
+            x = yield "g2 more spam"
+            trace.append("g2 received %s" % (x,))
+            trace.append("Finishing g2")
+        def run():
+            g = g1()
+            y = next(g)
+            x = 1
+            try:
+                while 1:
+                    y = g.send(x)
+                    trace.append("Yielded %s" % (y,))
+                    x += 1
+            except StopIteration:
+                trace.append("StopIteration")
+        self.assertRaises(ValueError,run)
+        self.assertEqual(trace,[
+            "Starting g1",
+            "g1 received 1",
+            "Starting g2",
+            "Yielded g2 spam",
+            "g2 received 2",
+        ])
+
+    def test_delegating_close(self):
+        """
+        Test delegating 'close'
+        """
+        trace = []
+        def g1():
+            try:
+                trace.append("Starting g1")
+                yield "g1 ham"
+                yield from g2()
+                yield "g1 eggs"
+            finally:
+                trace.append("Finishing g1")
+        def g2():
+            try:
+                trace.append("Starting g2")
+                yield "g2 spam"
+                yield "g2 more spam"
+            finally:
+                trace.append("Finishing g2")
+        g = g1()
+        for i in range(2):
+            x = next(g)
+            trace.append("Yielded %s" % (x,))
+        g.close()
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Yielded g1 ham",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Finishing g2",
+            "Finishing g1"
+        ])
+
+    def test_handing_exception_while_delegating_close(self):
+        """
+        Test handling exception while delegating 'close'
+        """
+        trace = []
+        def g1():
+            try:
+                trace.append("Starting g1")
+                yield "g1 ham"
+                yield from g2()
+                yield "g1 eggs"
+            finally:
+                trace.append("Finishing g1")
+        def g2():
+            try:
+                trace.append("Starting g2")
+                yield "g2 spam"
+                yield "g2 more spam"
+            finally:
+                trace.append("Finishing g2")
+                raise ValueError("nybbles have exploded with delight")
+        try:
+            g = g1()
+            for i in range(2):
+                x = next(g)
+                trace.append("Yielded %s" % (x,))
+            g.close()
+        except ValueError as e:
+            self.assertEqual(e.args[0], "nybbles have exploded with delight")
+            self.assertIsInstance(e.__context__, GeneratorExit)
+        else:
+            self.fail("subgenerator failed to raise ValueError")
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Yielded g1 ham",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Finishing g2",
+            "Finishing g1",
+        ])
+
+    def test_delegating_throw(self):
+        """
+        Test delegating 'throw'
+        """
+        trace = []
+        def g1():
+            try:
+                trace.append("Starting g1")
+                yield "g1 ham"
+                yield from g2()
+                yield "g1 eggs"
+            finally:
+                trace.append("Finishing g1")
+        def g2():
+            try:
+                trace.append("Starting g2")
+                yield "g2 spam"
+                yield "g2 more spam"
+            finally:
+                trace.append("Finishing g2")
+        try:
+            g = g1()
+            for i in range(2):
+                x = next(g)
+                trace.append("Yielded %s" % (x,))
+            e = ValueError("tomato ejected")
+            g.throw(e)
+        except ValueError as e:
+            self.assertEqual(e.args[0], "tomato ejected")
+        else:
+            self.fail("subgenerator failed to raise ValueError")
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Yielded g1 ham",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Finishing g2",
+            "Finishing g1",
+        ])
+
+    def test_value_attribute_of_StopIteration_exception(self):
+        """
+        Test 'value' attribute of StopIteration exception
+        """
+        trace = []
+        def pex(e):
+            trace.append("%s: %s" % (e.__class__.__name__, e))
+            trace.append("value = %s" % (e.value,))
+        e = StopIteration()
+        pex(e)
+        e = StopIteration("spam")
+        pex(e)
+        e.value = "eggs"
+        pex(e)
+        self.assertEqual(trace,[
+            "StopIteration: ",
+            "value = None",
+            "StopIteration: spam",
+            "value = spam",
+            "StopIteration: spam",
+            "value = eggs",
+        ])
+
+
+    def test_exception_value_crash(self):
+        # There used to be a refcount error when the return value
+        # stored in the StopIteration has a refcount of 1.
+        def g1():
+            yield from g2()
+        def g2():
+            yield "g2"
+            return [42]
+        self.assertEqual(list(g1()), ["g2"])
+
+
+    def test_generator_return_value(self):
+        """
+        Test generator return value
+        """
+        trace = []
+        def g1():
+            trace.append("Starting g1")
+            yield "g1 ham"
+            ret = yield from g2()
+            trace.append("g2 returned %s" % (ret,))
+            ret = yield from g2(42)
+            trace.append("g2 returned %s" % (ret,))
+            yield "g1 eggs"
+            trace.append("Finishing g1")
+        def g2(v = None):
+            trace.append("Starting g2")
+            yield "g2 spam"
+            yield "g2 more spam"
+            trace.append("Finishing g2")
+            if v:
+                return v
+        for x in g1():
+            trace.append("Yielded %s" % (x,))
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Yielded g1 ham",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Yielded g2 more spam",
+            "Finishing g2",
+            "g2 returned None",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Yielded g2 more spam",
+            "Finishing g2",
+            "g2 returned 42",
+            "Yielded g1 eggs",
+            "Finishing g1",
+        ])
+
+    def test_delegation_of_next_to_non_generator(self):
+        """
+        Test delegation of next() to non-generator
+        """
+        trace = []
+        def g():
+            yield from range(3)
+        for x in g():
+            trace.append("Yielded %s" % (x,))
+        self.assertEqual(trace,[
+            "Yielded 0",
+            "Yielded 1",
+            "Yielded 2",
+        ])
+
+
+    def test_conversion_of_sendNone_to_next(self):
+        """
+        Test conversion of send(None) to next()
+        """
+        trace = []
+        def g():
+            yield from range(3)
+        gi = g()
+        for x in range(3):
+            y = gi.send(None)
+            trace.append("Yielded: %s" % (y,))
+        self.assertEqual(trace,[
+            "Yielded: 0",
+            "Yielded: 1",
+            "Yielded: 2",
+        ])
+
+    def test_delegation_of_close_to_non_generator(self):
+        """
+        Test delegation of close() to non-generator
+        """
+        trace = []
+        def g():
+            try:
+                trace.append("starting g")
+                yield from range(3)
+                trace.append("g should not be here")
+            finally:
+                trace.append("finishing g")
+        gi = g()
+        next(gi)
+        with captured_stderr() as output:
+            gi.close()
+        self.assertEqual(output.getvalue(), '')
+        self.assertEqual(trace,[
+            "starting g",
+            "finishing g",
+        ])
+
+    def test_delegating_throw_to_non_generator(self):
+        """
+        Test delegating 'throw' to non-generator
+        """
+        trace = []
+        def g():
+            try:
+                trace.append("Starting g")
+                yield from range(10)
+            finally:
+                trace.append("Finishing g")
+        try:
+            gi = g()
+            for i in range(5):
+                x = next(gi)
+                trace.append("Yielded %s" % (x,))
+            e = ValueError("tomato ejected")
+            gi.throw(e)
+        except ValueError as e:
+            self.assertEqual(e.args[0],"tomato ejected")
+        else:
+            self.fail("subgenerator failed to raise ValueError")
+        self.assertEqual(trace,[
+            "Starting g",
+            "Yielded 0",
+            "Yielded 1",
+            "Yielded 2",
+            "Yielded 3",
+            "Yielded 4",
+            "Finishing g",
+        ])
+
+    def test_attempting_to_send_to_non_generator(self):
+        """
+        Test attempting to send to non-generator
+        """
+        trace = []
+        def g():
+            try:
+                trace.append("starting g")
+                yield from range(3)
+                trace.append("g should not be here")
+            finally:
+                trace.append("finishing g")
+        try:
+            gi = g()
+            next(gi)
+            for x in range(3):
+                y = gi.send(42)
+                trace.append("Should not have yielded: %s" % (y,))
+        except AttributeError as e:
+            self.assertIn("send", e.args[0])
+        else:
+            self.fail("was able to send into non-generator")
+        self.assertEqual(trace,[
+            "starting g",
+            "finishing g",
+        ])
+
+    def test_broken_getattr_handling(self):
+        """
+        Test subiterator with a broken getattr implementation
+        """
+        class Broken:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                return 1
+            def __getattr__(self, attr):
+                1/0
+
+        def g():
+            yield from Broken()
+
+        with self.assertRaises(ZeroDivisionError):
+            gi = g()
+            self.assertEqual(next(gi), 1)
+            gi.send(1)
+
+        with self.assertRaises(ZeroDivisionError):
+            gi = g()
+            self.assertEqual(next(gi), 1)
+            gi.throw(AttributeError)
+
+        with captured_stderr() as output:
+            gi = g()
+            self.assertEqual(next(gi), 1)
+            gi.close()
+        self.assertIn('ZeroDivisionError', output.getvalue())
+
+    def test_exception_in_initial_next_call(self):
+        """
+        Test exception in initial next() call
+        """
+        trace = []
+        def g1():
+            trace.append("g1 about to yield from g2")
+            yield from g2()
+            trace.append("g1 should not be here")
+        def g2():
+            yield 1/0
+        def run():
+            gi = g1()
+            next(gi)
+        self.assertRaises(ZeroDivisionError,run)
+        self.assertEqual(trace,[
+            "g1 about to yield from g2"
+        ])
+
+    def test_attempted_yield_from_loop(self):
+        """
+        Test attempted yield-from loop
+        """
+        trace = []
+        def g1():
+            trace.append("g1: starting")
+            yield "y1"
+            trace.append("g1: about to yield from g2")
+            yield from g2()
+            trace.append("g1 should not be here")
+
+        def g2():
+            trace.append("g2: starting")
+            yield "y2"
+            trace.append("g2: about to yield from g1")
+            yield from gi
+            trace.append("g2 should not be here")
+        try:
+            gi = g1()
+            for y in gi:
+                trace.append("Yielded: %s" % (y,))
+        except ValueError as e:
+            self.assertEqual(e.args[0],"generator already executing")
+        else:
+            self.fail("subgenerator didn't raise ValueError")
+        self.assertEqual(trace,[
+            "g1: starting",
+            "Yielded: y1",
+            "g1: about to yield from g2",
+            "g2: starting",
+            "Yielded: y2",
+            "g2: about to yield from g1",
+        ])
+
+    def test_returning_value_from_delegated_throw(self):
+        """
+        Test returning value from delegated 'throw'
+        """
+        trace = []
+        def g1():
+            try:
+                trace.append("Starting g1")
+                yield "g1 ham"
+                yield from g2()
+                yield "g1 eggs"
+            finally:
+                trace.append("Finishing g1")
+        def g2():
+            try:
+                trace.append("Starting g2")
+                yield "g2 spam"
+                yield "g2 more spam"
+            except LunchError:
+                trace.append("Caught LunchError in g2")
+                yield "g2 lunch saved"
+                yield "g2 yet more spam"
+        class LunchError(Exception):
+            pass
+        g = g1()
+        for i in range(2):
+            x = next(g)
+            trace.append("Yielded %s" % (x,))
+        e = LunchError("tomato ejected")
+        g.throw(e)
+        for x in g:
+            trace.append("Yielded %s" % (x,))
+        self.assertEqual(trace,[
+            "Starting g1",
+            "Yielded g1 ham",
+            "Starting g2",
+            "Yielded g2 spam",
+            "Caught LunchError in g2",
+            "Yielded g2 yet more spam",
+            "Yielded g1 eggs",
+            "Finishing g1",
+        ])
+
+    def test_next_and_return_with_value(self):
+        """
+        Test next and return with value
+        """
+        trace = []
+        def f(r):
+            gi = g(r)
+            next(gi)
+            try:
+                trace.append("f resuming g")
+                next(gi)
+                trace.append("f SHOULD NOT BE HERE")
+            except StopIteration as e:
+                trace.append("f caught %s" % (repr(e),))
+        def g(r):
+            trace.append("g starting")
+            yield
+            trace.append("g returning %s" % (r,))
+            return r
+        f(None)
+        f(42)
+        self.assertEqual(trace,[
+            "g starting",
+            "f resuming g",
+            "g returning None",
+            "f caught StopIteration()",
+            "g starting",
+            "f resuming g",
+            "g returning 42",
+            "f caught StopIteration(42,)",
+        ])
+
+    def test_send_and_return_with_value(self):
+        """
+        Test send and return with value
+        """
+        trace = []
+        def f(r):
+            gi = g(r)
+            next(gi)
+            try:
+                trace.append("f sending spam to g")
+                gi.send("spam")
+                trace.append("f SHOULD NOT BE HERE")
+            except StopIteration as e:
+                trace.append("f caught %r" % (e,))
+        def g(r):
+            trace.append("g starting")
+            x = yield
+            trace.append("g received %s" % (x,))
+            trace.append("g returning %s" % (r,))
+            return r
+        f(None)
+        f(42)
+        self.assertEqual(trace,[
+            "g starting",
+            "f sending spam to g",
+            "g received spam",
+            "g returning None",
+            "f caught StopIteration()",
+            "g starting",
+            "f sending spam to g",
+            "g received spam",
+            "g returning 42",
+            "f caught StopIteration(42,)",
+        ])
+
+    def test_catching_exception_from_subgen_and_returning(self):
+        """
+        Test catching an exception thrown into a
+        subgenerator and returning a value
+        """
+        trace = []
+        def inner():
+            try:
+                yield 1
+            except ValueError:
+                trace.append("inner caught ValueError")
+            return 2
+
+        def outer():
+            v = yield from inner()
+            trace.append("inner returned %r to outer" % v)
+            yield v
+        g = outer()
+        trace.append(next(g))
+        trace.append(g.throw(ValueError))
+        self.assertEqual(trace,[
+            1,
+            "inner caught ValueError",
+            "inner returned 2 to outer",
+            2,
+        ])
+
+    def test_throwing_GeneratorExit_into_subgen_that_returns(self):
+        """
+        Test throwing GeneratorExit into a subgenerator that
+        catches it and returns normally.
+        """
+        trace = []
+        def f():
+            try:
+                trace.append("Enter f")
+                yield
+                trace.append("Exit f")
+            except GeneratorExit:
+                return
+        def g():
+            trace.append("Enter g")
+            yield from f()
+            trace.append("Exit g")
+        try:
+            gi = g()
+            next(gi)
+            gi.throw(GeneratorExit)
+        except GeneratorExit:
+            pass
+        else:
+            self.fail("subgenerator failed to raise GeneratorExit")
+        self.assertEqual(trace,[
+            "Enter g",
+            "Enter f",
+        ])
+
+    def test_throwing_GeneratorExit_into_subgenerator_that_yields(self):
+        """
+        Test throwing GeneratorExit into a subgenerator that
+        catches it and yields.
+        """
+        trace = []
+        def f():
+            try:
+                trace.append("Enter f")
+                yield
+                trace.append("Exit f")
+            except GeneratorExit:
+                yield
+        def g():
+            trace.append("Enter g")
+            yield from f()
+            trace.append("Exit g")
+        try:
+            gi = g()
+            next(gi)
+            gi.throw(GeneratorExit)
+        except RuntimeError as e:
+            self.assertEqual(e.args[0], "generator ignored GeneratorExit")
+        else:
+            self.fail("subgenerator failed to raise GeneratorExit")
+        self.assertEqual(trace,[
+            "Enter g",
+            "Enter f",
+        ])
+
+    def test_throwing_GeneratorExit_into_subgen_that_raises(self):
+        """
+        Test throwing GeneratorExit into a subgenerator that
+        catches it and raises a different exception.
+        """
+        trace = []
+        def f():
+            try:
+                trace.append("Enter f")
+                yield
+                trace.append("Exit f")
+            except GeneratorExit:
+                raise ValueError("Vorpal bunny encountered")
+        def g():
+            trace.append("Enter g")
+            yield from f()
+            trace.append("Exit g")
+        try:
+            gi = g()
+            next(gi)
+            gi.throw(GeneratorExit)
+        except ValueError as e:
+            self.assertEqual(e.args[0], "Vorpal bunny encountered")
+            self.assertIsInstance(e.__context__, GeneratorExit)
+        else:
+            self.fail("subgenerator failed to raise ValueError")
+        self.assertEqual(trace,[
+            "Enter g",
+            "Enter f",
+        ])
+
+    def test_yield_from_empty(self):
+        def g():
+            yield from ()
+        self.assertRaises(StopIteration, next, g())
+
+    def test_delegating_generators_claim_to_be_running(self):
+        # Check with basic iteration
+        def one():
+            yield 0
+            yield from two()
+            yield 3
+        def two():
+            yield 1
+            try:
+                yield from g1
+            except ValueError:
+                pass
+            yield 2
+        g1 = one()
+        self.assertEqual(list(g1), [0, 1, 2, 3])
+        # Check with send
+        g1 = one()
+        res = [next(g1)]
+        try:
+            while True:
+                res.append(g1.send(42))
+        except StopIteration:
+            pass
+        self.assertEqual(res, [0, 1, 2, 3])
+        # Check with throw
+        class MyErr(Exception):
+            pass
+        def one():
+            try:
+                yield 0
+            except MyErr:
+                pass
+            yield from two()
+            try:
+                yield 3
+            except MyErr:
+                pass
+        def two():
+            try:
+                yield 1
+            except MyErr:
+                pass
+            try:
+                yield from g1
+            except ValueError:
+                pass
+            try:
+                yield 2
+            except MyErr:
+                pass
+        g1 = one()
+        res = [next(g1)]
+        try:
+            while True:
+                res.append(g1.throw(MyErr))
+        except StopIteration:
+            pass
+        # Check with close
+        class MyIt(object):
+            def __iter__(self):
+                return self
+            def __next__(self):
+                return 42
+            def close(self_):
+                self.assertTrue(g1.gi_running)
+                self.assertRaises(ValueError, next, g1)
+        def one():
+            yield from MyIt()
+        g1 = one()
+        next(g1)
+        g1.close()
+
+    def test_delegator_is_visible_to_debugger(self):
+        def call_stack():
+            return [f[3] for f in inspect.stack()]
+
+        def gen():
+            yield call_stack()
+            yield call_stack()
+            yield call_stack()
+
+        def spam(g):
+            yield from g
+
+        def eggs(g):
+            yield from g
+
+        for stack in spam(gen()):
+            self.assertTrue('spam' in stack)
+
+        for stack in spam(eggs(gen())):
+            self.assertTrue('spam' in stack and 'eggs' in stack)
+
+    def test_custom_iterator_return(self):
+        # See issue #15568
+        class MyIter:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                raise StopIteration(42)
+        def gen():
+            nonlocal ret
+            ret = yield from MyIter()
+        ret = None
+        list(gen())
+        self.assertEqual(ret, 42)
+
+
+def test_main():
+    from test import support
+    test_classes = [TestPEP380Operation]
+    support.run_unittest(*test_classes)
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index 9da2cae86b..f52d4bdee9 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -1,5 +1,6 @@
 import pickle
 import io
+import collections
 
 from test import support
 
@@ -7,6 +8,7 @@ from test.pickletester import AbstractPickleTests
 from test.pickletester import AbstractPickleModuleTests
 from test.pickletester import AbstractPersistentPicklerTests
 from test.pickletester import AbstractPicklerUnpicklerObjectTests
+from test.pickletester import AbstractDispatchTableTests
 from test.pickletester import BigmemPickleTests
 
 try:
@@ -80,6 +82,18 @@ class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
     unpickler_class = pickle._Unpickler
 
 
+class PyDispatchTableTests(AbstractDispatchTableTests):
+    pickler_class = pickle._Pickler
+    def get_dispatch_table(self):
+        return pickle.dispatch_table.copy()
+
+
+class PyChainDispatchTableTests(AbstractDispatchTableTests):
+    pickler_class = pickle._Pickler
+    def get_dispatch_table(self):
+        return collections.ChainMap({}, pickle.dispatch_table)
+
+
 if has_c_implementation:
     class CPicklerTests(PyPicklerTests):
         pickler = _pickle.Pickler
@@ -101,14 +115,26 @@ if has_c_implementation:
         pickler_class = _pickle.Pickler
         unpickler_class = _pickle.Unpickler
 
+    class CDispatchTableTests(AbstractDispatchTableTests):
+        pickler_class = pickle.Pickler
+        def get_dispatch_table(self):
+            return pickle.dispatch_table.copy()
+
+    class CChainDispatchTableTests(AbstractDispatchTableTests):
+        pickler_class = pickle.Pickler
+        def get_dispatch_table(self):
+            return collections.ChainMap({}, pickle.dispatch_table)
+
 
 def test_main():
-    tests = [PickleTests, PyPicklerTests, PyPersPicklerTests]
+    tests = [PickleTests, PyPicklerTests, PyPersPicklerTests,
+             PyDispatchTableTests, PyChainDispatchTableTests]
     if has_c_implementation:
         tests.extend([CPicklerTests, CPersPicklerTests,
                       CDumpPickle_LoadPickle, DumpPickle_CLoadPickle,
                       PyPicklerUnpicklerObjectTests,
                       CPicklerUnpicklerObjectTests,
+                      CDispatchTableTests, CChainDispatchTableTests,
                       InMemoryPickleTests])
     support.run_unittest(*tests)
     support.run_doctest(pickle)
diff --git a/Lib/test/test_pipes.py b/Lib/test/test_pipes.py
index d5b886f87e..6a7b45fb46 100644
--- a/Lib/test/test_pipes.py
+++ b/Lib/test/test_pipes.py
@@ -79,21 +79,6 @@ class SimplePipeTests(unittest.TestCase):
         with open(TESTFN) as f:
             self.assertEqual(f.read(), d)
 
-    def testQuoting(self):
-        safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
-        unicode_sample = '\xe9\xe0\xdf'  # e + acute accent, a + grave, sharp s
-        unsafe = '"`$\\!' + unicode_sample
-
-        self.assertEqual(pipes.quote(''), "''")
-        self.assertEqual(pipes.quote(safeunquoted), safeunquoted)
-        self.assertEqual(pipes.quote('test file name'), "'test file name'")
-        for u in unsafe:
-            self.assertEqual(pipes.quote('test%sname' % u),
-                              "'test%sname'" % u)
-        for u in unsafe:
-            self.assertEqual(pipes.quote("test%s'name'" % u),
-                             "'test%s'\"'\"'name'\"'\"''" % u)
-
     def testRepr(self):
         t = pipes.Template()
         self.assertEqual(repr(t), "<Template instance, steps=[]>")
diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py
index a4ddb15875..355efe7566 100644
--- a/Lib/test/test_pkg.py
+++ b/Lib/test/test_pkg.py
@@ -23,6 +23,8 @@ def cleanout(root):
 def fixdir(lst):
     if "__builtins__" in lst:
         lst.remove("__builtins__")
+    if "__initializing__" in lst:
+        lst.remove("__initializing__")
     return lst
 
 
@@ -196,14 +198,15 @@ class TestPkg(unittest.TestCase):
 
         import t5
         self.assertEqual(fixdir(dir(t5)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__', 'foo', 'string', 't5'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__', 'foo',
+                          'string', 't5'])
         self.assertEqual(fixdir(dir(t5.foo)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', 'string'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', 'string'])
         self.assertEqual(fixdir(dir(t5.string)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', 'spam'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', 'spam'])
 
     def test_6(self):
         hier = [
@@ -219,14 +222,14 @@ class TestPkg(unittest.TestCase):
         import t6
         self.assertEqual(fixdir(dir(t6)),
                          ['__all__', '__cached__', '__doc__', '__file__',
-                          '__name__', '__package__', '__path__'])
+                          '__loader__', '__name__', '__package__', '__path__'])
         s = """
             import t6
             from t6 import *
             self.assertEqual(fixdir(dir(t6)),
                              ['__all__', '__cached__', '__doc__', '__file__',
-                              '__name__', '__package__', '__path__',
-                              'eggs', 'ham', 'spam'])
+                              '__loader__', '__name__', '__package__',
+                              '__path__', 'eggs', 'ham', 'spam'])
             self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
             """
         self.run_code(s)
@@ -252,19 +255,19 @@ class TestPkg(unittest.TestCase):
         t7, sub, subsub = None, None, None
         import t7 as tas
         self.assertEqual(fixdir(dir(tas)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         from t7 import sub as subpar
         self.assertEqual(fixdir(dir(subpar)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         from t7.sub import subsub as subsubsub
         self.assertEqual(fixdir(dir(subsubsub)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                         '__package__', '__path__', 'spam'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__', 'spam'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         self.assertFalse(subsub)
diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py
index c37e9362b0..a8426b5c9a 100644
--- a/Lib/test/test_pkgimport.py
+++ b/Lib/test/test_pkgimport.py
@@ -7,7 +7,7 @@ import tempfile
 import unittest
 
 from imp import cache_from_source
-from test.support import run_unittest
+from test.support import run_unittest, create_empty_file
 
 class TestImport(unittest.TestCase):
 
@@ -29,7 +29,7 @@ class TestImport(unittest.TestCase):
         self.package_dir = os.path.join(self.test_dir,
                                         self.package_name)
         os.mkdir(self.package_dir)
-        open(os.path.join(self.package_dir, '__init__.py'), 'w').close()
+        create_empty_file(os.path.join(self.package_dir, '__init__.py'))
         self.module_path = os.path.join(self.package_dir, 'foo.py')
 
     def tearDown(self):
diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py
index 3b72f060f9..88e2d338dc 100644
--- a/Lib/test/test_pkgutil.py
+++ b/Lib/test/test_pkgutil.py
@@ -1,4 +1,4 @@
-from test.support import run_unittest, unload
+from test.support import run_unittest, unload, check_warnings
 import unittest
 import sys
 import imp
@@ -9,7 +9,11 @@ import tempfile
 import shutil
 import zipfile
 
-
+# Note: pkgutil.walk_packages is currently tested in test_runpy. This is
+# a hack to get a major issue resolved for 3.3b2. Longer term, it should
+# be moved back here, perhaps by factoring out the helper code for
+# creating interesting package layouts to a separate module.
+# Issue #15348 declares this is indeed a dodgy hack ;)
 
 class PkgutilTests(unittest.TestCase):
 
@@ -138,10 +142,11 @@ class PkgutilPEP302Tests(unittest.TestCase):
         del sys.modules['foo']
 
 
+# These tests, especially the setup and cleanup, are hideous. They
+# need to be cleaned up once issue 14715 is addressed.
 class ExtendPathTests(unittest.TestCase):
     def create_init(self, pkgname):
         dirname = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, dirname)
         sys.path.insert(0, dirname)
 
         pkgdir = os.path.join(dirname, pkgname)
@@ -156,22 +161,40 @@ class ExtendPathTests(unittest.TestCase):
         with open(module_name, 'w') as fl:
             print('value={}'.format(value), file=fl)
 
-    def setUp(self):
-        # Create 2 directories on sys.path
-        self.pkgname = 'foo'
-        self.dirname_0 = self.create_init(self.pkgname)
-        self.dirname_1 = self.create_init(self.pkgname)
+    def test_simple(self):
+        pkgname = 'foo'
+        dirname_0 = self.create_init(pkgname)
+        dirname_1 = self.create_init(pkgname)
+        self.create_submodule(dirname_0, pkgname, 'bar', 0)
+        self.create_submodule(dirname_1, pkgname, 'baz', 1)
+        import foo.bar
+        import foo.baz
+        # Ensure we read the expected values
+        self.assertEqual(foo.bar.value, 0)
+        self.assertEqual(foo.baz.value, 1)
 
-    def tearDown(self):
+        # Ensure the path is set up correctly
+        self.assertEqual(sorted(foo.__path__),
+                         sorted([os.path.join(dirname_0, pkgname),
+                                 os.path.join(dirname_1, pkgname)]))
+
+        # Cleanup
+        shutil.rmtree(dirname_0)
+        shutil.rmtree(dirname_1)
         del sys.path[0]
         del sys.path[0]
         del sys.modules['foo']
         del sys.modules['foo.bar']
         del sys.modules['foo.baz']
 
-    def test_simple(self):
-        self.create_submodule(self.dirname_0, self.pkgname, 'bar', 0)
-        self.create_submodule(self.dirname_1, self.pkgname, 'baz', 1)
+    def test_mixed_namespace(self):
+        pkgname = 'foo'
+        dirname_0 = self.create_init(pkgname)
+        dirname_1 = self.create_init(pkgname)
+        self.create_submodule(dirname_0, pkgname, 'bar', 0)
+        # Turn this into a PEP 420 namespace package
+        os.unlink(os.path.join(dirname_0, pkgname, '__init__.py'))
+        self.create_submodule(dirname_1, pkgname, 'baz', 1)
         import foo.bar
         import foo.baz
         # Ensure we read the expected values
@@ -180,8 +203,17 @@ class ExtendPathTests(unittest.TestCase):
 
         # Ensure the path is set up correctly
         self.assertEqual(sorted(foo.__path__),
-                         sorted([os.path.join(self.dirname_0, self.pkgname),
-                                 os.path.join(self.dirname_1, self.pkgname)]))
+                         sorted([os.path.join(dirname_0, pkgname),
+                                 os.path.join(dirname_1, pkgname)]))
+
+        # Cleanup
+        shutil.rmtree(dirname_0)
+        shutil.rmtree(dirname_1)
+        del sys.path[0]
+        del sys.path[0]
+        del sys.modules['foo']
+        del sys.modules['foo.bar']
+        del sys.modules['foo.baz']
 
     # XXX: test .pkg files
 
@@ -227,12 +259,52 @@ class NestedNamespacePackageTest(unittest.TestCase):
         self.assertEqual(d, 2)
 
 
+class ImportlibMigrationTests(unittest.TestCase):
+    # With full PEP 302 support in the standard import machinery, the
+    # PEP 302 emulation in this module is in the process of being
+    # deprecated in favour of importlib proper
+
+    def check_deprecated(self):
+        return check_warnings(
+            ("This emulation is deprecated, use 'importlib' instead",
+             DeprecationWarning))
+
+    def test_importer_deprecated(self):
+        with self.check_deprecated():
+            x = pkgutil.ImpImporter("")
+
+    def test_loader_deprecated(self):
+        with self.check_deprecated():
+            x = pkgutil.ImpLoader("", "", "", "")
+
+    def test_get_loader_avoids_emulation(self):
+        with check_warnings() as w:
+            self.assertIsNotNone(pkgutil.get_loader("sys"))
+            self.assertIsNotNone(pkgutil.get_loader("os"))
+            self.assertIsNotNone(pkgutil.get_loader("test.support"))
+            self.assertEqual(len(w.warnings), 0)
+
+    def test_get_importer_avoids_emulation(self):
+        # We use an illegal path so *none* of the path hooks should fire
+        with check_warnings() as w:
+            self.assertIsNone(pkgutil.get_importer("*??"))
+            self.assertEqual(len(w.warnings), 0)
+
+    def test_iter_importers_avoids_emulation(self):
+        with check_warnings() as w:
+            for importer in pkgutil.iter_importers(): pass
+            self.assertEqual(len(w.warnings), 0)
+
+
 def test_main():
     run_unittest(PkgutilTests, PkgutilPEP302Tests, ExtendPathTests,
-                 NestedNamespacePackageTest)
+                 NestedNamespacePackageTest, ImportlibMigrationTests)
     # this is necessary if test is run repeated (like when finding leaks)
     import zipimport
+    import importlib
     zipimport._zip_directory_cache.clear()
+    importlib.invalidate_caches()
+
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py
index 8751aa86ba..6abf44342a 100644
--- a/Lib/test/test_platform.py
+++ b/Lib/test/test_platform.py
@@ -1,8 +1,9 @@
-import sys
 import os
-import unittest
 import platform
 import subprocess
+import sys
+import unittest
+import warnings
 
 from test import support
 
@@ -56,13 +57,11 @@ class PlatformTest(unittest.TestCase):
 
     def setUp(self):
         self.save_version = sys.version
-        self.save_subversion = sys.subversion
         self.save_mercurial = sys._mercurial
         self.save_platform = sys.platform
 
     def tearDown(self):
         sys.version = self.save_version
-        sys.subversion = self.save_subversion
         sys._mercurial = self.save_mercurial
         sys.platform = self.save_platform
 
@@ -77,7 +76,7 @@ class PlatformTest(unittest.TestCase):
              ('IronPython', '1.0.0', '', '', '', '', '.NET 2.0.50727.42')),
             ):
             # branch and revision are not "parsed", but fetched
-            # from sys.subversion.  Ignore them
+            # from sys._mercurial.  Ignore them
             (name, version, branch, revision, buildno, builddate, compiler) \
                    = platform._sys_version(input)
             self.assertEqual(
@@ -113,8 +112,6 @@ class PlatformTest(unittest.TestCase):
             if subversion is None:
                 if hasattr(sys, "_mercurial"):
                     del sys._mercurial
-                if hasattr(sys, "subversion"):
-                    del sys.subversion
             else:
                 sys._mercurial = subversion
             if sys_platform is not None:
@@ -136,6 +133,12 @@ class PlatformTest(unittest.TestCase):
     def test_uname(self):
         res = platform.uname()
         self.assertTrue(any(res))
+        self.assertEqual(res[0], res.system)
+        self.assertEqual(res[1], res.node)
+        self.assertEqual(res[2], res.release)
+        self.assertEqual(res[3], res.version)
+        self.assertEqual(res[4], res.machine)
+        self.assertEqual(res[5], res.processor)
 
     @unittest.skipUnless(sys.platform.startswith('win'), "windows only test")
     def test_uname_win32_ARCHITEW6432(self):
@@ -169,7 +172,7 @@ class PlatformTest(unittest.TestCase):
     def test_mac_ver(self):
         res = platform.mac_ver()
 
-        if platform.uname()[0] == 'Darwin':
+        if platform.uname().system == 'Darwin':
             # We're on a MacOSX system, check that
             # the right version information is returned
             fd = os.popen('sw_vers', 'r')
@@ -247,6 +250,38 @@ class PlatformTest(unittest.TestCase):
             ):
             self.assertEqual(platform._parse_release_file(input), output)
 
+    def test_popen(self):
+        mswindows = (sys.platform == "win32")
+
+        if mswindows:
+            command = '"{}" -c "print(\'Hello\')"'.format(sys.executable)
+        else:
+            command = "'{}' -c 'print(\"Hello\")'".format(sys.executable)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            with platform.popen(command) as stdout:
+                hello = stdout.read().strip()
+                stdout.close()
+                self.assertEqual(hello, "Hello")
+
+        data = 'plop'
+        if mswindows:
+            command = '"{}" -c "import sys; data=sys.stdin.read(); exit(len(data))"'
+        else:
+            command = "'{}' -c 'import sys; data=sys.stdin.read(); exit(len(data))'"
+        command = command.format(sys.executable)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            with platform.popen(command, 'w') as stdin:
+                stdout = stdin.write(data)
+                ret = stdin.close()
+                self.assertIsNotNone(ret)
+                if os.name == 'nt':
+                    returncode = ret
+                else:
+                    returncode = ret >> 8
+                self.assertEqual(returncode, len(data))
+
 
 def test_main():
     support.run_unittest(
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
index 5f980d0a2d..a9e343eb15 100644
--- a/Lib/test/test_plistlib.py
+++ b/Lib/test/test_plistlib.py
@@ -55,6 +55,10 @@ TESTDATA = b"""<?xml version="1.0" encoding="UTF-8"?>
         </array>
         <key>aString</key>
         <string>Doodah</string>
+        <key>anEmptyDict</key>
+        <dict/>
+        <key>anEmptyList</key>
+        <array/>
         <key>anInt</key>
         <integer>728</integer>
         <key>nestedData</key>
@@ -112,6 +116,8 @@ class TestPlistlib(unittest.TestCase):
             someMoreData = plistlib.Data(b"<lots of binary gunk>\0\1\2\3" * 10),
             nestedData = [plistlib.Data(b"<lots of binary gunk>\0\1\2\3" * 10)],
             aDate = datetime.datetime(2004, 10, 26, 10, 33, 33),
+            anEmptyDict = dict(),
+            anEmptyList = list()
         )
         pl['\xc5benraa'] = "That was a unicode key."
         return pl
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index e3901b886c..c0929a06b3 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -108,6 +108,10 @@ class DummyPOP3Handler(asynchat.async_chat):
     def cmd_apop(self, arg):
         self.push('+OK done nothing.')
 
+    def cmd_quit(self, arg):
+        self.push('+OK closing.')
+        self.close_when_done()
+
 
 class DummyPOP3Server(asyncore.dispatcher, threading.Thread):
 
@@ -165,10 +169,10 @@ class TestPOP3Class(TestCase):
     def setUp(self):
         self.server = DummyPOP3Server((HOST, PORT))
         self.server.start()
-        self.client = poplib.POP3(self.server.host, self.server.port)
+        self.client = poplib.POP3(self.server.host, self.server.port, timeout=3)
 
     def tearDown(self):
-        self.client.quit()
+        self.client.close()
         self.server.stop()
 
     def test_getwelcome(self):
@@ -228,6 +232,12 @@ class TestPOP3Class(TestCase):
         self.client.uidl()
         self.client.uidl('foo')
 
+    def test_quit(self):
+        resp = self.client.quit()
+        self.assertTrue(resp)
+        self.assertIsNone(self.client.sock)
+        self.assertIsNone(self.client.file)
+
 
 SUPPORTS_SSL = False
 if hasattr(poplib, 'POP3_SSL'):
@@ -274,6 +284,7 @@ if hasattr(poplib, 'POP3_SSL'):
             else:
                 DummyPOP3Handler.handle_read(self)
 
+
     class TestPOP3_SSLClass(TestPOP3Class):
         # repeat previous tests by using poplib.POP3_SSL
 
diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py
index 865209d7c9..4d9e1f599b 100644
--- a/Lib/test/test_posix.py
+++ b/Lib/test/test_posix.py
@@ -9,6 +9,7 @@ import errno
 import sys
 import time
 import os
+import fcntl
 import platform
 import pwd
 import shutil
@@ -16,6 +17,7 @@ import stat
 import tempfile
 import unittest
 import warnings
+import _testcapi
 
 _DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(),
                               support.TESTFN + '-dummy-symlink')
@@ -43,7 +45,7 @@ class PosixTester(unittest.TestCase):
         NO_ARG_FUNCTIONS = [ "ctermid", "getcwd", "getcwdb", "uname",
                              "times", "getloadavg",
                              "getegid", "geteuid", "getgid", "getgroups",
-                             "getpid", "getpgrp", "getppid", "getuid",
+                             "getpid", "getpgrp", "getppid", "getuid", "sync",
                            ]
 
         for name in NO_ARG_FUNCTIONS:
@@ -128,6 +130,7 @@ class PosixTester(unittest.TestCase):
             fp = open(support.TESTFN)
             try:
                 self.assertTrue(posix.fstatvfs(fp.fileno()))
+                self.assertTrue(posix.statvfs(fp.fileno()))
             finally:
                 fp.close()
 
@@ -142,6 +145,151 @@ class PosixTester(unittest.TestCase):
             finally:
                 fp.close()
 
+    @unittest.skipUnless(hasattr(posix, 'truncate'), "test needs posix.truncate()")
+    def test_truncate(self):
+        with open(support.TESTFN, 'w') as fp:
+            fp.write('test')
+            fp.flush()
+        posix.truncate(support.TESTFN, 0)
+
+    @unittest.skipUnless(getattr(os, 'execve', None) in os.supports_fd, "test needs execve() to support the fd parameter")
+    @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()")
+    @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()")
+    def test_fexecve(self):
+        fp = os.open(sys.executable, os.O_RDONLY)
+        try:
+            pid = os.fork()
+            if pid == 0:
+                os.chdir(os.path.split(sys.executable)[0])
+                posix.execve(fp, [sys.executable, '-c', 'pass'], os.environ)
+            else:
+                self.assertEqual(os.waitpid(pid, 0), (pid, 0))
+        finally:
+            os.close(fp)
+
+    @unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()")
+    @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()")
+    def test_waitid(self):
+        pid = os.fork()
+        if pid == 0:
+            os.chdir(os.path.split(sys.executable)[0])
+            posix.execve(sys.executable, [sys.executable, '-c', 'pass'], os.environ)
+        else:
+            res = posix.waitid(posix.P_PID, pid, posix.WEXITED)
+            self.assertEqual(pid, res.si_pid)
+
+    @unittest.skipUnless(hasattr(posix, 'lockf'), "test needs posix.lockf()")
+    def test_lockf(self):
+        fd = os.open(support.TESTFN, os.O_WRONLY | os.O_CREAT)
+        try:
+            os.write(fd, b'test')
+            os.lseek(fd, 0, os.SEEK_SET)
+            posix.lockf(fd, posix.F_LOCK, 4)
+            # section is locked
+            posix.lockf(fd, posix.F_ULOCK, 4)
+        finally:
+            os.close(fd)
+
+    @unittest.skipUnless(hasattr(posix, 'pread'), "test needs posix.pread()")
+    def test_pread(self):
+        fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT)
+        try:
+            os.write(fd, b'test')
+            os.lseek(fd, 0, os.SEEK_SET)
+            self.assertEqual(b'es', posix.pread(fd, 2, 1))
+            # the first pread() shouldn't disturb the file offset
+            self.assertEqual(b'te', posix.read(fd, 2))
+        finally:
+            os.close(fd)
+
+    @unittest.skipUnless(hasattr(posix, 'pwrite'), "test needs posix.pwrite()")
+    def test_pwrite(self):
+        fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT)
+        try:
+            os.write(fd, b'test')
+            os.lseek(fd, 0, os.SEEK_SET)
+            posix.pwrite(fd, b'xx', 1)
+            self.assertEqual(b'txxt', posix.read(fd, 4))
+        finally:
+            os.close(fd)
+
+    @unittest.skipUnless(hasattr(posix, 'posix_fallocate'),
+        "test needs posix.posix_fallocate()")
+    def test_posix_fallocate(self):
+        fd = os.open(support.TESTFN, os.O_WRONLY | os.O_CREAT)
+        try:
+            posix.posix_fallocate(fd, 0, 10)
+        except OSError as inst:
+            # issue10812, ZFS doesn't appear to support posix_fallocate,
+            # so skip Solaris-based since they are likely to have ZFS.
+            if inst.errno != errno.EINVAL or not sys.platform.startswith("sunos"):
+                raise
+        finally:
+            os.close(fd)
+
+    @unittest.skipUnless(hasattr(posix, 'posix_fadvise'),
+        "test needs posix.posix_fadvise()")
+    def test_posix_fadvise(self):
+        fd = os.open(support.TESTFN, os.O_RDONLY)
+        try:
+            posix.posix_fadvise(fd, 0, 0, posix.POSIX_FADV_WILLNEED)
+        finally:
+            os.close(fd)
+
+    @unittest.skipUnless(os.utime in os.supports_fd, "test needs fd support in os.utime")
+    def test_utime_with_fd(self):
+        now = time.time()
+        fd = os.open(support.TESTFN, os.O_RDONLY)
+        try:
+            posix.utime(fd)
+            posix.utime(fd, None)
+            self.assertRaises(TypeError, posix.utime, fd, (None, None))
+            self.assertRaises(TypeError, posix.utime, fd, (now, None))
+            self.assertRaises(TypeError, posix.utime, fd, (None, now))
+            posix.utime(fd, (int(now), int(now)))
+            posix.utime(fd, (now, now))
+            self.assertRaises(ValueError, posix.utime, fd, (now, now), ns=(now, now))
+            self.assertRaises(ValueError, posix.utime, fd, (now, 0), ns=(None, None))
+            self.assertRaises(ValueError, posix.utime, fd, (None, None), ns=(now, 0))
+            posix.utime(fd, (int(now), int((now - int(now)) * 1e9)))
+            posix.utime(fd, ns=(int(now), int((now - int(now)) * 1e9)))
+
+        finally:
+            os.close(fd)
+
+    @unittest.skipUnless(os.utime in os.supports_follow_symlinks, "test needs follow_symlinks support in os.utime")
+    def test_utime_nofollow_symlinks(self):
+        now = time.time()
+        posix.utime(support.TESTFN, None, follow_symlinks=False)
+        self.assertRaises(TypeError, posix.utime, support.TESTFN, (None, None), follow_symlinks=False)
+        self.assertRaises(TypeError, posix.utime, support.TESTFN, (now, None), follow_symlinks=False)
+        self.assertRaises(TypeError, posix.utime, support.TESTFN, (None, now), follow_symlinks=False)
+        posix.utime(support.TESTFN, (int(now), int(now)), follow_symlinks=False)
+        posix.utime(support.TESTFN, (now, now), follow_symlinks=False)
+        posix.utime(support.TESTFN, follow_symlinks=False)
+
+    @unittest.skipUnless(hasattr(posix, 'writev'), "test needs posix.writev()")
+    def test_writev(self):
+        fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT)
+        try:
+            os.writev(fd, (b'test1', b'tt2', b't3'))
+            os.lseek(fd, 0, os.SEEK_SET)
+            self.assertEqual(b'test1tt2t3', posix.read(fd, 10))
+        finally:
+            os.close(fd)
+
+    @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()")
+    def test_readv(self):
+        fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT)
+        try:
+            os.write(fd, b'test1tt2t3')
+            os.lseek(fd, 0, os.SEEK_SET)
+            buf = [bytearray(i) for i in [5, 3, 2]]
+            self.assertEqual(posix.readv(fd, buf), 10)
+            self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf])
+        finally:
+            os.close(fd)
+
     def test_dup(self):
         if hasattr(posix, 'dup'):
             fp = open(support.TESTFN)
@@ -167,6 +315,13 @@ class PosixTester(unittest.TestCase):
                 fp1.close()
                 fp2.close()
 
+    @unittest.skipUnless(hasattr(os, 'O_CLOEXEC'), "needs os.O_CLOEXEC")
+    @support.requires_linux_version(2, 6, 23)
+    def test_oscloexec(self):
+        fd = os.open(support.TESTFN, os.O_RDONLY|os.O_CLOEXEC)
+        self.addCleanup(os.close, fd)
+        self.assertTrue(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC)
+
     def test_osexlock(self):
         if hasattr(posix, "O_EXLOCK"):
             fd = os.open(support.TESTFN,
@@ -203,12 +358,29 @@ class PosixTester(unittest.TestCase):
             fp = open(support.TESTFN)
             try:
                 self.assertTrue(posix.fstat(fp.fileno()))
+                self.assertTrue(posix.stat(fp.fileno()))
+
+                self.assertRaisesRegex(TypeError,
+                        'should be string, bytes or integer, not',
+                        posix.stat, float(fp.fileno()))
             finally:
                 fp.close()
 
     def test_stat(self):
         if hasattr(posix, 'stat'):
             self.assertTrue(posix.stat(support.TESTFN))
+            self.assertTrue(posix.stat(os.fsencode(support.TESTFN)))
+            self.assertTrue(posix.stat(bytearray(os.fsencode(support.TESTFN))))
+
+            self.assertRaisesRegex(TypeError,
+                    'can\'t specify None for path argument',
+                    posix.stat, None)
+            self.assertRaisesRegex(TypeError,
+                    'should be string, bytes or integer, not',
+                    posix.stat, list(support.TESTFN))
+            self.assertRaisesRegex(TypeError,
+                    'should be string, bytes or integer, not',
+                    posix.stat, list(os.fsencode(support.TESTFN)))
 
     @unittest.skipUnless(hasattr(posix, 'mkfifo'), "don't have mkfifo()")
     def test_mkfifo(self):
@@ -298,7 +470,7 @@ class PosixTester(unittest.TestCase):
         self.assertRaises(OSError, posix.chown, support.TESTFN, -1, -1)
 
         # re-create the file
-        open(support.TESTFN, 'w').close()
+        support.create_empty_file(support.TESTFN)
         self._test_all_chown_common(posix.chown, support.TESTFN,
                                     getattr(posix, 'stat', None))
 
@@ -329,13 +501,32 @@ class PosixTester(unittest.TestCase):
             self.assertRaises(OSError, posix.chdir, support.TESTFN)
 
     def test_listdir(self):
-        if hasattr(posix, 'listdir'):
-            self.assertTrue(support.TESTFN in posix.listdir(os.curdir))
+        self.assertTrue(support.TESTFN in posix.listdir(os.curdir))
 
     def test_listdir_default(self):
-        # When listdir is called without argument, it's the same as listdir(os.curdir)
-        if hasattr(posix, 'listdir'):
-            self.assertTrue(support.TESTFN in posix.listdir())
+        # When listdir is called without argument,
+        # it's the same as listdir(os.curdir).
+        self.assertTrue(support.TESTFN in posix.listdir())
+
+    def test_listdir_bytes(self):
+        # When listdir is called with a bytes object,
+        # the returned strings are of type bytes.
+        self.assertTrue(os.fsencode(support.TESTFN) in posix.listdir(b'.'))
+
+    @unittest.skipUnless(posix.listdir in os.supports_fd,
+                         "test needs fd support for posix.listdir()")
+    def test_listdir_fd(self):
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        self.addCleanup(posix.close, f)
+        self.assertEqual(
+            sorted(posix.listdir('.')),
+            sorted(posix.listdir(f))
+            )
+        # Check that the fd offset was reset (issue #13739)
+        self.assertEqual(
+            sorted(posix.listdir('.')),
+            sorted(posix.listdir(f))
+            )
 
     def test_access(self):
         if hasattr(posix, 'access'):
@@ -357,6 +548,36 @@ class PosixTester(unittest.TestCase):
             os.close(reader)
             os.close(writer)
 
+    @unittest.skipUnless(hasattr(os, 'pipe2'), "test needs os.pipe2()")
+    @support.requires_linux_version(2, 6, 27)
+    def test_pipe2(self):
+        self.assertRaises(TypeError, os.pipe2, 'DEADBEEF')
+        self.assertRaises(TypeError, os.pipe2, 0, 0)
+
+        # try calling with flags = 0, like os.pipe()
+        r, w = os.pipe2(0)
+        os.close(r)
+        os.close(w)
+
+        # test flags
+        r, w = os.pipe2(os.O_CLOEXEC|os.O_NONBLOCK)
+        self.addCleanup(os.close, r)
+        self.addCleanup(os.close, w)
+        self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFD) & fcntl.FD_CLOEXEC)
+        self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFD) & fcntl.FD_CLOEXEC)
+        # try reading from an empty pipe: this should fail, not block
+        self.assertRaises(OSError, os.read, r, 1)
+        # try a write big enough to fill-up the pipe: this should either
+        # fail or perform a partial write, not block
+        try:
+            os.write(w, b'x' * support.PIPE_MAX_SIZE)
+        except OSError:
+            pass
+
+        # Issue 15989
+        self.assertRaises(OverflowError, os.pipe2, _testcapi.INT_MAX + 1)
+        self.assertRaises(OverflowError, os.pipe2, _testcapi.UINT_MAX + 1)
+
     def test_utime(self):
         if hasattr(posix, 'utime'):
             now = time.time()
@@ -367,13 +588,14 @@ class PosixTester(unittest.TestCase):
             posix.utime(support.TESTFN, (int(now), int(now)))
             posix.utime(support.TESTFN, (now, now))
 
-    def _test_chflags_regular_file(self, chflags_func, target_file):
+    def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs):
         st = os.stat(target_file)
         self.assertTrue(hasattr(st, 'st_flags'))
 
         # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE.
+        flags = st.st_flags | stat.UF_IMMUTABLE
         try:
-            chflags_func(target_file, st.st_flags | stat.UF_IMMUTABLE)
+            chflags_func(target_file, flags, **kwargs)
         except OSError as err:
             if err.errno != errno.EOPNOTSUPP:
                 raise
@@ -397,6 +619,7 @@ class PosixTester(unittest.TestCase):
     @unittest.skipUnless(hasattr(posix, 'lchflags'), 'test needs os.lchflags()')
     def test_lchflags_regular_file(self):
         self._test_chflags_regular_file(posix.lchflags, support.TESTFN)
+        self._test_chflags_regular_file(posix.chflags, support.TESTFN, follow_symlinks=False)
 
     @unittest.skipUnless(hasattr(posix, 'lchflags'), 'test needs os.lchflags()')
     def test_lchflags_symlink(self):
@@ -408,25 +631,28 @@ class PosixTester(unittest.TestCase):
         self.teardown_files.append(_DUMMY_SYMLINK)
         dummy_symlink_st = os.lstat(_DUMMY_SYMLINK)
 
-        # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE.
-        try:
-            posix.lchflags(_DUMMY_SYMLINK,
-                           dummy_symlink_st.st_flags | stat.UF_IMMUTABLE)
-        except OSError as err:
-            if err.errno != errno.EOPNOTSUPP:
-                raise
-            msg = 'chflag UF_IMMUTABLE not supported by underlying fs'
-            self.skipTest(msg)
+        def chflags_nofollow(path, flags):
+            return posix.chflags(path, flags, follow_symlinks=False)
 
-        try:
-            new_testfn_st = os.stat(support.TESTFN)
-            new_dummy_symlink_st = os.lstat(_DUMMY_SYMLINK)
+        for fn in (posix.lchflags, chflags_nofollow):
+            # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE.
+            flags = dummy_symlink_st.st_flags | stat.UF_IMMUTABLE
+            try:
+                fn(_DUMMY_SYMLINK, flags)
+            except OSError as err:
+                if err.errno != errno.EOPNOTSUPP:
+                    raise
+                msg = 'chflag UF_IMMUTABLE not supported by underlying fs'
+                self.skipTest(msg)
+            try:
+                new_testfn_st = os.stat(support.TESTFN)
+                new_dummy_symlink_st = os.lstat(_DUMMY_SYMLINK)
 
-            self.assertEqual(testfn_st.st_flags, new_testfn_st.st_flags)
-            self.assertEqual(dummy_symlink_st.st_flags | stat.UF_IMMUTABLE,
-                             new_dummy_symlink_st.st_flags)
-        finally:
-            posix.lchflags(_DUMMY_SYMLINK, dummy_symlink_st.st_flags)
+                self.assertEqual(testfn_st.st_flags, new_testfn_st.st_flags)
+                self.assertEqual(dummy_symlink_st.st_flags | stat.UF_IMMUTABLE,
+                                 new_dummy_symlink_st.st_flags)
+            finally:
+                fn(_DUMMY_SYMLINK, dummy_symlink_st.st_flags)
 
     def test_environ(self):
         if os.name == "nt":
@@ -474,13 +700,22 @@ class PosixTester(unittest.TestCase):
                 os.chdir(curdir)
                 support.rmtree(base_path)
 
+    @unittest.skipUnless(hasattr(posix, 'getgrouplist'), "test needs posix.getgrouplist()")
+    @unittest.skipUnless(hasattr(pwd, 'getpwuid'), "test needs pwd.getpwuid()")
+    @unittest.skipUnless(hasattr(os, 'getuid'), "test needs os.getuid()")
+    def test_getgrouplist(self):
+        user = pwd.getpwuid(os.getuid())[0]
+        group = pwd.getpwuid(os.getuid())[3]
+        self.assertIn(group, posix.getgrouplist(user, group))
+
+
     @unittest.skipUnless(hasattr(os, 'getegid'), "test needs os.getegid()")
     def test_getgroups(self):
         with os.popen('id -G') as idg:
             groups = idg.read().strip()
             ret = idg.close()
 
-        if ret != None or not groups:
+        if ret is not None or not groups:
             raise unittest.SkipTest("need working 'id -G'")
 
         # Issues 16698: OS X ABIs prior to 10.6 have limits on getgroups()
@@ -498,6 +733,348 @@ class PosixTester(unittest.TestCase):
                 set([int(x) for x in groups.split()]),
                 set(posix.getgroups() + [posix.getegid()]))
 
+    # tests for the posix *at functions follow
+
+    @unittest.skipUnless(os.access in os.supports_dir_fd, "test needs dir_fd support for os.access()")
+    def test_access_dir_fd(self):
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            self.assertTrue(posix.access(support.TESTFN, os.R_OK, dir_fd=f))
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.chmod in os.supports_dir_fd, "test needs dir_fd support in os.chmod()")
+    def test_chmod_dir_fd(self):
+        os.chmod(support.TESTFN, stat.S_IRUSR)
+
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.chmod(support.TESTFN, stat.S_IRUSR | stat.S_IWUSR, dir_fd=f)
+
+            s = posix.stat(support.TESTFN)
+            self.assertEqual(s[0] & stat.S_IRWXU, stat.S_IRUSR | stat.S_IWUSR)
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.chown in os.supports_dir_fd, "test needs dir_fd support in os.chown()")
+    def test_chown_dir_fd(self):
+        support.unlink(support.TESTFN)
+        support.create_empty_file(support.TESTFN)
+
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.chown(support.TESTFN, os.getuid(), os.getgid(), dir_fd=f)
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()")
+    def test_stat_dir_fd(self):
+        support.unlink(support.TESTFN)
+        with open(support.TESTFN, 'w') as outfile:
+            outfile.write("testline\n")
+
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            s1 = posix.stat(support.TESTFN)
+            s2 = posix.stat(support.TESTFN, dir_fd=f)
+            self.assertEqual(s1, s2)
+            s2 = posix.stat(support.TESTFN, dir_fd=None)
+            self.assertEqual(s1, s2)
+            self.assertRaisesRegex(TypeError, 'should be integer, not',
+                    posix.stat, support.TESTFN, dir_fd=posix.getcwd())
+            self.assertRaisesRegex(TypeError, 'should be integer, not',
+                    posix.stat, support.TESTFN, dir_fd=float(f))
+            self.assertRaises(OverflowError,
+                    posix.stat, support.TESTFN, dir_fd=10**20)
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()")
+    def test_utime_dir_fd(self):
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            now = time.time()
+            posix.utime(support.TESTFN, None, dir_fd=f)
+            posix.utime(support.TESTFN, dir_fd=f)
+            self.assertRaises(TypeError, posix.utime, support.TESTFN, now, dir_fd=f)
+            self.assertRaises(TypeError, posix.utime, support.TESTFN, (None, None), dir_fd=f)
+            self.assertRaises(TypeError, posix.utime, support.TESTFN, (now, None), dir_fd=f)
+            self.assertRaises(TypeError, posix.utime, support.TESTFN, (None, now), dir_fd=f)
+            self.assertRaises(TypeError, posix.utime, support.TESTFN, (now, "x"), dir_fd=f)
+            posix.utime(support.TESTFN, (int(now), int(now)), dir_fd=f)
+            posix.utime(support.TESTFN, (now, now), dir_fd=f)
+            posix.utime(support.TESTFN,
+                    (int(now), int((now - int(now)) * 1e9)), dir_fd=f)
+            posix.utime(support.TESTFN, dir_fd=f,
+                            times=(int(now), int((now - int(now)) * 1e9)))
+
+            # try dir_fd and follow_symlinks together
+            if os.utime in os.supports_follow_symlinks:
+                try:
+                    posix.utime(support.TESTFN, follow_symlinks=False, dir_fd=f)
+                except ValueError:
+                    # whoops!  using both together not supported on this platform.
+                    pass
+
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.link in os.supports_dir_fd, "test needs dir_fd support in os.link()")
+    def test_link_dir_fd(self):
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.link(support.TESTFN, support.TESTFN + 'link', src_dir_fd=f, dst_dir_fd=f)
+            # should have same inodes
+            self.assertEqual(posix.stat(support.TESTFN)[1],
+                posix.stat(support.TESTFN + 'link')[1])
+        finally:
+            posix.close(f)
+            support.unlink(support.TESTFN + 'link')
+
+    @unittest.skipUnless(os.mkdir in os.supports_dir_fd, "test needs dir_fd support in os.mkdir()")
+    def test_mkdir_dir_fd(self):
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.mkdir(support.TESTFN + 'dir', dir_fd=f)
+            posix.stat(support.TESTFN + 'dir') # should not raise exception
+        finally:
+            posix.close(f)
+            support.rmtree(support.TESTFN + 'dir')
+
+    @unittest.skipUnless((os.mknod in os.supports_dir_fd) and hasattr(stat, 'S_IFIFO'),
+                         "test requires both stat.S_IFIFO and dir_fd support for os.mknod()")
+    def test_mknod_dir_fd(self):
+        # Test using mknodat() to create a FIFO (the only use specified
+        # by POSIX).
+        support.unlink(support.TESTFN)
+        mode = stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.mknod(support.TESTFN, mode, 0, dir_fd=f)
+        except OSError as e:
+            # Some old systems don't allow unprivileged users to use
+            # mknod(), or only support creating device nodes.
+            self.assertIn(e.errno, (errno.EPERM, errno.EINVAL))
+        else:
+            self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode))
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.open in os.supports_dir_fd, "test needs dir_fd support in os.open()")
+    def test_open_dir_fd(self):
+        support.unlink(support.TESTFN)
+        with open(support.TESTFN, 'w') as outfile:
+            outfile.write("testline\n")
+        a = posix.open(posix.getcwd(), posix.O_RDONLY)
+        b = posix.open(support.TESTFN, posix.O_RDONLY, dir_fd=a)
+        try:
+            res = posix.read(b, 9).decode(encoding="utf-8")
+            self.assertEqual("testline\n", res)
+        finally:
+            posix.close(a)
+            posix.close(b)
+
+    @unittest.skipUnless(os.readlink in os.supports_dir_fd, "test needs dir_fd support in os.readlink()")
+    def test_readlink_dir_fd(self):
+        os.symlink(support.TESTFN, support.TESTFN + 'link')
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            self.assertEqual(posix.readlink(support.TESTFN + 'link'),
+                posix.readlink(support.TESTFN + 'link', dir_fd=f))
+        finally:
+            support.unlink(support.TESTFN + 'link')
+            posix.close(f)
+
+    @unittest.skipUnless(os.rename in os.supports_dir_fd, "test needs dir_fd support in os.rename()")
+    def test_rename_dir_fd(self):
+        support.unlink(support.TESTFN)
+        support.create_empty_file(support.TESTFN + 'ren')
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.rename(support.TESTFN + 'ren', support.TESTFN, src_dir_fd=f, dst_dir_fd=f)
+        except:
+            posix.rename(support.TESTFN + 'ren', support.TESTFN)
+            raise
+        else:
+            posix.stat(support.TESTFN) # should not raise exception
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.symlink in os.supports_dir_fd, "test needs dir_fd support in os.symlink()")
+    def test_symlink_dir_fd(self):
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.symlink(support.TESTFN, support.TESTFN + 'link', dir_fd=f)
+            self.assertEqual(posix.readlink(support.TESTFN + 'link'), support.TESTFN)
+        finally:
+            posix.close(f)
+            support.unlink(support.TESTFN + 'link')
+
+    @unittest.skipUnless(os.unlink in os.supports_dir_fd, "test needs dir_fd support in os.unlink()")
+    def test_unlink_dir_fd(self):
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        support.create_empty_file(support.TESTFN + 'del')
+        posix.stat(support.TESTFN + 'del') # should not raise exception
+        try:
+            posix.unlink(support.TESTFN + 'del', dir_fd=f)
+        except:
+            support.unlink(support.TESTFN + 'del')
+            raise
+        else:
+            self.assertRaises(OSError, posix.stat, support.TESTFN + 'link')
+        finally:
+            posix.close(f)
+
+    @unittest.skipUnless(os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()")
+    def test_mkfifo_dir_fd(self):
+        support.unlink(support.TESTFN)
+        f = posix.open(posix.getcwd(), posix.O_RDONLY)
+        try:
+            posix.mkfifo(support.TESTFN, stat.S_IRUSR | stat.S_IWUSR, dir_fd=f)
+            self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode))
+        finally:
+            posix.close(f)
+
+    requires_sched_h = unittest.skipUnless(hasattr(posix, 'sched_yield'),
+                                           "don't have scheduling support")
+    requires_sched_affinity = unittest.skipUnless(hasattr(posix, 'sched_setaffinity'),
+                                                  "don't have sched affinity support")
+
+    @requires_sched_h
+    def test_sched_yield(self):
+        # This has no error conditions (at least on Linux).
+        posix.sched_yield()
+
+    @requires_sched_h
+    @unittest.skipUnless(hasattr(posix, 'sched_get_priority_max'),
+                         "requires sched_get_priority_max()")
+    def test_sched_priority(self):
+        # Round-robin usually has interesting priorities.
+        pol = posix.SCHED_RR
+        lo = posix.sched_get_priority_min(pol)
+        hi = posix.sched_get_priority_max(pol)
+        self.assertIsInstance(lo, int)
+        self.assertIsInstance(hi, int)
+        self.assertGreaterEqual(hi, lo)
+        # OSX evidently just returns 15 without checking the argument.
+        if sys.platform != "darwin":
+            self.assertRaises(OSError, posix.sched_get_priority_min, -23)
+            self.assertRaises(OSError, posix.sched_get_priority_max, -23)
+
+    @unittest.skipUnless(hasattr(posix, 'sched_setscheduler'), "can't change scheduler")
+    def test_get_and_set_scheduler_and_param(self):
+        possible_schedulers = [sched for name, sched in posix.__dict__.items()
+                               if name.startswith("SCHED_")]
+        mine = posix.sched_getscheduler(0)
+        self.assertIn(mine, possible_schedulers)
+        try:
+            parent = posix.sched_getscheduler(os.getppid())
+        except OSError as e:
+            if e.errno != errno.EPERM:
+                raise
+        else:
+            self.assertIn(parent, possible_schedulers)
+        self.assertRaises(OSError, posix.sched_getscheduler, -1)
+        self.assertRaises(OSError, posix.sched_getparam, -1)
+        param = posix.sched_getparam(0)
+        self.assertIsInstance(param.sched_priority, int)
+
+        # POSIX states that calling sched_setparam() or sched_setscheduler() on
+        # a process with a scheduling policy other than SCHED_FIFO or SCHED_RR
+        # is implementation-defined: NetBSD and FreeBSD can return EINVAL.
+        if not sys.platform.startswith(('freebsd', 'netbsd')):
+            try:
+                posix.sched_setscheduler(0, mine, param)
+                posix.sched_setparam(0, param)
+            except OSError as e:
+                if e.errno != errno.EPERM:
+                    raise
+            self.assertRaises(OSError, posix.sched_setparam, -1, param)
+
+        self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param)
+        self.assertRaises(TypeError, posix.sched_setscheduler, 0, mine, None)
+        self.assertRaises(TypeError, posix.sched_setparam, 0, 43)
+        param = posix.sched_param(None)
+        self.assertRaises(TypeError, posix.sched_setparam, 0, param)
+        large = 214748364700
+        param = posix.sched_param(large)
+        self.assertRaises(OverflowError, posix.sched_setparam, 0, param)
+        param = posix.sched_param(sched_priority=-large)
+        self.assertRaises(OverflowError, posix.sched_setparam, 0, param)
+
+    @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function")
+    def test_sched_rr_get_interval(self):
+        try:
+            interval = posix.sched_rr_get_interval(0)
+        except OSError as e:
+            # This likely means that sched_rr_get_interval is only valid for
+            # processes with the SCHED_RR scheduler in effect.
+            if e.errno != errno.EINVAL:
+                raise
+            self.skipTest("only works on SCHED_RR processes")
+        self.assertIsInstance(interval, float)
+        # Reasonable constraints, I think.
+        self.assertGreaterEqual(interval, 0.)
+        self.assertLess(interval, 1.)
+
+    @requires_sched_affinity
+    def test_sched_getaffinity(self):
+        mask = posix.sched_getaffinity(0)
+        self.assertIsInstance(mask, set)
+        self.assertGreaterEqual(len(mask), 1)
+        self.assertRaises(OSError, posix.sched_getaffinity, -1)
+        for cpu in mask:
+            self.assertIsInstance(cpu, int)
+            self.assertGreaterEqual(cpu, 0)
+            self.assertLess(cpu, 1 << 32)
+
+    @requires_sched_affinity
+    def test_sched_setaffinity(self):
+        mask = posix.sched_getaffinity(0)
+        if len(mask) > 1:
+            # Empty masks are forbidden
+            mask.pop()
+        posix.sched_setaffinity(0, mask)
+        self.assertEqual(posix.sched_getaffinity(0), mask)
+        self.assertRaises(OSError, posix.sched_setaffinity, 0, [])
+        self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10])
+        self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128])
+        self.assertRaises(OSError, posix.sched_setaffinity, -1, mask)
+
+    def test_rtld_constants(self):
+        # check presence of major RTLD_* constants
+        posix.RTLD_LAZY
+        posix.RTLD_NOW
+        posix.RTLD_GLOBAL
+        posix.RTLD_LOCAL
+
+    @unittest.skipUnless(hasattr(os, 'SEEK_HOLE'),
+                         "test needs an OS that reports file holes")
+    def test_fs_holes(self):
+        # Even if the filesystem doesn't report holes,
+        # if the OS supports it the SEEK_* constants
+        # will be defined and will have a consistent
+        # behaviour:
+        # os.SEEK_DATA = current position
+        # os.SEEK_HOLE = end of file position
+        with open(support.TESTFN, 'r+b') as fp:
+            fp.write(b"hello")
+            fp.flush()
+            size = fp.tell()
+            fno = fp.fileno()
+            try :
+                for i in range(size):
+                    self.assertEqual(i, os.lseek(fno, i, os.SEEK_DATA))
+                    self.assertLessEqual(size, os.lseek(fno, i, os.SEEK_HOLE))
+                self.assertRaises(OSError, os.lseek, fno, size, os.SEEK_DATA)
+                self.assertRaises(OSError, os.lseek, fno, size, os.SEEK_HOLE)
+            except OSError :
+                # Some OSs claim to support SEEK_HOLE/SEEK_DATA
+                # but it is not true.
+                # For instance:
+                # http://lists.freebsd.org/pipermail/freebsd-amd64/2012-January/014332.html
+                raise unittest.SkipTest("OSError raised!")
+
 class PosixGroupsTester(unittest.TestCase):
 
     def setUp(self):
@@ -533,9 +1110,11 @@ class PosixGroupsTester(unittest.TestCase):
             posix.setgroups(groups)
             self.assertListEqual(groups, posix.getgroups())
 
-
 def test_main():
-    support.run_unittest(PosixTester, PosixGroupsTester)
+    try:
+        support.run_unittest(PosixTester, PosixGroupsTester)
+    finally:
+        support.reap_children()
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py
index 262e09dfdb..724c530261 100644
--- a/Lib/test/test_posixpath.py
+++ b/Lib/test/test_posixpath.py
@@ -1,11 +1,11 @@
-import unittest
-from test import support, test_genericpath
-
 import itertools
-import posixpath
 import os
+import posixpath
 import sys
+import unittest
+import warnings
 from posixpath import realpath, abspath, dirname, basename
+from test import support, test_genericpath
 
 try:
     import posix
@@ -245,7 +245,9 @@ class PosixPathTest(unittest.TestCase):
 
     def test_ismount(self):
         self.assertIs(posixpath.ismount("/"), True)
-        self.assertIs(posixpath.ismount(b"/"), True)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            self.assertIs(posixpath.ismount(b"/"), True)
 
     def test_ismount_non_existent(self):
         # Non-existent mountpoint.
@@ -598,14 +600,10 @@ class PosixPathTest(unittest.TestCase):
             self.assertTrue(posixpath.sameopenfile(a.fileno(), b.fileno()))
 
 
-class PosixCommonTest(test_genericpath.CommonTest):
+class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase):
     pathmodule = posixpath
     attributes = ['relpath', 'samefile', 'sameopenfile', 'samestat']
 
 
-def test_main():
-    support.run_unittest(PosixPathTest, PosixCommonTest)
-
-
 if __name__=="__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py
index 0c93d0e59b..d492d75b08 100644
--- a/Lib/test/test_pprint.py
+++ b/Lib/test/test_pprint.py
@@ -219,6 +219,8 @@ class QueryTestCase(unittest.TestCase):
  others.should.not.be: like.this}"""
         self.assertEqual(DottedPrettyPrinter().pformat(o), exp)
 
+    @unittest.expectedFailure
+    #See http://bugs.python.org/issue13907
     @test.support.cpython_only
     def test_set_reprs(self):
         # This test creates a complex arrangement of frozensets and
@@ -241,10 +243,12 @@ class QueryTestCase(unittest.TestCase):
         # Consequently, this test is fragile and
         # implementation-dependent.  Small changes to Python's sort
         # algorithm cause the test to fail when it should pass.
+        # XXX Or changes to the dictionary implmentation...
 
         self.assertEqual(pprint.pformat(set()), 'set()')
         self.assertEqual(pprint.pformat(set(range(3))), '{0, 1, 2}')
         self.assertEqual(pprint.pformat(frozenset()), 'frozenset()')
+
         self.assertEqual(pprint.pformat(frozenset(range(3))), 'frozenset({0, 1, 2})')
         cube_repr_tgt = """\
 {frozenset(): frozenset({frozenset({2}), frozenset({0}), frozenset({1})}),
diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py
index 8d37bbc4d6..9d6dbea46b 100644
--- a/Lib/test/test_print.py
+++ b/Lib/test/test_print.py
@@ -111,6 +111,32 @@ class TestPrint(unittest.TestCase):
         self.assertRaises(TypeError, print, '', end=3)
         self.assertRaises(AttributeError, print, '', file='')
 
+    def test_print_flush(self):
+        # operation of the flush flag
+        class filelike():
+            def __init__(self):
+                self.written = ''
+                self.flushed = 0
+            def write(self, str):
+                self.written += str
+            def flush(self):
+                self.flushed += 1
+
+        f = filelike()
+        print(1, file=f, end='', flush=True)
+        print(2, file=f, end='', flush=True)
+        print(3, file=f, flush=False)
+        self.assertEqual(f.written, '123\n')
+        self.assertEqual(f.flushed, 2)
+
+        # ensure exceptions from flush are passed through
+        class noflush():
+            def write(self, str):
+                pass
+            def flush(self):
+                raise RuntimeError
+        self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True)
+
 def test_main():
     support.run_unittest(TestPrint)
 
diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py
index b58270f294..96eeb88b89 100644
--- a/Lib/test/test_property.py
+++ b/Lib/test/test_property.py
@@ -128,6 +128,29 @@ class PropertyTests(unittest.TestCase):
         self.assertEqual(newgetter.spam, 8)
         self.assertEqual(newgetter.__class__.spam.__doc__, "new docstring")
 
+    def test_property___isabstractmethod__descriptor(self):
+        for val in (True, False, [], [1], '', '1'):
+            class C(object):
+                def foo(self):
+                    pass
+                foo.__isabstractmethod__ = val
+                foo = property(foo)
+            self.assertIs(C.foo.__isabstractmethod__, bool(val))
+
+        # check that the property's __isabstractmethod__ descriptor does the
+        # right thing when presented with a value that fails truth testing:
+        class NotBool(object):
+            def __nonzero__(self):
+                raise ValueError()
+            __len__ = __nonzero__
+        with self.assertRaises(ValueError):
+            class C(object):
+                def foo(self):
+                    pass
+                foo.__isabstractmethod__ = NotBool()
+                foo = property(foo)
+            C.foo.__isabstractmethod__
+
 
 # Issue 5890: subclasses of property do not preserve method __doc__ strings
 class PropertySub(property):
diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py
index 4d471d534e..29297f8841 100644
--- a/Lib/test/test_pty.py
+++ b/Lib/test/test_pty.py
@@ -205,6 +205,7 @@ class SmallPtyTests(unittest.TestCase):
         self.orig_stdout_fileno = pty.STDOUT_FILENO
         self.orig_pty_select = pty.select
         self.fds = []  # A list of file descriptors to close.
+        self.files = []
         self.select_rfds_lengths = []
         self.select_rfds_results = []
 
@@ -212,6 +213,11 @@ class SmallPtyTests(unittest.TestCase):
         pty.STDIN_FILENO = self.orig_stdin_fileno
         pty.STDOUT_FILENO = self.orig_stdout_fileno
         pty.select = self.orig_pty_select
+        for file in self.files:
+            try:
+                file.close()
+            except OSError:
+                pass
         for fd in self.fds:
             try:
                 os.close(fd)
@@ -223,6 +229,11 @@ class SmallPtyTests(unittest.TestCase):
         self.fds.extend(pipe_fds)
         return pipe_fds
 
+    def _socketpair(self):
+        socketpair = socket.socketpair()
+        self.files.extend(socketpair)
+        return socketpair
+
     def _mock_select(self, rfds, wfds, xfds):
         # This will raise IndexError when no more expected calls exist.
         self.assertEqual(self.select_rfds_lengths.pop(0), len(rfds))
@@ -234,9 +245,7 @@ class SmallPtyTests(unittest.TestCase):
         pty.STDOUT_FILENO = mock_stdout_fd
         mock_stdin_fd, write_to_stdin_fd = self._pipe()
         pty.STDIN_FILENO = mock_stdin_fd
-        socketpair = socket.socketpair()
-        for s in socketpair:
-            self.addCleanup(s.close)
+        socketpair = self._socketpair()
         masters = [s.fileno() for s in socketpair]
 
         # Feed data.  Smaller than PIPEBUF.  These writes will not block.
@@ -264,9 +273,7 @@ class SmallPtyTests(unittest.TestCase):
         pty.STDOUT_FILENO = mock_stdout_fd
         mock_stdin_fd, write_to_stdin_fd = self._pipe()
         pty.STDIN_FILENO = mock_stdin_fd
-        socketpair = socket.socketpair()
-        for s in socketpair:
-            self.addCleanup(s.close)
+        socketpair = self._socketpair()
         masters = [s.fileno() for s in socketpair]
 
         os.close(masters[1])
diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py
new file mode 100644
index 0000000000..b81a595f69
--- /dev/null
+++ b/Lib/test/test_pulldom.py
@@ -0,0 +1,347 @@
+import io
+import unittest
+import sys
+import xml.sax
+
+from xml.sax.xmlreader import AttributesImpl
+from xml.dom import pulldom
+
+from test.support import run_unittest, findfile
+
+
+tstfile = findfile("test.xml", subdir="xmltestdata")
+
+# A handy XML snippet, containing attributes, a namespace prefix, and a
+# self-closing tag:
+SMALL_SAMPLE = """<?xml version="1.0"?>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xdc="http://www.xml.com/books">
+<!-- A comment -->
+<title>Introduction to XSL</title>
+<hr/>
+<p><xdc:author xdc:attrib="prefixed attribute" attrib="other attrib">A. Namespace</xdc:author></p>
+</html>"""
+
+
+class PullDOMTestCase(unittest.TestCase):
+
+    def test_parse(self):
+        """Minimal test of DOMEventStream.parse()"""
+
+        # This just tests that parsing from a stream works. Actual parser
+        # semantics are tested using parseString with a more focused XML
+        # fragment.
+
+        # Test with a filename:
+        handler = pulldom.parse(tstfile)
+        self.addCleanup(handler.stream.close)
+        list(handler)
+
+        # Test with a file object:
+        with open(tstfile, "rb") as fin:
+            list(pulldom.parse(fin))
+
+    def test_parse_semantics(self):
+        """Test DOMEventStream parsing semantics."""
+
+        items = pulldom.parseString(SMALL_SAMPLE)
+        evt, node = next(items)
+        # Just check the node is a Document:
+        self.assertTrue(hasattr(node, "createElement"))
+        self.assertEqual(pulldom.START_DOCUMENT, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("html", node.tagName)
+        self.assertEqual(2, len(node.attributes))
+        self.assertEqual(node.attributes.getNamedItem("xmlns:xdc").value,
+              "http://www.xml.com/books")
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt) # Line break
+        evt, node = next(items)
+        # XXX - A comment should be reported here!
+        # self.assertEqual(pulldom.COMMENT, evt)
+        # Line break after swallowed comment:
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual("title", node.tagName)
+        title_node = node
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        self.assertEqual("Introduction to XSL", node.data)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("title", node.tagName)
+        self.assertTrue(title_node is node)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("hr", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("hr", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("p", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("xdc:author", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("xdc:author", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        # XXX No END_DOCUMENT item is ever obtained:
+        #evt, node = next(items)
+        #self.assertEqual(pulldom.END_DOCUMENT, evt)
+
+    def test_expandItem(self):
+        """Ensure expandItem works as expected."""
+        items = pulldom.parseString(SMALL_SAMPLE)
+        # Loop through the nodes until we get to a "title" start tag:
+        for evt, item in items:
+            if evt == pulldom.START_ELEMENT and item.tagName == "title":
+                items.expandNode(item)
+                self.assertEqual(1, len(item.childNodes))
+                break
+        else:
+            self.fail("No \"title\" element detected in SMALL_SAMPLE!")
+        # Loop until we get to the next start-element:
+        for evt, node in items:
+            if evt == pulldom.START_ELEMENT:
+                break
+        self.assertEqual("hr", node.tagName,
+            "expandNode did not leave DOMEventStream in the correct state.")
+        # Attempt to expand a standalone element:
+        items.expandNode(node)
+        self.assertEqual(next(items)[0], pulldom.CHARACTERS)
+        evt, node = next(items)
+        self.assertEqual(node.tagName, "p")
+        items.expandNode(node)
+        next(items) # Skip character data
+        evt, node = next(items)
+        self.assertEqual(node.tagName, "html")
+        with self.assertRaises(StopIteration):
+            next(items)
+        items.clear()
+        self.assertIsNone(items.parser)
+        self.assertIsNone(items.stream)
+
+    @unittest.expectedFailure
+    def test_comment(self):
+        """PullDOM does not receive "comment" events."""
+        items = pulldom.parseString(SMALL_SAMPLE)
+        for evt, _ in items:
+            if evt == pulldom.COMMENT:
+                break
+        else:
+            self.fail("No comment was encountered")
+
+    @unittest.expectedFailure
+    def test_end_document(self):
+        """PullDOM does not receive "end-document" events."""
+        items = pulldom.parseString(SMALL_SAMPLE)
+        # Read all of the nodes up to and including </html>:
+        for evt, node in items:
+            if evt == pulldom.END_ELEMENT and node.tagName == "html":
+                break
+        try:
+            # Assert that the next node is END_DOCUMENT:
+            evt, node = next(items)
+            self.assertEqual(pulldom.END_DOCUMENT, evt)
+        except StopIteration:
+            self.fail(
+                "Ran out of events, but should have received END_DOCUMENT")
+
+
+class ThoroughTestCase(unittest.TestCase):
+    """Test the hard-to-reach parts of pulldom."""
+
+    def test_thorough_parse(self):
+        """Test some of the hard-to-reach parts of PullDOM."""
+        self._test_thorough(pulldom.parse(None, parser=SAXExerciser()))
+
+    @unittest.expectedFailure
+    def test_sax2dom_fail(self):
+        """SAX2DOM can"t handle a PI before the root element."""
+        pd = SAX2DOMTestHelper(None, SAXExerciser(), 12)
+        self._test_thorough(pd)
+
+    def test_thorough_sax2dom(self):
+        """Test some of the hard-to-reach parts of SAX2DOM."""
+        pd = SAX2DOMTestHelper(None, SAX2DOMExerciser(), 12)
+        self._test_thorough(pd, False)
+
+    def _test_thorough(self, pd, before_root=True):
+        """Test some of the hard-to-reach parts of the parser, using a mock
+        parser."""
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.START_DOCUMENT, evt)
+        # Just check the node is a Document:
+        self.assertTrue(hasattr(node, "createElement"))
+
+        if before_root:
+            evt, node = next(pd)
+            self.assertEqual(pulldom.COMMENT, evt)
+            self.assertEqual("a comment", node.data)
+            evt, node = next(pd)
+            self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt)
+            self.assertEqual("target", node.target)
+            self.assertEqual("data", node.data)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("html", node.tagName)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.COMMENT, evt)
+        self.assertEqual("a comment", node.data)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt)
+        self.assertEqual("target", node.target)
+        self.assertEqual("data", node.data)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("p", node.tagName)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        self.assertEqual("text", node.data)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("p", node.tagName)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("html", node.tagName)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.END_DOCUMENT, evt)
+
+
+class SAXExerciser(object):
+    """A fake sax parser that calls some of the harder-to-reach sax methods to
+    ensure it emits the correct events"""
+
+    def setContentHandler(self, handler):
+        self._handler = handler
+
+    def parse(self, _):
+        h = self._handler
+        h.startDocument()
+
+        # The next two items ensure that items preceding the first
+        # start_element are properly stored and emitted:
+        h.comment("a comment")
+        h.processingInstruction("target", "data")
+
+        h.startElement("html", AttributesImpl({}))
+
+        h.comment("a comment")
+        h.processingInstruction("target", "data")
+
+        h.startElement("p", AttributesImpl({"class": "paraclass"}))
+        h.characters("text")
+        h.endElement("p")
+        h.endElement("html")
+        h.endDocument()
+
+    def stub(self, *args, **kwargs):
+        """Stub method. Does nothing."""
+        pass
+    setProperty = stub
+    setFeature = stub
+
+
+class SAX2DOMExerciser(SAXExerciser):
+    """The same as SAXExerciser, but without the processing instruction and
+    comment before the root element, because S2D can"t handle it"""
+
+    def parse(self, _):
+        h = self._handler
+        h.startDocument()
+        h.startElement("html", AttributesImpl({}))
+        h.comment("a comment")
+        h.processingInstruction("target", "data")
+        h.startElement("p", AttributesImpl({"class": "paraclass"}))
+        h.characters("text")
+        h.endElement("p")
+        h.endElement("html")
+        h.endDocument()
+
+
+class SAX2DOMTestHelper(pulldom.DOMEventStream):
+    """Allows us to drive SAX2DOM from a DOMEventStream."""
+
+    def reset(self):
+        self.pulldom = pulldom.SAX2DOM()
+        # This content handler relies on namespace support
+        self.parser.setFeature(xml.sax.handler.feature_namespaces, 1)
+        self.parser.setContentHandler(self.pulldom)
+
+
+class SAX2DOMTestCase(unittest.TestCase):
+
+    def confirm(self, test, testname="Test"):
+        self.assertTrue(test, testname)
+
+    def test_basic(self):
+        """Ensure SAX2DOM can parse from a stream."""
+        with io.StringIO(SMALL_SAMPLE) as fin:
+            sd = SAX2DOMTestHelper(fin, xml.sax.make_parser(),
+                                   len(SMALL_SAMPLE))
+            for evt, node in sd:
+                if evt == pulldom.START_ELEMENT and node.tagName == "html":
+                    break
+            # Because the buffer is the same length as the XML, all the
+            # nodes should have been parsed and added:
+            self.assertGreater(len(node.childNodes), 0)
+
+    def testSAX2DOM(self):
+        """Ensure SAX2DOM expands nodes as expected."""
+        sax2dom = pulldom.SAX2DOM()
+        sax2dom.startDocument()
+        sax2dom.startElement("doc", {})
+        sax2dom.characters("text")
+        sax2dom.startElement("subelm", {})
+        sax2dom.characters("text")
+        sax2dom.endElement("subelm")
+        sax2dom.characters("text")
+        sax2dom.endElement("doc")
+        sax2dom.endDocument()
+
+        doc = sax2dom.document
+        root = doc.documentElement
+        (text1, elm1, text2) = root.childNodes
+        text3 = elm1.childNodes[0]
+
+        self.assertIsNone(text1.previousSibling)
+        self.assertIs(text1.nextSibling, elm1)
+        self.assertIs(elm1.previousSibling, text1)
+        self.assertIs(elm1.nextSibling, text2)
+        self.assertIs(text2.previousSibling, elm1)
+        self.assertIsNone(text2.nextSibling)
+        self.assertIsNone(text3.previousSibling)
+        self.assertIsNone(text3.nextSibling)
+
+        self.assertIs(root.parentNode, doc)
+        self.assertIs(text1.parentNode, root)
+        self.assertIs(elm1.parentNode, root)
+        self.assertIs(text2.parentNode, root)
+        self.assertIs(text3.parentNode, elm1)
+        doc.unlink()
+
+
+def test_main():
+    run_unittest(PullDOMTestCase, ThoroughTestCase, SAX2DOMTestCase)
+
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
index c7318ff613..d98a526a59 100644
--- a/Lib/test/test_pydoc.py
+++ b/Lib/test/test_pydoc.py
@@ -207,7 +207,7 @@ expected_html_data_docstrings = tuple(s.replace(' ', '&nbsp;')
 missing_pattern = "no Python documentation found for '%s'"
 
 # output pattern for module with bad imports
-badimport_pattern = "problem in %s - ImportError: No module named %s"
+badimport_pattern = "problem in %s - ImportError: No module named %r"
 
 def run_pydoc(module_name, *args, **env):
     """
@@ -245,8 +245,8 @@ def get_pydoc_text(module):
 def print_diffs(text1, text2):
     "Prints unified diffs for two texts"
     # XXX now obsolete, use unittest built-in support
-    lines1 = text1.splitlines(True)
-    lines2 = text2.splitlines(True)
+    lines1 = text1.splitlines(keepends=True)
+    lines2 = text2.splitlines(keepends=True)
     diffs = difflib.unified_diff(lines1, lines2, n=0, fromfile='expected',
                                  tofile='got')
     print('\n' + ''.join(diffs))
@@ -263,6 +263,8 @@ class PydocDocTest(unittest.TestCase):
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
     def test_html_doc(self):
         result, doc_loc = get_pydoc_html(pydoc_mod)
         mod_file = inspect.getabsfile(pydoc_mod)
@@ -280,6 +282,8 @@ class PydocDocTest(unittest.TestCase):
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
     def test_text_doc(self):
         result, doc_loc = get_pydoc_text(pydoc_mod)
         expected_text = expected_text_pattern % (
@@ -334,6 +338,8 @@ class PydocDocTest(unittest.TestCase):
 
     @unittest.skipIf(sys.flags.optimize >= 2,
                      'Docstrings are omitted with -O2 and above')
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'trace function introduces __locals__ unexpectedly')
     def test_help_output_redirect(self):
         # issue 940286, if output is set in Helper, then all output from
         # Helper.help should be redirected
@@ -403,11 +409,10 @@ class PydocImportTest(unittest.TestCase):
         modname = 'testmod_xyzzy'
         testpairs = (
             ('i_am_not_here', 'i_am_not_here'),
-            ('test.i_am_not_here_either', 'i_am_not_here_either'),
-            ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
-            ('i_am_not_here.{}'.format(modname),
-             'i_am_not_here.{}'.format(modname)),
-            ('test.{}'.format(modname), modname),
+            ('test.i_am_not_here_either', 'test.i_am_not_here_either'),
+            ('test.i_am_not_here.neither_am_i', 'test.i_am_not_here'),
+            ('i_am_not_here.{}'.format(modname), 'i_am_not_here'),
+            ('test.{}'.format(modname), 'test.{}'.format(modname)),
             )
 
         sourcefn = os.path.join(TESTFN, modname) + os.extsep + "py"
diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py
index e02c1af131..be5c1c6cc8 100644
--- a/Lib/test/test_raise.py
+++ b/Lib/test/test_raise.py
@@ -4,6 +4,7 @@
 """Tests for the raise statement."""
 
 from test import support
+import re
 import sys
 import types
 import unittest
@@ -77,6 +78,16 @@ class TestRaise(unittest.TestCase):
                 nested_reraise()
         self.assertRaises(TypeError, reraise)
 
+    def test_raise_from_None(self):
+        try:
+            try:
+                raise TypeError("foo")
+            except:
+                raise ValueError() from None
+        except ValueError as e:
+            self.assertTrue(isinstance(e.__context__, TypeError))
+            self.assertIsNone(e.__cause__)
+
     def test_with_reraise1(self):
         def reraise():
             try:
@@ -130,8 +141,35 @@ class TestRaise(unittest.TestCase):
         with self.assertRaises(TypeError):
             raise MyException
 
+    def test_assert_with_tuple_arg(self):
+        try:
+            assert False, (3,)
+        except AssertionError as e:
+            self.assertEqual(str(e), "(3,)")
+
+
 
 class TestCause(unittest.TestCase):
+
+    def testCauseSyntax(self):
+        try:
+            try:
+                try:
+                    raise TypeError
+                except Exception:
+                    raise ValueError from None
+            except ValueError as exc:
+                self.assertIsNone(exc.__cause__)
+                self.assertTrue(exc.__suppress_context__)
+                exc.__suppress_context__ = False
+                raise exc
+        except ValueError as exc:
+            e = exc
+
+        self.assertIsNone(e.__cause__)
+        self.assertFalse(e.__suppress_context__)
+        self.assertIsInstance(e.__context__, TypeError)
+
     def test_invalid_cause(self):
         try:
             raise IndexError from 5
@@ -171,6 +209,7 @@ class TestCause(unittest.TestCase):
 
 
 class TestTraceback(unittest.TestCase):
+
     def test_sets_traceback(self):
         try:
             raise IndexError()
diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py
index 97d8098704..c3ab7d2321 100644
--- a/Lib/test/test_random.py
+++ b/Lib/test/test_random.py
@@ -34,8 +34,12 @@ class TestBasicOps(unittest.TestCase):
         self.assertEqual(randseq, self.randomlist(N))
 
     def test_seedargs(self):
+        # Seed value with a negative hash.
+        class MySeed(object):
+            def __hash__(self):
+                return -1729
         for arg in [None, 0, 0, 1, 1, -1, -1, 10**20, -(10**20),
-                    3.14, 1+2j, 'a', tuple('abc')]:
+                    3.14, 1+2j, 'a', tuple('abc'), MySeed()]:
             self.gen.seed(arg)
         for arg in [list(range(3)), dict(one=1)]:
             self.assertRaises(TypeError, self.gen.seed, arg)
diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py
index 4dab448d5a..2a13bfeabd 100644
--- a/Lib/test/test_range.py
+++ b/Lib/test/test_range.py
@@ -350,13 +350,35 @@ class RangeTest(unittest.TestCase):
 
     def test_pickling(self):
         testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1),
-                     (13, 21, 3), (-2, 2, 2)]
+                     (13, 21, 3), (-2, 2, 2), (2**65, 2**65+2)]
         for proto in range(pickle.HIGHEST_PROTOCOL + 1):
             for t in testcases:
                 r = range(*t)
                 self.assertEqual(list(pickle.loads(pickle.dumps(r, proto))),
                                  list(r))
 
+    def test_iterator_pickling(self):
+        testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1),
+                     (13, 21, 3), (-2, 2, 2), (2**65, 2**65+2)]
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            for t in testcases:
+                it = itorg = iter(range(*t))
+                data = list(range(*t))
+
+                d = pickle.dumps(it)
+                it = pickle.loads(d)
+                self.assertEqual(type(itorg), type(it))
+                self.assertEqual(list(it), data)
+
+                it = pickle.loads(d)
+                try:
+                    next(it)
+                except StopIteration:
+                    continue
+                d = pickle.dumps(it)
+                it = pickle.loads(d)
+                self.assertEqual(list(it), data[1:])
+
     def test_odd_bug(self):
         # This used to raise a "SystemError: NULL result without error"
         # because the range validation step was eating the exception
@@ -516,6 +538,87 @@ class RangeTest(unittest.TestCase):
                 for k in values - {0}:
                     r[i:j:k]
 
+    def test_comparison(self):
+        test_ranges = [range(0), range(0, -1), range(1, 1, 3),
+                       range(1), range(5, 6), range(5, 6, 2),
+                       range(5, 7, 2), range(2), range(0, 4, 2),
+                       range(0, 5, 2), range(0, 6, 2)]
+        test_tuples = list(map(tuple, test_ranges))
+
+        # Check that equality of ranges matches equality of the corresponding
+        # tuples for each pair from the test lists above.
+        ranges_eq = [a == b for a in test_ranges for b in test_ranges]
+        tuples_eq = [a == b for a in test_tuples for b in test_tuples]
+        self.assertEqual(ranges_eq, tuples_eq)
+
+        # Check that != correctly gives the logical negation of ==
+        ranges_ne = [a != b for a in test_ranges for b in test_ranges]
+        self.assertEqual(ranges_ne, [not x for x in ranges_eq])
+
+        # Equal ranges should have equal hashes.
+        for a in test_ranges:
+            for b in test_ranges:
+                if a == b:
+                    self.assertEqual(hash(a), hash(b))
+
+        # Ranges are unequal to other types (even sequence types)
+        self.assertIs(range(0) == (), False)
+        self.assertIs(() == range(0), False)
+        self.assertIs(range(2) == [0, 1], False)
+
+        # Huge integers aren't a problem.
+        self.assertEqual(range(0, 2**100 - 1, 2),
+                         range(0, 2**100, 2))
+        self.assertEqual(hash(range(0, 2**100 - 1, 2)),
+                         hash(range(0, 2**100, 2)))
+        self.assertNotEqual(range(0, 2**100, 2),
+                            range(0, 2**100 + 1, 2))
+        self.assertEqual(range(2**200, 2**201 - 2**99, 2**100),
+                         range(2**200, 2**201, 2**100))
+        self.assertEqual(hash(range(2**200, 2**201 - 2**99, 2**100)),
+                         hash(range(2**200, 2**201, 2**100)))
+        self.assertNotEqual(range(2**200, 2**201, 2**100),
+                            range(2**200, 2**201 + 1, 2**100))
+
+        # Order comparisons are not implemented for ranges.
+        with self.assertRaises(TypeError):
+            range(0) < range(0)
+        with self.assertRaises(TypeError):
+            range(0) > range(0)
+        with self.assertRaises(TypeError):
+            range(0) <= range(0)
+        with self.assertRaises(TypeError):
+            range(0) >= range(0)
+
+
+    def test_attributes(self):
+        # test the start, stop and step attributes of range objects
+        self.assert_attrs(range(0), 0, 0, 1)
+        self.assert_attrs(range(10), 0, 10, 1)
+        self.assert_attrs(range(-10), 0, -10, 1)
+        self.assert_attrs(range(0, 10, 1), 0, 10, 1)
+        self.assert_attrs(range(0, 10, 3), 0, 10, 3)
+        self.assert_attrs(range(10, 0, -1), 10, 0, -1)
+        self.assert_attrs(range(10, 0, -3), 10, 0, -3)
+
+    def assert_attrs(self, rangeobj, start, stop, step):
+        self.assertEqual(rangeobj.start, start)
+        self.assertEqual(rangeobj.stop, stop)
+        self.assertEqual(rangeobj.step, step)
+
+        with self.assertRaises(AttributeError):
+            rangeobj.start = 0
+        with self.assertRaises(AttributeError):
+            rangeobj.stop = 10
+        with self.assertRaises(AttributeError):
+            rangeobj.step = 1
+
+        with self.assertRaises(AttributeError):
+            del rangeobj.start
+        with self.assertRaises(AttributeError):
+            del rangeobj.stop
+        with self.assertRaises(AttributeError):
+            del rangeobj.step
 
 def test_main():
     test.support.run_unittest(RangeTest)
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
index 0c95f4e6ae..ef19164ed8 100644
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -8,9 +8,6 @@ import string
 import traceback
 from weakref import proxy
 
-from test.test_bigmem import character_size
-
-
 # Misc tests from Tim Peters' re.doc
 
 # WARNING: Don't change details in these tests if you don't know
@@ -496,7 +493,7 @@ class ReTests(unittest.TestCase):
         self.assertEqual(m.span(), span)
 
     def test_re_escape(self):
-        alnum_chars = string.ascii_letters + string.digits
+        alnum_chars = string.ascii_letters + string.digits + '_'
         p = ''.join(chr(i) for i in range(256))
         for c in p:
             if c in alnum_chars:
@@ -509,7 +506,7 @@ class ReTests(unittest.TestCase):
         self.assertMatch(re.escape(p), p)
 
     def test_re_escape_byte(self):
-        alnum_chars = (string.ascii_letters + string.digits).encode('ascii')
+        alnum_chars = (string.ascii_letters + string.digits + '_').encode('ascii')
         p = bytes(range(256))
         for i in p:
             b = bytes([i])
@@ -556,24 +553,93 @@ class ReTests(unittest.TestCase):
             self.assertNotEqual(re.compile('^pattern$', flag), None)
 
     def test_sre_character_literals(self):
-        for i in [0, 8, 16, 32, 64, 127, 128, 255]:
-            self.assertNotEqual(re.match(r"\%03o" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"\%03o0" % i, chr(i)+"0"), None)
-            self.assertNotEqual(re.match(r"\%03o8" % i, chr(i)+"8"), None)
-            self.assertNotEqual(re.match(r"\x%02x" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"\x%02x0" % i, chr(i)+"0"), None)
-            self.assertNotEqual(re.match(r"\x%02xz" % i, chr(i)+"z"), None)
-        self.assertRaises(re.error, re.match, "\911", "")
+        for i in [0, 8, 16, 32, 64, 127, 128, 255, 256, 0xFFFF, 0x10000, 0x10FFFF]:
+            if i < 256:
+                self.assertIsNotNone(re.match(r"\%03o" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"\%03o0" % i, chr(i)+"0"))
+                self.assertIsNotNone(re.match(r"\%03o8" % i, chr(i)+"8"))
+                self.assertIsNotNone(re.match(r"\x%02x" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"\x%02x0" % i, chr(i)+"0"))
+                self.assertIsNotNone(re.match(r"\x%02xz" % i, chr(i)+"z"))
+            if i < 0x10000:
+                self.assertIsNotNone(re.match(r"\u%04x" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"\u%04x0" % i, chr(i)+"0"))
+                self.assertIsNotNone(re.match(r"\u%04xz" % i, chr(i)+"z"))
+            self.assertIsNotNone(re.match(r"\U%08x" % i, chr(i)))
+            self.assertIsNotNone(re.match(r"\U%08x0" % i, chr(i)+"0"))
+            self.assertIsNotNone(re.match(r"\U%08xz" % i, chr(i)+"z"))
+        self.assertIsNotNone(re.match(r"\0", "\000"))
+        self.assertIsNotNone(re.match(r"\08", "\0008"))
+        self.assertIsNotNone(re.match(r"\01", "\001"))
+        self.assertIsNotNone(re.match(r"\018", "\0018"))
+        self.assertIsNotNone(re.match(r"\567", chr(0o167)))
+        self.assertRaises(re.error, re.match, r"\911", "")
+        self.assertRaises(re.error, re.match, r"\x1", "")
+        self.assertRaises(re.error, re.match, r"\x1z", "")
+        self.assertRaises(re.error, re.match, r"\u123", "")
+        self.assertRaises(re.error, re.match, r"\u123z", "")
+        self.assertRaises(re.error, re.match, r"\U0001234", "")
+        self.assertRaises(re.error, re.match, r"\U0001234z", "")
+        self.assertRaises(re.error, re.match, r"\U00110000", "")
 
     def test_sre_character_class_literals(self):
+        for i in [0, 8, 16, 32, 64, 127, 128, 255, 256, 0xFFFF, 0x10000, 0x10FFFF]:
+            if i < 256:
+                self.assertIsNotNone(re.match(r"[\%o]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\%o8]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\%03o]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\%03o0]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\%03o8]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\x%02x]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\x%02x0]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\x%02xz]" % i, chr(i)))
+            if i < 0x10000:
+                self.assertIsNotNone(re.match(r"[\u%04x]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\u%04x0]" % i, chr(i)))
+                self.assertIsNotNone(re.match(r"[\u%04xz]" % i, chr(i)))
+            self.assertIsNotNone(re.match(r"[\U%08x]" % i, chr(i)))
+            self.assertIsNotNone(re.match(r"[\U%08x0]" % i, chr(i)+"0"))
+            self.assertIsNotNone(re.match(r"[\U%08xz]" % i, chr(i)+"z"))
+        self.assertIsNotNone(re.match(r"[\U0001d49c-\U0001d4b5]", "\U0001d49e"))
+        self.assertRaises(re.error, re.match, r"[\911]", "")
+        self.assertRaises(re.error, re.match, r"[\x1z]", "")
+        self.assertRaises(re.error, re.match, r"[\u123z]", "")
+        self.assertRaises(re.error, re.match, r"[\U0001234z]", "")
+        self.assertRaises(re.error, re.match, r"[\U00110000]", "")
+
+    def test_sre_byte_literals(self):
+        for i in [0, 8, 16, 32, 64, 127, 128, 255]:
+            self.assertIsNotNone(re.match((r"\%03o" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"\%03o0" % i).encode(), bytes([i])+b"0"))
+            self.assertIsNotNone(re.match((r"\%03o8" % i).encode(), bytes([i])+b"8"))
+            self.assertIsNotNone(re.match((r"\x%02x" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0"))
+            self.assertIsNotNone(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z"))
+        self.assertIsNotNone(re.match(br"\u", b'u'))
+        self.assertIsNotNone(re.match(br"\U", b'U'))
+        self.assertIsNotNone(re.match(br"\0", b"\000"))
+        self.assertIsNotNone(re.match(br"\08", b"\0008"))
+        self.assertIsNotNone(re.match(br"\01", b"\001"))
+        self.assertIsNotNone(re.match(br"\018", b"\0018"))
+        self.assertIsNotNone(re.match(br"\567", bytes([0o167])))
+        self.assertRaises(re.error, re.match, br"\911", b"")
+        self.assertRaises(re.error, re.match, br"\x1", b"")
+        self.assertRaises(re.error, re.match, br"\x1z", b"")
+
+    def test_sre_byte_class_literals(self):
         for i in [0, 8, 16, 32, 64, 127, 128, 255]:
-            self.assertNotEqual(re.match(r"[\%03o]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\%03o0]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\%03o8]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\x%02x]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\x%02x0]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\x%02xz]" % i, chr(i)), None)
-        self.assertRaises(re.error, re.match, "[\911]", "")
+            self.assertIsNotNone(re.match((r"[\%o]" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"[\%o8]" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"[\%03o]" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"[\%03o0]" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"[\%03o8]" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"[\x%02x]" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"[\x%02x0]" % i).encode(), bytes([i])))
+            self.assertIsNotNone(re.match((r"[\x%02xz]" % i).encode(), bytes([i])))
+        self.assertIsNotNone(re.match(br"[\u]", b'u'))
+        self.assertIsNotNone(re.match(br"[\U]", b'U'))
+        self.assertRaises(re.error, re.match, br"[\911]", "")
+        self.assertRaises(re.error, re.match, br"[\x1z]", "")
 
     def test_bug_113254(self):
         self.assertEqual(re.match(r'(a)|(b)', 'b').start(1), -1)
@@ -691,6 +757,26 @@ class ReTests(unittest.TestCase):
         self.assertEqual([item.group(0) for item in iter],
                          [":", "::", ":::"])
 
+        pat = re.compile(r":+")
+        iter = pat.finditer("a:b::c:::d", 1, 10)
+        self.assertEqual([item.group(0) for item in iter],
+                         [":", "::", ":::"])
+
+        pat = re.compile(r":+")
+        iter = pat.finditer("a:b::c:::d", pos=1, endpos=10)
+        self.assertEqual([item.group(0) for item in iter],
+                         [":", "::", ":::"])
+
+        pat = re.compile(r":+")
+        iter = pat.finditer("a:b::c:::d", endpos=10, pos=1)
+        self.assertEqual([item.group(0) for item in iter],
+                         [":", "::", ":::"])
+
+        pat = re.compile(r":+")
+        iter = pat.finditer("a:b::c:::d", pos=3, endpos=8)
+        self.assertEqual([item.group(0) for item in iter],
+                         ["::", "::"])
+
     def test_bug_926075(self):
         self.assertTrue(re.compile('bug_926075') is not
                      re.compile(b'bug_926075'))
@@ -857,6 +943,13 @@ class ReTests(unittest.TestCase):
         self.assertRaises(OverflowError, _sre.compile, "abc", 0, [long_overflow])
         self.assertRaises(TypeError, _sre.compile, {}, 0, [])
 
+    def test_search_dot_unicode(self):
+        self.assertIsNotNone(re.search("123.*-", '123abc-'))
+        self.assertIsNotNone(re.search("123.*-", '123\xe9-'))
+        self.assertIsNotNone(re.search("123.*-", '123\u20ac-'))
+        self.assertIsNotNone(re.search("123.*-", '123\U0010ffff-'))
+        self.assertIsNotNone(re.search("123.*-", '123\xe9\u20ac\U0010ffff-'))
+
     def test_compile(self):
         # Test return value when given string and pattern as parameter
         pattern = re.compile('random pattern')
@@ -873,7 +966,7 @@ class ReTests(unittest.TestCase):
         self.assertEqual(re.findall(r'[\A\B\b\C\Z]', 'AB\bCZ'),
                          ['A', 'B', '\b', 'C', 'Z'])
 
-    @bigmemtest(size=_2G, memuse=character_size)
+    @bigmemtest(size=_2G, memuse=1)
     def test_large_search(self, size):
         # Issue #10182: indices were 32-bit-truncated.
         s = 'a' * size
@@ -884,7 +977,7 @@ class ReTests(unittest.TestCase):
 
     # The huge memuse is because of re.sub() using a list and a join()
     # to create the replacement result.
-    @bigmemtest(size=_2G, memuse=16 + 2 * character_size)
+    @bigmemtest(size=_2G, memuse=16 + 2)
     def test_large_subn(self, size):
         # Issue #10182: indices were 32-bit-truncated.
         s = 'a' * size
@@ -892,6 +985,11 @@ class ReTests(unittest.TestCase):
         self.assertEqual(r, s)
         self.assertEqual(n, size + 1)
 
+    def test_bug_16688(self):
+        # Issue 16688: Backreferences make case-insensitive regex fail on
+        # non-ASCII strings.
+        self.assertEqual(re.findall(r"(?i)(a)\1", "aa \u0100"), ['a'])
+        self.assertEqual(re.match(r"(?s).{1,3}", "\u0100\u0100").span(), (0, 2))
 
     def test_repeat_minmax_overflow(self):
         # Issue #13169
diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py
index b0dc4d7813..589ecdd4da 100644
--- a/Lib/test/test_reprlib.py
+++ b/Lib/test/test_reprlib.py
@@ -3,12 +3,14 @@
   Nick Mathewson
 """
 
+import imp
 import sys
 import os
 import shutil
+import importlib
 import unittest
 
-from test.support import run_unittest
+from test.support import run_unittest, create_empty_file, verbose
 from reprlib import repr as r # Don't shadow builtin repr
 from reprlib import Repr
 from reprlib import recursive_repr
@@ -129,8 +131,8 @@ class ReprTests(unittest.TestCase):
         self.assertIn(s.find("..."), [12, 13])
 
     def test_lambda(self):
-        self.assertTrue(repr(lambda x: x).startswith(
-            "<function <lambda"))
+        r = repr(lambda x: x)
+        self.assertTrue(r.startswith("<function ReprTests.test_lambda.<locals>.<lambda"), r)
         # XXX anonymous functions?  see func_repr
 
     def test_builtin_function(self):
@@ -193,26 +195,29 @@ class ReprTests(unittest.TestCase):
         r(y)
         r(z)
 
-def touch(path, text=''):
-    fp = open(path, 'w')
-    fp.write(text)
-    fp.close()
+def write_file(path, text):
+    with open(path, 'w', encoding='ASCII') as fp:
+        fp.write(text)
 
 class LongReprTest(unittest.TestCase):
+    longname = 'areallylongpackageandmodulenametotestreprtruncation'
+
     def setUp(self):
-        longname = 'areallylongpackageandmodulenametotestreprtruncation'
-        self.pkgname = os.path.join(longname)
-        self.subpkgname = os.path.join(longname, longname)
+        self.pkgname = os.path.join(self.longname)
+        self.subpkgname = os.path.join(self.longname, self.longname)
         # Make the package and subpackage
         shutil.rmtree(self.pkgname, ignore_errors=True)
         os.mkdir(self.pkgname)
-        touch(os.path.join(self.pkgname, '__init__.py'))
+        create_empty_file(os.path.join(self.pkgname, '__init__.py'))
         shutil.rmtree(self.subpkgname, ignore_errors=True)
         os.mkdir(self.subpkgname)
-        touch(os.path.join(self.subpkgname, '__init__.py'))
+        create_empty_file(os.path.join(self.subpkgname, '__init__.py'))
         # Remember where we are
         self.here = os.getcwd()
         sys.path.insert(0, self.here)
+        # When regrtest is run with its -j option, this command alone is not
+        # enough.
+        importlib.invalidate_caches()
 
     def tearDown(self):
         actions = []
@@ -229,20 +234,40 @@ class LongReprTest(unittest.TestCase):
                 os.remove(p)
         del sys.path[0]
 
+    def _check_path_limitations(self, module_name):
+        # base directory
+        source_path_len = len(self.here)
+        # a path separator + `longname` (twice)
+        source_path_len += 2 * (len(self.longname) + 1)
+        # a path separator + `module_name` + ".py"
+        source_path_len += len(module_name) + 1 + len(".py")
+        cached_path_len = source_path_len + len(imp.cache_from_source("x.py")) - len("x.py")
+        if os.name == 'nt' and cached_path_len >= 258:
+            # Under Windows, the max path len is 260 including C's terminating
+            # NUL character.
+            # (see http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx#maxpath)
+            self.skipTest("test paths too long (%d characters) for Windows' 260 character limit"
+                          % cached_path_len)
+        elif os.name == 'nt' and verbose:
+            print("cached_path_len =", cached_path_len)
+
     def test_module(self):
-        eq = self.assertEqual
-        touch(os.path.join(self.subpkgname, self.pkgname + '.py'))
+        self._check_path_limitations(self.pkgname)
+        create_empty_file(os.path.join(self.subpkgname, self.pkgname + '.py'))
+        importlib.invalidate_caches()
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation
-        eq(repr(areallylongpackageandmodulenametotestreprtruncation),
-           "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
-        eq(repr(sys), "<module 'sys' (built-in)>")
+        module = areallylongpackageandmodulenametotestreprtruncation
+        self.assertEqual(repr(module), "<module %r from %r>" % (module.__name__, module.__file__))
+        self.assertEqual(repr(sys), "<module 'sys' (built-in)>")
 
     def test_type(self):
+        self._check_path_limitations('foo')
         eq = self.assertEqual
-        touch(os.path.join(self.subpkgname, 'foo.py'), '''\
+        write_file(os.path.join(self.subpkgname, 'foo.py'), '''\
 class foo(object):
     pass
 ''')
+        importlib.invalidate_caches()
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import foo
         eq(repr(foo.foo),
                "<class '%s.foo'>" % foo.__name__)
@@ -253,39 +278,46 @@ class foo(object):
         pass
 
     def test_class(self):
-        touch(os.path.join(self.subpkgname, 'bar.py'), '''\
+        self._check_path_limitations('bar')
+        write_file(os.path.join(self.subpkgname, 'bar.py'), '''\
 class bar:
     pass
 ''')
+        importlib.invalidate_caches()
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import bar
         # Module name may be prefixed with "test.", depending on how run.
         self.assertEqual(repr(bar.bar), "<class '%s.bar'>" % bar.__name__)
 
     def test_instance(self):
-        touch(os.path.join(self.subpkgname, 'baz.py'), '''\
+        self._check_path_limitations('baz')
+        write_file(os.path.join(self.subpkgname, 'baz.py'), '''\
 class baz:
     pass
 ''')
+        importlib.invalidate_caches()
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import baz
         ibaz = baz.baz()
         self.assertTrue(repr(ibaz).startswith(
             "<%s.baz object at 0x" % baz.__name__))
 
     def test_method(self):
+        self._check_path_limitations('qux')
         eq = self.assertEqual
-        touch(os.path.join(self.subpkgname, 'qux.py'), '''\
+        write_file(os.path.join(self.subpkgname, 'qux.py'), '''\
 class aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:
     def amethod(self): pass
 ''')
+        importlib.invalidate_caches()
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import qux
         # Unbound methods first
-        self.assertTrue(repr(qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod).startswith(
-        '<function amethod'))
+        r = repr(qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod)
+        self.assertTrue(r.startswith('<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod'), r)
         # Bound method next
         iqux = qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa()
-        self.assertTrue(repr(iqux.amethod).startswith(
+        r = repr(iqux.amethod)
+        self.assertTrue(r.startswith(
             '<bound method aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod of <%s.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa object at 0x' \
-            % (qux.__name__,) ))
+            % (qux.__name__,) ), r)
 
     def test_builtin_function(self):
         # XXX test built-in functions and methods with really long names
diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py
index f8f3717b64..0b629dc5bc 100644
--- a/Lib/test/test_richcmp.py
+++ b/Lib/test/test_richcmp.py
@@ -220,6 +220,7 @@ class MiscTest(unittest.TestCase):
         for func in (do, operator.not_):
             self.assertRaises(Exc, func, Bad())
 
+    @support.no_tracing
     def test_recursion(self):
         # Check that comparison for recursive objects fails gracefully
         from collections import UserList
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index 96d2bebbc4..fefefc4285 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -5,11 +5,15 @@ import os.path
 import sys
 import re
 import tempfile
+import importlib
 import py_compile
-from test.support import forget, make_legacy_pyc, run_unittest, unload, verbose
+from test.support import (
+    forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
+    create_empty_file)
 from test.script_helper import (
     make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
 
+
 import runpy
 from runpy import _run_code, _run_module_code, run_module, run_path
 # Note: This module can't safely test _run_module_as_main as it
@@ -145,7 +149,7 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin):
                                     mod_package)
         self.check_code_execution(create_ns, expected_ns)
 
-
+# TODO: Use self.addCleanup to get rid of a lot of try-finally blocks
 class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
     """Unit tests for runpy.run_module"""
 
@@ -175,8 +179,7 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
     def _add_pkg_dir(self, pkg_dir):
         os.mkdir(pkg_dir)
         pkg_fname = os.path.join(pkg_dir, "__init__.py")
-        pkg_file = open(pkg_fname, "w")
-        pkg_file.close()
+        create_empty_file(pkg_fname)
         return pkg_fname
 
     def _make_pkg(self, source, depth, mod_base="runpy_test"):
@@ -252,10 +255,12 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
         try:
             if verbose > 1: print("Running from source:", mod_name)
             self.check_code_execution(create_ns, expected_ns)
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
+            importlib.invalidate_caches()
             if verbose > 1: print("Running from compiled:", mod_name)
             self._fix_ns_for_legacy_pyc(expected_ns, alter_sys)
             self.check_code_execution(create_ns, expected_ns)
@@ -285,11 +290,13 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
         try:
             if verbose > 1: print("Running from source:", pkg_name)
             self.check_code_execution(create_ns, expected_ns)
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
             if verbose > 1: print("Running from compiled:", pkg_name)
+            importlib.invalidate_caches()
             self._fix_ns_for_legacy_pyc(expected_ns, alter_sys)
             self.check_code_execution(create_ns, expected_ns)
         finally:
@@ -306,8 +313,7 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
             module_dir = os.path.join(module_dir, pkg_name)
         # Add sibling module
         sibling_fname = os.path.join(module_dir, "sibling.py")
-        sibling_file = open(sibling_fname, "w")
-        sibling_file.close()
+        create_empty_file(sibling_fname)
         if verbose > 1: print("  Added sibling module:", sibling_fname)
         # Add nephew module
         uncle_dir = os.path.join(parent_dir, "uncle")
@@ -317,8 +323,7 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
         self._add_pkg_dir(cousin_dir)
         if verbose > 1: print("  Added cousin package:", cousin_dir)
         nephew_fname = os.path.join(cousin_dir, "nephew.py")
-        nephew_file = open(nephew_fname, "w")
-        nephew_file.close()
+        create_empty_file(nephew_fname)
         if verbose > 1: print("  Added nephew module:", nephew_fname)
 
     def _check_relative_imports(self, depth, run_name=None):
@@ -343,11 +348,13 @@ from ..uncle.cousin import nephew
             self.assertIn("sibling", d1)
             self.assertIn("nephew", d1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case the loader caches paths
             if verbose > 1: print("Running from compiled:", mod_name)
+            importlib.invalidate_caches()
             d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
             self.assertEqual(d2["__name__"], expected_name)
             self.assertEqual(d2["__package__"], pkg_name)
@@ -407,6 +414,40 @@ from ..uncle.cousin import nephew
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
 
+    def test_pkgutil_walk_packages(self):
+        # This is a dodgy hack to use the test_runpy infrastructure to test
+        # issue #15343. Issue #15348 declares this is indeed a dodgy hack ;)
+        import pkgutil
+        max_depth = 4
+        base_name = "__runpy_pkg__"
+        package_suffixes = ["uncle", "uncle.cousin"]
+        module_suffixes = ["uncle.cousin.nephew", base_name + ".sibling"]
+        expected_packages = set()
+        expected_modules = set()
+        for depth in range(1, max_depth):
+            pkg_name = ".".join([base_name] * depth)
+            expected_packages.add(pkg_name)
+            for name in package_suffixes:
+                expected_packages.add(pkg_name + "." + name)
+            for name in module_suffixes:
+                expected_modules.add(pkg_name + "." + name)
+        pkg_name = ".".join([base_name] * max_depth)
+        expected_packages.add(pkg_name)
+        expected_modules.add(pkg_name + ".runpy_test")
+        pkg_dir, mod_fname, mod_name = (
+               self._make_pkg("", max_depth))
+        self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name)
+        for depth in range(2, max_depth+1):
+            self._add_relative_modules(pkg_dir, "", depth)
+        for finder, mod_name, ispkg in pkgutil.walk_packages([pkg_dir]):
+            self.assertIsInstance(finder,
+                                  importlib.machinery.FileFinder)
+            if ispkg:
+                expected_packages.remove(mod_name)
+            else:
+                expected_modules.remove(mod_name)
+        self.assertEqual(len(expected_packages), 0, expected_packages)
+        self.assertEqual(len(expected_modules), 0, expected_modules)
 
 class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
     """Unit tests for runpy.run_path"""
@@ -507,6 +548,7 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             msg = "can't find '__main__' module in %r" % zip_name
             self._check_import_error(zip_name, msg)
 
+    @no_tracing
     def test_main_recursion_error(self):
         with temp_dir() as script_dir, temp_dir() as dummy_dir:
             mod_name = '__main__'
diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py
index bfa7d72d16..05f66fdca1 100644
--- a/Lib/test/test_sax.py
+++ b/Lib/test/test_sax.py
@@ -23,8 +23,8 @@ import unittest
 TEST_XMLFILE = findfile("test.xml", subdir="xmltestdata")
 TEST_XMLFILE_OUT = findfile("test.xml.out", subdir="xmltestdata")
 try:
-    TEST_XMLFILE.encode("utf8")
-    TEST_XMLFILE_OUT.encode("utf8")
+    TEST_XMLFILE.encode("utf-8")
+    TEST_XMLFILE_OUT.encode("utf-8")
 except UnicodeEncodeError:
     raise unittest.SkipTest("filename is not encodable to utf8")
 
diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py
index 91b8f0c939..1fe6ad442c 100644
--- a/Lib/test/test_sched.py
+++ b/Lib/test/test_sched.py
@@ -1,9 +1,44 @@
 #!/usr/bin/env python
 
+import queue
 import sched
 import time
 import unittest
 from test import support
+try:
+    import threading
+except ImportError:
+    threading = None
+
+TIMEOUT = 10
+
+
+class Timer:
+    def __init__(self):
+        self._cond = threading.Condition()
+        self._time = 0
+        self._stop = 0
+
+    def time(self):
+        with self._cond:
+            return self._time
+
+    # increase the time but not beyond the established limit
+    def sleep(self, t):
+        assert t >= 0
+        with self._cond:
+            t += self._time
+            while self._stop < t:
+                self._time = self._stop
+                self._cond.wait()
+            self._time = t
+
+    # advance time limit for user code
+    def advance(self, t):
+        assert t >= 0
+        with self._cond:
+            self._stop += t
+            self._cond.notify_all()
 
 
 class TestCase(unittest.TestCase):
@@ -26,6 +61,37 @@ class TestCase(unittest.TestCase):
         scheduler.run()
         self.assertEqual(l, [0.01, 0.02, 0.03, 0.04, 0.05])
 
+    @unittest.skipUnless(threading, 'Threading required for this test.')
+    def test_enter_concurrent(self):
+        q = queue.Queue()
+        fun = q.put
+        timer = Timer()
+        scheduler = sched.scheduler(timer.time, timer.sleep)
+        scheduler.enter(1, 1, fun, (1,))
+        scheduler.enter(3, 1, fun, (3,))
+        t = threading.Thread(target=scheduler.run)
+        t.start()
+        timer.advance(1)
+        self.assertEqual(q.get(timeout=TIMEOUT), 1)
+        self.assertTrue(q.empty())
+        for x in [4, 5, 2]:
+            z = scheduler.enter(x - 1, 1, fun, (x,))
+        timer.advance(2)
+        self.assertEqual(q.get(timeout=TIMEOUT), 2)
+        self.assertEqual(q.get(timeout=TIMEOUT), 3)
+        self.assertTrue(q.empty())
+        timer.advance(1)
+        self.assertEqual(q.get(timeout=TIMEOUT), 4)
+        self.assertTrue(q.empty())
+        timer.advance(1)
+        self.assertEqual(q.get(timeout=TIMEOUT), 5)
+        self.assertTrue(q.empty())
+        timer.advance(1000)
+        t.join(timeout=TIMEOUT)
+        self.assertFalse(t.is_alive())
+        self.assertTrue(q.empty())
+        self.assertEqual(timer.time(), 5)
+
     def test_priority(self):
         l = []
         fun = lambda x: l.append(x)
@@ -50,6 +116,39 @@ class TestCase(unittest.TestCase):
         scheduler.run()
         self.assertEqual(l, [0.02, 0.03, 0.04])
 
+    @unittest.skipUnless(threading, 'Threading required for this test.')
+    def test_cancel_concurrent(self):
+        q = queue.Queue()
+        fun = q.put
+        timer = Timer()
+        scheduler = sched.scheduler(timer.time, timer.sleep)
+        now = timer.time()
+        event1 = scheduler.enterabs(now + 1, 1, fun, (1,))
+        event2 = scheduler.enterabs(now + 2, 1, fun, (2,))
+        event4 = scheduler.enterabs(now + 4, 1, fun, (4,))
+        event5 = scheduler.enterabs(now + 5, 1, fun, (5,))
+        event3 = scheduler.enterabs(now + 3, 1, fun, (3,))
+        t = threading.Thread(target=scheduler.run)
+        t.start()
+        timer.advance(1)
+        self.assertEqual(q.get(timeout=TIMEOUT), 1)
+        self.assertTrue(q.empty())
+        scheduler.cancel(event2)
+        scheduler.cancel(event5)
+        timer.advance(1)
+        self.assertTrue(q.empty())
+        timer.advance(1)
+        self.assertEqual(q.get(timeout=TIMEOUT), 3)
+        self.assertTrue(q.empty())
+        timer.advance(1)
+        self.assertEqual(q.get(timeout=TIMEOUT), 4)
+        self.assertTrue(q.empty())
+        timer.advance(1000)
+        t.join(timeout=TIMEOUT)
+        self.assertFalse(t.is_alive())
+        self.assertTrue(q.empty())
+        self.assertEqual(timer.time(), 4)
+
     def test_empty(self):
         l = []
         fun = lambda x: l.append(x)
@@ -63,15 +162,39 @@ class TestCase(unittest.TestCase):
 
     def test_queue(self):
         l = []
-        events = []
         fun = lambda x: l.append(x)
         scheduler = sched.scheduler(time.time, time.sleep)
-        self.assertEqual(scheduler._queue, [])
-        for x in [0.05, 0.04, 0.03, 0.02, 0.01]:
-            events.append(scheduler.enterabs(x, 1, fun, (x,)))
-        self.assertEqual(scheduler._queue.sort(), events.sort())
+        now = time.time()
+        e5 = scheduler.enterabs(now + 0.05, 1, fun)
+        e1 = scheduler.enterabs(now + 0.01, 1, fun)
+        e2 = scheduler.enterabs(now + 0.02, 1, fun)
+        e4 = scheduler.enterabs(now + 0.04, 1, fun)
+        e3 = scheduler.enterabs(now + 0.03, 1, fun)
+        # queue property is supposed to return an order list of
+        # upcoming events
+        self.assertEqual(list(scheduler.queue), [e1, e2, e3, e4, e5])
+
+    def test_args_kwargs(self):
+        flag = []
+
+        def fun(*a, **b):
+            flag.append(None)
+            self.assertEqual(a, (1,2,3))
+            self.assertEqual(b, {"foo":1})
+
+        scheduler = sched.scheduler(time.time, time.sleep)
+        z = scheduler.enterabs(0.01, 1, fun, argument=(1,2,3), kwargs={"foo":1})
         scheduler.run()
-        self.assertEqual(scheduler._queue, [])
+        self.assertEqual(flag, [None])
+
+    def test_run_non_blocking(self):
+        l = []
+        fun = lambda x: l.append(x)
+        scheduler = sched.scheduler(time.time, time.sleep)
+        for x in [10, 9, 8, 7, 6]:
+            scheduler.enter(x, 1, fun, (x,))
+        scheduler.run(blocking=False)
+        self.assertEqual(l, [])
 
 
 def test_main():
diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py
index fbc87aa80c..129a18aade 100644
--- a/Lib/test/test_scope.py
+++ b/Lib/test/test_scope.py
@@ -1,5 +1,5 @@
 import unittest
-from test.support import check_syntax_error, run_unittest
+from test.support import check_syntax_error, cpython_only, run_unittest
 
 
 class ScopeTests(unittest.TestCase):
@@ -496,23 +496,22 @@ class ScopeTests(unittest.TestCase):
         self.assertNotIn("x", varnames)
         self.assertIn("y", varnames)
 
+    @cpython_only
     def testLocalsClass_WithTrace(self):
         # Issue23728: after the trace function returns, the locals()
         # dictionary is used to update all variables, this used to
         # include free variables. But in class statements, free
         # variables are not inserted...
         import sys
+        self.addCleanup(sys.settrace, sys.gettrace())
         sys.settrace(lambda a,b,c:None)
-        try:
-            x = 12
+        x = 12
 
-            class C:
-                def f(self):
-                    return x
+        class C:
+            def f(self):
+                return x
 
-            self.assertEqual(x, 12) # Used to raise UnboundLocalError
-        finally:
-            sys.settrace(None)
+        self.assertEqual(x, 12) # Used to raise UnboundLocalError
 
     def testBoundAndFree(self):
         # var is bound and free in class
@@ -527,6 +526,7 @@ class ScopeTests(unittest.TestCase):
         inst = f(3)()
         self.assertEqual(inst.a, inst.m())
 
+    @cpython_only
     def testInteractionWithTraceFunc(self):
 
         import sys
@@ -543,6 +543,7 @@ class ScopeTests(unittest.TestCase):
         class TestClass:
             pass
 
+        self.addCleanup(sys.settrace, sys.gettrace())
         sys.settrace(tracer)
         adaptgetter("foo", TestClass, (1, ""))
         sys.settrace(None)
diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py
index 4ddc2176ca..ddb9a0f67e 100644
--- a/Lib/test/test_select.py
+++ b/Lib/test/test_select.py
@@ -1,8 +1,9 @@
-from test import support
-import unittest
-import select
+import errno
 import os
+import select
 import sys
+import unittest
+from test import support
 
 @unittest.skipIf(sys.platform[:3] in ('win', 'os2', 'riscos'),
                  "can't easily test on this system")
@@ -20,6 +21,21 @@ class SelectTestCase(unittest.TestCase):
         self.assertRaises(TypeError, select.select, [self.Nope()], [], [])
         self.assertRaises(TypeError, select.select, [self.Almost()], [], [])
         self.assertRaises(TypeError, select.select, [], [], [], "not a number")
+        self.assertRaises(ValueError, select.select, [], [], [], -1)
+
+    # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606
+    @unittest.skipIf(sys.platform.startswith('freebsd'),
+                     'skip because of a FreeBSD bug: kern/155606')
+    def test_errno(self):
+        with open(__file__, 'rb') as fp:
+            fd = fp.fileno()
+            fp.close()
+            try:
+                select.select([fd], [], [], 0)
+            except select.error as err:
+                self.assertEqual(err.errno, errno.EBADF)
+            else:
+                self.fail("exception not raised")
 
     def test_returned_list_identity(self):
         # See issue #8329
diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py
index 6642440dea..8e9e5878b2 100644
--- a/Lib/test/test_set.py
+++ b/Lib/test/test_set.py
@@ -9,6 +9,7 @@ from random import randrange, shuffle
 import sys
 import warnings
 import collections
+import collections.abc
 
 class PassThru(Exception):
     pass
@@ -234,6 +235,26 @@ class TestJointOps(unittest.TestCase):
                 dup = pickle.loads(p)
                 self.assertEqual(self.s.x, dup.x)
 
+    def test_iterator_pickling(self):
+        itorg = iter(self.s)
+        data = self.thetype(self.s)
+        d = pickle.dumps(itorg)
+        it = pickle.loads(d)
+        # Set iterators unpickle as list iterators due to the
+        # undefined order of set items.
+        # self.assertEqual(type(itorg), type(it))
+        self.assertTrue(isinstance(it, collections.abc.Iterator))
+        self.assertEqual(self.thetype(it), data)
+
+        it = pickle.loads(d)
+        try:
+            drop = next(it)
+        except StopIteration:
+            return
+        d = pickle.dumps(it)
+        it = pickle.loads(d)
+        self.assertEqual(self.thetype(it), data - self.thetype((drop,)))
+
     def test_deepcopy(self):
         class Tracer:
             def __init__(self, value):
diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py
index 3e73f52b87..13c126566d 100644
--- a/Lib/test/test_shelve.py
+++ b/Lib/test/test_shelve.py
@@ -2,7 +2,7 @@ import unittest
 import shelve
 import glob
 from test import support
-from collections import MutableMapping
+from collections.abc import MutableMapping
 from test.test_dbm import dbm_iterator
 
 def L1(s):
@@ -129,8 +129,8 @@ class TestCase(unittest.TestCase):
         shelve.Shelf(d)[key] = [1]
         self.assertIn(key.encode('utf-8'), d)
         # but a different one can be given
-        shelve.Shelf(d, keyencoding='latin1')[key] = [1]
-        self.assertIn(key.encode('latin1'), d)
+        shelve.Shelf(d, keyencoding='latin-1')[key] = [1]
+        self.assertIn(key.encode('latin-1'), d)
         # with all consequences
         s = shelve.Shelf(d, keyencoding='ascii')
         self.assertRaises(UnicodeEncodeError, s.__setitem__, key, [1])
diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py
index 25e4b6df6c..d2809aede2 100644
--- a/Lib/test/test_shlex.py
+++ b/Lib/test/test_shlex.py
@@ -1,6 +1,7 @@
-import unittest
-import os, sys, io
+import io
 import shlex
+import string
+import unittest
 
 from test import support
 
@@ -173,6 +174,21 @@ class ShlexTest(unittest.TestCase):
                              "%s: %s != %s" %
                              (self.data[i][0], l, self.data[i][1:]))
 
+    def testQuote(self):
+        safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
+        unicode_sample = '\xe9\xe0\xdf'  # e + acute accent, a + grave, sharp s
+        unsafe = '"`$\\!' + unicode_sample
+
+        self.assertEqual(shlex.quote(''), "''")
+        self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
+        self.assertEqual(shlex.quote('test file name'), "'test file name'")
+        for u in unsafe:
+            self.assertEqual(shlex.quote('test%sname' % u),
+                             "'test%sname'" % u)
+        for u in unsafe:
+            self.assertEqual(shlex.quote("test%s'name'" % u),
+                             "'test%s'\"'\"'name'\"'\"''" % u)
+
 # Allow this test to be used with old shlex.py
 if not getattr(shlex, "split", None):
     for methname in dir(ShlexTest):
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index a9b4676dff..3b1b6949d0 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -7,8 +7,9 @@ import sys
 import stat
 import os
 import os.path
-import functools
 import errno
+import functools
+import subprocess
 from test import support
 from test.support import TESTFN
 from os.path import splitdrive
@@ -22,7 +23,7 @@ import tarfile
 import warnings
 
 from test import support
-from test.support import TESTFN, check_warnings, captured_stdout
+from test.support import TESTFN, check_warnings, captured_stdout, requires_zlib
 
 try:
     import bz2
@@ -40,11 +41,6 @@ except ImportError:
     UID_GID_SUPPORT = False
 
 try:
-    import zlib
-except ImportError:
-    zlib = None
-
-try:
     import zipfile
     ZIP_SUPPORT = True
 except ImportError:
@@ -52,7 +48,7 @@ except ImportError:
 
 def _fake_rename(*args, **kwargs):
     # Pretend the destination path is on a different filesystem.
-    raise OSError()
+    raise OSError(getattr(errno, 'EXDEV', 18), "Invalid cross-device link")
 
 def mock_rename(func):
     @functools.wraps(func)
@@ -65,6 +61,31 @@ def mock_rename(func):
             os.rename = builtin_rename
     return wrap
 
+def write_file(path, content, binary=False):
+    """Write *content* to a file located at *path*.
+
+    If *path* is a tuple instead of a string, os.path.join will be used to
+    make a path.  If *binary* is true, the file will be opened in binary
+    mode.
+    """
+    if isinstance(path, tuple):
+        path = os.path.join(*path)
+    with open(path, 'wb' if binary else 'w') as fp:
+        fp.write(content)
+
+def read_file(path, binary=False):
+    """Return contents from a file located at *path*.
+
+    If *path* is a tuple instead of a string, os.path.join will be used to
+    make a path.  If *binary* is true, the file will be opened in binary
+    mode.
+    """
+    if isinstance(path, tuple):
+        path = os.path.join(*path)
+    with open(path, 'rb' if binary else 'r') as fp:
+        return fp.read()
+
+
 class TestShutil(unittest.TestCase):
 
     def setUp(self):
@@ -77,19 +98,6 @@ class TestShutil(unittest.TestCase):
             d = self.tempdirs.pop()
             shutil.rmtree(d, os.name in ('nt', 'cygwin'))
 
-    def write_file(self, path, content='xxx'):
-        """Writes a file in the given path.
-
-
-        path can be a string or a sequence.
-        """
-        if isinstance(path, (list, tuple)):
-            path = os.path.join(*path)
-        f = open(path, 'w')
-        try:
-            f.write(content)
-        finally:
-            f.close()
 
     def mkdtemp(self):
         """Create a temporary directory that will be cleaned up.
@@ -100,6 +108,15 @@ class TestShutil(unittest.TestCase):
         self.tempdirs.append(d)
         return d
 
+    def test_rmtree_works_on_bytes(self):
+        tmp = self.mkdtemp()
+        victim = os.path.join(tmp, 'killme')
+        os.mkdir(victim)
+        write_file(os.path.join(victim, 'somefile'), 'foo')
+        victim = os.fsencode(victim)
+        self.assertIsInstance(victim, bytes)
+        shutil.rmtree(victim)
+
     @support.skip_unless_symlink
     def test_rmtree_fails_on_symlink(self):
         tmp = self.mkdtemp()
@@ -119,18 +136,40 @@ class TestShutil(unittest.TestCase):
         self.assertEqual(errors[0][1], link)
         self.assertIsInstance(errors[0][2][1], OSError)
 
+    @support.skip_unless_symlink
+    def test_rmtree_works_on_symlinks(self):
+        tmp = self.mkdtemp()
+        dir1 = os.path.join(tmp, 'dir1')
+        dir2 = os.path.join(dir1, 'dir2')
+        dir3 = os.path.join(tmp, 'dir3')
+        for d in dir1, dir2, dir3:
+            os.mkdir(d)
+        file1 = os.path.join(tmp, 'file1')
+        write_file(file1, 'foo')
+        link1 = os.path.join(dir1, 'link1')
+        os.symlink(dir2, link1)
+        link2 = os.path.join(dir1, 'link2')
+        os.symlink(dir3, link2)
+        link3 = os.path.join(dir1, 'link3')
+        os.symlink(file1, link3)
+        # make sure symlinks are removed but not followed
+        shutil.rmtree(dir1)
+        self.assertFalse(os.path.exists(dir1))
+        self.assertTrue(os.path.exists(dir3))
+        self.assertTrue(os.path.exists(file1))
+
     def test_rmtree_errors(self):
         # filename is guaranteed not to exist
         filename = tempfile.mktemp()
-        self.assertRaises(OSError, shutil.rmtree, filename)
-        # test that ignore_errors option is honoured
+        self.assertRaises(FileNotFoundError, shutil.rmtree, filename)
+        # test that ignore_errors option is honored
         shutil.rmtree(filename, ignore_errors=True)
 
         # existing file
         tmpdir = self.mkdtemp()
-        self.write_file((tmpdir, "tstfile"), "")
+        write_file((tmpdir, "tstfile"), "")
         filename = os.path.join(tmpdir, "tstfile")
-        with self.assertRaises(OSError) as cm:
+        with self.assertRaises(NotADirectoryError) as cm:
             shutil.rmtree(filename)
         # The reason for this rather odd construct is that Windows sprinkles
         # a \*.* at the end of file names. But only sometimes on some buildbots
@@ -147,13 +186,14 @@ class TestShutil(unittest.TestCase):
         self.assertEqual(len(errors), 2)
         self.assertIs(errors[0][0], os.listdir)
         self.assertEqual(errors[0][1], filename)
-        self.assertIsInstance(errors[0][2][1], OSError)
+        self.assertIsInstance(errors[0][2][1], NotADirectoryError)
         self.assertIn(errors[0][2][1].filename, possible_args)
         self.assertIs(errors[1][0], os.rmdir)
         self.assertEqual(errors[1][1], filename)
-        self.assertIsInstance(errors[1][2][1], OSError)
+        self.assertIsInstance(errors[1][2][1], NotADirectoryError)
         self.assertIn(errors[1][2][1].filename, possible_args)
 
+
     # See bug #1071513 for why we don't run this on cygwin
     # and bug #1076467 for why we don't run this as root.
     if (hasattr(os, 'chmod') and sys.platform[:6] != 'cygwin'
@@ -161,30 +201,34 @@ class TestShutil(unittest.TestCase):
         def test_on_error(self):
             self.errorState = 0
             os.mkdir(TESTFN)
-            self.childpath = os.path.join(TESTFN, 'a')
-            f = open(self.childpath, 'w')
-            f.close()
+            self.addCleanup(shutil.rmtree, TESTFN)
+
+            self.child_file_path = os.path.join(TESTFN, 'a')
+            self.child_dir_path = os.path.join(TESTFN, 'b')
+            support.create_empty_file(self.child_file_path)
+            os.mkdir(self.child_dir_path)
             old_dir_mode = os.stat(TESTFN).st_mode
-            old_child_mode = os.stat(self.childpath).st_mode
+            old_child_file_mode = os.stat(self.child_file_path).st_mode
+            old_child_dir_mode = os.stat(self.child_dir_path).st_mode
             # Make unwritable.
-            os.chmod(self.childpath, stat.S_IREAD)
-            os.chmod(TESTFN, stat.S_IREAD)
+            new_mode = stat.S_IREAD|stat.S_IEXEC
+            os.chmod(self.child_file_path, new_mode)
+            os.chmod(self.child_dir_path, new_mode)
+            os.chmod(TESTFN, new_mode)
+
+            self.addCleanup(os.chmod, TESTFN, old_dir_mode)
+            self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode)
+            self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode)
 
             shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror)
             # Test whether onerror has actually been called.
-            self.assertEqual(self.errorState, 2,
-                             "Expected call to onerror function did not happen.")
-
-            # Make writable again.
-            os.chmod(TESTFN, old_dir_mode)
-            os.chmod(self.childpath, old_child_mode)
-
-            # Clean up.
-            shutil.rmtree(TESTFN)
+            self.assertEqual(self.errorState, 3,
+                             "Expected call to onerror function did not "
+                             "happen.")
 
     def check_args_to_onerror(self, func, arg, exc):
         # test_rmtree_errors deliberately runs rmtree
-        # on a directory that is chmod 400, which will fail.
+        # on a directory that is chmod 500, which will fail.
         # This function is run when shutil.rmtree fails.
         # 99.9% of the time it initially fails to remove
         # a file in the directory, so the first time through
@@ -193,99 +237,447 @@ class TestShutil(unittest.TestCase):
         # FUSE experienced a failure earlier in the process
         # at os.listdir.  The first failure may legally
         # be either.
-        if self.errorState == 0:
-            if func is os.remove:
-                self.assertEqual(arg, self.childpath)
+        if self.errorState < 2:
+            if func is os.unlink:
+                self.assertEqual(arg, self.child_file_path)
+            elif func is os.rmdir:
+                self.assertEqual(arg, self.child_dir_path)
             else:
-                self.assertIs(func, os.listdir,
-                              "func must be either os.remove or os.listdir")
-                self.assertEqual(arg, TESTFN)
+                self.assertIs(func, os.listdir)
+                self.assertIn(arg, [TESTFN, self.child_dir_path])
             self.assertTrue(issubclass(exc[0], OSError))
-            self.errorState = 1
+            self.errorState += 1
         else:
             self.assertEqual(func, os.rmdir)
             self.assertEqual(arg, TESTFN)
             self.assertTrue(issubclass(exc[0], OSError))
-            self.errorState = 2
+            self.errorState = 3
+
+    def test_rmtree_does_not_choke_on_failing_lstat(self):
+        try:
+            orig_lstat = os.lstat
+            def raiser(fn, *args, **kwargs):
+                if fn != TESTFN:
+                    raise OSError()
+                else:
+                    return orig_lstat(fn)
+            os.lstat = raiser
+
+            os.mkdir(TESTFN)
+            write_file((TESTFN, 'foo'), 'foo')
+            shutil.rmtree(TESTFN)
+        finally:
+            os.lstat = orig_lstat
+
+    @unittest.skipUnless(hasattr(os, 'chmod'), 'requires os.chmod')
+    @support.skip_unless_symlink
+    def test_copymode_follow_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'quux')
+        write_file(src, 'foo')
+        write_file(dst, 'foo')
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        os.chmod(src, stat.S_IRWXU|stat.S_IRWXG)
+        # file to file
+        os.chmod(dst, stat.S_IRWXO)
+        self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        shutil.copymode(src, dst)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # follow src link
+        os.chmod(dst, stat.S_IRWXO)
+        shutil.copymode(src_link, dst)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # follow dst link
+        os.chmod(dst, stat.S_IRWXO)
+        shutil.copymode(src, dst_link)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # follow both links
+        os.chmod(dst, stat.S_IRWXO)
+        shutil.copymode(src_link, dst)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+
+    @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod')
+    @support.skip_unless_symlink
+    def test_copymode_symlink_to_symlink(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'quux')
+        write_file(src, 'foo')
+        write_file(dst, 'foo')
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        os.chmod(src, stat.S_IRWXU|stat.S_IRWXG)
+        os.chmod(dst, stat.S_IRWXU)
+        os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG)
+        # link to link
+        os.lchmod(dst_link, stat.S_IRWXO)
+        shutil.copymode(src_link, dst_link, follow_symlinks=False)
+        self.assertEqual(os.lstat(src_link).st_mode,
+                         os.lstat(dst_link).st_mode)
+        self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # src link - use chmod
+        os.lchmod(dst_link, stat.S_IRWXO)
+        shutil.copymode(src_link, dst, follow_symlinks=False)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+        # dst link - use chmod
+        os.lchmod(dst_link, stat.S_IRWXO)
+        shutil.copymode(src, dst_link, follow_symlinks=False)
+        self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode)
+
+    @unittest.skipIf(hasattr(os, 'lchmod'), 'requires os.lchmod to be missing')
+    @support.skip_unless_symlink
+    def test_copymode_symlink_to_symlink_wo_lchmod(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'quux')
+        write_file(src, 'foo')
+        write_file(dst, 'foo')
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        shutil.copymode(src_link, dst_link, follow_symlinks=False)  # silent fail
+
+    @support.skip_unless_symlink
+    def test_copystat_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        dst_link = os.path.join(tmp_dir, 'qux')
+        write_file(src, 'foo')
+        src_stat = os.stat(src)
+        os.utime(src, (src_stat.st_atime,
+                       src_stat.st_mtime - 42.0))  # ensure different mtimes
+        write_file(dst, 'bar')
+        self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime)
+        os.symlink(src, src_link)
+        os.symlink(dst, dst_link)
+        if hasattr(os, 'lchmod'):
+            os.lchmod(src_link, stat.S_IRWXO)
+        if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
+            os.lchflags(src_link, stat.UF_NODUMP)
+        src_link_stat = os.lstat(src_link)
+        # follow
+        if hasattr(os, 'lchmod'):
+            shutil.copystat(src_link, dst_link, follow_symlinks=True)
+            self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode)
+        # don't follow
+        shutil.copystat(src_link, dst_link, follow_symlinks=False)
+        dst_link_stat = os.lstat(dst_link)
+        if os.utime in os.supports_follow_symlinks:
+            for attr in 'st_atime', 'st_mtime':
+                # The modification times may be truncated in the new file.
+                self.assertLessEqual(getattr(src_link_stat, attr),
+                                     getattr(dst_link_stat, attr) + 1)
+        if hasattr(os, 'lchmod'):
+            self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode)
+        if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'):
+            self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags)
+        # tell to follow but dst is not a link
+        shutil.copystat(src_link, dst, follow_symlinks=False)
+        self.assertTrue(abs(os.stat(src).st_mtime - os.stat(dst).st_mtime) <
+                        00000.1)
+
+    @unittest.skipUnless(hasattr(os, 'chflags') and
+                         hasattr(errno, 'EOPNOTSUPP') and
+                         hasattr(errno, 'ENOTSUP'),
+                         "requires os.chflags, EOPNOTSUPP & ENOTSUP")
+    def test_copystat_handles_harmless_chflags_errors(self):
+        tmpdir = self.mkdtemp()
+        file1 = os.path.join(tmpdir, 'file1')
+        file2 = os.path.join(tmpdir, 'file2')
+        write_file(file1, 'xxx')
+        write_file(file2, 'xxx')
+
+        def make_chflags_raiser(err):
+            ex = OSError()
+
+            def _chflags_raiser(path, flags, *, follow_symlinks=True):
+                ex.errno = err
+                raise ex
+            return _chflags_raiser
+        old_chflags = os.chflags
+        try:
+            for err in errno.EOPNOTSUPP, errno.ENOTSUP:
+                os.chflags = make_chflags_raiser(err)
+                shutil.copystat(file1, file2)
+            # assert others errors break it
+            os.chflags = make_chflags_raiser(errno.EOPNOTSUPP + errno.ENOTSUP)
+            self.assertRaises(OSError, shutil.copystat, file1, file2)
+        finally:
+            os.chflags = old_chflags
+
+    @support.skip_unless_xattr
+    def test_copyxattr(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        write_file(src, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        write_file(dst, 'bar')
+
+        # no xattr == no problem
+        shutil._copyxattr(src, dst)
+        # common case
+        os.setxattr(src, 'user.foo', b'42')
+        os.setxattr(src, 'user.bar', b'43')
+        shutil._copyxattr(src, dst)
+        self.assertEqual(os.listxattr(src), os.listxattr(dst))
+        self.assertEqual(
+                os.getxattr(src, 'user.foo'),
+                os.getxattr(dst, 'user.foo'))
+        # check errors don't affect other attrs
+        os.remove(dst)
+        write_file(dst, 'bar')
+        os_error = OSError(errno.EPERM, 'EPERM')
+
+        def _raise_on_user_foo(fname, attr, val, **kwargs):
+            if attr == 'user.foo':
+                raise os_error
+            else:
+                orig_setxattr(fname, attr, val, **kwargs)
+        try:
+            orig_setxattr = os.setxattr
+            os.setxattr = _raise_on_user_foo
+            shutil._copyxattr(src, dst)
+            self.assertIn('user.bar', os.listxattr(dst))
+        finally:
+            os.setxattr = orig_setxattr
+        # the source filesystem not supporting xattrs should be ok, too.
+        def _raise_on_src(fname, *, follow_symlinks=True):
+            if fname == src:
+                raise OSError(errno.ENOTSUP, 'Operation not supported')
+            return orig_listxattr(fname, follow_symlinks=follow_symlinks)
+        try:
+            orig_listxattr = os.listxattr
+            os.listxattr = _raise_on_src
+            shutil._copyxattr(src, dst)
+        finally:
+            os.listxattr = orig_listxattr
+
+        # test that shutil.copystat copies xattrs
+        src = os.path.join(tmp_dir, 'the_original')
+        write_file(src, src)
+        os.setxattr(src, 'user.the_value', b'fiddly')
+        dst = os.path.join(tmp_dir, 'the_copy')
+        write_file(dst, dst)
+        shutil.copystat(src, dst)
+        self.assertEqual(os.getxattr(dst, 'user.the_value'), b'fiddly')
+
+    @support.skip_unless_symlink
+    @support.skip_unless_xattr
+    @unittest.skipUnless(hasattr(os, 'geteuid') and os.geteuid() == 0,
+                         'root privileges required')
+    def test_copyxattr_symlinks(self):
+        # On Linux, it's only possible to access non-user xattr for symlinks;
+        # which in turn require root privileges. This test should be expanded
+        # as soon as other platforms gain support for extended attributes.
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        src_link = os.path.join(tmp_dir, 'baz')
+        write_file(src, 'foo')
+        os.symlink(src, src_link)
+        os.setxattr(src, 'trusted.foo', b'42')
+        os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False)
+        dst = os.path.join(tmp_dir, 'bar')
+        dst_link = os.path.join(tmp_dir, 'qux')
+        write_file(dst, 'bar')
+        os.symlink(dst, dst_link)
+        shutil._copyxattr(src_link, dst_link, follow_symlinks=False)
+        self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43')
+        self.assertRaises(OSError, os.getxattr, dst, 'trusted.foo')
+        shutil._copyxattr(src_link, dst, follow_symlinks=False)
+        self.assertEqual(os.getxattr(dst, 'trusted.foo'), b'43')
+
+    @support.skip_unless_symlink
+    def test_copy_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        write_file(src, 'foo')
+        os.symlink(src, src_link)
+        if hasattr(os, 'lchmod'):
+            os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
+        # don't follow
+        shutil.copy(src_link, dst, follow_symlinks=True)
+        self.assertFalse(os.path.islink(dst))
+        self.assertEqual(read_file(src), read_file(dst))
+        os.remove(dst)
+        # follow
+        shutil.copy(src_link, dst, follow_symlinks=False)
+        self.assertTrue(os.path.islink(dst))
+        self.assertEqual(os.readlink(dst), os.readlink(src_link))
+        if hasattr(os, 'lchmod'):
+            self.assertEqual(os.lstat(src_link).st_mode,
+                             os.lstat(dst).st_mode)
+
+    @support.skip_unless_symlink
+    def test_copy2_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        src_link = os.path.join(tmp_dir, 'baz')
+        write_file(src, 'foo')
+        os.symlink(src, src_link)
+        if hasattr(os, 'lchmod'):
+            os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
+        if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
+            os.lchflags(src_link, stat.UF_NODUMP)
+        src_stat = os.stat(src)
+        src_link_stat = os.lstat(src_link)
+        # follow
+        shutil.copy2(src_link, dst, follow_symlinks=True)
+        self.assertFalse(os.path.islink(dst))
+        self.assertEqual(read_file(src), read_file(dst))
+        os.remove(dst)
+        # don't follow
+        shutil.copy2(src_link, dst, follow_symlinks=False)
+        self.assertTrue(os.path.islink(dst))
+        self.assertEqual(os.readlink(dst), os.readlink(src_link))
+        dst_stat = os.lstat(dst)
+        if os.utime in os.supports_follow_symlinks:
+            for attr in 'st_atime', 'st_mtime':
+                # The modification times may be truncated in the new file.
+                self.assertLessEqual(getattr(src_link_stat, attr),
+                                     getattr(dst_stat, attr) + 1)
+        if hasattr(os, 'lchmod'):
+            self.assertEqual(src_link_stat.st_mode, dst_stat.st_mode)
+            self.assertNotEqual(src_stat.st_mode, dst_stat.st_mode)
+        if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'):
+            self.assertEqual(src_link_stat.st_flags, dst_stat.st_flags)
+
+    @support.skip_unless_xattr
+    def test_copy2_xattr(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'foo')
+        dst = os.path.join(tmp_dir, 'bar')
+        write_file(src, 'foo')
+        os.setxattr(src, 'user.foo', b'42')
+        shutil.copy2(src, dst)
+        self.assertEqual(
+                os.getxattr(src, 'user.foo'),
+                os.getxattr(dst, 'user.foo'))
+        os.remove(dst)
+
+    @support.skip_unless_symlink
+    def test_copyfile_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src = os.path.join(tmp_dir, 'src')
+        dst = os.path.join(tmp_dir, 'dst')
+        dst_link = os.path.join(tmp_dir, 'dst_link')
+        link = os.path.join(tmp_dir, 'link')
+        write_file(src, 'foo')
+        os.symlink(src, link)
+        # don't follow
+        shutil.copyfile(link, dst_link, follow_symlinks=False)
+        self.assertTrue(os.path.islink(dst_link))
+        self.assertEqual(os.readlink(link), os.readlink(dst_link))
+        # follow
+        shutil.copyfile(link, dst)
+        self.assertFalse(os.path.islink(dst))
+
+    def test_rmtree_uses_safe_fd_version_if_available(self):
+        _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <=
+                             os.supports_dir_fd and
+                             os.listdir in os.supports_fd and
+                             os.stat in os.supports_follow_symlinks)
+        if _use_fd_functions:
+            self.assertTrue(shutil._use_fd_functions)
+            self.assertTrue(shutil.rmtree.avoids_symlink_attacks)
+            tmp_dir = self.mkdtemp()
+            d = os.path.join(tmp_dir, 'a')
+            os.mkdir(d)
+            try:
+                real_rmtree = shutil._rmtree_safe_fd
+                class Called(Exception): pass
+                def _raiser(*args, **kwargs):
+                    raise Called
+                shutil._rmtree_safe_fd = _raiser
+                self.assertRaises(Called, shutil.rmtree, d)
+            finally:
+                shutil._rmtree_safe_fd = real_rmtree
+        else:
+            self.assertFalse(shutil._use_fd_functions)
+            self.assertFalse(shutil.rmtree.avoids_symlink_attacks)
 
     def test_rmtree_dont_delete_file(self):
         # When called on a file instead of a directory, don't delete it.
         handle, path = tempfile.mkstemp()
-        os.fdopen(handle).close()
-        self.assertRaises(OSError, shutil.rmtree, path)
+        os.close(handle)
+        self.assertRaises(NotADirectoryError, shutil.rmtree, path)
         os.remove(path)
 
-    def _write_data(self, path, data):
-        f = open(path, "w")
-        f.write(data)
-        f.close()
-
     def test_copytree_simple(self):
-
-        def read_data(path):
-            f = open(path)
-            data = f.read()
-            f.close()
-            return data
-
         src_dir = tempfile.mkdtemp()
         dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')
-        self._write_data(os.path.join(src_dir, 'test.txt'), '123')
+        self.addCleanup(shutil.rmtree, src_dir)
+        self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir))
+        write_file((src_dir, 'test.txt'), '123')
         os.mkdir(os.path.join(src_dir, 'test_dir'))
-        self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456')
+        write_file((src_dir, 'test_dir', 'test.txt'), '456')
+
+        shutil.copytree(src_dir, dst_dir)
+        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt')))
+        self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir')))
+        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir',
+                                                    'test.txt')))
+        actual = read_file((dst_dir, 'test.txt'))
+        self.assertEqual(actual, '123')
+        actual = read_file((dst_dir, 'test_dir', 'test.txt'))
+        self.assertEqual(actual, '456')
 
-        try:
-            shutil.copytree(src_dir, dst_dir)
-            self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt')))
-            self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir')))
-            self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir',
-                                                        'test.txt')))
-            actual = read_data(os.path.join(dst_dir, 'test.txt'))
-            self.assertEqual(actual, '123')
-            actual = read_data(os.path.join(dst_dir, 'test_dir', 'test.txt'))
-            self.assertEqual(actual, '456')
-        finally:
-            for path in (
-                    os.path.join(src_dir, 'test.txt'),
-                    os.path.join(dst_dir, 'test.txt'),
-                    os.path.join(src_dir, 'test_dir', 'test.txt'),
-                    os.path.join(dst_dir, 'test_dir', 'test.txt'),
-                ):
-                if os.path.exists(path):
-                    os.remove(path)
-            for path in (src_dir,
-                    os.path.dirname(dst_dir)
-                ):
-                if os.path.exists(path):
-                    shutil.rmtree(path)
+    @support.skip_unless_symlink
+    def test_copytree_symlinks(self):
+        tmp_dir = self.mkdtemp()
+        src_dir = os.path.join(tmp_dir, 'src')
+        dst_dir = os.path.join(tmp_dir, 'dst')
+        sub_dir = os.path.join(src_dir, 'sub')
+        os.mkdir(src_dir)
+        os.mkdir(sub_dir)
+        write_file((src_dir, 'file.txt'), 'foo')
+        src_link = os.path.join(sub_dir, 'link')
+        dst_link = os.path.join(dst_dir, 'sub/link')
+        os.symlink(os.path.join(src_dir, 'file.txt'),
+                   src_link)
+        if hasattr(os, 'lchmod'):
+            os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
+        if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
+            os.lchflags(src_link, stat.UF_NODUMP)
+        src_stat = os.lstat(src_link)
+        shutil.copytree(src_dir, dst_dir, symlinks=True)
+        self.assertTrue(os.path.islink(os.path.join(dst_dir, 'sub', 'link')))
+        self.assertEqual(os.readlink(os.path.join(dst_dir, 'sub', 'link')),
+                         os.path.join(src_dir, 'file.txt'))
+        dst_stat = os.lstat(dst_link)
+        if hasattr(os, 'lchmod'):
+            self.assertEqual(dst_stat.st_mode, src_stat.st_mode)
+        if hasattr(os, 'lchflags'):
+            self.assertEqual(dst_stat.st_flags, src_stat.st_flags)
 
     def test_copytree_with_exclude(self):
-
-        def read_data(path):
-            f = open(path)
-            data = f.read()
-            f.close()
-            return data
-
         # creating data
         join = os.path.join
         exists = os.path.exists
         src_dir = tempfile.mkdtemp()
         try:
             dst_dir = join(tempfile.mkdtemp(), 'destination')
-            self._write_data(join(src_dir, 'test.txt'), '123')
-            self._write_data(join(src_dir, 'test.tmp'), '123')
+            write_file((src_dir, 'test.txt'), '123')
+            write_file((src_dir, 'test.tmp'), '123')
             os.mkdir(join(src_dir, 'test_dir'))
-            self._write_data(join(src_dir, 'test_dir', 'test.txt'), '456')
+            write_file((src_dir, 'test_dir', 'test.txt'), '456')
             os.mkdir(join(src_dir, 'test_dir2'))
-            self._write_data(join(src_dir, 'test_dir2', 'test.txt'), '456')
+            write_file((src_dir, 'test_dir2', 'test.txt'), '456')
             os.mkdir(join(src_dir, 'test_dir2', 'subdir'))
             os.mkdir(join(src_dir, 'test_dir2', 'subdir2'))
-            self._write_data(join(src_dir, 'test_dir2', 'subdir', 'test.txt'),
-                             '456')
-            self._write_data(join(src_dir, 'test_dir2', 'subdir2', 'test.py'),
-                             '456')
-
+            write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456')
+            write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456')
 
             # testing glob-like patterns
             try:
@@ -293,21 +685,19 @@ class TestShutil(unittest.TestCase):
                 shutil.copytree(src_dir, dst_dir, ignore=patterns)
                 # checking the result: some elements should not be copied
                 self.assertTrue(exists(join(dst_dir, 'test.txt')))
-                self.assertTrue(not exists(join(dst_dir, 'test.tmp')))
-                self.assertTrue(not exists(join(dst_dir, 'test_dir2')))
+                self.assertFalse(exists(join(dst_dir, 'test.tmp')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2')))
             finally:
-                if os.path.exists(dst_dir):
-                    shutil.rmtree(dst_dir)
+                shutil.rmtree(dst_dir)
             try:
                 patterns = shutil.ignore_patterns('*.tmp', 'subdir*')
                 shutil.copytree(src_dir, dst_dir, ignore=patterns)
                 # checking the result: some elements should not be copied
-                self.assertTrue(not exists(join(dst_dir, 'test.tmp')))
-                self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2')))
-                self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir')))
+                self.assertFalse(exists(join(dst_dir, 'test.tmp')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir')))
             finally:
-                if os.path.exists(dst_dir):
-                    shutil.rmtree(dst_dir)
+                shutil.rmtree(dst_dir)
 
             # testing callable-style
             try:
@@ -326,13 +716,12 @@ class TestShutil(unittest.TestCase):
                 shutil.copytree(src_dir, dst_dir, ignore=_filter)
 
                 # checking the result: some elements should not be copied
-                self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2',
-                                        'test.py')))
-                self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2',
+                                             'test.py')))
+                self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir')))
 
             finally:
-                if os.path.exists(dst_dir):
-                    shutil.rmtree(dst_dir)
+                shutil.rmtree(dst_dir)
         finally:
             shutil.rmtree(src_dir)
             shutil.rmtree(os.path.dirname(dst_dir))
@@ -357,35 +746,6 @@ class TestShutil(unittest.TestCase):
         finally:
             shutil.rmtree(TESTFN, ignore_errors=True)
 
-    @unittest.skipUnless(hasattr(os, 'chflags') and
-                         hasattr(errno, 'EOPNOTSUPP') and
-                         hasattr(errno, 'ENOTSUP'),
-                         "requires os.chflags, EOPNOTSUPP & ENOTSUP")
-    def test_copystat_handles_harmless_chflags_errors(self):
-        tmpdir = self.mkdtemp()
-        file1 = os.path.join(tmpdir, 'file1')
-        file2 = os.path.join(tmpdir, 'file2')
-        self.write_file(file1, 'xxx')
-        self.write_file(file2, 'xxx')
-
-        def make_chflags_raiser(err):
-            ex = OSError()
-
-            def _chflags_raiser(path, flags):
-                ex.errno = err
-                raise ex
-            return _chflags_raiser
-        old_chflags = os.chflags
-        try:
-            for err in errno.EOPNOTSUPP, errno.ENOTSUP:
-                os.chflags = make_chflags_raiser(err)
-                shutil.copystat(file1, file2)
-            # assert others errors break it
-            os.chflags = make_chflags_raiser(errno.EOPNOTSUPP + errno.ENOTSUP)
-            self.assertRaises(OSError, shutil.copystat, file1, file2)
-        finally:
-            os.chflags = old_chflags
-
     @support.skip_unless_symlink
     def test_dont_copy_file_onto_symlink_to_itself(self):
         # bug 851123.
@@ -416,6 +776,7 @@ class TestShutil(unittest.TestCase):
             os.mkdir(src)
             os.symlink(src, dst)
             self.assertRaises(OSError, shutil.rmtree, dst)
+            shutil.rmtree(dst, ignore_errors=True)
         finally:
             shutil.rmtree(TESTFN, ignore_errors=True)
 
@@ -456,9 +817,9 @@ class TestShutil(unittest.TestCase):
 
         src_dir = self.mkdtemp()
         dst_dir = os.path.join(self.mkdtemp(), 'destination')
-        self._write_data(os.path.join(src_dir, 'test.txt'), '123')
+        write_file((src_dir, 'test.txt'), '123')
         os.mkdir(os.path.join(src_dir, 'test_dir'))
-        self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456')
+        write_file((src_dir, 'test_dir', 'test.txt'), '456')
 
         copied = []
         def _copy(src, dst):
@@ -475,7 +836,7 @@ class TestShutil(unittest.TestCase):
         dst_dir = os.path.join(self.mkdtemp(), 'destination')
         os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt'))
         os.mkdir(os.path.join(src_dir, 'test_dir'))
-        self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456')
+        write_file((src_dir, 'test_dir', 'test.txt'), '456')
         self.assertRaises(Error, shutil.copytree, src_dir, dst_dir)
 
         # a dangling symlink is ignored with the proper flag
@@ -491,7 +852,7 @@ class TestShutil(unittest.TestCase):
     def _copy_file(self, method):
         fname = 'test.txt'
         tmpdir = self.mkdtemp()
-        self.write_file([tmpdir, fname])
+        write_file((tmpdir, fname), 'xxx')
         file1 = os.path.join(tmpdir, fname)
         tmpdir2 = self.mkdtemp()
         method(file1, tmpdir2)
@@ -523,14 +884,14 @@ class TestShutil(unittest.TestCase):
             self.assertEqual(getattr(file1_stat, 'st_flags'),
                              getattr(file2_stat, 'st_flags'))
 
-    @unittest.skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_make_tarball(self):
         # creating something to tar
         tmpdir = self.mkdtemp()
-        self.write_file([tmpdir, 'file1'], 'xxx')
-        self.write_file([tmpdir, 'file2'], 'xxx')
+        write_file((tmpdir, 'file1'), 'xxx')
+        write_file((tmpdir, 'file2'), 'xxx')
         os.mkdir(os.path.join(tmpdir, 'sub'))
-        self.write_file([tmpdir, 'sub', 'file3'], 'xxx')
+        write_file((tmpdir, 'sub', 'file3'), 'xxx')
 
         tmpdir2 = self.mkdtemp()
         # force shutil to create the directory
@@ -577,16 +938,16 @@ class TestShutil(unittest.TestCase):
         tmpdir = self.mkdtemp()
         dist = os.path.join(tmpdir, 'dist')
         os.mkdir(dist)
-        self.write_file([dist, 'file1'], 'xxx')
-        self.write_file([dist, 'file2'], 'xxx')
+        write_file((dist, 'file1'), 'xxx')
+        write_file((dist, 'file2'), 'xxx')
         os.mkdir(os.path.join(dist, 'sub'))
-        self.write_file([dist, 'sub', 'file3'], 'xxx')
+        write_file((dist, 'sub', 'file3'), 'xxx')
         os.mkdir(os.path.join(dist, 'sub2'))
         tmpdir2 = self.mkdtemp()
         base_name = os.path.join(tmpdir2, 'archive')
         return tmpdir, tmpdir2, base_name
 
-    @unittest.skipUnless(zlib, "Requires zlib")
+    @requires_zlib
     @unittest.skipUnless(find_executable('tar') and find_executable('gzip'),
                          'Need the tar command to run')
     def test_tarfile_vs_tar(self):
@@ -641,13 +1002,13 @@ class TestShutil(unittest.TestCase):
         tarball = base_name + '.tar'
         self.assertTrue(os.path.exists(tarball))
 
-    @unittest.skipUnless(zlib, "Requires zlib")
+    @requires_zlib
     @unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run')
     def test_make_zipfile(self):
         # creating something to tar
         tmpdir = self.mkdtemp()
-        self.write_file([tmpdir, 'file1'], 'xxx')
-        self.write_file([tmpdir, 'file2'], 'xxx')
+        write_file((tmpdir, 'file1'), 'xxx')
+        write_file((tmpdir, 'file2'), 'xxx')
 
         tmpdir2 = self.mkdtemp()
         # force shutil to create the directory
@@ -665,7 +1026,7 @@ class TestShutil(unittest.TestCase):
         base_name = os.path.join(tmpdir, 'archive')
         self.assertRaises(ValueError, make_archive, base_name, 'xxx')
 
-    @unittest.skipUnless(zlib, "Requires zlib")
+    @requires_zlib
     def test_make_archive_owner_group(self):
         # testing make_archive with owner and group, with various combinations
         # this works even if there's not gid/uid support
@@ -693,7 +1054,7 @@ class TestShutil(unittest.TestCase):
         self.assertTrue(os.path.exists(res))
 
 
-    @unittest.skipUnless(zlib, "Requires zlib")
+    @requires_zlib
     @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
     def test_tarfile_root_owner(self):
         tmpdir, tmpdir2, base_name =  self._create_files()
@@ -762,7 +1123,7 @@ class TestShutil(unittest.TestCase):
                     diff.append(file_)
         return diff
 
-    @unittest.skipUnless(zlib, "Requires zlib")
+    @requires_zlib
     def test_unpack_archive(self):
         formats = ['tar', 'gztar', 'zip']
         if BZ2_SUPPORTED:
@@ -813,6 +1174,184 @@ class TestShutil(unittest.TestCase):
         unregister_unpack_format('Boo2')
         self.assertEqual(get_unpack_formats(), formats)
 
+    @unittest.skipUnless(hasattr(shutil, 'disk_usage'),
+                         "disk_usage not available on this platform")
+    def test_disk_usage(self):
+        usage = shutil.disk_usage(os.getcwd())
+        self.assertGreater(usage.total, 0)
+        self.assertGreater(usage.used, 0)
+        self.assertGreaterEqual(usage.free, 0)
+        self.assertGreaterEqual(usage.total, usage.used)
+        self.assertGreater(usage.total, usage.free)
+
+    @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
+    @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown')
+    def test_chown(self):
+
+        # cleaned-up automatically by TestShutil.tearDown method
+        dirname = self.mkdtemp()
+        filename = tempfile.mktemp(dir=dirname)
+        write_file(filename, 'testing chown function')
+
+        with self.assertRaises(ValueError):
+            shutil.chown(filename)
+
+        with self.assertRaises(LookupError):
+            shutil.chown(filename, user='non-exising username')
+
+        with self.assertRaises(LookupError):
+            shutil.chown(filename, group='non-exising groupname')
+
+        with self.assertRaises(TypeError):
+            shutil.chown(filename, b'spam')
+
+        with self.assertRaises(TypeError):
+            shutil.chown(filename, 3.14)
+
+        uid = os.getuid()
+        gid = os.getgid()
+
+        def check_chown(path, uid=None, gid=None):
+            s = os.stat(filename)
+            if uid is not None:
+                self.assertEqual(uid, s.st_uid)
+            if gid is not None:
+                self.assertEqual(gid, s.st_gid)
+
+        shutil.chown(filename, uid, gid)
+        check_chown(filename, uid, gid)
+        shutil.chown(filename, uid)
+        check_chown(filename, uid)
+        shutil.chown(filename, user=uid)
+        check_chown(filename, uid)
+        shutil.chown(filename, group=gid)
+        check_chown(filename, gid=gid)
+
+        shutil.chown(dirname, uid, gid)
+        check_chown(dirname, uid, gid)
+        shutil.chown(dirname, uid)
+        check_chown(dirname, uid)
+        shutil.chown(dirname, user=uid)
+        check_chown(dirname, uid)
+        shutil.chown(dirname, group=gid)
+        check_chown(dirname, gid=gid)
+
+        user = pwd.getpwuid(uid)[0]
+        group = grp.getgrgid(gid)[0]
+        shutil.chown(filename, user, group)
+        check_chown(filename, uid, gid)
+        shutil.chown(dirname, user, group)
+        check_chown(dirname, uid, gid)
+
+    def test_copy_return_value(self):
+        # copy and copy2 both return their destination path.
+        for fn in (shutil.copy, shutil.copy2):
+            src_dir = self.mkdtemp()
+            dst_dir = self.mkdtemp()
+            src = os.path.join(src_dir, 'foo')
+            write_file(src, 'foo')
+            rv = fn(src, dst_dir)
+            self.assertEqual(rv, os.path.join(dst_dir, 'foo'))
+            rv = fn(src, os.path.join(dst_dir, 'bar'))
+            self.assertEqual(rv, os.path.join(dst_dir, 'bar'))
+
+    def test_copyfile_return_value(self):
+        # copytree returns its destination path.
+        src_dir = self.mkdtemp()
+        dst_dir = self.mkdtemp()
+        dst_file = os.path.join(dst_dir, 'bar')
+        src_file = os.path.join(src_dir, 'foo')
+        write_file(src_file, 'foo')
+        rv = shutil.copyfile(src_file, dst_file)
+        self.assertTrue(os.path.exists(rv))
+        self.assertEqual(read_file(src_file), read_file(dst_file))
+
+    def test_copytree_return_value(self):
+        # copytree returns its destination path.
+        src_dir = self.mkdtemp()
+        dst_dir = src_dir + "dest"
+        self.addCleanup(shutil.rmtree, dst_dir, True)
+        src = os.path.join(src_dir, 'foo')
+        write_file(src, 'foo')
+        rv = shutil.copytree(src_dir, dst_dir)
+        self.assertEqual(['foo'], os.listdir(rv))
+
+
+class TestWhich(unittest.TestCase):
+
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp(prefix="Tmp")
+        self.addCleanup(shutil.rmtree, self.temp_dir, True)
+        # Give the temp_file an ".exe" suffix for all.
+        # It's needed on Windows and not harmful on other platforms.
+        self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir,
+                                                     prefix="Tmp",
+                                                     suffix=".Exe")
+        os.chmod(self.temp_file.name, stat.S_IXUSR)
+        self.addCleanup(self.temp_file.close)
+        self.dir, self.file = os.path.split(self.temp_file.name)
+
+    def test_basic(self):
+        # Given an EXE in a directory, it should be returned.
+        rv = shutil.which(self.file, path=self.dir)
+        self.assertEqual(rv, self.temp_file.name)
+
+    def test_absolute_cmd(self):
+        # When given the fully qualified path to an executable that exists,
+        # it should be returned.
+        rv = shutil.which(self.temp_file.name, path=self.temp_dir)
+        self.assertEqual(rv, self.temp_file.name)
+
+    def test_relative_cmd(self):
+        # When given the relative path with a directory part to an executable
+        # that exists, it should be returned.
+        base_dir, tail_dir = os.path.split(self.dir)
+        relpath = os.path.join(tail_dir, self.file)
+        with support.temp_cwd(path=base_dir):
+            rv = shutil.which(relpath, path=self.temp_dir)
+            self.assertEqual(rv, relpath)
+        # But it shouldn't be searched in PATH directories (issue #16957).
+        with support.temp_cwd(path=self.dir):
+            rv = shutil.which(relpath, path=base_dir)
+            self.assertIsNone(rv)
+
+    def test_cwd(self):
+        # Issue #16957
+        base_dir = os.path.dirname(self.dir)
+        with support.temp_cwd(path=self.dir):
+            rv = shutil.which(self.file, path=base_dir)
+            if sys.platform == "win32":
+                # Windows: current directory implicitly on PATH
+                self.assertEqual(rv, os.path.join(os.curdir, self.file))
+            else:
+                # Other platforms: shouldn't match in the current directory.
+                self.assertIsNone(rv)
+
+    def test_non_matching_mode(self):
+        # Set the file read-only and ask for writeable files.
+        os.chmod(self.temp_file.name, stat.S_IREAD)
+        rv = shutil.which(self.file, path=self.dir, mode=os.W_OK)
+        self.assertIsNone(rv)
+
+    def test_relative_path(self):
+        base_dir, tail_dir = os.path.split(self.dir)
+        with support.temp_cwd(path=base_dir):
+            rv = shutil.which(self.file, path=tail_dir)
+            self.assertEqual(rv, os.path.join(tail_dir, self.file))
+
+    def test_nonexistent_file(self):
+        # Return None when no matching executable file is found on the path.
+        rv = shutil.which("foo.exe", path=self.dir)
+        self.assertIsNone(rv)
+
+    @unittest.skipUnless(sys.platform == "win32",
+                         "pathext check is Windows-only")
+    def test_pathext_checking(self):
+        # Ask for the file without the ".exe" extension, then ensure that
+        # it gets found properly with the extension.
+        rv = shutil.which(self.file[:-4], path=self.dir)
+        self.assertEqual(rv, self.temp_file.name[:-4] + ".EXE")
+
 
 class TestMove(unittest.TestCase):
 
@@ -926,6 +1465,58 @@ class TestMove(unittest.TestCase):
         finally:
             shutil.rmtree(TESTFN, ignore_errors=True)
 
+    @support.skip_unless_symlink
+    @mock_rename
+    def test_move_file_symlink(self):
+        dst = os.path.join(self.src_dir, 'bar')
+        os.symlink(self.src_file, dst)
+        shutil.move(dst, self.dst_file)
+        self.assertTrue(os.path.islink(self.dst_file))
+        self.assertTrue(os.path.samefile(self.src_file, self.dst_file))
+
+    @support.skip_unless_symlink
+    @mock_rename
+    def test_move_file_symlink_to_dir(self):
+        filename = "bar"
+        dst = os.path.join(self.src_dir, filename)
+        os.symlink(self.src_file, dst)
+        shutil.move(dst, self.dst_dir)
+        final_link = os.path.join(self.dst_dir, filename)
+        self.assertTrue(os.path.islink(final_link))
+        self.assertTrue(os.path.samefile(self.src_file, final_link))
+
+    @support.skip_unless_symlink
+    @mock_rename
+    def test_move_dangling_symlink(self):
+        src = os.path.join(self.src_dir, 'baz')
+        dst = os.path.join(self.src_dir, 'bar')
+        os.symlink(src, dst)
+        dst_link = os.path.join(self.dst_dir, 'quux')
+        shutil.move(dst, dst_link)
+        self.assertTrue(os.path.islink(dst_link))
+        self.assertEqual(os.path.realpath(src), os.path.realpath(dst_link))
+
+    @support.skip_unless_symlink
+    @mock_rename
+    def test_move_dir_symlink(self):
+        src = os.path.join(self.src_dir, 'baz')
+        dst = os.path.join(self.src_dir, 'bar')
+        os.mkdir(src)
+        os.symlink(src, dst)
+        dst_link = os.path.join(self.dst_dir, 'quux')
+        shutil.move(dst, dst_link)
+        self.assertTrue(os.path.islink(dst_link))
+        self.assertTrue(os.path.samefile(src, dst_link))
+
+    def test_move_return_value(self):
+        rv = shutil.move(self.src_file, self.dst_dir)
+        self.assertEqual(rv,
+                os.path.join(self.dst_dir, os.path.basename(self.src_file)))
+
+    def test_move_as_rename_return_value(self):
+        rv = shutil.move(self.src_file, os.path.join(self.dst_dir, 'bar'))
+        self.assertEqual(rv, os.path.join(self.dst_dir, 'bar'))
+
 
 class TestCopyFile(unittest.TestCase):
 
@@ -1035,6 +1626,7 @@ class TestCopyFile(unittest.TestCase):
         # but a different case.
 
         self.src_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.src_dir, True)
         dst_dir = os.path.join(
                 os.path.dirname(self.src_dir),
                 os.path.basename(self.src_dir).upper())
@@ -1044,13 +1636,57 @@ class TestCopyFile(unittest.TestCase):
             shutil.move(self.src_dir, dst_dir)
             self.assertTrue(os.path.isdir(dst_dir))
         finally:
-            if os.path.exists(dst_dir):
-                os.rmdir(dst_dir)
+            os.rmdir(dst_dir)
+
+class TermsizeTests(unittest.TestCase):
+    def test_does_not_crash(self):
+        """Check if get_terminal_size() returns a meaningful value.
+
+        There's no easy portable way to actually check the size of the
+        terminal, so let's check if it returns something sensible instead.
+        """
+        size = shutil.get_terminal_size()
+        self.assertGreaterEqual(size.columns, 0)
+        self.assertGreaterEqual(size.lines, 0)
+
+    def test_os_environ_first(self):
+        "Check if environment variables have precedence"
+
+        with support.EnvironmentVarGuard() as env:
+            env['COLUMNS'] = '777'
+            size = shutil.get_terminal_size()
+        self.assertEqual(size.columns, 777)
+
+        with support.EnvironmentVarGuard() as env:
+            env['LINES'] = '888'
+            size = shutil.get_terminal_size()
+        self.assertEqual(size.lines, 888)
+
+    @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty")
+    def test_stty_match(self):
+        """Check if stty returns the same results ignoring env
+
+        This test will fail if stdin and stdout are connected to
+        different terminals with different sizes. Nevertheless, such
+        situations should be pretty rare.
+        """
+        try:
+            size = subprocess.check_output(['stty', 'size']).decode().split()
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            self.skipTest("stty invocation failed")
+        expected = (int(size[1]), int(size[0])) # reversed order
+
+        with support.EnvironmentVarGuard() as env:
+            del env['LINES']
+            del env['COLUMNS']
+            actual = shutil.get_terminal_size()
 
+        self.assertEqual(expected, actual)
 
 
 def test_main():
-    support.run_unittest(TestShutil, TestMove, TestCopyFile)
+    support.run_unittest(TestShutil, TestMove, TestCopyFile,
+                         TermsizeTests, TestWhich)
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py
index 4a1b4a654f..99243dffe1 100644
--- a/Lib/test/test_signal.py
+++ b/Lib/test/test_signal.py
@@ -1,17 +1,19 @@
-import errno
+import unittest
+from test import support
+from contextlib import closing
 import gc
-import os
 import pickle
 import select
 import signal
+import struct
 import subprocess
-import sys
-import time
 import traceback
-import unittest
-from test import support
-from contextlib import closing
+import sys, os, time, errno
 from test.script_helper import assert_python_ok, spawn_python
+try:
+    import threading
+except ImportError:
+    threading = None
 
 if sys.platform in ('os2', 'riscos'):
     raise unittest.SkipTest("Can't test signal on %s" % sys.platform)
@@ -57,15 +59,9 @@ class InterProcessSignalTests(unittest.TestCase):
 
     def handlerA(self, signum, frame):
         self.a_called = True
-        if support.verbose:
-            print("handlerA invoked from signal %s at:\n%s" % (
-                signum, self.format_frame(frame, limit=1)))
 
     def handlerB(self, signum, frame):
         self.b_called = True
-        if support.verbose:
-            print ("handlerB invoked from signal %s at:\n%s" % (
-                signum, self.format_frame(frame, limit=1)))
         raise HandlerBCalled(signum, self.format_frame(frame))
 
     def wait(self, child):
@@ -92,8 +88,6 @@ class InterProcessSignalTests(unittest.TestCase):
 
         # Let the sub-processes know who to send signals to.
         pid = os.getpid()
-        if support.verbose:
-            print("test runner's pid is", pid)
 
         child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)])
         if child:
@@ -117,8 +111,6 @@ class InterProcessSignalTests(unittest.TestCase):
         except HandlerBCalled:
             self.assertTrue(self.b_called)
             self.assertFalse(self.a_called)
-            if support.verbose:
-                print("HandlerBCalled exception caught")
 
         child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)])
         if child:
@@ -134,8 +126,7 @@ class InterProcessSignalTests(unittest.TestCase):
             # may return early.
             time.sleep(1)
         except KeyboardInterrupt:
-            if support.verbose:
-                print("KeyboardInterrupt (the alarm() went off)")
+            pass
         except:
             self.fail("Some other exception woke us from pause: %s" %
                       traceback.format_exc())
@@ -191,7 +182,7 @@ class InterProcessSignalTests(unittest.TestCase):
 
 
 @unittest.skipIf(sys.platform == "win32", "Not valid on Windows")
-class BasicSignalTests(unittest.TestCase):
+class PosixTests(unittest.TestCase):
     def trivial_signal_handler(self, *args):
         pass
 
@@ -231,33 +222,53 @@ class WindowsSignalTests(unittest.TestCase):
             signal.signal(7, handler)
 
 
+class WakeupFDTests(unittest.TestCase):
+
+    def test_invalid_fd(self):
+        fd = support.make_bad_fd()
+        self.assertRaises(ValueError, signal.set_wakeup_fd, fd)
+
+
 @unittest.skipIf(sys.platform == "win32", "Not valid on Windows")
 class WakeupSignalTests(unittest.TestCase):
-    def check_wakeup(self, test_body):
-        # use a subprocess to have only one thread and to not change signal
-        # handling of the parent process
+    def check_wakeup(self, test_body, *signals, ordered=True):
+        # use a subprocess to have only one thread
         code = """if 1:
         import fcntl
         import os
         import signal
+        import struct
+
+        signals = {!r}
 
         def handler(signum, frame):
             pass
 
+        def check_signum(signals):
+            data = os.read(read, len(signals)+1)
+            raised = struct.unpack('%uB' % len(data), data)
+            if not {!r}:
+                raised = set(raised)
+                signals = set(signals)
+            if raised != signals:
+                raise Exception("%r != %r" % (raised, signals))
+
         {}
 
         signal.signal(signal.SIGALRM, handler)
         read, write = os.pipe()
-        flags = fcntl.fcntl(write, fcntl.F_GETFL, 0)
-        flags = flags | os.O_NONBLOCK
-        fcntl.fcntl(write, fcntl.F_SETFL, flags)
+        for fd in (read, write):
+            flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
+            flags = flags | os.O_NONBLOCK
+            fcntl.fcntl(fd, fcntl.F_SETFL, flags)
         signal.set_wakeup_fd(write)
 
         test()
+        check_signum(signals)
 
         os.close(read)
         os.close(write)
-        """.format(test_body)
+        """.format(signals, ordered, test_body)
 
         assert_python_ok('-c', code)
 
@@ -283,7 +294,7 @@ class WakeupSignalTests(unittest.TestCase):
             dt = after_time - mid_time
             if dt >= TIMEOUT_HALF:
                 raise Exception("%s >= %s" % (dt, TIMEOUT_HALF))
-        """)
+        """, signal.SIGALRM)
 
     def test_wakeup_fd_during(self):
         self.check_wakeup("""def test():
@@ -306,7 +317,32 @@ class WakeupSignalTests(unittest.TestCase):
             dt = after_time - before_time
             if dt >= TIMEOUT_HALF:
                 raise Exception("%s >= %s" % (dt, TIMEOUT_HALF))
-        """)
+        """, signal.SIGALRM)
+
+    def test_signum(self):
+        self.check_wakeup("""def test():
+            signal.signal(signal.SIGUSR1, handler)
+            os.kill(os.getpid(), signal.SIGUSR1)
+            os.kill(os.getpid(), signal.SIGALRM)
+        """, signal.SIGUSR1, signal.SIGALRM)
+
+    @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'),
+                         'need signal.pthread_sigmask()')
+    def test_pending(self):
+        self.check_wakeup("""def test():
+            signum1 = signal.SIGUSR1
+            signum2 = signal.SIGUSR2
+
+            signal.signal(signum1, handler)
+            signal.signal(signum2, handler)
+
+            signal.pthread_sigmask(signal.SIG_BLOCK, (signum1, signum2))
+            os.kill(os.getpid(), signum1)
+            os.kill(os.getpid(), signum2)
+            # Unblocking the 2 signals calls the C signal handler twice
+            signal.pthread_sigmask(signal.SIG_UNBLOCK, (signum1, signum2))
+        """,  signal.SIGUSR1, signal.SIGUSR2, ordered=False)
+
 
 @unittest.skipIf(sys.platform == "win32", "Not valid on Windows")
 class SiginterruptTest(unittest.TestCase):
@@ -316,9 +352,6 @@ class SiginterruptTest(unittest.TestCase):
         read is interrupted by the signal and raises an exception.  Return False
         if it returns normally.
         """
-        class Timeout(Exception):
-            pass
-
         # use a subprocess to have only one thread, to have a timeout on the
         # blocking read and to not touch signal handling in this process
         code = """if 1:
@@ -359,18 +392,8 @@ class SiginterruptTest(unittest.TestCase):
                 # wait until the child process is loaded and has started
                 first_line = process.stdout.readline()
 
-                # Wait the process with a timeout of 5 seconds
-                timeout = time.time() + 5.0
-                while True:
-                    if timeout < time.time():
-                        raise Timeout()
-                    status = process.poll()
-                    if status is not None:
-                        break
-                    time.sleep(0.1)
-
-                stdout, stderr = process.communicate()
-            except Timeout:
+                stdout, stderr = process.communicate(timeout=5.0)
+            except subprocess.TimeoutExpired:
                 process.kill()
                 return False
             else:
@@ -419,8 +442,6 @@ class ItimerTest(unittest.TestCase):
 
     def sig_alrm(self, *args):
         self.hndl_called = True
-        if support.verbose:
-            print("SIGALRM handler invoked", args)
 
     def sig_vtalrm(self, *args):
         self.hndl_called = True
@@ -432,21 +453,13 @@ class ItimerTest(unittest.TestCase):
         elif self.hndl_count == 3:
             # disable ITIMER_VIRTUAL, this function shouldn't be called anymore
             signal.setitimer(signal.ITIMER_VIRTUAL, 0)
-            if support.verbose:
-                print("last SIGVTALRM handler call")
 
         self.hndl_count += 1
 
-        if support.verbose:
-            print("SIGVTALRM handler invoked", args)
-
     def sig_prof(self, *args):
         self.hndl_called = True
         signal.setitimer(signal.ITIMER_PROF, 0)
 
-        if support.verbose:
-            print("SIGPROF handler invoked", args)
-
     def test_itimer_exc(self):
         # XXX I'm assuming -1 is an invalid itimer, but maybe some platform
         # defines it ?
@@ -459,10 +472,7 @@ class ItimerTest(unittest.TestCase):
     def test_itimer_real(self):
         self.itimer = signal.ITIMER_REAL
         signal.setitimer(self.itimer, 1.0)
-        if support.verbose:
-            print("\ncall pause()...")
         signal.pause()
-
         self.assertEqual(self.hndl_called, True)
 
     # Issue 3864, unknown if this affects earlier versions of freebsd also
@@ -511,11 +521,359 @@ class ItimerTest(unittest.TestCase):
         # and the handler should have been called
         self.assertEqual(self.hndl_called, True)
 
+
+class PendingSignalsTests(unittest.TestCase):
+    """
+    Test pthread_sigmask(), pthread_kill(), sigpending() and sigwait()
+    functions.
+    """
+    @unittest.skipUnless(hasattr(signal, 'sigpending'),
+                         'need signal.sigpending()')
+    def test_sigpending_empty(self):
+        self.assertEqual(signal.sigpending(), set())
+
+    @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'),
+                         'need signal.pthread_sigmask()')
+    @unittest.skipUnless(hasattr(signal, 'sigpending'),
+                         'need signal.sigpending()')
+    def test_sigpending(self):
+        code = """if 1:
+            import os
+            import signal
+
+            def handler(signum, frame):
+                1/0
+
+            signum = signal.SIGUSR1
+            signal.signal(signum, handler)
+
+            signal.pthread_sigmask(signal.SIG_BLOCK, [signum])
+            os.kill(os.getpid(), signum)
+            pending = signal.sigpending()
+            if pending != {signum}:
+                raise Exception('%s != {%s}' % (pending, signum))
+            try:
+                signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum])
+            except ZeroDivisionError:
+                pass
+            else:
+                raise Exception("ZeroDivisionError not raised")
+        """
+        assert_python_ok('-c', code)
+
+    @unittest.skipUnless(hasattr(signal, 'pthread_kill'),
+                         'need signal.pthread_kill()')
+    def test_pthread_kill(self):
+        code = """if 1:
+            import signal
+            import threading
+            import sys
+
+            signum = signal.SIGUSR1
+
+            def handler(signum, frame):
+                1/0
+
+            signal.signal(signum, handler)
+
+            if sys.platform == 'freebsd6':
+                # Issue #12392 and #12469: send a signal to the main thread
+                # doesn't work before the creation of the first thread on
+                # FreeBSD 6
+                def noop():
+                    pass
+                thread = threading.Thread(target=noop)
+                thread.start()
+                thread.join()
+
+            tid = threading.get_ident()
+            try:
+                signal.pthread_kill(tid, signum)
+            except ZeroDivisionError:
+                pass
+            else:
+                raise Exception("ZeroDivisionError not raised")
+        """
+        assert_python_ok('-c', code)
+
+    @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'),
+                         'need signal.pthread_sigmask()')
+    def wait_helper(self, blocked, test):
+        """
+        test: body of the "def test(signum):" function.
+        blocked: number of the blocked signal
+        """
+        code = '''if 1:
+        import signal
+        import sys
+
+        def handler(signum, frame):
+            1/0
+
+        %s
+
+        blocked = %s
+        signum = signal.SIGALRM
+
+        # child: block and wait the signal
+        try:
+            signal.signal(signum, handler)
+            signal.pthread_sigmask(signal.SIG_BLOCK, [blocked])
+
+            # Do the tests
+            test(signum)
+
+            # The handler must not be called on unblock
+            try:
+                signal.pthread_sigmask(signal.SIG_UNBLOCK, [blocked])
+            except ZeroDivisionError:
+                print("the signal handler has been called",
+                      file=sys.stderr)
+                sys.exit(1)
+        except BaseException as err:
+            print("error: {}".format(err), file=sys.stderr)
+            sys.stderr.flush()
+            sys.exit(1)
+        ''' % (test.strip(), blocked)
+
+        # sig*wait* must be called with the signal blocked: since the current
+        # process might have several threads running, use a subprocess to have
+        # a single thread.
+        assert_python_ok('-c', code)
+
+    @unittest.skipUnless(hasattr(signal, 'sigwait'),
+                         'need signal.sigwait()')
+    def test_sigwait(self):
+        self.wait_helper(signal.SIGALRM, '''
+        def test(signum):
+            signal.alarm(1)
+            received = signal.sigwait([signum])
+            if received != signum:
+                raise Exception('received %s, not %s' % (received, signum))
+        ''')
+
+    @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'),
+                         'need signal.sigwaitinfo()')
+    def test_sigwaitinfo(self):
+        self.wait_helper(signal.SIGALRM, '''
+        def test(signum):
+            signal.alarm(1)
+            info = signal.sigwaitinfo([signum])
+            if info.si_signo != signum:
+                raise Exception("info.si_signo != %s" % signum)
+        ''')
+
+    @unittest.skipUnless(hasattr(signal, 'sigtimedwait'),
+                         'need signal.sigtimedwait()')
+    def test_sigtimedwait(self):
+        self.wait_helper(signal.SIGALRM, '''
+        def test(signum):
+            signal.alarm(1)
+            info = signal.sigtimedwait([signum], 10.1000)
+            if info.si_signo != signum:
+                raise Exception('info.si_signo != %s' % signum)
+        ''')
+
+    @unittest.skipUnless(hasattr(signal, 'sigtimedwait'),
+                         'need signal.sigtimedwait()')
+    def test_sigtimedwait_poll(self):
+        # check that polling with sigtimedwait works
+        self.wait_helper(signal.SIGALRM, '''
+        def test(signum):
+            import os
+            os.kill(os.getpid(), signum)
+            info = signal.sigtimedwait([signum], 0)
+            if info.si_signo != signum:
+                raise Exception('info.si_signo != %s' % signum)
+        ''')
+
+    @unittest.skipUnless(hasattr(signal, 'sigtimedwait'),
+                         'need signal.sigtimedwait()')
+    def test_sigtimedwait_timeout(self):
+        self.wait_helper(signal.SIGALRM, '''
+        def test(signum):
+            received = signal.sigtimedwait([signum], 1.0)
+            if received is not None:
+                raise Exception("received=%r" % (received,))
+        ''')
+
+    @unittest.skipUnless(hasattr(signal, 'sigtimedwait'),
+                         'need signal.sigtimedwait()')
+    def test_sigtimedwait_negative_timeout(self):
+        signum = signal.SIGALRM
+        self.assertRaises(ValueError, signal.sigtimedwait, [signum], -1.0)
+
+    @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'),
+                         'need signal.sigwaitinfo()')
+    def test_sigwaitinfo_interrupted(self):
+        self.wait_helper(signal.SIGUSR1, '''
+        def test(signum):
+            import errno
+
+            hndl_called = True
+            def alarm_handler(signum, frame):
+                hndl_called = False
+
+            signal.signal(signal.SIGALRM, alarm_handler)
+            signal.alarm(1)
+            try:
+                signal.sigwaitinfo([signal.SIGUSR1])
+            except OSError as e:
+                if e.errno == errno.EINTR:
+                    if not hndl_called:
+                        raise Exception("SIGALRM handler not called")
+                else:
+                    raise Exception("Expected EINTR to be raised by sigwaitinfo")
+            else:
+                raise Exception("Expected EINTR to be raised by sigwaitinfo")
+        ''')
+
+    @unittest.skipUnless(hasattr(signal, 'sigwait'),
+                         'need signal.sigwait()')
+    @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'),
+                         'need signal.pthread_sigmask()')
+    @unittest.skipIf(threading is None, "test needs threading module")
+    def test_sigwait_thread(self):
+        # Check that calling sigwait() from a thread doesn't suspend the whole
+        # process. A new interpreter is spawned to avoid problems when mixing
+        # threads and fork(): only async-safe functions are allowed between
+        # fork() and exec().
+        assert_python_ok("-c", """if True:
+            import os, threading, sys, time, signal
+
+            # the default handler terminates the process
+            signum = signal.SIGUSR1
+
+            def kill_later():
+                # wait until the main thread is waiting in sigwait()
+                time.sleep(1)
+                os.kill(os.getpid(), signum)
+
+            # the signal must be blocked by all the threads
+            signal.pthread_sigmask(signal.SIG_BLOCK, [signum])
+            killer = threading.Thread(target=kill_later)
+            killer.start()
+            received = signal.sigwait([signum])
+            if received != signum:
+                print("sigwait() received %s, not %s" % (received, signum),
+                      file=sys.stderr)
+                sys.exit(1)
+            killer.join()
+            # unblock the signal, which should have been cleared by sigwait()
+            signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum])
+        """)
+
+    @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'),
+                         'need signal.pthread_sigmask()')
+    def test_pthread_sigmask_arguments(self):
+        self.assertRaises(TypeError, signal.pthread_sigmask)
+        self.assertRaises(TypeError, signal.pthread_sigmask, 1)
+        self.assertRaises(TypeError, signal.pthread_sigmask, 1, 2, 3)
+        self.assertRaises(OSError, signal.pthread_sigmask, 1700, [])
+
+    @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'),
+                         'need signal.pthread_sigmask()')
+    def test_pthread_sigmask(self):
+        code = """if 1:
+        import signal
+        import os; import threading
+
+        def handler(signum, frame):
+            1/0
+
+        def kill(signum):
+            os.kill(os.getpid(), signum)
+
+        def read_sigmask():
+            return signal.pthread_sigmask(signal.SIG_BLOCK, [])
+
+        signum = signal.SIGUSR1
+
+        # Install our signal handler
+        old_handler = signal.signal(signum, handler)
+
+        # Unblock SIGUSR1 (and copy the old mask) to test our signal handler
+        old_mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum])
+        try:
+            kill(signum)
+        except ZeroDivisionError:
+            pass
+        else:
+            raise Exception("ZeroDivisionError not raised")
+
+        # Block and then raise SIGUSR1. The signal is blocked: the signal
+        # handler is not called, and the signal is now pending
+        signal.pthread_sigmask(signal.SIG_BLOCK, [signum])
+        kill(signum)
+
+        # Check the new mask
+        blocked = read_sigmask()
+        if signum not in blocked:
+            raise Exception("%s not in %s" % (signum, blocked))
+        if old_mask ^ blocked != {signum}:
+            raise Exception("%s ^ %s != {%s}" % (old_mask, blocked, signum))
+
+        # Unblock SIGUSR1
+        try:
+            # unblock the pending signal calls immediatly the signal handler
+            signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum])
+        except ZeroDivisionError:
+            pass
+        else:
+            raise Exception("ZeroDivisionError not raised")
+        try:
+            kill(signum)
+        except ZeroDivisionError:
+            pass
+        else:
+            raise Exception("ZeroDivisionError not raised")
+
+        # Check the new mask
+        unblocked = read_sigmask()
+        if signum in unblocked:
+            raise Exception("%s in %s" % (signum, unblocked))
+        if blocked ^ unblocked != {signum}:
+            raise Exception("%s ^ %s != {%s}" % (blocked, unblocked, signum))
+        if old_mask != unblocked:
+            raise Exception("%s != %s" % (old_mask, unblocked))
+        """
+        assert_python_ok('-c', code)
+
+    @unittest.skipIf(sys.platform == 'freebsd6',
+        "issue #12392: send a signal to the main thread doesn't work "
+        "before the creation of the first thread on FreeBSD 6")
+    @unittest.skipUnless(hasattr(signal, 'pthread_kill'),
+                         'need signal.pthread_kill()')
+    def test_pthread_kill_main_thread(self):
+        # Test that a signal can be sent to the main thread with pthread_kill()
+        # before any other thread has been created (see issue #12392).
+        code = """if True:
+            import threading
+            import signal
+            import sys
+
+            def handler(signum, frame):
+                sys.exit(3)
+
+            signal.signal(signal.SIGUSR1, handler)
+            signal.pthread_kill(threading.get_ident(), signal.SIGUSR1)
+            sys.exit(2)
+        """
+
+        with spawn_python('-c', code) as process:
+            stdout, stderr = process.communicate()
+            exitcode = process.wait()
+            if exitcode != 3:
+                raise Exception("Child error (exit code %s): %s" %
+                                (exitcode, stdout))
+
+
 def test_main():
     try:
-        support.run_unittest(BasicSignalTests, InterProcessSignalTests,
-                             WakeupSignalTests, SiginterruptTest,
-                             ItimerTest, WindowsSignalTests)
+        support.run_unittest(PosixTests, InterProcessSignalTests,
+                             WakeupFDTests, WakeupSignalTests,
+                             SiginterruptTest, ItimerTest, WindowsSignalTests,
+                             PendingSignalsTests)
     finally:
         support.reap_children()
 
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index ba426494fb..29286c710b 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -39,6 +39,7 @@ class HelperFunctionsTests(unittest.TestCase):
         self.old_base = site.USER_BASE
         self.old_site = site.USER_SITE
         self.old_prefixes = site.PREFIXES
+        self.original_vars = sysconfig._CONFIG_VARS
         self.old_vars = copy(sysconfig._CONFIG_VARS)
 
     def tearDown(self):
@@ -47,7 +48,9 @@ class HelperFunctionsTests(unittest.TestCase):
         site.USER_BASE = self.old_base
         site.USER_SITE = self.old_site
         site.PREFIXES = self.old_prefixes
-        sysconfig._CONFIG_VARS = self.old_vars
+        sysconfig._CONFIG_VARS = self.original_vars
+        sysconfig._CONFIG_VARS.clear()
+        sysconfig._CONFIG_VARS.update(self.old_vars)
 
     def test_makepath(self):
         # Test makepath() have an absolute path for its first return value
diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py
index 68ccc29036..93f14c4562 100644
--- a/Lib/test/test_smtpd.py
+++ b/Lib/test/test_smtpd.py
@@ -1,4 +1,4 @@
-from unittest import TestCase
+import unittest
 from test import support, mock_socket
 import socket
 import io
@@ -26,7 +26,7 @@ class BrokenDummyServer(DummyServer):
         raise DummyDispatcherBroken()
 
 
-class SMTPDServerTest(TestCase):
+class SMTPDServerTest(unittest.TestCase):
     def setUp(self):
         smtpd.socket = asyncore.socket = mock_socket
 
@@ -39,6 +39,7 @@ class SMTPDServerTest(TestCase):
             channel.socket.queue_recv(line)
             channel.handle_read()
 
+        write_line(b'HELO example')
         write_line(b'MAIL From:eggs@example')
         write_line(b'RCPT To:spam@example')
         write_line(b'DATA')
@@ -49,7 +50,7 @@ class SMTPDServerTest(TestCase):
         asyncore.socket = smtpd.socket = socket
 
 
-class SMTPDChannelTest(TestCase):
+class SMTPDChannelTest(unittest.TestCase):
     def setUp(self):
         smtpd.socket = asyncore.socket = mock_socket
         self.old_debugstream = smtpd.DEBUGSTREAM
@@ -78,31 +79,94 @@ class SMTPDChannelTest(TestCase):
         self.assertEqual(self.channel.socket.last,
                          b'500 Error: bad syntax\r\n')
 
-    def test_EHLO_not_implemented(self):
-        self.write_line(b'EHLO test.example')
+    def test_EHLO(self):
+        self.write_line(b'EHLO example')
+        self.assertEqual(self.channel.socket.last, b'250 HELP\r\n')
+
+    def test_EHLO_bad_syntax(self):
+        self.write_line(b'EHLO')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Syntax: EHLO hostname\r\n')
+
+    def test_EHLO_duplicate(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'EHLO example')
         self.assertEqual(self.channel.socket.last,
-                         b'502 Error: command "EHLO" not implemented\r\n')
+                         b'503 Duplicate HELO/EHLO\r\n')
+
+    def test_EHLO_HELO_duplicate(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'HELO example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Duplicate HELO/EHLO\r\n')
 
     def test_HELO(self):
         name = smtpd.socket.getfqdn()
-        self.write_line(b'HELO test.example')
+        self.write_line(b'HELO example')
         self.assertEqual(self.channel.socket.last,
                          '250 {}\r\n'.format(name).encode('ascii'))
 
+    def test_HELO_EHLO_duplicate(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'EHLO example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Duplicate HELO/EHLO\r\n')
+
+    def test_HELP(self):
+        self.write_line(b'HELP')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 Supported commands: EHLO HELO MAIL RCPT ' + \
+                         b'DATA RSET NOOP QUIT VRFY\r\n')
+
+    def test_HELP_command(self):
+        self.write_line(b'HELP MAIL')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 Syntax: MAIL FROM: <address>\r\n')
+
+    def test_HELP_command_unknown(self):
+        self.write_line(b'HELP SPAM')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Supported commands: EHLO HELO MAIL RCPT ' + \
+                         b'DATA RSET NOOP QUIT VRFY\r\n')
+
     def test_HELO_bad_syntax(self):
         self.write_line(b'HELO')
         self.assertEqual(self.channel.socket.last,
                          b'501 Syntax: HELO hostname\r\n')
 
     def test_HELO_duplicate(self):
-        self.write_line(b'HELO test.example')
-        self.write_line(b'HELO test.example')
+        self.write_line(b'HELO example')
+        self.write_line(b'HELO example')
         self.assertEqual(self.channel.socket.last,
                          b'503 Duplicate HELO/EHLO\r\n')
 
+    def test_HELO_parameter_rejected_when_extensions_not_enabled(self):
+        self.extended_smtp = False
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from:<foo@example.com> SIZE=1234')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Syntax: MAIL FROM: <address>\r\n')
+
+    def test_MAIL_allows_space_after_colon(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from:   <foo@example.com>')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 OK\r\n')
+
+    def test_extended_MAIL_allows_space_after_colon(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from:   <foo@example.com> size=20')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 OK\r\n')
+
     def test_NOOP(self):
         self.write_line(b'NOOP')
-        self.assertEqual(self.channel.socket.last, b'250 Ok\r\n')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_HELO_NOOP(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'NOOP')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
 
     def test_NOOP_bad_syntax(self):
         self.write_line(b'NOOP hi')
@@ -113,26 +177,47 @@ class SMTPDChannelTest(TestCase):
         self.write_line(b'QUIT')
         self.assertEqual(self.channel.socket.last, b'221 Bye\r\n')
 
+    def test_HELO_QUIT(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'QUIT')
+        self.assertEqual(self.channel.socket.last, b'221 Bye\r\n')
+
     def test_QUIT_arg_ignored(self):
         self.write_line(b'QUIT bye bye')
         self.assertEqual(self.channel.socket.last, b'221 Bye\r\n')
 
     def test_bad_state(self):
         self.channel.smtp_state = 'BAD STATE'
-        self.write_line(b'HELO')
+        self.write_line(b'HELO example')
         self.assertEqual(self.channel.socket.last,
                          b'451 Internal confusion\r\n')
 
     def test_command_too_long(self):
-        self.write_line(b'MAIL from ' +
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from: ' +
                         b'a' * self.channel.command_size_limit +
                         b'@example')
         self.assertEqual(self.channel.socket.last,
                          b'500 Error: line too long\r\n')
 
-    def test_data_too_long(self):
-        # Small hack. Setting limit to 2K octets here will save us some time.
-        self.channel.data_size_limit = 2048
+    def test_MAIL_command_limit_extended_with_SIZE(self):
+        self.write_line(b'EHLO example')
+        fill_len = self.channel.command_size_limit - len('MAIL from:<@example>')
+        self.write_line(b'MAIL from:<' +
+                        b'a' * fill_len +
+                        b'@example> SIZE=1234')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+        self.write_line(b'MAIL from:<' +
+                        b'a' * (fill_len + 26) +
+                        b'@example> SIZE=1234')
+        self.assertEqual(self.channel.socket.last,
+                         b'500 Error: line too long\r\n')
+
+    def test_data_longer_than_default_data_size_limit(self):
+        # Hack the default so we don't have to generate so much data.
+        self.channel.data_size_limit = 1048
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
         self.write_line(b'RCPT To:spam@example')
         self.write_line(b'DATA')
@@ -141,64 +226,183 @@ class SMTPDChannelTest(TestCase):
         self.assertEqual(self.channel.socket.last,
                          b'552 Error: Too much mail data\r\n')
 
+    def test_MAIL_size_parameter(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs@example> SIZE=512')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 OK\r\n')
+
+    def test_MAIL_invalid_size_parameter(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs@example> SIZE=invalid')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: MAIL FROM: <address> [SP <mail-parameters>]\r\n')
+
+    def test_MAIL_RCPT_unknown_parameters(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs@example> ham=green')
+        self.assertEqual(self.channel.socket.last,
+            b'555 MAIL FROM parameters not recognized or not implemented\r\n')
+
+        self.write_line(b'MAIL FROM:<eggs@example>')
+        self.write_line(b'RCPT TO:<eggs@example> ham=green')
+        self.assertEqual(self.channel.socket.last,
+            b'555 RCPT TO parameters not recognized or not implemented\r\n')
+
+    def test_MAIL_size_parameter_larger_than_default_data_size_limit(self):
+        self.channel.data_size_limit = 1048
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs@example> SIZE=2096')
+        self.assertEqual(self.channel.socket.last,
+            b'552 Error: message size exceeds fixed maximum message size\r\n')
+
     def test_need_MAIL(self):
+        self.write_line(b'HELO example')
         self.write_line(b'RCPT to:spam@example')
         self.assertEqual(self.channel.socket.last,
             b'503 Error: need MAIL command\r\n')
 
-    def test_MAIL_syntax(self):
+    def test_MAIL_syntax_HELO(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL from eggs@example')
         self.assertEqual(self.channel.socket.last,
-            b'501 Syntax: MAIL FROM:<address>\r\n')
+            b'501 Syntax: MAIL FROM: <address>\r\n')
 
-    def test_MAIL_missing_from(self):
+    def test_MAIL_syntax_EHLO(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from eggs@example')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: MAIL FROM: <address> [SP <mail-parameters>]\r\n')
+
+    def test_MAIL_missing_address(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL from:')
         self.assertEqual(self.channel.socket.last,
-            b'501 Syntax: MAIL FROM:<address>\r\n')
+            b'501 Syntax: MAIL FROM: <address>\r\n')
 
     def test_MAIL_chevrons(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL from:<eggs@example>')
-        self.assertEqual(self.channel.socket.last, b'250 Ok\r\n')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_MAIL_empty_chevrons(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from:<>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_MAIL_quoted_localpart(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: <"Fred Blogs"@example.com>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
+
+    def test_MAIL_quoted_localpart_no_angles(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: "Fred Blogs"@example.com')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
+
+    def test_MAIL_quoted_localpart_with_size(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
+
+    def test_MAIL_quoted_localpart_with_size_no_angles(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
 
     def test_nested_MAIL(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL from:eggs@example')
         self.write_line(b'MAIL from:spam@example')
         self.assertEqual(self.channel.socket.last,
             b'503 Error: nested MAIL command\r\n')
 
+    def test_VRFY(self):
+        self.write_line(b'VRFY eggs@example')
+        self.assertEqual(self.channel.socket.last,
+            b'252 Cannot VRFY user, but will accept message and attempt ' + \
+            b'delivery\r\n')
+
+    def test_VRFY_syntax(self):
+        self.write_line(b'VRFY')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: VRFY <address>\r\n')
+
+    def test_EXPN_not_implemented(self):
+        self.write_line(b'EXPN')
+        self.assertEqual(self.channel.socket.last,
+            b'502 EXPN not implemented\r\n')
+
+    def test_no_HELO_MAIL(self):
+        self.write_line(b'MAIL from:<foo@example.com>')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Error: send HELO first\r\n')
+
     def test_need_RCPT(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
         self.write_line(b'DATA')
         self.assertEqual(self.channel.socket.last,
             b'503 Error: need RCPT command\r\n')
 
-    def test_RCPT_syntax(self):
-        self.write_line(b'MAIL From:eggs@example')
+    def test_RCPT_syntax_HELO(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From: eggs@example')
         self.write_line(b'RCPT to eggs@example')
         self.assertEqual(self.channel.socket.last,
             b'501 Syntax: RCPT TO: <address>\r\n')
 
+    def test_RCPT_syntax_EHLO(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL From: eggs@example')
+        self.write_line(b'RCPT to eggs@example')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: RCPT TO: <address> [SP <mail-parameters>]\r\n')
+
+    def test_RCPT_lowercase_to_OK(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From: eggs@example')
+        self.write_line(b'RCPT to: <eggs@example>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_no_HELO_RCPT(self):
+        self.write_line(b'RCPT to eggs@example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Error: send HELO first\r\n')
+
     def test_data_dialog(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
-        self.assertEqual(self.channel.socket.last, b'250 Ok\r\n')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
         self.write_line(b'RCPT To:spam@example')
-        self.assertEqual(self.channel.socket.last, b'250 Ok\r\n')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
 
         self.write_line(b'DATA')
         self.assertEqual(self.channel.socket.last,
             b'354 End data with <CR><LF>.<CR><LF>\r\n')
         self.write_line(b'data\r\nmore\r\n.')
-        self.assertEqual(self.channel.socket.last, b'250 Ok\r\n')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
         self.assertEqual(self.server.messages,
             [('peer', 'eggs@example', ['spam@example'], 'data\nmore')])
 
     def test_DATA_syntax(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
         self.write_line(b'RCPT To:spam@example')
         self.write_line(b'DATA spam')
         self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n')
 
+    def test_no_HELO_DATA(self):
+        self.write_line(b'DATA spam')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Error: send HELO first\r\n')
+
     def test_data_transparency_section_4_5_2(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
         self.write_line(b'RCPT To:spam@example')
         self.write_line(b'DATA')
@@ -206,6 +410,7 @@ class SMTPDChannelTest(TestCase):
         self.assertEqual(self.channel.received_data, '.')
 
     def test_multiple_RCPT(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
         self.write_line(b'RCPT To:spam@example')
         self.write_line(b'RCPT To:ham@example')
@@ -216,6 +421,7 @@ class SMTPDChannelTest(TestCase):
 
     def test_manual_status(self):
         # checks that the Channel is able to return a custom status message
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
         self.write_line(b'RCPT To:spam@example')
         self.write_line(b'DATA')
@@ -223,10 +429,11 @@ class SMTPDChannelTest(TestCase):
         self.assertEqual(self.channel.socket.last, b'250 Okish\r\n')
 
     def test_RSET(self):
+        self.write_line(b'HELO example')
         self.write_line(b'MAIL From:eggs@example')
         self.write_line(b'RCPT To:spam@example')
         self.write_line(b'RSET')
-        self.assertEqual(self.channel.socket.last, b'250 Ok\r\n')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
         self.write_line(b'MAIL From:foo@example')
         self.write_line(b'RCPT To:eggs@example')
         self.write_line(b'DATA')
@@ -234,58 +441,117 @@ class SMTPDChannelTest(TestCase):
         self.assertEqual(self.server.messages,
             [('peer', 'foo@example', ['eggs@example'], 'data')])
 
+    def test_HELO_RSET(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'RSET')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
     def test_RSET_syntax(self):
         self.write_line(b'RSET hi')
         self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n')
 
+    def test_unknown_command(self):
+        self.write_line(b'UNKNOWN_CMD')
+        self.assertEqual(self.channel.socket.last,
+                         b'500 Error: command "UNKNOWN_CMD" not ' + \
+                         b'recognized\r\n')
+
     def test_attribute_deprecations(self):
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__server
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__server = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__line
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__line = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__state
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__state = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__greeting
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__greeting = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__mailfrom
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__mailfrom = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__rcpttos
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__rcpttos = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__data
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__data = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__fqdn
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__fqdn = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__peer
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__peer = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__conn
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__conn = 'spam'
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             spam = self.channel._SMTPChannel__addr
-        with support.check_warnings(('', PendingDeprecationWarning)):
+        with support.check_warnings(('', DeprecationWarning)):
             self.channel._SMTPChannel__addr = 'spam'
 
-def test_main():
-    support.run_unittest(SMTPDServerTest, SMTPDChannelTest)
+
+class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase):
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+        self.server = DummyServer('a', 'b')
+        conn, addr = self.server.accept()
+        # Set DATA size limit to 32 bytes for easy testing
+        self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32)
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, line):
+        self.channel.socket.queue_recv(line)
+        self.channel.handle_read()
+
+    def test_data_limit_dialog(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs@example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.write_line(b'RCPT To:spam@example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+        self.write_line(b'DATA')
+        self.assertEqual(self.channel.socket.last,
+            b'354 End data with <CR><LF>.<CR><LF>\r\n')
+        self.write_line(b'data\r\nmore\r\n.')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.server.messages,
+            [('peer', 'eggs@example', ['spam@example'], 'data\nmore')])
+
+    def test_data_limit_dialog_too_much_data(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs@example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.write_line(b'RCPT To:spam@example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+        self.write_line(b'DATA')
+        self.assertEqual(self.channel.socket.last,
+            b'354 End data with <CR><LF>.<CR><LF>\r\n')
+        self.write_line(b'This message is longer than 32 bytes\r\n.')
+        self.assertEqual(self.channel.socket.last,
+                         b'552 Error: Too much mail data\r\n')
+
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index 2cb0d1ab52..befc49e9d3 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -9,6 +9,7 @@ import re
 import sys
 import time
 import select
+import errno
 
 import unittest
 from test import support, mock_socket
@@ -72,6 +73,14 @@ class GeneralTests(unittest.TestCase):
         smtp = smtplib.SMTP(HOST, self.port)
         smtp.close()
 
+    def testSourceAddress(self):
+        mock_socket.reply_with(b"220 Hola mundo")
+        # connects
+        smtp = smtplib.SMTP(HOST, self.port,
+                source_address=('127.0.0.1',19876))
+        self.assertEqual(smtp.source_address, ('127.0.0.1', 19876))
+        smtp.close()
+
     def testBasic2(self):
         mock_socket.reply_with(b"220 Hola mundo")
         # connects, include port in host name
@@ -204,15 +213,29 @@ class DebuggingServerTests(unittest.TestCase):
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
         smtp.quit()
 
+    def testSourceAddress(self):
+        # connect
+        port = support.find_unused_port()
+        try:
+            smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost',
+                    timeout=3, source_address=('127.0.0.1', port))
+            self.assertEqual(smtp.source_address, ('127.0.0.1', port))
+            self.assertEqual(smtp.local_hostname, 'localhost')
+            smtp.quit()
+        except IOError as e:
+            if e.errno == errno.EADDRINUSE:
+                self.skipTest("couldn't bind to port %d" % port)
+            raise
+
     def testNOOP(self):
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
-        expected = (250, b'Ok')
+        expected = (250, b'OK')
         self.assertEqual(smtp.noop(), expected)
         smtp.quit()
 
     def testRSET(self):
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
-        expected = (250, b'Ok')
+        expected = (250, b'OK')
         self.assertEqual(smtp.rset(), expected)
         smtp.quit()
 
@@ -223,10 +246,18 @@ class DebuggingServerTests(unittest.TestCase):
         self.assertEqual(smtp.ehlo(), expected)
         smtp.quit()
 
+    def testNotImplemented(self):
+        # EXPN isn't implemented in DebuggingServer
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        expected = (502, b'EXPN not implemented')
+        smtp.putcmd('EXPN')
+        self.assertEqual(smtp.getreply(), expected)
+        smtp.quit()
+
     def testVRFY(self):
-        # VRFY isn't implemented in DebuggingServer
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
-        expected = (502, b'Error: command "VRFY" not implemented')
+        expected = (252, b'Cannot VRFY user, but will accept message ' + \
+                         b'and attempt delivery')
         self.assertEqual(smtp.vrfy('nobody@nowhere.com'), expected)
         self.assertEqual(smtp.verify('nobody@nowhere.com'), expected)
         smtp.quit()
@@ -242,7 +273,8 @@ class DebuggingServerTests(unittest.TestCase):
 
     def testHELP(self):
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
-        self.assertEqual(smtp.help(), b'Error: command "HELP" not implemented')
+        self.assertEqual(smtp.help(), b'Supported commands: EHLO HELO MAIL ' + \
+                                      b'RCPT DATA RSET NOOP QUIT VRFY')
         smtp.quit()
 
     def testSend(self):
@@ -560,6 +592,9 @@ sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'],
 # Simulated SMTP channel & server
 class SimSMTPChannel(smtpd.SMTPChannel):
 
+    # For testing failures in QUIT when using the context manager API.
+    quit_response = None
+
     def __init__(self, extra_features, *args, **kw):
         self._extrafeatures = ''.join(
             [ "250-{0}\r\n".format(x) for x in extra_features ])
@@ -610,19 +645,31 @@ class SimSMTPChannel(smtpd.SMTPChannel):
         else:
             self.push('550 No access for you!')
 
+    def smtp_QUIT(self, arg):
+        # args is ignored
+        if self.quit_response is None:
+            super(SimSMTPChannel, self).smtp_QUIT(arg)
+        else:
+            self.push(self.quit_response)
+            self.close_when_done()
+
     def handle_error(self):
         raise
 
 
 class SimSMTPServer(smtpd.SMTPServer):
 
+    # For testing failures in QUIT when using the context manager API.
+    quit_response = None
+
     def __init__(self, *args, **kw):
         self._extra_features = []
         smtpd.SMTPServer.__init__(self, *args, **kw)
 
     def handle_accepted(self, conn, addr):
-        self._SMTPchannel = SimSMTPChannel(self._extra_features,
-                                           self, conn, addr)
+        self._SMTPchannel = SimSMTPChannel(
+            self._extra_features, self, conn, addr)
+        self._SMTPchannel.quit_response = self.quit_response
 
     def process_message(self, peer, mailfrom, rcpttos, data):
         pass
@@ -752,6 +799,25 @@ class SMTPSimTests(unittest.TestCase):
             self.assertIn(sim_auth_credentials['cram-md5'], str(err))
         smtp.close()
 
+    def test_with_statement(self):
+        with smtplib.SMTP(HOST, self.port) as smtp:
+            code, message = smtp.noop()
+            self.assertEqual(code, 250)
+        self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo')
+        with smtplib.SMTP(HOST, self.port) as smtp:
+            smtp.close()
+        self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo')
+
+    def test_with_statement_QUIT_failure(self):
+        self.serv.quit_response = '421 QUIT FAILED'
+        with self.assertRaises(smtplib.SMTPResponseException) as error:
+            with smtplib.SMTP(HOST, self.port) as smtp:
+                smtp.noop()
+        self.assertEqual(error.exception.smtp_code, 421)
+        self.assertEqual(error.exception.smtp_error, b'QUIT FAILED')
+        # We don't need to clean up self.serv.quit_response because a new
+        # server is always instantiated in the setUp().
+
     #TODO: add tests for correct AUTH method fallback now that the
     #test infrastructure can support it.
 
diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py
index 0198ab669e..86224ef2e4 100644
--- a/Lib/test/test_smtpnet.py
+++ b/Lib/test/test_smtpnet.py
@@ -4,28 +4,60 @@ import unittest
 from test import support
 import smtplib
 
+ssl = support.import_module("ssl")
+
 support.requires("network")
 
+
+class SmtpTest(unittest.TestCase):
+    testServer = 'smtp.gmail.com'
+    remotePort = 25
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+
+    def test_connect_starttls(self):
+        support.get_attribute(smtplib, 'SMTP_SSL')
+        with support.transient_internet(self.testServer):
+            server = smtplib.SMTP(self.testServer, self.remotePort)
+            try:
+                server.starttls(context=self.context)
+            except smtplib.SMTPException as e:
+                if e.args[0] == 'STARTTLS extension not supported by server.':
+                    unittest.skip(e.args[0])
+                else:
+                    raise
+            server.ehlo()
+            server.quit()
+
+
 class SmtpSSLTest(unittest.TestCase):
     testServer = 'smtp.gmail.com'
     remotePort = 465
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
 
     def test_connect(self):
         support.get_attribute(smtplib, 'SMTP_SSL')
         with support.transient_internet(self.testServer):
             server = smtplib.SMTP_SSL(self.testServer, self.remotePort)
-        server.ehlo()
-        server.quit()
+            server.ehlo()
+            server.quit()
 
     def test_connect_default_port(self):
         support.get_attribute(smtplib, 'SMTP_SSL')
         with support.transient_internet(self.testServer):
             server = smtplib.SMTP_SSL(self.testServer)
-        server.ehlo()
-        server.quit()
+            server.ehlo()
+            server.quit()
+
+    def test_connect_using_sslcontext(self):
+        support.get_attribute(smtplib, 'SMTP_SSL')
+        with support.transient_internet(self.testServer):
+            server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=self.context)
+            server.ehlo()
+            server.quit()
+
 
 def test_main():
-    support.run_unittest(SmtpSSLTest)
+    support.run_unittest(SmtpTest, SmtpSSLTest)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
index 5573421728..da6ef059f2 100644
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -2,11 +2,13 @@
 
 import unittest
 from test import support
+from unittest.case import _ExpectedFailure
 
 import errno
 import io
 import socket
 import select
+import tempfile
 import _testcapi
 import time
 import traceback
@@ -19,34 +21,19 @@ import contextlib
 from weakref import proxy
 import signal
 import math
+import pickle
+import struct
 try:
     import fcntl
 except ImportError:
     fcntl = False
-
-def try_address(host, port=0, family=socket.AF_INET):
-    """Try to bind a socket on the given host:port and return True
-    if that has been possible."""
-    try:
-        sock = socket.socket(family, socket.SOCK_STREAM)
-        sock.bind((host, port))
-    except (socket.error, socket.gaierror):
-        return False
-    else:
-        sock.close()
-        return True
-
-def linux_version():
-    try:
-        # platform.release() is something like '2.6.33.7-desktop-2mnb'
-        version_string = platform.release().split('-')[0]
-        return tuple(map(int, version_string.split('.')))
-    except ValueError:
-        return 0, 0, 0
+try:
+    import multiprocessing
+except ImportError:
+    multiprocessing = False
 
 HOST = support.HOST
-MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf8') ## test unicode string and carriage return
-SUPPORTS_IPV6 = socket.has_ipv6 and try_address('::1', family=socket.AF_INET6)
+MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return
 
 try:
     import _thread as thread
@@ -55,6 +42,33 @@ except ImportError:
     thread = None
     threading = None
 
+def _have_socket_can():
+    """Check whether CAN sockets are supported on this host."""
+    try:
+        s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW)
+    except (AttributeError, socket.error, OSError):
+        return False
+    else:
+        s.close()
+    return True
+
+def _have_socket_rds():
+    """Check whether RDS sockets are supported on this host."""
+    try:
+        s = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0)
+    except (AttributeError, OSError):
+        return False
+    else:
+        s.close()
+    return True
+
+HAVE_SOCKET_CAN = _have_socket_can()
+
+HAVE_SOCKET_RDS = _have_socket_rds()
+
+# Size in bytes of the int type
+SIZEOF_INT = array.array("i").itemsize
+
 class SocketTCPTest(unittest.TestCase):
 
     def setUp(self):
@@ -76,6 +90,63 @@ class SocketUDPTest(unittest.TestCase):
         self.serv.close()
         self.serv = None
 
+class ThreadSafeCleanupTestCase(unittest.TestCase):
+    """Subclass of unittest.TestCase with thread-safe cleanup methods.
+
+    This subclass protects the addCleanup() and doCleanups() methods
+    with a recursive lock.
+    """
+
+    if threading:
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            self._cleanup_lock = threading.RLock()
+
+        def addCleanup(self, *args, **kwargs):
+            with self._cleanup_lock:
+                return super().addCleanup(*args, **kwargs)
+
+        def doCleanups(self, *args, **kwargs):
+            with self._cleanup_lock:
+                return super().doCleanups(*args, **kwargs)
+
+class SocketCANTest(unittest.TestCase):
+
+    """To be able to run this test, a `vcan0` CAN interface can be created with
+    the following commands:
+    # modprobe vcan
+    # ip link add dev vcan0 type vcan
+    # ifconfig vcan0 up
+    """
+    interface = 'vcan0'
+    bufsize = 128
+
+    def setUp(self):
+        self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW)
+        self.addCleanup(self.s.close)
+        try:
+            self.s.bind((self.interface,))
+        except socket.error:
+            self.skipTest('network interface `%s` does not exist' %
+                           self.interface)
+
+
+class SocketRDSTest(unittest.TestCase):
+
+    """To be able to run this test, the `rds` kernel module must be loaded:
+    # modprobe rds
+    """
+    bufsize = 8192
+
+    def setUp(self):
+        self.serv = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0)
+        self.addCleanup(self.serv.close)
+        try:
+            self.port = support.bind_port(self.serv)
+        except OSError:
+            self.skipTest('unable to bind RDS socket')
+
+
 class ThreadableTest:
     """Threadable Test class
 
@@ -133,6 +204,7 @@ class ThreadableTest:
         self.client_ready = threading.Event()
         self.done = threading.Event()
         self.queue = queue.Queue(1)
+        self.server_crashed = False
 
         # Do some munging to start the client test.
         methodname = self.id()
@@ -142,8 +214,12 @@ class ThreadableTest:
         self.client_thread = thread.start_new_thread(
             self.clientRun, (test_method,))
 
-        self.__setUp()
-        if not self.server_ready.is_set():
+        try:
+            self.__setUp()
+        except:
+            self.server_crashed = True
+            raise
+        finally:
             self.server_ready.set()
         self.client_ready.wait()
 
@@ -159,10 +235,16 @@ class ThreadableTest:
         self.server_ready.wait()
         self.clientSetUp()
         self.client_ready.set()
+        if self.server_crashed:
+            self.clientTearDown()
+            return
         if not hasattr(test_func, '__call__'):
             raise TypeError("test_func must be a callable function")
         try:
             test_func()
+        except _ExpectedFailure:
+            # We deliberately ignore expected failures
+            pass
         except BaseException as e:
             self.queue.put(e)
         finally:
@@ -203,6 +285,48 @@ class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest):
         self.cli = None
         ThreadableTest.clientTearDown(self)
 
+class ThreadedCANSocketTest(SocketCANTest, ThreadableTest):
+
+    def __init__(self, methodName='runTest'):
+        SocketCANTest.__init__(self, methodName=methodName)
+        ThreadableTest.__init__(self)
+
+    def clientSetUp(self):
+        self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW)
+        try:
+            self.cli.bind((self.interface,))
+        except socket.error:
+            # skipTest should not be called here, and will be called in the
+            # server instead
+            pass
+
+    def clientTearDown(self):
+        self.cli.close()
+        self.cli = None
+        ThreadableTest.clientTearDown(self)
+
+class ThreadedRDSSocketTest(SocketRDSTest, ThreadableTest):
+
+    def __init__(self, methodName='runTest'):
+        SocketRDSTest.__init__(self, methodName=methodName)
+        ThreadableTest.__init__(self)
+
+    def clientSetUp(self):
+        self.cli = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0)
+        try:
+            # RDS sockets must be bound explicitly to send or receive data
+            self.cli.bind((HOST, 0))
+            self.cli_addr = self.cli.getsockname()
+        except OSError:
+            # skipTest should not be called here, and will be called in the
+            # server instead
+            pass
+
+    def clientTearDown(self):
+        self.cli.close()
+        self.cli = None
+        ThreadableTest.clientTearDown(self)
+
 class SocketConnectedTest(ThreadedTCPSocketTest):
     """Socket tests for client-server connection.
 
@@ -258,6 +382,243 @@ class SocketPairTest(unittest.TestCase, ThreadableTest):
         ThreadableTest.clientTearDown(self)
 
 
+# The following classes are used by the sendmsg()/recvmsg() tests.
+# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase
+# gives a drop-in replacement for SocketConnectedTest, but different
+# address families can be used, and the attributes serv_addr and
+# cli_addr will be set to the addresses of the endpoints.
+
+class SocketTestBase(unittest.TestCase):
+    """A base class for socket tests.
+
+    Subclasses must provide methods newSocket() to return a new socket
+    and bindSock(sock) to bind it to an unused address.
+
+    Creates a socket self.serv and sets self.serv_addr to its address.
+    """
+
+    def setUp(self):
+        self.serv = self.newSocket()
+        self.bindServer()
+
+    def bindServer(self):
+        """Bind server socket and set self.serv_addr to its address."""
+        self.bindSock(self.serv)
+        self.serv_addr = self.serv.getsockname()
+
+    def tearDown(self):
+        self.serv.close()
+        self.serv = None
+
+
+class SocketListeningTestMixin(SocketTestBase):
+    """Mixin to listen on the server socket."""
+
+    def setUp(self):
+        super().setUp()
+        self.serv.listen(1)
+
+
+class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase,
+                              ThreadableTest):
+    """Mixin to add client socket and allow client/server tests.
+
+    Client socket is self.cli and its address is self.cli_addr.  See
+    ThreadableTest for usage information.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        ThreadableTest.__init__(self)
+
+    def clientSetUp(self):
+        self.cli = self.newClientSocket()
+        self.bindClient()
+
+    def newClientSocket(self):
+        """Return a new socket for use as client."""
+        return self.newSocket()
+
+    def bindClient(self):
+        """Bind client socket and set self.cli_addr to its address."""
+        self.bindSock(self.cli)
+        self.cli_addr = self.cli.getsockname()
+
+    def clientTearDown(self):
+        self.cli.close()
+        self.cli = None
+        ThreadableTest.clientTearDown(self)
+
+
+class ConnectedStreamTestMixin(SocketListeningTestMixin,
+                               ThreadedSocketTestMixin):
+    """Mixin to allow client/server stream tests with connected client.
+
+    Server's socket representing connection to client is self.cli_conn
+    and client's connection to server is self.serv_conn.  (Based on
+    SocketConnectedTest.)
+    """
+
+    def setUp(self):
+        super().setUp()
+        # Indicate explicitly we're ready for the client thread to
+        # proceed and then perform the blocking call to accept
+        self.serverExplicitReady()
+        conn, addr = self.serv.accept()
+        self.cli_conn = conn
+
+    def tearDown(self):
+        self.cli_conn.close()
+        self.cli_conn = None
+        super().tearDown()
+
+    def clientSetUp(self):
+        super().clientSetUp()
+        self.cli.connect(self.serv_addr)
+        self.serv_conn = self.cli
+
+    def clientTearDown(self):
+        self.serv_conn.close()
+        self.serv_conn = None
+        super().clientTearDown()
+
+
+class UnixSocketTestBase(SocketTestBase):
+    """Base class for Unix-domain socket tests."""
+
+    # This class is used for file descriptor passing tests, so we
+    # create the sockets in a private directory so that other users
+    # can't send anything that might be problematic for a privileged
+    # user running the tests.
+
+    def setUp(self):
+        self.dir_path = tempfile.mkdtemp()
+        self.addCleanup(os.rmdir, self.dir_path)
+        super().setUp()
+
+    def bindSock(self, sock):
+        path = tempfile.mktemp(dir=self.dir_path)
+        sock.bind(path)
+        self.addCleanup(support.unlink, path)
+
+class UnixStreamBase(UnixSocketTestBase):
+    """Base class for Unix-domain SOCK_STREAM tests."""
+
+    def newSocket(self):
+        return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+
+class InetTestBase(SocketTestBase):
+    """Base class for IPv4 socket tests."""
+
+    host = HOST
+
+    def setUp(self):
+        super().setUp()
+        self.port = self.serv_addr[1]
+
+    def bindSock(self, sock):
+        support.bind_port(sock, host=self.host)
+
+class TCPTestBase(InetTestBase):
+    """Base class for TCP-over-IPv4 tests."""
+
+    def newSocket(self):
+        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+class UDPTestBase(InetTestBase):
+    """Base class for UDP-over-IPv4 tests."""
+
+    def newSocket(self):
+        return socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+class SCTPStreamBase(InetTestBase):
+    """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode."""
+
+    def newSocket(self):
+        return socket.socket(socket.AF_INET, socket.SOCK_STREAM,
+                             socket.IPPROTO_SCTP)
+
+
+class Inet6TestBase(InetTestBase):
+    """Base class for IPv6 socket tests."""
+
+    # Don't use "localhost" here - it may not have an IPv6 address
+    # assigned to it by default (e.g. in /etc/hosts), and if someone
+    # has assigned it an IPv4-mapped address, then it's unlikely to
+    # work with the full IPv6 API.
+    host = "::1"
+
+class UDP6TestBase(Inet6TestBase):
+    """Base class for UDP-over-IPv6 tests."""
+
+    def newSocket(self):
+        return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+
+
+# Test-skipping decorators for use with ThreadableTest.
+
+def skipWithClientIf(condition, reason):
+    """Skip decorated test if condition is true, add client_skip decorator.
+
+    If the decorated object is not a class, sets its attribute
+    "client_skip" to a decorator which will return an empty function
+    if the test is to be skipped, or the original function if it is
+    not.  This can be used to avoid running the client part of a
+    skipped test when using ThreadableTest.
+    """
+    def client_pass(*args, **kwargs):
+        pass
+    def skipdec(obj):
+        retval = unittest.skip(reason)(obj)
+        if not isinstance(obj, type):
+            retval.client_skip = lambda f: client_pass
+        return retval
+    def noskipdec(obj):
+        if not (isinstance(obj, type) or hasattr(obj, "client_skip")):
+            obj.client_skip = lambda f: f
+        return obj
+    return skipdec if condition else noskipdec
+
+
+def requireAttrs(obj, *attributes):
+    """Skip decorated test if obj is missing any of the given attributes.
+
+    Sets client_skip attribute as skipWithClientIf() does.
+    """
+    missing = [name for name in attributes if not hasattr(obj, name)]
+    return skipWithClientIf(
+        missing, "don't have " + ", ".join(name for name in missing))
+
+
+def requireSocket(*args):
+    """Skip decorated test if a socket cannot be created with given arguments.
+
+    When an argument is given as a string, will use the value of that
+    attribute of the socket module, or skip the test if it doesn't
+    exist.  Sets client_skip attribute as skipWithClientIf() does.
+    """
+    err = None
+    missing = [obj for obj in args if
+               isinstance(obj, str) and not hasattr(socket, obj)]
+    if missing:
+        err = "don't have " + ", ".join(name for name in missing)
+    else:
+        callargs = [getattr(socket, obj) if isinstance(obj, str) else obj
+                    for obj in args]
+        try:
+            s = socket.socket(*callargs)
+        except socket.error as e:
+            # XXX: check errno?
+            err = str(e)
+        else:
+            s.close()
+    return skipWithClientIf(
+        err is not None,
+        "can't create socket({0}): {1}".format(
+            ", ".join(str(o) for o in args), err))
+
+
 #######################################################################
 ## Begin Tests
 
@@ -283,18 +644,13 @@ class GeneralModuleTests(unittest.TestCase):
 
     def testSocketError(self):
         # Testing socket module exceptions
-        def raise_error(*args, **kwargs):
+        msg = "Error raising socket exception (%s)."
+        with self.assertRaises(socket.error, msg=msg % 'socket.error'):
             raise socket.error
-        def raise_herror(*args, **kwargs):
+        with self.assertRaises(socket.error, msg=msg % 'socket.herror'):
             raise socket.herror
-        def raise_gaierror(*args, **kwargs):
+        with self.assertRaises(socket.error, msg=msg % 'socket.gaierror'):
             raise socket.gaierror
-        self.assertRaises(socket.error, raise_error,
-                              "Error raising socket exception.")
-        self.assertRaises(socket.error, raise_herror,
-                              "Error raising socket exception.")
-        self.assertRaises(socket.error, raise_gaierror,
-                              "Error raising socket exception.")
 
     def testSendtoErrors(self):
         # Testing that sendto doens't masks failures. See #10169.
@@ -370,6 +726,52 @@ class GeneralModuleTests(unittest.TestCase):
         if not fqhn in all_host_names:
             self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names)))
 
+    @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()")
+    @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()")
+    def test_sethostname(self):
+        oldhn = socket.gethostname()
+        try:
+            socket.sethostname('new')
+        except socket.error as e:
+            if e.errno == errno.EPERM:
+                self.skipTest("test should be run as root")
+            else:
+                raise
+        try:
+            # running test as root!
+            self.assertEqual(socket.gethostname(), 'new')
+            # Should work with bytes objects too
+            socket.sethostname(b'bar')
+            self.assertEqual(socket.gethostname(), 'bar')
+        finally:
+            socket.sethostname(oldhn)
+
+    @unittest.skipUnless(hasattr(socket, 'if_nameindex'),
+                         'socket.if_nameindex() not available.')
+    def testInterfaceNameIndex(self):
+        interfaces = socket.if_nameindex()
+        for index, name in interfaces:
+            self.assertIsInstance(index, int)
+            self.assertIsInstance(name, str)
+            # interface indices are non-zero integers
+            self.assertGreater(index, 0)
+            _index = socket.if_nametoindex(name)
+            self.assertIsInstance(_index, int)
+            self.assertEqual(index, _index)
+            _name = socket.if_indextoname(index)
+            self.assertIsInstance(_name, str)
+            self.assertEqual(name, _name)
+
+    @unittest.skipUnless(hasattr(socket, 'if_nameindex'),
+                         'socket.if_nameindex() not available.')
+    def testInvalidInterfaceNameIndex(self):
+        # test nonexistent interface index/name
+        self.assertRaises(socket.error, socket.if_indextoname, 0)
+        self.assertRaises(socket.error, socket.if_nametoindex, '_DEADBEEF')
+        # test with invalid values
+        self.assertRaises(TypeError, socket.if_nametoindex, 0)
+        self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF')
+
     def testRefCountGetNameInfo(self):
         # Testing reference count for getnameinfo
         if hasattr(sys, "getrefcount"):
@@ -422,10 +824,8 @@ class GeneralModuleTests(unittest.TestCase):
         # Find one service that exists, then check all the related interfaces.
         # I've ordered this by protocols that have both a tcp and udp
         # protocol, at least for modern Linuxes.
-        if (sys.platform.startswith('linux') or
-            sys.platform.startswith('freebsd') or
-            sys.platform.startswith('netbsd') or
-            sys.platform == 'darwin'):
+        if (sys.platform.startswith(('freebsd', 'netbsd'))
+            or sys.platform in ('linux', 'darwin')):
             # avoid the 'echo' service on this platform, as there is an
             # assumption breaking non-standard port/protocol entry
             services = ('daytime', 'qotd', 'domain')
@@ -720,7 +1120,7 @@ class GeneralModuleTests(unittest.TestCase):
         socket.getaddrinfo('localhost', 80)
         socket.getaddrinfo('127.0.0.1', 80)
         socket.getaddrinfo(None, 80)
-        if SUPPORTS_IPV6:
+        if support.IPV6_ENABLED:
             socket.getaddrinfo('::1', 80)
         # port can be a string service name such as "http", a numeric
         # port number or None
@@ -773,7 +1173,12 @@ class GeneralModuleTests(unittest.TestCase):
     @unittest.skipUnless(support.is_resource_enabled('network'),
                          'network is not enabled')
     def test_idna(self):
-        support.requires('network')
+        # Check for internet access before running test (issue #12804).
+        try:
+            socket.gethostbyname('python.org')
+        except socket.gaierror as e:
+            if e.errno == socket.EAI_NODATA:
+                self.skipTest('internet access required for this test')
         # these should all be successful
         socket.gethostbyname('и�пытание.python.org')
         socket.gethostbyname_ex('и�пытание.python.org')
@@ -851,6 +1256,12 @@ class GeneralModuleTests(unittest.TestCase):
             self.assertRaises(ValueError, fp.writable)
             self.assertRaises(ValueError, fp.seekable)
 
+    def test_pickle(self):
+        sock = socket.socket()
+        with sock:
+            for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
+                self.assertRaises(TypeError, pickle.dumps, sock, protocol)
+
     def test_listen_backlog(self):
         for backlog in 0, -1:
             srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -864,7 +1275,7 @@ class GeneralModuleTests(unittest.TestCase):
         self.assertRaises(OverflowError, srv.listen, _testcapi.INT_MAX + 1)
         srv.close()
 
-    @unittest.skipUnless(SUPPORTS_IPV6, 'IPv6 required for this test.')
+    @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')
     def test_flowinfo(self):
         self.assertRaises(OverflowError, socket.getnameinfo,
                           ('::1',0, 0xffffffff), 0)
@@ -872,6 +1283,222 @@ class GeneralModuleTests(unittest.TestCase):
             self.assertRaises(OverflowError, s.bind, ('::1', 0, -10))
 
 
+@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.')
+class BasicCANTest(unittest.TestCase):
+
+    def testCrucialConstants(self):
+        socket.AF_CAN
+        socket.PF_CAN
+        socket.CAN_RAW
+
+    def testCreateSocket(self):
+        with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s:
+            pass
+
+    def testBindAny(self):
+        with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s:
+            s.bind(('', ))
+
+    def testTooLongInterfaceName(self):
+        # most systems limit IFNAMSIZ to 16, take 1024 to be sure
+        with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s:
+            self.assertRaisesRegex(socket.error, 'interface name too long',
+                                   s.bind, ('x' * 1024,))
+
+    @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"),
+                         'socket.CAN_RAW_LOOPBACK required for this test.')
+    def testLoopback(self):
+        with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s:
+            for loopback in (0, 1):
+                s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK,
+                             loopback)
+                self.assertEqual(loopback,
+                    s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK))
+
+    @unittest.skipUnless(hasattr(socket, "CAN_RAW_FILTER"),
+                         'socket.CAN_RAW_FILTER required for this test.')
+    def testFilter(self):
+        can_id, can_mask = 0x200, 0x700
+        can_filter = struct.pack("=II", can_id, can_mask)
+        with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s:
+            s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, can_filter)
+            self.assertEqual(can_filter,
+                    s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, 8))
+
+
+@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.')
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class CANTest(ThreadedCANSocketTest):
+
+    """The CAN frame structure is defined in <linux/can.h>:
+
+    struct can_frame {
+        canid_t can_id;  /* 32 bit CAN_ID + EFF/RTR/ERR flags */
+        __u8    can_dlc; /* data length code: 0 .. 8 */
+        __u8    data[8] __attribute__((aligned(8)));
+    };
+    """
+    can_frame_fmt = "=IB3x8s"
+
+    def __init__(self, methodName='runTest'):
+        ThreadedCANSocketTest.__init__(self, methodName=methodName)
+
+    @classmethod
+    def build_can_frame(cls, can_id, data):
+        """Build a CAN frame."""
+        can_dlc = len(data)
+        data = data.ljust(8, b'\x00')
+        return struct.pack(cls.can_frame_fmt, can_id, can_dlc, data)
+
+    @classmethod
+    def dissect_can_frame(cls, frame):
+        """Dissect a CAN frame."""
+        can_id, can_dlc, data = struct.unpack(cls.can_frame_fmt, frame)
+        return (can_id, can_dlc, data[:can_dlc])
+
+    def testSendFrame(self):
+        cf, addr = self.s.recvfrom(self.bufsize)
+        self.assertEqual(self.cf, cf)
+        self.assertEqual(addr[0], self.interface)
+        self.assertEqual(addr[1], socket.AF_CAN)
+
+    def _testSendFrame(self):
+        self.cf = self.build_can_frame(0x00, b'\x01\x02\x03\x04\x05')
+        self.cli.send(self.cf)
+
+    def testSendMaxFrame(self):
+        cf, addr = self.s.recvfrom(self.bufsize)
+        self.assertEqual(self.cf, cf)
+
+    def _testSendMaxFrame(self):
+        self.cf = self.build_can_frame(0x00, b'\x07' * 8)
+        self.cli.send(self.cf)
+
+    def testSendMultiFrames(self):
+        cf, addr = self.s.recvfrom(self.bufsize)
+        self.assertEqual(self.cf1, cf)
+
+        cf, addr = self.s.recvfrom(self.bufsize)
+        self.assertEqual(self.cf2, cf)
+
+    def _testSendMultiFrames(self):
+        self.cf1 = self.build_can_frame(0x07, b'\x44\x33\x22\x11')
+        self.cli.send(self.cf1)
+
+        self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33')
+        self.cli.send(self.cf2)
+
+
+@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.')
+class BasicRDSTest(unittest.TestCase):
+
+    def testCrucialConstants(self):
+        socket.AF_RDS
+        socket.PF_RDS
+
+    def testCreateSocket(self):
+        with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s:
+            pass
+
+    def testSocketBufferSize(self):
+        bufsize = 16384
+        with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s:
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, bufsize)
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, bufsize)
+
+
+@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.')
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RDSTest(ThreadedRDSSocketTest):
+
+    def __init__(self, methodName='runTest'):
+        ThreadedRDSSocketTest.__init__(self, methodName=methodName)
+
+    def setUp(self):
+        super().setUp()
+        self.evt = threading.Event()
+
+    def testSendAndRecv(self):
+        data, addr = self.serv.recvfrom(self.bufsize)
+        self.assertEqual(self.data, data)
+        self.assertEqual(self.cli_addr, addr)
+
+    def _testSendAndRecv(self):
+        self.data = b'spam'
+        self.cli.sendto(self.data, 0, (HOST, self.port))
+
+    def testPeek(self):
+        data, addr = self.serv.recvfrom(self.bufsize, socket.MSG_PEEK)
+        self.assertEqual(self.data, data)
+        data, addr = self.serv.recvfrom(self.bufsize)
+        self.assertEqual(self.data, data)
+
+    def _testPeek(self):
+        self.data = b'spam'
+        self.cli.sendto(self.data, 0, (HOST, self.port))
+
+    @requireAttrs(socket.socket, 'recvmsg')
+    def testSendAndRecvMsg(self):
+        data, ancdata, msg_flags, addr = self.serv.recvmsg(self.bufsize)
+        self.assertEqual(self.data, data)
+
+    @requireAttrs(socket.socket, 'sendmsg')
+    def _testSendAndRecvMsg(self):
+        self.data = b'hello ' * 10
+        self.cli.sendmsg([self.data], (), 0, (HOST, self.port))
+
+    def testSendAndRecvMulti(self):
+        data, addr = self.serv.recvfrom(self.bufsize)
+        self.assertEqual(self.data1, data)
+
+        data, addr = self.serv.recvfrom(self.bufsize)
+        self.assertEqual(self.data2, data)
+
+    def _testSendAndRecvMulti(self):
+        self.data1 = b'bacon'
+        self.cli.sendto(self.data1, 0, (HOST, self.port))
+
+        self.data2 = b'egg'
+        self.cli.sendto(self.data2, 0, (HOST, self.port))
+
+    def testSelect(self):
+        r, w, x = select.select([self.serv], [], [], 3.0)
+        self.assertIn(self.serv, r)
+        data, addr = self.serv.recvfrom(self.bufsize)
+        self.assertEqual(self.data, data)
+
+    def _testSelect(self):
+        self.data = b'select'
+        self.cli.sendto(self.data, 0, (HOST, self.port))
+
+    def testCongestion(self):
+        # wait until the sender is done
+        self.evt.wait()
+
+    def _testCongestion(self):
+        # test the behavior in case of congestion
+        self.data = b'fill'
+        self.cli.setblocking(False)
+        try:
+            # try to lower the receiver's socket buffer size
+            self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384)
+        except OSError:
+            pass
+        with self.assertRaises(OSError) as cm:
+            try:
+                # fill the receiver's socket buffer
+                while True:
+                    self.cli.sendto(self.data, 0, (HOST, self.port))
+            finally:
+                # signal the receiver we're done
+                self.evt.set()
+        # sendto() should have failed with ENOBUFS
+        self.assertEqual(cm.exception.errno, errno.ENOBUFS)
+        # and we should have received a congestion notification through poll
+        r, w, x = select.select([self.serv], [], [], 3.0)
+        self.assertIn(self.serv, r)
+
+
 @unittest.skipUnless(thread, 'Threading required for this test.')
 class BasicTCPTest(SocketConnectedTest):
 
@@ -1016,6 +1643,1866 @@ class BasicUDPTest(ThreadedUDPSocketTest):
     def _testRecvFromNegative(self):
         self.cli.sendto(MSG, 0, (HOST, self.port))
 
+# Tests for the sendmsg()/recvmsg() interface.  Where possible, the
+# same test code is used with different families and types of socket
+# (e.g. stream, datagram), and tests using recvmsg() are repeated
+# using recvmsg_into().
+#
+# The generic test classes such as SendmsgTests and
+# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be
+# supplied with sockets cli_sock and serv_sock representing the
+# client's and the server's end of the connection respectively, and
+# attributes cli_addr and serv_addr holding their (numeric where
+# appropriate) addresses.
+#
+# The final concrete test classes combine these with subclasses of
+# SocketTestBase which set up client and server sockets of a specific
+# type, and with subclasses of SendrecvmsgBase such as
+# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these
+# sockets to cli_sock and serv_sock and override the methods and
+# attributes of SendrecvmsgBase to fill in destination addresses if
+# needed when sending, check for specific flags in msg_flags, etc.
+#
+# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using
+# recvmsg_into().
+
+# XXX: like the other datagram (UDP) tests in this module, the code
+# here assumes that datagram delivery on the local machine will be
+# reliable.
+
+class SendrecvmsgBase(ThreadSafeCleanupTestCase):
+    # Base class for sendmsg()/recvmsg() tests.
+
+    # Time in seconds to wait before considering a test failed, or
+    # None for no timeout.  Not all tests actually set a timeout.
+    fail_timeout = 3.0
+
+    def setUp(self):
+        self.misc_event = threading.Event()
+        super().setUp()
+
+    def sendToServer(self, msg):
+        # Send msg to the server.
+        return self.cli_sock.send(msg)
+
+    # Tuple of alternative default arguments for sendmsg() when called
+    # via sendmsgToServer() (e.g. to include a destination address).
+    sendmsg_to_server_defaults = ()
+
+    def sendmsgToServer(self, *args):
+        # Call sendmsg() on self.cli_sock with the given arguments,
+        # filling in any arguments which are not supplied with the
+        # corresponding items of self.sendmsg_to_server_defaults, if
+        # any.
+        return self.cli_sock.sendmsg(
+            *(args + self.sendmsg_to_server_defaults[len(args):]))
+
+    def doRecvmsg(self, sock, bufsize, *args):
+        # Call recvmsg() on sock with given arguments and return its
+        # result.  Should be used for tests which can use either
+        # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides
+        # this method with one which emulates it using recvmsg_into(),
+        # thus allowing the same test to be used for both methods.
+        result = sock.recvmsg(bufsize, *args)
+        self.registerRecvmsgResult(result)
+        return result
+
+    def registerRecvmsgResult(self, result):
+        # Called by doRecvmsg() with the return value of recvmsg() or
+        # recvmsg_into().  Can be overridden to arrange cleanup based
+        # on the returned ancillary data, for instance.
+        pass
+
+    def checkRecvmsgAddress(self, addr1, addr2):
+        # Called to compare the received address with the address of
+        # the peer.
+        self.assertEqual(addr1, addr2)
+
+    # Flags that are normally unset in msg_flags
+    msg_flags_common_unset = 0
+    for name in ("MSG_CTRUNC", "MSG_OOB"):
+        msg_flags_common_unset |= getattr(socket, name, 0)
+
+    # Flags that are normally set
+    msg_flags_common_set = 0
+
+    # Flags set when a complete record has been received (e.g. MSG_EOR
+    # for SCTP)
+    msg_flags_eor_indicator = 0
+
+    # Flags set when a complete record has not been received
+    # (e.g. MSG_TRUNC for datagram sockets)
+    msg_flags_non_eor_indicator = 0
+
+    def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0):
+        # Method to check the value of msg_flags returned by recvmsg[_into]().
+        #
+        # Checks that all bits in msg_flags_common_set attribute are
+        # set in "flags" and all bits in msg_flags_common_unset are
+        # unset.
+        #
+        # The "eor" argument specifies whether the flags should
+        # indicate that a full record (or datagram) has been received.
+        # If "eor" is None, no checks are done; otherwise, checks
+        # that:
+        #
+        #  * if "eor" is true, all bits in msg_flags_eor_indicator are
+        #    set and all bits in msg_flags_non_eor_indicator are unset
+        #
+        #  * if "eor" is false, all bits in msg_flags_non_eor_indicator
+        #    are set and all bits in msg_flags_eor_indicator are unset
+        #
+        # If "checkset" and/or "checkunset" are supplied, they require
+        # the given bits to be set or unset respectively, overriding
+        # what the attributes require for those bits.
+        #
+        # If any bits are set in "ignore", they will not be checked,
+        # regardless of the other inputs.
+        #
+        # Will raise Exception if the inputs require a bit to be both
+        # set and unset, and it is not ignored.
+
+        defaultset = self.msg_flags_common_set
+        defaultunset = self.msg_flags_common_unset
+
+        if eor:
+            defaultset |= self.msg_flags_eor_indicator
+            defaultunset |= self.msg_flags_non_eor_indicator
+        elif eor is not None:
+            defaultset |= self.msg_flags_non_eor_indicator
+            defaultunset |= self.msg_flags_eor_indicator
+
+        # Function arguments override defaults
+        defaultset &= ~checkunset
+        defaultunset &= ~checkset
+
+        # Merge arguments with remaining defaults, and check for conflicts
+        checkset |= defaultset
+        checkunset |= defaultunset
+        inboth = checkset & checkunset & ~ignore
+        if inboth:
+            raise Exception("contradictory set, unset requirements for flags "
+                            "{0:#x}".format(inboth))
+
+        # Compare with given msg_flags value
+        mask = (checkset | checkunset) & ~ignore
+        self.assertEqual(flags & mask, checkset & mask)
+
+
+class RecvmsgIntoMixin(SendrecvmsgBase):
+    # Mixin to implement doRecvmsg() using recvmsg_into().
+
+    def doRecvmsg(self, sock, bufsize, *args):
+        buf = bytearray(bufsize)
+        result = sock.recvmsg_into([buf], *args)
+        self.registerRecvmsgResult(result)
+        self.assertGreaterEqual(result[0], 0)
+        self.assertLessEqual(result[0], bufsize)
+        return (bytes(buf[:result[0]]),) + result[1:]
+
+
+class SendrecvmsgDgramFlagsBase(SendrecvmsgBase):
+    # Defines flags to be checked in msg_flags for datagram sockets.
+
+    @property
+    def msg_flags_non_eor_indicator(self):
+        return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC
+
+
+class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase):
+    # Defines flags to be checked in msg_flags for SCTP sockets.
+
+    @property
+    def msg_flags_eor_indicator(self):
+        return super().msg_flags_eor_indicator | socket.MSG_EOR
+
+
+class SendrecvmsgConnectionlessBase(SendrecvmsgBase):
+    # Base class for tests on connectionless-mode sockets.  Users must
+    # supply sockets on attributes cli and serv to be mapped to
+    # cli_sock and serv_sock respectively.
+
+    @property
+    def serv_sock(self):
+        return self.serv
+
+    @property
+    def cli_sock(self):
+        return self.cli
+
+    @property
+    def sendmsg_to_server_defaults(self):
+        return ([], [], 0, self.serv_addr)
+
+    def sendToServer(self, msg):
+        return self.cli_sock.sendto(msg, self.serv_addr)
+
+
+class SendrecvmsgConnectedBase(SendrecvmsgBase):
+    # Base class for tests on connected sockets.  Users must supply
+    # sockets on attributes serv_conn and cli_conn (representing the
+    # connections *to* the server and the client), to be mapped to
+    # cli_sock and serv_sock respectively.
+
+    @property
+    def serv_sock(self):
+        return self.cli_conn
+
+    @property
+    def cli_sock(self):
+        return self.serv_conn
+
+    def checkRecvmsgAddress(self, addr1, addr2):
+        # Address is currently "unspecified" for a connected socket,
+        # so we don't examine it
+        pass
+
+
+class SendrecvmsgServerTimeoutBase(SendrecvmsgBase):
+    # Base class to set a timeout on server's socket.
+
+    def setUp(self):
+        super().setUp()
+        self.serv_sock.settimeout(self.fail_timeout)
+
+
+class SendmsgTests(SendrecvmsgServerTimeoutBase):
+    # Tests for sendmsg() which can use any socket type and do not
+    # involve recvmsg() or recvmsg_into().
+
+    def testSendmsg(self):
+        # Send a simple message with sendmsg().
+        self.assertEqual(self.serv_sock.recv(len(MSG)), MSG)
+
+    def _testSendmsg(self):
+        self.assertEqual(self.sendmsgToServer([MSG]), len(MSG))
+
+    def testSendmsgDataGenerator(self):
+        # Send from buffer obtained from a generator (not a sequence).
+        self.assertEqual(self.serv_sock.recv(len(MSG)), MSG)
+
+    def _testSendmsgDataGenerator(self):
+        self.assertEqual(self.sendmsgToServer((o for o in [MSG])),
+                         len(MSG))
+
+    def testSendmsgAncillaryGenerator(self):
+        # Gather (empty) ancillary data from a generator.
+        self.assertEqual(self.serv_sock.recv(len(MSG)), MSG)
+
+    def _testSendmsgAncillaryGenerator(self):
+        self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])),
+                         len(MSG))
+
+    def testSendmsgArray(self):
+        # Send data from an array instead of the usual bytes object.
+        self.assertEqual(self.serv_sock.recv(len(MSG)), MSG)
+
+    def _testSendmsgArray(self):
+        self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]),
+                         len(MSG))
+
+    def testSendmsgGather(self):
+        # Send message data from more than one buffer (gather write).
+        self.assertEqual(self.serv_sock.recv(len(MSG)), MSG)
+
+    def _testSendmsgGather(self):
+        self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG))
+
+    def testSendmsgBadArgs(self):
+        # Check that sendmsg() rejects invalid arguments.
+        self.assertEqual(self.serv_sock.recv(1000), b"done")
+
+    def _testSendmsgBadArgs(self):
+        self.assertRaises(TypeError, self.cli_sock.sendmsg)
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          b"not in an iterable")
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          object())
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [object()])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG, object()])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], object())
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [], object())
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [], 0, object())
+        self.sendToServer(b"done")
+
+    def testSendmsgBadCmsg(self):
+        # Check that invalid ancillary data items are rejected.
+        self.assertEqual(self.serv_sock.recv(1000), b"done")
+
+    def _testSendmsgBadCmsg(self):
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [object()])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [(object(), 0, b"data")])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [(0, object(), b"data")])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [(0, 0, object())])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [(0, 0)])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [(0, 0, b"data", 42)])
+        self.sendToServer(b"done")
+
+    @requireAttrs(socket, "CMSG_SPACE")
+    def testSendmsgBadMultiCmsg(self):
+        # Check that invalid ancillary data items are rejected when
+        # more than one item is present.
+        self.assertEqual(self.serv_sock.recv(1000), b"done")
+
+    @testSendmsgBadMultiCmsg.client_skip
+    def _testSendmsgBadMultiCmsg(self):
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [0, 0, b""])
+        self.assertRaises(TypeError, self.sendmsgToServer,
+                          [MSG], [(0, 0, b""), object()])
+        self.sendToServer(b"done")
+
+    def testSendmsgExcessCmsgReject(self):
+        # Check that sendmsg() rejects excess ancillary data items
+        # when the number that can be sent is limited.
+        self.assertEqual(self.serv_sock.recv(1000), b"done")
+
+    def _testSendmsgExcessCmsgReject(self):
+        if not hasattr(socket, "CMSG_SPACE"):
+            # Can only send one item
+            with self.assertRaises(socket.error) as cm:
+                self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")])
+            self.assertIsNone(cm.exception.errno)
+        self.sendToServer(b"done")
+
+    def testSendmsgAfterClose(self):
+        # Check that sendmsg() fails on a closed socket.
+        pass
+
+    def _testSendmsgAfterClose(self):
+        self.cli_sock.close()
+        self.assertRaises(socket.error, self.sendmsgToServer, [MSG])
+
+
+class SendmsgStreamTests(SendmsgTests):
+    # Tests for sendmsg() which require a stream socket and do not
+    # involve recvmsg() or recvmsg_into().
+
+    def testSendmsgExplicitNoneAddr(self):
+        # Check that peer address can be specified as None.
+        self.assertEqual(self.serv_sock.recv(len(MSG)), MSG)
+
+    def _testSendmsgExplicitNoneAddr(self):
+        self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG))
+
+    def testSendmsgTimeout(self):
+        # Check that timeout works with sendmsg().
+        self.assertEqual(self.serv_sock.recv(512), b"a"*512)
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+
+    def _testSendmsgTimeout(self):
+        try:
+            self.cli_sock.settimeout(0.03)
+            with self.assertRaises(socket.timeout):
+                while True:
+                    self.sendmsgToServer([b"a"*512])
+        finally:
+            self.misc_event.set()
+
+    # XXX: would be nice to have more tests for sendmsg flags argument.
+
+    # Linux supports MSG_DONTWAIT when sending, but in general, it
+    # only works when receiving.  Could add other platforms if they
+    # support it too.
+    @skipWithClientIf(sys.platform not in {"linux2"},
+                      "MSG_DONTWAIT not known to work on this platform when "
+                      "sending")
+    def testSendmsgDontWait(self):
+        # Check that MSG_DONTWAIT in flags causes non-blocking behaviour.
+        self.assertEqual(self.serv_sock.recv(512), b"a"*512)
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+
+    @testSendmsgDontWait.client_skip
+    def _testSendmsgDontWait(self):
+        try:
+            with self.assertRaises(socket.error) as cm:
+                while True:
+                    self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT)
+            self.assertIn(cm.exception.errno,
+                          (errno.EAGAIN, errno.EWOULDBLOCK))
+        finally:
+            self.misc_event.set()
+
+
+class SendmsgConnectionlessTests(SendmsgTests):
+    # Tests for sendmsg() which require a connectionless-mode
+    # (e.g. datagram) socket, and do not involve recvmsg() or
+    # recvmsg_into().
+
+    def testSendmsgNoDestAddr(self):
+        # Check that sendmsg() fails when no destination address is
+        # given for unconnected socket.
+        pass
+
+    def _testSendmsgNoDestAddr(self):
+        self.assertRaises(socket.error, self.cli_sock.sendmsg,
+                          [MSG])
+        self.assertRaises(socket.error, self.cli_sock.sendmsg,
+                          [MSG], [], 0, None)
+
+
+class RecvmsgGenericTests(SendrecvmsgBase):
+    # Tests for recvmsg() which can also be emulated using
+    # recvmsg_into(), and can use any socket type.
+
+    def testRecvmsg(self):
+        # Receive a simple message with recvmsg[_into]().
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG))
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsg(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgExplicitDefaults(self):
+        # Test recvmsg[_into]() with default arguments provided explicitly.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), 0, 0)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgExplicitDefaults(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgShorter(self):
+        # Receive a message smaller than buffer.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG) + 42)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgShorter(self):
+        self.sendToServer(MSG)
+
+    # FreeBSD < 8 doesn't always set the MSG_TRUNC flag when a truncated
+    # datagram is received (issue #13001).
+    @support.requires_freebsd_version(8)
+    def testRecvmsgTrunc(self):
+        # Receive part of message, check for truncation indicators.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG) - 3)
+        self.assertEqual(msg, MSG[:-3])
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=False)
+
+    @support.requires_freebsd_version(8)
+    def _testRecvmsgTrunc(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgShortAncillaryBuf(self):
+        # Test ancillary data buffer too small to hold any ancillary data.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), 1)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgShortAncillaryBuf(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgLongAncillaryBuf(self):
+        # Test large ancillary data buffer.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), 10240)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgLongAncillaryBuf(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgAfterClose(self):
+        # Check that recvmsg[_into]() fails on a closed socket.
+        self.serv_sock.close()
+        self.assertRaises(socket.error, self.doRecvmsg, self.serv_sock, 1024)
+
+    def _testRecvmsgAfterClose(self):
+        pass
+
+    def testRecvmsgTimeout(self):
+        # Check that timeout works.
+        try:
+            self.serv_sock.settimeout(0.03)
+            self.assertRaises(socket.timeout,
+                              self.doRecvmsg, self.serv_sock, len(MSG))
+        finally:
+            self.misc_event.set()
+
+    def _testRecvmsgTimeout(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+
+    @requireAttrs(socket, "MSG_PEEK")
+    def testRecvmsgPeek(self):
+        # Check that MSG_PEEK in flags enables examination of pending
+        # data without consuming it.
+
+        # Receive part of data with MSG_PEEK.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG) - 3, 0,
+                                                   socket.MSG_PEEK)
+        self.assertEqual(msg, MSG[:-3])
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        # Ignoring MSG_TRUNC here (so this test is the same for stream
+        # and datagram sockets).  Some wording in POSIX seems to
+        # suggest that it needn't be set when peeking, but that may
+        # just be a slip.
+        self.checkFlags(flags, eor=False,
+                        ignore=getattr(socket, "MSG_TRUNC", 0))
+
+        # Receive all data with MSG_PEEK.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), 0,
+                                                   socket.MSG_PEEK)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+        # Check that the same data can still be received normally.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG))
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    @testRecvmsgPeek.client_skip
+    def _testRecvmsgPeek(self):
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket.socket, "sendmsg")
+    def testRecvmsgFromSendmsg(self):
+        # Test receiving with recvmsg[_into]() when message is sent
+        # using sendmsg().
+        self.serv_sock.settimeout(self.fail_timeout)
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG))
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    @testRecvmsgFromSendmsg.client_skip
+    def _testRecvmsgFromSendmsg(self):
+        self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG))
+
+
+class RecvmsgGenericStreamTests(RecvmsgGenericTests):
+    # Tests which require a stream socket and can use either recvmsg()
+    # or recvmsg_into().
+
+    def testRecvmsgEOF(self):
+        # Receive end-of-stream indicator (b"", peer socket closed).
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024)
+        self.assertEqual(msg, b"")
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=None) # Might not have end-of-record marker
+
+    def _testRecvmsgEOF(self):
+        self.cli_sock.close()
+
+    def testRecvmsgOverflow(self):
+        # Receive a message in more than one chunk.
+        seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                    len(MSG) - 3)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=False)
+
+        seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+        msg = seg1 + seg2
+        self.assertEqual(msg, MSG)
+
+    def _testRecvmsgOverflow(self):
+        self.sendToServer(MSG)
+
+
+class RecvmsgTests(RecvmsgGenericTests):
+    # Tests for recvmsg() which can use any socket type.
+
+    def testRecvmsgBadArgs(self):
+        # Check that recvmsg() rejects invalid arguments.
+        self.assertRaises(TypeError, self.serv_sock.recvmsg)
+        self.assertRaises(ValueError, self.serv_sock.recvmsg,
+                          -1, 0, 0)
+        self.assertRaises(ValueError, self.serv_sock.recvmsg,
+                          len(MSG), -1, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg,
+                          [bytearray(10)], 0, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg,
+                          object(), 0, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg,
+                          len(MSG), object(), 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg,
+                          len(MSG), 0, object())
+
+        msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgBadArgs(self):
+        self.sendToServer(MSG)
+
+
+class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests):
+    # Tests for recvmsg_into() which can use any socket type.
+
+    def testRecvmsgIntoBadArgs(self):
+        # Check that recvmsg_into() rejects invalid arguments.
+        buf = bytearray(len(MSG))
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into,
+                          len(MSG), 0, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into,
+                          buf, 0, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into,
+                          [object()], 0, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into,
+                          [b"I'm not writable"], 0, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into,
+                          [buf, object()], 0, 0)
+        self.assertRaises(ValueError, self.serv_sock.recvmsg_into,
+                          [buf], -1, 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into,
+                          [buf], object(), 0)
+        self.assertRaises(TypeError, self.serv_sock.recvmsg_into,
+                          [buf], 0, object())
+
+        nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0)
+        self.assertEqual(nbytes, len(MSG))
+        self.assertEqual(buf, bytearray(MSG))
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgIntoBadArgs(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgIntoGenerator(self):
+        # Receive into buffer obtained from a generator (not a sequence).
+        buf = bytearray(len(MSG))
+        nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into(
+            (o for o in [buf]))
+        self.assertEqual(nbytes, len(MSG))
+        self.assertEqual(buf, bytearray(MSG))
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgIntoGenerator(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgIntoArray(self):
+        # Receive into an array rather than the usual bytearray.
+        buf = array.array("B", [0] * len(MSG))
+        nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf])
+        self.assertEqual(nbytes, len(MSG))
+        self.assertEqual(buf.tobytes(), MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgIntoArray(self):
+        self.sendToServer(MSG)
+
+    def testRecvmsgIntoScatter(self):
+        # Receive into multiple buffers (scatter write).
+        b1 = bytearray(b"----")
+        b2 = bytearray(b"0123456789")
+        b3 = bytearray(b"--------------")
+        nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into(
+            [b1, memoryview(b2)[2:9], b3])
+        self.assertEqual(nbytes, len(b"Mary had a little lamb"))
+        self.assertEqual(b1, bytearray(b"Mary"))
+        self.assertEqual(b2, bytearray(b"01 had a 9"))
+        self.assertEqual(b3, bytearray(b"little lamb---"))
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True)
+
+    def _testRecvmsgIntoScatter(self):
+        self.sendToServer(b"Mary had a little lamb")
+
+
+class CmsgMacroTests(unittest.TestCase):
+    # Test the functions CMSG_LEN() and CMSG_SPACE().  Tests
+    # assumptions used by sendmsg() and recvmsg[_into](), which share
+    # code with these functions.
+
+    # Match the definition in socketmodule.c
+    socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX)
+
+    @requireAttrs(socket, "CMSG_LEN")
+    def testCMSG_LEN(self):
+        # Test CMSG_LEN() with various valid and invalid values,
+        # checking the assumptions used by recvmsg() and sendmsg().
+        toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1
+        values = list(range(257)) + list(range(toobig - 257, toobig))
+
+        # struct cmsghdr has at least three members, two of which are ints
+        self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2)
+        for n in values:
+            ret = socket.CMSG_LEN(n)
+            # This is how recvmsg() calculates the data size
+            self.assertEqual(ret - socket.CMSG_LEN(0), n)
+            self.assertLessEqual(ret, self.socklen_t_limit)
+
+        self.assertRaises(OverflowError, socket.CMSG_LEN, -1)
+        # sendmsg() shares code with these functions, and requires
+        # that it reject values over the limit.
+        self.assertRaises(OverflowError, socket.CMSG_LEN, toobig)
+        self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize)
+
+    @requireAttrs(socket, "CMSG_SPACE")
+    def testCMSG_SPACE(self):
+        # Test CMSG_SPACE() with various valid and invalid values,
+        # checking the assumptions used by sendmsg().
+        toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1
+        values = list(range(257)) + list(range(toobig - 257, toobig))
+
+        last = socket.CMSG_SPACE(0)
+        # struct cmsghdr has at least three members, two of which are ints
+        self.assertGreater(last, array.array("i").itemsize * 2)
+        for n in values:
+            ret = socket.CMSG_SPACE(n)
+            self.assertGreaterEqual(ret, last)
+            self.assertGreaterEqual(ret, socket.CMSG_LEN(n))
+            self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0))
+            self.assertLessEqual(ret, self.socklen_t_limit)
+            last = ret
+
+        self.assertRaises(OverflowError, socket.CMSG_SPACE, -1)
+        # sendmsg() shares code with these functions, and requires
+        # that it reject values over the limit.
+        self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig)
+        self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize)
+
+
+class SCMRightsTest(SendrecvmsgServerTimeoutBase):
+    # Tests for file descriptor passing on Unix-domain sockets.
+
+    # Invalid file descriptor value that's unlikely to evaluate to a
+    # real FD even if one of its bytes is replaced with a different
+    # value (which shouldn't actually happen).
+    badfd = -0x5555
+
+    def newFDs(self, n):
+        # Return a list of n file descriptors for newly-created files
+        # containing their list indices as ASCII numbers.
+        fds = []
+        for i in range(n):
+            fd, path = tempfile.mkstemp()
+            self.addCleanup(os.unlink, path)
+            self.addCleanup(os.close, fd)
+            os.write(fd, str(i).encode())
+            fds.append(fd)
+        return fds
+
+    def checkFDs(self, fds):
+        # Check that the file descriptors in the given list contain
+        # their correct list indices as ASCII numbers.
+        for n, fd in enumerate(fds):
+            os.lseek(fd, 0, os.SEEK_SET)
+            self.assertEqual(os.read(fd, 1024), str(n).encode())
+
+    def registerRecvmsgResult(self, result):
+        self.addCleanup(self.closeRecvmsgFDs, result)
+
+    def closeRecvmsgFDs(self, recvmsg_result):
+        # Close all file descriptors specified in the ancillary data
+        # of the given return value from recvmsg() or recvmsg_into().
+        for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]:
+            if (cmsg_level == socket.SOL_SOCKET and
+                    cmsg_type == socket.SCM_RIGHTS):
+                fds = array.array("i")
+                fds.frombytes(cmsg_data[:
+                        len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+                for fd in fds:
+                    os.close(fd)
+
+    def createAndSendFDs(self, n):
+        # Send n new file descriptors created by newFDs() to the
+        # server, with the constant MSG as the non-ancillary data.
+        self.assertEqual(
+            self.sendmsgToServer([MSG],
+                                 [(socket.SOL_SOCKET,
+                                   socket.SCM_RIGHTS,
+                                   array.array("i", self.newFDs(n)))]),
+            len(MSG))
+
+    def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0):
+        # Check that constant MSG was received with numfds file
+        # descriptors in a maximum of maxcmsgs control messages (which
+        # must contain only complete integers).  By default, check
+        # that MSG_CTRUNC is unset, but ignore any flags in
+        # ignoreflags.
+        msg, ancdata, flags, addr = result
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC,
+                        ignore=ignoreflags)
+
+        self.assertIsInstance(ancdata, list)
+        self.assertLessEqual(len(ancdata), maxcmsgs)
+        fds = array.array("i")
+        for item in ancdata:
+            self.assertIsInstance(item, tuple)
+            cmsg_level, cmsg_type, cmsg_data = item
+            self.assertEqual(cmsg_level, socket.SOL_SOCKET)
+            self.assertEqual(cmsg_type, socket.SCM_RIGHTS)
+            self.assertIsInstance(cmsg_data, bytes)
+            self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0)
+            fds.frombytes(cmsg_data)
+
+        self.assertEqual(len(fds), numfds)
+        self.checkFDs(fds)
+
+    def testFDPassSimple(self):
+        # Pass a single FD (array read from bytes object).
+        self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock,
+                                               len(MSG), 10240))
+
+    def _testFDPassSimple(self):
+        self.assertEqual(
+            self.sendmsgToServer(
+                [MSG],
+                [(socket.SOL_SOCKET,
+                  socket.SCM_RIGHTS,
+                  array.array("i", self.newFDs(1)).tobytes())]),
+            len(MSG))
+
+    def testMultipleFDPass(self):
+        # Pass multiple FDs in a single array.
+        self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock,
+                                               len(MSG), 10240))
+
+    def _testMultipleFDPass(self):
+        self.createAndSendFDs(4)
+
+    @requireAttrs(socket, "CMSG_SPACE")
+    def testFDPassCMSG_SPACE(self):
+        # Test using CMSG_SPACE() to calculate ancillary buffer size.
+        self.checkRecvmsgFDs(
+            4, self.doRecvmsg(self.serv_sock, len(MSG),
+                              socket.CMSG_SPACE(4 * SIZEOF_INT)))
+
+    @testFDPassCMSG_SPACE.client_skip
+    def _testFDPassCMSG_SPACE(self):
+        self.createAndSendFDs(4)
+
+    def testFDPassCMSG_LEN(self):
+        # Test using CMSG_LEN() to calculate ancillary buffer size.
+        self.checkRecvmsgFDs(1,
+                             self.doRecvmsg(self.serv_sock, len(MSG),
+                                            socket.CMSG_LEN(4 * SIZEOF_INT)),
+                             # RFC 3542 says implementations may set
+                             # MSG_CTRUNC if there isn't enough space
+                             # for trailing padding.
+                             ignoreflags=socket.MSG_CTRUNC)
+
+    def _testFDPassCMSG_LEN(self):
+        self.createAndSendFDs(1)
+
+    # Issue #12958: The following test has problems on Mac OS X
+    @support.anticipate_failure(sys.platform == "darwin")
+    @requireAttrs(socket, "CMSG_SPACE")
+    def testFDPassSeparate(self):
+        # Pass two FDs in two separate arrays.  Arrays may be combined
+        # into a single control message by the OS.
+        self.checkRecvmsgFDs(2,
+                             self.doRecvmsg(self.serv_sock, len(MSG), 10240),
+                             maxcmsgs=2)
+
+    @testFDPassSeparate.client_skip
+    @support.anticipate_failure(sys.platform == "darwin")
+    def _testFDPassSeparate(self):
+        fd0, fd1 = self.newFDs(2)
+        self.assertEqual(
+            self.sendmsgToServer([MSG], [(socket.SOL_SOCKET,
+                                          socket.SCM_RIGHTS,
+                                          array.array("i", [fd0])),
+                                         (socket.SOL_SOCKET,
+                                          socket.SCM_RIGHTS,
+                                          array.array("i", [fd1]))]),
+            len(MSG))
+
+    # Issue #12958: The following test has problems on Mac OS X
+    @support.anticipate_failure(sys.platform == "darwin")
+    @requireAttrs(socket, "CMSG_SPACE")
+    def testFDPassSeparateMinSpace(self):
+        # Pass two FDs in two separate arrays, receiving them into the
+        # minimum space for two arrays.
+        self.checkRecvmsgFDs(2,
+                             self.doRecvmsg(self.serv_sock, len(MSG),
+                                            socket.CMSG_SPACE(SIZEOF_INT) +
+                                            socket.CMSG_LEN(SIZEOF_INT)),
+                             maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC)
+
+    @testFDPassSeparateMinSpace.client_skip
+    @support.anticipate_failure(sys.platform == "darwin")
+    def _testFDPassSeparateMinSpace(self):
+        fd0, fd1 = self.newFDs(2)
+        self.assertEqual(
+            self.sendmsgToServer([MSG], [(socket.SOL_SOCKET,
+                                          socket.SCM_RIGHTS,
+                                          array.array("i", [fd0])),
+                                         (socket.SOL_SOCKET,
+                                          socket.SCM_RIGHTS,
+                                          array.array("i", [fd1]))]),
+            len(MSG))
+
+    def sendAncillaryIfPossible(self, msg, ancdata):
+        # Try to send msg and ancdata to server, but if the system
+        # call fails, just send msg with no ancillary data.
+        try:
+            nbytes = self.sendmsgToServer([msg], ancdata)
+        except socket.error as e:
+            # Check that it was the system call that failed
+            self.assertIsInstance(e.errno, int)
+            nbytes = self.sendmsgToServer([msg])
+        self.assertEqual(nbytes, len(msg))
+
+    def testFDPassEmpty(self):
+        # Try to pass an empty FD array.  Can receive either no array
+        # or an empty array.
+        self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock,
+                                               len(MSG), 10240),
+                             ignoreflags=socket.MSG_CTRUNC)
+
+    def _testFDPassEmpty(self):
+        self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET,
+                                            socket.SCM_RIGHTS,
+                                            b"")])
+
+    def testFDPassPartialInt(self):
+        # Try to pass a truncated FD array.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), 10240)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC)
+        self.assertLessEqual(len(ancdata), 1)
+        for cmsg_level, cmsg_type, cmsg_data in ancdata:
+            self.assertEqual(cmsg_level, socket.SOL_SOCKET)
+            self.assertEqual(cmsg_type, socket.SCM_RIGHTS)
+            self.assertLess(len(cmsg_data), SIZEOF_INT)
+
+    def _testFDPassPartialInt(self):
+        self.sendAncillaryIfPossible(
+            MSG,
+            [(socket.SOL_SOCKET,
+              socket.SCM_RIGHTS,
+              array.array("i", [self.badfd]).tobytes()[:-1])])
+
+    @requireAttrs(socket, "CMSG_SPACE")
+    def testFDPassPartialIntInMiddle(self):
+        # Try to pass two FD arrays, the first of which is truncated.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), 10240)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC)
+        self.assertLessEqual(len(ancdata), 2)
+        fds = array.array("i")
+        # Arrays may have been combined in a single control message
+        for cmsg_level, cmsg_type, cmsg_data in ancdata:
+            self.assertEqual(cmsg_level, socket.SOL_SOCKET)
+            self.assertEqual(cmsg_type, socket.SCM_RIGHTS)
+            fds.frombytes(cmsg_data[:
+                    len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+        self.assertLessEqual(len(fds), 2)
+        self.checkFDs(fds)
+
+    @testFDPassPartialIntInMiddle.client_skip
+    def _testFDPassPartialIntInMiddle(self):
+        fd0, fd1 = self.newFDs(2)
+        self.sendAncillaryIfPossible(
+            MSG,
+            [(socket.SOL_SOCKET,
+              socket.SCM_RIGHTS,
+              array.array("i", [fd0, self.badfd]).tobytes()[:-1]),
+             (socket.SOL_SOCKET,
+              socket.SCM_RIGHTS,
+              array.array("i", [fd1]))])
+
+    def checkTruncatedHeader(self, result, ignoreflags=0):
+        # Check that no ancillary data items are returned when data is
+        # truncated inside the cmsghdr structure.
+        msg, ancdata, flags, addr = result
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC,
+                        ignore=ignoreflags)
+
+    def testCmsgTruncNoBufSize(self):
+        # Check that no ancillary data is received when no buffer size
+        # is specified.
+        self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)),
+                                  # BSD seems to set MSG_CTRUNC only
+                                  # if an item has been partially
+                                  # received.
+                                  ignoreflags=socket.MSG_CTRUNC)
+
+    def _testCmsgTruncNoBufSize(self):
+        self.createAndSendFDs(1)
+
+    def testCmsgTrunc0(self):
+        # Check that no ancillary data is received when buffer size is 0.
+        self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0),
+                                  ignoreflags=socket.MSG_CTRUNC)
+
+    def _testCmsgTrunc0(self):
+        self.createAndSendFDs(1)
+
+    # Check that no ancillary data is returned for various non-zero
+    # (but still too small) buffer sizes.
+
+    def testCmsgTrunc1(self):
+        self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1))
+
+    def _testCmsgTrunc1(self):
+        self.createAndSendFDs(1)
+
+    def testCmsgTrunc2Int(self):
+        # The cmsghdr structure has at least three members, two of
+        # which are ints, so we still shouldn't see any ancillary
+        # data.
+        self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG),
+                                                 SIZEOF_INT * 2))
+
+    def _testCmsgTrunc2Int(self):
+        self.createAndSendFDs(1)
+
+    def testCmsgTruncLen0Minus1(self):
+        self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG),
+                                                 socket.CMSG_LEN(0) - 1))
+
+    def _testCmsgTruncLen0Minus1(self):
+        self.createAndSendFDs(1)
+
+    # The following tests try to truncate the control message in the
+    # middle of the FD array.
+
+    def checkTruncatedArray(self, ancbuf, maxdata, mindata=0):
+        # Check that file descriptor data is truncated to between
+        # mindata and maxdata bytes when received with buffer size
+        # ancbuf, and that any complete file descriptor numbers are
+        # valid.
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), ancbuf)
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC)
+
+        if mindata == 0 and ancdata == []:
+            return
+        self.assertEqual(len(ancdata), 1)
+        cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+        self.assertEqual(cmsg_level, socket.SOL_SOCKET)
+        self.assertEqual(cmsg_type, socket.SCM_RIGHTS)
+        self.assertGreaterEqual(len(cmsg_data), mindata)
+        self.assertLessEqual(len(cmsg_data), maxdata)
+        fds = array.array("i")
+        fds.frombytes(cmsg_data[:
+                len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+        self.checkFDs(fds)
+
+    def testCmsgTruncLen0(self):
+        self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0)
+
+    def _testCmsgTruncLen0(self):
+        self.createAndSendFDs(1)
+
+    def testCmsgTruncLen0Plus1(self):
+        self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1)
+
+    def _testCmsgTruncLen0Plus1(self):
+        self.createAndSendFDs(2)
+
+    def testCmsgTruncLen1(self):
+        self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT),
+                                 maxdata=SIZEOF_INT)
+
+    def _testCmsgTruncLen1(self):
+        self.createAndSendFDs(2)
+
+    def testCmsgTruncLen2Minus1(self):
+        self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1,
+                                 maxdata=(2 * SIZEOF_INT) - 1)
+
+    def _testCmsgTruncLen2Minus1(self):
+        self.createAndSendFDs(2)
+
+
+class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase):
+    # Test sendmsg() and recvmsg[_into]() using the ancillary data
+    # features of the RFC 3542 Advanced Sockets API for IPv6.
+    # Currently we can only handle certain data items (e.g. traffic
+    # class, hop limit, MTU discovery and fragmentation settings)
+    # without resorting to unportable means such as the struct module,
+    # but the tests here are aimed at testing the ancillary data
+    # handling in sendmsg() and recvmsg() rather than the IPv6 API
+    # itself.
+
+    # Test value to use when setting hop limit of packet
+    hop_limit = 2
+
+    # Test value to use when setting traffic class of packet.
+    # -1 means "use kernel default".
+    traffic_class = -1
+
+    def ancillaryMapping(self, ancdata):
+        # Given ancillary data list ancdata, return a mapping from
+        # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data.
+        # Check that no (level, type) pair appears more than once.
+        d = {}
+        for cmsg_level, cmsg_type, cmsg_data in ancdata:
+            self.assertNotIn((cmsg_level, cmsg_type), d)
+            d[(cmsg_level, cmsg_type)] = cmsg_data
+        return d
+
+    def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0):
+        # Receive hop limit into ancbufsize bytes of ancillary data
+        # space.  Check that data is MSG, ancillary data is not
+        # truncated (but ignore any flags in ignoreflags), and hop
+        # limit is between 0 and maxhop inclusive.
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVHOPLIMIT, 1)
+        self.misc_event.set()
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), ancbufsize)
+
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC,
+                        ignore=ignoreflags)
+
+        self.assertEqual(len(ancdata), 1)
+        self.assertIsInstance(ancdata[0], tuple)
+        cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+        self.assertEqual(cmsg_level, socket.IPPROTO_IPV6)
+        self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT)
+        self.assertIsInstance(cmsg_data, bytes)
+        self.assertEqual(len(cmsg_data), SIZEOF_INT)
+        a = array.array("i")
+        a.frombytes(cmsg_data)
+        self.assertGreaterEqual(a[0], 0)
+        self.assertLessEqual(a[0], maxhop)
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testRecvHopLimit(self):
+        # Test receiving the packet hop limit as ancillary data.
+        self.checkHopLimit(ancbufsize=10240)
+
+    @testRecvHopLimit.client_skip
+    def _testRecvHopLimit(self):
+        # Need to wait until server has asked to receive ancillary
+        # data, as implementations are not required to buffer it
+        # otherwise.
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testRecvHopLimitCMSG_SPACE(self):
+        # Test receiving hop limit, using CMSG_SPACE to calculate buffer size.
+        self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT))
+
+    @testRecvHopLimitCMSG_SPACE.client_skip
+    def _testRecvHopLimitCMSG_SPACE(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    # Could test receiving into buffer sized using CMSG_LEN, but RFC
+    # 3542 says portable applications must provide space for trailing
+    # padding.  Implementations may set MSG_CTRUNC if there isn't
+    # enough space for the padding.
+
+    @requireAttrs(socket.socket, "sendmsg")
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testSetHopLimit(self):
+        # Test setting hop limit on outgoing packet and receiving it
+        # at the other end.
+        self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit)
+
+    @testSetHopLimit.client_skip
+    def _testSetHopLimit(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.assertEqual(
+            self.sendmsgToServer([MSG],
+                                 [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT,
+                                   array.array("i", [self.hop_limit]))]),
+            len(MSG))
+
+    def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255,
+                                     ignoreflags=0):
+        # Receive traffic class and hop limit into ancbufsize bytes of
+        # ancillary data space.  Check that data is MSG, ancillary
+        # data is not truncated (but ignore any flags in ignoreflags),
+        # and traffic class and hop limit are in range (hop limit no
+        # more than maxhop).
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVHOPLIMIT, 1)
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVTCLASS, 1)
+        self.misc_event.set()
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), ancbufsize)
+
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC,
+                        ignore=ignoreflags)
+        self.assertEqual(len(ancdata), 2)
+        ancmap = self.ancillaryMapping(ancdata)
+
+        tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)]
+        self.assertEqual(len(tcdata), SIZEOF_INT)
+        a = array.array("i")
+        a.frombytes(tcdata)
+        self.assertGreaterEqual(a[0], 0)
+        self.assertLessEqual(a[0], 255)
+
+        hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)]
+        self.assertEqual(len(hldata), SIZEOF_INT)
+        a = array.array("i")
+        a.frombytes(hldata)
+        self.assertGreaterEqual(a[0], 0)
+        self.assertLessEqual(a[0], maxhop)
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testRecvTrafficClassAndHopLimit(self):
+        # Test receiving traffic class and hop limit as ancillary data.
+        self.checkTrafficClassAndHopLimit(ancbufsize=10240)
+
+    @testRecvTrafficClassAndHopLimit.client_skip
+    def _testRecvTrafficClassAndHopLimit(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testRecvTrafficClassAndHopLimitCMSG_SPACE(self):
+        # Test receiving traffic class and hop limit, using
+        # CMSG_SPACE() to calculate buffer size.
+        self.checkTrafficClassAndHopLimit(
+            ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2)
+
+    @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip
+    def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket.socket, "sendmsg")
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testSetTrafficClassAndHopLimit(self):
+        # Test setting traffic class and hop limit on outgoing packet,
+        # and receiving them at the other end.
+        self.checkTrafficClassAndHopLimit(ancbufsize=10240,
+                                          maxhop=self.hop_limit)
+
+    @testSetTrafficClassAndHopLimit.client_skip
+    def _testSetTrafficClassAndHopLimit(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.assertEqual(
+            self.sendmsgToServer([MSG],
+                                 [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS,
+                                   array.array("i", [self.traffic_class])),
+                                  (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT,
+                                   array.array("i", [self.hop_limit]))]),
+            len(MSG))
+
+    @requireAttrs(socket.socket, "sendmsg")
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testOddCmsgSize(self):
+        # Try to send ancillary data with first item one byte too
+        # long.  Fall back to sending with correct size if this fails,
+        # and check that second item was handled correctly.
+        self.checkTrafficClassAndHopLimit(ancbufsize=10240,
+                                          maxhop=self.hop_limit)
+
+    @testOddCmsgSize.client_skip
+    def _testOddCmsgSize(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        try:
+            nbytes = self.sendmsgToServer(
+                [MSG],
+                [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS,
+                  array.array("i", [self.traffic_class]).tobytes() + b"\x00"),
+                 (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT,
+                  array.array("i", [self.hop_limit]))])
+        except socket.error as e:
+            self.assertIsInstance(e.errno, int)
+            nbytes = self.sendmsgToServer(
+                [MSG],
+                [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS,
+                  array.array("i", [self.traffic_class])),
+                 (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT,
+                  array.array("i", [self.hop_limit]))])
+            self.assertEqual(nbytes, len(MSG))
+
+    # Tests for proper handling of truncated ancillary data
+
+    def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0):
+        # Receive hop limit into ancbufsize bytes of ancillary data
+        # space, which should be too small to contain the ancillary
+        # data header (if ancbufsize is None, pass no second argument
+        # to recvmsg()).  Check that data is MSG, MSG_CTRUNC is set
+        # (unless included in ignoreflags), and no ancillary data is
+        # returned.
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVHOPLIMIT, 1)
+        self.misc_event.set()
+        args = () if ancbufsize is None else (ancbufsize,)
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), *args)
+
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.assertEqual(ancdata, [])
+        self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC,
+                        ignore=ignoreflags)
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testCmsgTruncNoBufSize(self):
+        # Check that no ancillary data is received when no ancillary
+        # buffer size is provided.
+        self.checkHopLimitTruncatedHeader(ancbufsize=None,
+                                          # BSD seems to set
+                                          # MSG_CTRUNC only if an item
+                                          # has been partially
+                                          # received.
+                                          ignoreflags=socket.MSG_CTRUNC)
+
+    @testCmsgTruncNoBufSize.client_skip
+    def _testCmsgTruncNoBufSize(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testSingleCmsgTrunc0(self):
+        # Check that no ancillary data is received when ancillary
+        # buffer size is zero.
+        self.checkHopLimitTruncatedHeader(ancbufsize=0,
+                                          ignoreflags=socket.MSG_CTRUNC)
+
+    @testSingleCmsgTrunc0.client_skip
+    def _testSingleCmsgTrunc0(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    # Check that no ancillary data is returned for various non-zero
+    # (but still too small) buffer sizes.
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testSingleCmsgTrunc1(self):
+        self.checkHopLimitTruncatedHeader(ancbufsize=1)
+
+    @testSingleCmsgTrunc1.client_skip
+    def _testSingleCmsgTrunc1(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testSingleCmsgTrunc2Int(self):
+        self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT)
+
+    @testSingleCmsgTrunc2Int.client_skip
+    def _testSingleCmsgTrunc2Int(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testSingleCmsgTruncLen0Minus1(self):
+        self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1)
+
+    @testSingleCmsgTruncLen0Minus1.client_skip
+    def _testSingleCmsgTruncLen0Minus1(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT")
+    def testSingleCmsgTruncInData(self):
+        # Test truncation of a control message inside its associated
+        # data.  The message may be returned with its data truncated,
+        # or not returned at all.
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVHOPLIMIT, 1)
+        self.misc_event.set()
+        msg, ancdata, flags, addr = self.doRecvmsg(
+            self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1)
+
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC)
+
+        self.assertLessEqual(len(ancdata), 1)
+        if ancdata:
+            cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+            self.assertEqual(cmsg_level, socket.IPPROTO_IPV6)
+            self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT)
+            self.assertLess(len(cmsg_data), SIZEOF_INT)
+
+    @testSingleCmsgTruncInData.client_skip
+    def _testSingleCmsgTruncInData(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0):
+        # Receive traffic class and hop limit into ancbufsize bytes of
+        # ancillary data space, which should be large enough to
+        # contain the first item, but too small to contain the header
+        # of the second.  Check that data is MSG, MSG_CTRUNC is set
+        # (unless included in ignoreflags), and only one ancillary
+        # data item is returned.
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVHOPLIMIT, 1)
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVTCLASS, 1)
+        self.misc_event.set()
+        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                   len(MSG), ancbufsize)
+
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC,
+                        ignore=ignoreflags)
+
+        self.assertEqual(len(ancdata), 1)
+        cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+        self.assertEqual(cmsg_level, socket.IPPROTO_IPV6)
+        self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT})
+        self.assertEqual(len(cmsg_data), SIZEOF_INT)
+        a = array.array("i")
+        a.frombytes(cmsg_data)
+        self.assertGreaterEqual(a[0], 0)
+        self.assertLessEqual(a[0], 255)
+
+    # Try the above test with various buffer sizes.
+
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testSecondCmsgTrunc0(self):
+        self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT),
+                                        ignoreflags=socket.MSG_CTRUNC)
+
+    @testSecondCmsgTrunc0.client_skip
+    def _testSecondCmsgTrunc0(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testSecondCmsgTrunc1(self):
+        self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1)
+
+    @testSecondCmsgTrunc1.client_skip
+    def _testSecondCmsgTrunc1(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testSecondCmsgTrunc2Int(self):
+        self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) +
+                                        2 * SIZEOF_INT)
+
+    @testSecondCmsgTrunc2Int.client_skip
+    def _testSecondCmsgTrunc2Int(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testSecondCmsgTruncLen0Minus1(self):
+        self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) +
+                                        socket.CMSG_LEN(0) - 1)
+
+    @testSecondCmsgTruncLen0Minus1.client_skip
+    def _testSecondCmsgTruncLen0Minus1(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+    @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT",
+                  "IPV6_RECVTCLASS", "IPV6_TCLASS")
+    def testSecomdCmsgTruncInData(self):
+        # Test truncation of the second of two control messages inside
+        # its associated data.
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVHOPLIMIT, 1)
+        self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
+                                  socket.IPV6_RECVTCLASS, 1)
+        self.misc_event.set()
+        msg, ancdata, flags, addr = self.doRecvmsg(
+            self.serv_sock, len(MSG),
+            socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1)
+
+        self.assertEqual(msg, MSG)
+        self.checkRecvmsgAddress(addr, self.cli_addr)
+        self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC)
+
+        cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}
+
+        cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0)
+        self.assertEqual(cmsg_level, socket.IPPROTO_IPV6)
+        cmsg_types.remove(cmsg_type)
+        self.assertEqual(len(cmsg_data), SIZEOF_INT)
+        a = array.array("i")
+        a.frombytes(cmsg_data)
+        self.assertGreaterEqual(a[0], 0)
+        self.assertLessEqual(a[0], 255)
+
+        if ancdata:
+            cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0)
+            self.assertEqual(cmsg_level, socket.IPPROTO_IPV6)
+            cmsg_types.remove(cmsg_type)
+            self.assertLess(len(cmsg_data), SIZEOF_INT)
+
+        self.assertEqual(ancdata, [])
+
+    @testSecomdCmsgTruncInData.client_skip
+    def _testSecomdCmsgTruncInData(self):
+        self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout))
+        self.sendToServer(MSG)
+
+
+# Derive concrete test classes for different socket types.
+
+class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase,
+                             SendrecvmsgConnectionlessBase,
+                             ThreadedSocketTestMixin, UDPTestBase):
+    pass
+
+@requireAttrs(socket.socket, "sendmsg")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg_into")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase):
+    pass
+
+
+class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase,
+                              SendrecvmsgConnectionlessBase,
+                              ThreadedSocketTestMixin, UDP6TestBase):
+    pass
+
+@requireAttrs(socket.socket, "sendmsg")
+@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')
+@requireSocket("AF_INET6", "SOCK_DGRAM")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg")
+@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')
+@requireSocket("AF_INET6", "SOCK_DGRAM")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg_into")
+@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')
+@requireSocket("AF_INET6", "SOCK_DGRAM")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg")
+@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')
+@requireAttrs(socket, "IPPROTO_IPV6")
+@requireSocket("AF_INET6", "SOCK_DGRAM")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest,
+                                      SendrecvmsgUDP6TestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg_into")
+@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')
+@requireAttrs(socket, "IPPROTO_IPV6")
+@requireSocket("AF_INET6", "SOCK_DGRAM")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin,
+                                          RFC3542AncillaryTest,
+                                          SendrecvmsgUDP6TestBase):
+    pass
+
+
+class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase,
+                             ConnectedStreamTestMixin, TCPTestBase):
+    pass
+
+@requireAttrs(socket.socket, "sendmsg")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests,
+                     SendrecvmsgTCPTestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg_into")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests,
+                         SendrecvmsgTCPTestBase):
+    pass
+
+
+class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase,
+                                    SendrecvmsgConnectedBase,
+                                    ConnectedStreamTestMixin, SCTPStreamBase):
+    pass
+
+@requireAttrs(socket.socket, "sendmsg")
+@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg")
+@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests,
+                            SendrecvmsgSCTPStreamTestBase):
+
+    def testRecvmsgEOF(self):
+        try:
+            super(RecvmsgSCTPStreamTest, self).testRecvmsgEOF()
+        except OSError as e:
+            if e.errno != errno.ENOTCONN:
+                raise
+            self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876")
+
+@requireAttrs(socket.socket, "recvmsg_into")
+@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests,
+                                SendrecvmsgSCTPStreamTestBase):
+
+    def testRecvmsgEOF(self):
+        try:
+            super(RecvmsgIntoSCTPStreamTest, self).testRecvmsgEOF()
+        except OSError as e:
+            if e.errno != errno.ENOTCONN:
+                raise
+            self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876")
+
+
+class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase,
+                                    ConnectedStreamTestMixin, UnixStreamBase):
+    pass
+
+@requireAttrs(socket.socket, "sendmsg")
+@requireAttrs(socket, "AF_UNIX")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg")
+@requireAttrs(socket, "AF_UNIX")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests,
+                            SendrecvmsgUnixStreamTestBase):
+    pass
+
+@requireAttrs(socket.socket, "recvmsg_into")
+@requireAttrs(socket, "AF_UNIX")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests,
+                                SendrecvmsgUnixStreamTestBase):
+    pass
+
+@requireAttrs(socket.socket, "sendmsg", "recvmsg")
+@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase):
+    pass
+
+@requireAttrs(socket.socket, "sendmsg", "recvmsg_into")
+@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest,
+                                     SendrecvmsgUnixStreamTestBase):
+    pass
+
+
+# Test interrupting the interruptible send/receive methods with a
+# signal when a timeout is set.  These tests avoid having multiple
+# threads alive during the test so that the OS cannot deliver the
+# signal to the wrong one.
+
+class InterruptedTimeoutBase(unittest.TestCase):
+    # Base class for interrupted send/receive tests.  Installs an
+    # empty handler for SIGALRM and removes it on teardown, along with
+    # any scheduled alarms.
+
+    def setUp(self):
+        super().setUp()
+        orig_alrm_handler = signal.signal(signal.SIGALRM,
+                                          lambda signum, frame: None)
+        self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler)
+        self.addCleanup(self.setAlarm, 0)
+
+    # Timeout for socket operations
+    timeout = 4.0
+
+    # Provide setAlarm() method to schedule delivery of SIGALRM after
+    # given number of seconds, or cancel it if zero, and an
+    # appropriate time value to use.  Use setitimer() if available.
+    if hasattr(signal, "setitimer"):
+        alarm_time = 0.05
+
+        def setAlarm(self, seconds):
+            signal.setitimer(signal.ITIMER_REAL, seconds)
+    else:
+        # Old systems may deliver the alarm up to one second early
+        alarm_time = 2
+
+        def setAlarm(self, seconds):
+            signal.alarm(seconds)
+
+
+# Require siginterrupt() in order to ensure that system calls are
+# interrupted by default.
+@requireAttrs(signal, "siginterrupt")
+@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"),
+                     "Don't have signal.alarm or signal.setitimer")
+class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase):
+    # Test interrupting the recv*() methods with signals when a
+    # timeout is set.
+
+    def setUp(self):
+        super().setUp()
+        self.serv.settimeout(self.timeout)
+
+    def checkInterruptedRecv(self, func, *args, **kwargs):
+        # Check that func(*args, **kwargs) raises socket.error with an
+        # errno of EINTR when interrupted by a signal.
+        self.setAlarm(self.alarm_time)
+        with self.assertRaises(socket.error) as cm:
+            func(*args, **kwargs)
+        self.assertNotIsInstance(cm.exception, socket.timeout)
+        self.assertEqual(cm.exception.errno, errno.EINTR)
+
+    def testInterruptedRecvTimeout(self):
+        self.checkInterruptedRecv(self.serv.recv, 1024)
+
+    def testInterruptedRecvIntoTimeout(self):
+        self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024))
+
+    def testInterruptedRecvfromTimeout(self):
+        self.checkInterruptedRecv(self.serv.recvfrom, 1024)
+
+    def testInterruptedRecvfromIntoTimeout(self):
+        self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024))
+
+    @requireAttrs(socket.socket, "recvmsg")
+    def testInterruptedRecvmsgTimeout(self):
+        self.checkInterruptedRecv(self.serv.recvmsg, 1024)
+
+    @requireAttrs(socket.socket, "recvmsg_into")
+    def testInterruptedRecvmsgIntoTimeout(self):
+        self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)])
+
+
+# Require siginterrupt() in order to ensure that system calls are
+# interrupted by default.
+@requireAttrs(signal, "siginterrupt")
+@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"),
+                     "Don't have signal.alarm or signal.setitimer")
+@unittest.skipUnless(thread, 'Threading required for this test.')
+class InterruptedSendTimeoutTest(InterruptedTimeoutBase,
+                                 ThreadSafeCleanupTestCase,
+                                 SocketListeningTestMixin, TCPTestBase):
+    # Test interrupting the interruptible send*() methods with signals
+    # when a timeout is set.
+
+    def setUp(self):
+        super().setUp()
+        self.serv_conn = self.newSocket()
+        self.addCleanup(self.serv_conn.close)
+        # Use a thread to complete the connection, but wait for it to
+        # terminate before running the test, so that there is only one
+        # thread to accept the signal.
+        cli_thread = threading.Thread(target=self.doConnect)
+        cli_thread.start()
+        self.cli_conn, addr = self.serv.accept()
+        self.addCleanup(self.cli_conn.close)
+        cli_thread.join()
+        self.serv_conn.settimeout(self.timeout)
+
+    def doConnect(self):
+        self.serv_conn.connect(self.serv_addr)
+
+    def checkInterruptedSend(self, func, *args, **kwargs):
+        # Check that func(*args, **kwargs), run in a loop, raises
+        # socket.error with an errno of EINTR when interrupted by a
+        # signal.
+        with self.assertRaises(socket.error) as cm:
+            while True:
+                self.setAlarm(self.alarm_time)
+                func(*args, **kwargs)
+        self.assertNotIsInstance(cm.exception, socket.timeout)
+        self.assertEqual(cm.exception.errno, errno.EINTR)
+
+    # Issue #12958: The following tests have problems on Mac OS X
+    @support.anticipate_failure(sys.platform == "darwin")
+    def testInterruptedSendTimeout(self):
+        self.checkInterruptedSend(self.serv_conn.send, b"a"*512)
+
+    @support.anticipate_failure(sys.platform == "darwin")
+    def testInterruptedSendtoTimeout(self):
+        # Passing an actual address here as Python's wrapper for
+        # sendto() doesn't allow passing a zero-length one; POSIX
+        # requires that the address is ignored since the socket is
+        # connection-mode, however.
+        self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512,
+                                  self.serv_addr)
+
+    @support.anticipate_failure(sys.platform == "darwin")
+    @requireAttrs(socket.socket, "sendmsg")
+    def testInterruptedSendmsgTimeout(self):
+        self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512])
+
+
 @unittest.skipUnless(thread, 'Threading required for this test.')
 class TCPCloserTest(ThreadedTCPSocketTest):
 
@@ -1099,11 +3586,8 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest):
         pass
 
     if hasattr(socket, "SOCK_NONBLOCK"):
+        @support.requires_linux_version(2, 6, 28)
         def testInitNonBlocking(self):
-            v = linux_version()
-            if v < (2, 6, 28):
-                self.skipTest("Linux kernel 2.6.28 or higher required, not %s"
-                              % ".".join(map(str, v)))
             # reinit server socket
             self.serv.close()
             self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM |
@@ -1205,7 +3689,7 @@ class FileObjectClassTestCase(SocketConnectedTest):
     """
 
     bufsize = -1 # Use default buffer size
-    encoding = 'utf8'
+    encoding = 'utf-8'
     errors = 'strict'
     newline = None
 
@@ -1386,7 +3870,7 @@ class FileObjectInterruptedTestCase(unittest.TestCase):
 
     @staticmethod
     def _raise_eintr():
-        raise socket.error(errno.EINTR)
+        raise socket.error(errno.EINTR, "interrupted")
 
     def _textiowrap_mock_socket(self, mock, buffering=-1):
         raw = socket.SocketIO(mock, "r")
@@ -1426,7 +3910,7 @@ class FileObjectInterruptedTestCase(unittest.TestCase):
             data = b''
         else:
             data = ''
-            expecting = expecting.decode('utf8')
+            expecting = expecting.decode('utf-8')
         while len(data) != len(expecting):
             part = fo.read(size)
             if not part:
@@ -1588,7 +4072,7 @@ class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase):
     """Tests for socket.makefile() in text mode (rather than binary)"""
 
     read_mode = 'r'
-    read_msg = MSG.decode('utf8')
+    read_msg = MSG.decode('utf-8')
     write_mode = 'wb'
     write_msg = MSG
     newline = ''
@@ -1600,7 +4084,7 @@ class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase):
     read_mode = 'rb'
     read_msg = MSG
     write_mode = 'w'
-    write_msg = MSG.decode('utf8')
+    write_msg = MSG.decode('utf-8')
     newline = ''
 
 
@@ -1608,9 +4092,9 @@ class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase):
     """Tests for socket.makefile() in text mode (rather than binary)"""
 
     read_mode = 'r'
-    read_msg = MSG.decode('utf8')
+    read_msg = MSG.decode('utf-8')
     write_mode = 'w'
-    write_msg = MSG.decode('utf8')
+    write_msg = MSG.decode('utf-8')
     newline = ''
 
 
@@ -1901,6 +4385,78 @@ class TestLinuxAbstractNamespace(unittest.TestCase):
         with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
             self.assertRaises(socket.error, s.bind, address)
 
+    def testStrName(self):
+        # Check that an abstract name can be passed as a string.
+        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        try:
+            s.bind("\x00python\x00test\x00")
+            self.assertEqual(s.getsockname(), b"\x00python\x00test\x00")
+        finally:
+            s.close()
+
+class TestUnixDomain(unittest.TestCase):
+
+    def setUp(self):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+    def tearDown(self):
+        self.sock.close()
+
+    def encoded(self, path):
+        # Return the given path encoded in the file system encoding,
+        # or skip the test if this is not possible.
+        try:
+            return os.fsencode(path)
+        except UnicodeEncodeError:
+            self.skipTest(
+                "Pathname {0!a} cannot be represented in file "
+                "system encoding {1!r}".format(
+                    path, sys.getfilesystemencoding()))
+
+    def bind(self, sock, path):
+        # Bind the socket
+        try:
+            sock.bind(path)
+        except OSError as e:
+            if str(e) == "AF_UNIX path too long":
+                self.skipTest(
+                    "Pathname {0!a} is too long to serve as a AF_UNIX path"
+                    .format(path))
+            else:
+                raise
+
+    def testStrAddr(self):
+        # Test binding to and retrieving a normal string pathname.
+        path = os.path.abspath(support.TESTFN)
+        self.bind(self.sock, path)
+        self.addCleanup(support.unlink, path)
+        self.assertEqual(self.sock.getsockname(), path)
+
+    def testBytesAddr(self):
+        # Test binding to a bytes pathname.
+        path = os.path.abspath(support.TESTFN)
+        self.bind(self.sock, self.encoded(path))
+        self.addCleanup(support.unlink, path)
+        self.assertEqual(self.sock.getsockname(), path)
+
+    def testSurrogateescapeBind(self):
+        # Test binding to a valid non-ASCII pathname, with the
+        # non-ASCII bytes supplied using surrogateescape encoding.
+        path = os.path.abspath(support.TESTFN_UNICODE)
+        b = self.encoded(path)
+        self.bind(self.sock, b.decode("ascii", "surrogateescape"))
+        self.addCleanup(support.unlink, path)
+        self.assertEqual(self.sock.getsockname(), path)
+
+    def testUnencodableAddr(self):
+        # Test binding to a pathname that cannot be encoded in the
+        # file system encoding.
+        if support.TESTFN_UNENCODABLE is None:
+            self.skipTest("No unencodable filename available")
+        path = os.path.abspath(support.TESTFN_UNENCODABLE)
+        self.bind(self.sock, path)
+        self.addCleanup(support.unlink, path)
+        self.assertEqual(self.sock.getsockname(), path)
 
 @unittest.skipUnless(thread, 'Threading required for this test.')
 class BufferIOTest(SocketConnectedTest):
@@ -2101,11 +4657,8 @@ class ContextManagersTest(ThreadedTCPSocketTest):
                      "SOCK_CLOEXEC not defined")
 @unittest.skipUnless(fcntl, "module fcntl not available")
 class CloexecConstantTest(unittest.TestCase):
+    @support.requires_linux_version(2, 6, 28)
     def test_SOCK_CLOEXEC(self):
-        v = linux_version()
-        if v < (2, 6, 28):
-            self.skipTest("Linux kernel 2.6.28 or higher required, not %s"
-                          % ".".join(map(str, v)))
         with socket.socket(socket.AF_INET,
                            socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s:
             self.assertTrue(s.type & socket.SOCK_CLOEXEC)
@@ -2123,11 +4676,8 @@ class NonblockConstantTest(unittest.TestCase):
             self.assertFalse(s.type & socket.SOCK_NONBLOCK)
             self.assertEqual(s.gettimeout(), None)
 
+    @support.requires_linux_version(2, 6, 28)
     def test_SOCK_NONBLOCK(self):
-        v = linux_version()
-        if v < (2, 6, 28):
-            self.skipTest("Linux kernel 2.6.28 or higher required, not %s"
-                          % ".".join(map(str, v)))
         # a lot of it seems silly and redundant, but I wanted to test that
         # changing back and forth worked ok
         with socket.socket(socket.AF_INET,
@@ -2160,6 +4710,109 @@ class NonblockConstantTest(unittest.TestCase):
         socket.setdefaulttimeout(t)
 
 
+@unittest.skipUnless(os.name == "nt", "Windows specific")
+@unittest.skipUnless(multiprocessing, "need multiprocessing")
+class TestSocketSharing(SocketTCPTest):
+    # This must be classmethod and not staticmethod or multiprocessing
+    # won't be able to bootstrap it.
+    @classmethod
+    def remoteProcessServer(cls, q):
+        # Recreate socket from shared data
+        sdata = q.get()
+        message = q.get()
+
+        s = socket.fromshare(sdata)
+        s2, c = s.accept()
+
+        # Send the message
+        s2.sendall(message)
+        s2.close()
+        s.close()
+
+    def testShare(self):
+        # Transfer the listening server socket to another process
+        # and service it from there.
+
+        # Create process:
+        q = multiprocessing.Queue()
+        p = multiprocessing.Process(target=self.remoteProcessServer, args=(q,))
+        p.start()
+
+        # Get the shared socket data
+        data = self.serv.share(p.pid)
+
+        # Pass the shared socket to the other process
+        addr = self.serv.getsockname()
+        self.serv.close()
+        q.put(data)
+
+        # The data that the server will send us
+        message = b"slapmahfro"
+        q.put(message)
+
+        # Connect
+        s = socket.create_connection(addr)
+        #  listen for the data
+        m = []
+        while True:
+            data = s.recv(100)
+            if not data:
+                break
+            m.append(data)
+        s.close()
+        received = b"".join(m)
+        self.assertEqual(received, message)
+        p.join()
+
+    def testShareLength(self):
+        data = self.serv.share(os.getpid())
+        self.assertRaises(ValueError, socket.fromshare, data[:-1])
+        self.assertRaises(ValueError, socket.fromshare, data+b"foo")
+
+    def compareSockets(self, org, other):
+        # socket sharing is expected to work only for blocking socket
+        # since the internal python timout value isn't transfered.
+        self.assertEqual(org.gettimeout(), None)
+        self.assertEqual(org.gettimeout(), other.gettimeout())
+
+        self.assertEqual(org.family, other.family)
+        self.assertEqual(org.type, other.type)
+        # If the user specified "0" for proto, then
+        # internally windows will have picked the correct value.
+        # Python introspection on the socket however will still return
+        # 0.  For the shared socket, the python value is recreated
+        # from the actual value, so it may not compare correctly.
+        if org.proto != 0:
+            self.assertEqual(org.proto, other.proto)
+
+    def testShareLocal(self):
+        data = self.serv.share(os.getpid())
+        s = socket.fromshare(data)
+        try:
+            self.compareSockets(self.serv, s)
+        finally:
+            s.close()
+
+    def testTypes(self):
+        families = [socket.AF_INET, socket.AF_INET6]
+        types = [socket.SOCK_STREAM, socket.SOCK_DGRAM]
+        for f in families:
+            for t in types:
+                try:
+                    source = socket.socket(f, t)
+                except OSError:
+                    continue # This combination is not supported
+                try:
+                    data = source.share(os.getpid())
+                    shared = socket.fromshare(data)
+                    try:
+                        self.compareSockets(source, shared)
+                    finally:
+                        shared.close()
+                finally:
+                    source.close()
+
+
 def test_main():
     tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest,
              TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ]
@@ -2183,11 +4836,41 @@ def test_main():
     ])
     if hasattr(socket, "socketpair"):
         tests.append(BasicSocketPairTest)
-    if sys.platform == 'linux2':
+    if hasattr(socket, "AF_UNIX"):
+        tests.append(TestUnixDomain)
+    if sys.platform == 'linux':
         tests.append(TestLinuxAbstractNamespace)
     if isTipcAvailable():
         tests.append(TIPCTest)
         tests.append(TIPCThreadableTest)
+    tests.extend([BasicCANTest, CANTest])
+    tests.extend([BasicRDSTest, RDSTest])
+    tests.extend([
+        CmsgMacroTests,
+        SendmsgUDPTest,
+        RecvmsgUDPTest,
+        RecvmsgIntoUDPTest,
+        SendmsgUDP6Test,
+        RecvmsgUDP6Test,
+        RecvmsgRFC3542AncillaryUDP6Test,
+        RecvmsgIntoRFC3542AncillaryUDP6Test,
+        RecvmsgIntoUDP6Test,
+        SendmsgTCPTest,
+        RecvmsgTCPTest,
+        RecvmsgIntoTCPTest,
+        SendmsgSCTPStreamTest,
+        RecvmsgSCTPStreamTest,
+        RecvmsgIntoSCTPStreamTest,
+        SendmsgUnixStreamTest,
+        RecvmsgUnixStreamTest,
+        RecvmsgIntoUnixStreamTest,
+        RecvmsgSCMRightsStreamTest,
+        RecvmsgIntoSCMRightsStreamTest,
+        # These are slow when setitimer() is not available
+        InterruptedRecvTimeoutTest,
+        InterruptedSendTimeoutTest,
+        TestSocketSharing,
+    ])
 
     thread_info = support.threading_setup()
     support.run_unittest(*tests)
diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py
index 160f5b83dd..464057e19f 100644
--- a/Lib/test/test_socketserver.py
+++ b/Lib/test/test_socketserver.py
@@ -244,7 +244,7 @@ class SocketServerTest(unittest.TestCase):
                 self.called += 1
                 if self.called == 1:
                     # raise the exception on first call
-                    raise select.error(errno.EINTR, os.strerror(errno.EINTR))
+                    raise OSError(errno.EINTR, os.strerror(errno.EINTR))
                 else:
                     # Return real select value for consecutive calls
                     return old_select(*args)
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index d1cb3f1b84..815475ea4d 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -42,6 +42,9 @@ ONLYCERT = data_file("ssl_cert.pem")
 ONLYKEY = data_file("ssl_key.pem")
 BYTES_ONLYCERT = os.fsencode(ONLYCERT)
 BYTES_ONLYKEY = os.fsencode(ONLYKEY)
+CERTFILE_PROTECTED = data_file("keycert.passwd.pem")
+ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem")
+KEY_PASSWORD = "somepass"
 CAPATH = data_file("capath")
 BYTES_CAPATH = os.fsencode(CAPATH)
 
@@ -53,6 +56,8 @@ WRONGCERT = data_file("XXXnonexisting.pem")
 BADKEY = data_file("badkey.pem")
 NOKIACERT = data_file("nokia.pem")
 
+DHFILE = data_file("dh512.pem")
+BYTES_DHFILE = os.fsencode(DHFILE)
 
 def handle_error(prefix):
     exc_format = ' '.join(traceback.format_exception(*sys.exc_info()))
@@ -95,7 +100,14 @@ class BasicSocketTests(unittest.TestCase):
         ssl.CERT_NONE
         ssl.CERT_OPTIONAL
         ssl.CERT_REQUIRED
+        ssl.OP_CIPHER_SERVER_PREFERENCE
+        ssl.OP_SINGLE_DH_USE
+        if ssl.HAS_ECDH:
+            ssl.OP_SINGLE_ECDH_USE
+        if ssl.OPENSSL_VERSION_INFO >= (1, 0):
+            ssl.OP_NO_COMPRESSION
         self.assertIn(ssl.HAS_SNI, {True, False})
+        self.assertIn(ssl.HAS_ECDH, {True, False})
 
     def test_random(self):
         v = ssl.RAND_status()
@@ -103,6 +115,16 @@ class BasicSocketTests(unittest.TestCase):
             sys.stdout.write("\n RAND_status is %d (%s)\n"
                              % (v, (v and "sufficient randomness") or
                                 "insufficient randomness"))
+
+        data, is_cryptographic = ssl.RAND_pseudo_bytes(16)
+        self.assertEqual(len(data), 16)
+        self.assertEqual(is_cryptographic, v == 1)
+        if v:
+            data = ssl.RAND_bytes(16)
+            self.assertEqual(len(data), 16)
+        else:
+            self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16)
+
         self.assertRaises(TypeError, ssl.RAND_egd, 1)
         self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1)
         ssl.RAND_add("this is a random string", 75.0)
@@ -186,20 +208,21 @@ class BasicSocketTests(unittest.TestCase):
         s = socket.socket(socket.AF_INET)
         ss = ssl.wrap_socket(s)
         wr = weakref.ref(ss)
-        del ss
-        self.assertEqual(wr(), None)
+        with support.check_warnings(("", ResourceWarning)):
+            del ss
+            self.assertEqual(wr(), None)
 
     def test_wrapped_unconnected(self):
         # Methods on an unconnected SSLSocket propagate the original
         # socket.error raise by the underlying socket object.
         s = socket.socket(socket.AF_INET)
-        ss = ssl.wrap_socket(s)
-        self.assertRaises(socket.error, ss.recv, 1)
-        self.assertRaises(socket.error, ss.recv_into, bytearray(b'x'))
-        self.assertRaises(socket.error, ss.recvfrom, 1)
-        self.assertRaises(socket.error, ss.recvfrom_into, bytearray(b'x'), 1)
-        self.assertRaises(socket.error, ss.send, b'x')
-        self.assertRaises(socket.error, ss.sendto, b'x', ('0.0.0.0', 0))
+        with ssl.wrap_socket(s) as ss:
+            self.assertRaises(socket.error, ss.recv, 1)
+            self.assertRaises(socket.error, ss.recv_into, bytearray(b'x'))
+            self.assertRaises(socket.error, ss.recvfrom, 1)
+            self.assertRaises(socket.error, ss.recvfrom_into, bytearray(b'x'), 1)
+            self.assertRaises(socket.error, ss.send, b'x')
+            self.assertRaises(socket.error, ss.sendto, b'x', ('0.0.0.0', 0))
 
     def test_timeout(self):
         # Issue #8524: when creating an SSL socket, the timeout of the
@@ -207,8 +230,8 @@ class BasicSocketTests(unittest.TestCase):
         for timeout in (None, 0.0, 5.0):
             s = socket.socket(socket.AF_INET)
             s.settimeout(timeout)
-            ss = ssl.wrap_socket(s)
-            self.assertEqual(timeout, ss.gettimeout())
+            with ssl.wrap_socket(s) as ss:
+                self.assertEqual(timeout, ss.gettimeout())
 
     def test_errors(self):
         sock = socket.socket()
@@ -221,9 +244,9 @@ class BasicSocketTests(unittest.TestCase):
         self.assertRaisesRegex(ValueError,
                         "certfile must be specified for server-side operations",
                         ssl.wrap_socket, sock, server_side=True, certfile="")
-        s = ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE)
-        self.assertRaisesRegex(ValueError, "can't connect in server-side mode",
-                                s.connect, (HOST, 8080))
+        with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s:
+            self.assertRaisesRegex(ValueError, "can't connect in server-side mode",
+                                    s.connect, (HOST, 8080))
         with self.assertRaises(IOError) as cm:
             with socket.socket() as sock:
                 ssl.wrap_socket(sock, certfile=WRONGCERT)
@@ -333,6 +356,33 @@ class BasicSocketTests(unittest.TestCase):
             self.assertRaises(ValueError, ctx.wrap_socket, sock, True,
                               server_hostname="some.hostname")
 
+    def test_unknown_channel_binding(self):
+        # should raise ValueError for unknown type
+        s = socket.socket(socket.AF_INET)
+        with ssl.wrap_socket(s) as ss:
+            with self.assertRaises(ValueError):
+                ss.get_channel_binding("unknown-type")
+
+    @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES,
+                         "'tls-unique' channel binding not available")
+    def test_tls_unique_channel_binding(self):
+        # unconnected should return None for known type
+        s = socket.socket(socket.AF_INET)
+        with ssl.wrap_socket(s) as ss:
+            self.assertIsNone(ss.get_channel_binding("tls-unique"))
+        # the same for server-side
+        s = socket.socket(socket.AF_INET)
+        with ssl.wrap_socket(s, server_side=True, certfile=CERTFILE) as ss:
+            self.assertIsNone(ss.get_channel_binding("tls-unique"))
+
+    def test_dealloc_warn(self):
+        ss = ssl.wrap_socket(socket.socket(socket.AF_INET))
+        r = repr(ss)
+        with self.assertWarns(ResourceWarning) as cm:
+            ss = None
+            support.gc_collect()
+        self.assertIn(r, str(cm.warning.args[0]))
+
 class ContextTests(unittest.TestCase):
 
     @skip_if_broken_ubuntu_ssl
@@ -423,6 +473,60 @@ class ContextTests(unittest.TestCase):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"):
             ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY)
+        # Password protected key and cert
+        ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD)
+        ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode())
+        ctx.load_cert_chain(CERTFILE_PROTECTED,
+                            password=bytearray(KEY_PASSWORD.encode()))
+        ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD)
+        ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode())
+        ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED,
+                            bytearray(KEY_PASSWORD.encode()))
+        with self.assertRaisesRegex(TypeError, "should be a string"):
+            ctx.load_cert_chain(CERTFILE_PROTECTED, password=True)
+        with self.assertRaises(ssl.SSLError):
+            ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass")
+        with self.assertRaisesRegex(ValueError, "cannot be longer"):
+            # openssl has a fixed limit on the password buffer.
+            # PEM_BUFSIZE is generally set to 1kb.
+            # Return a string larger than this.
+            ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400)
+        # Password callback
+        def getpass_unicode():
+            return KEY_PASSWORD
+        def getpass_bytes():
+            return KEY_PASSWORD.encode()
+        def getpass_bytearray():
+            return bytearray(KEY_PASSWORD.encode())
+        def getpass_badpass():
+            return "badpass"
+        def getpass_huge():
+            return b'a' * (1024 * 1024)
+        def getpass_bad_type():
+            return 9
+        def getpass_exception():
+            raise Exception('getpass error')
+        class GetPassCallable:
+            def __call__(self):
+                return KEY_PASSWORD
+            def getpass(self):
+                return KEY_PASSWORD
+        ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode)
+        ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes)
+        ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray)
+        ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable())
+        ctx.load_cert_chain(CERTFILE_PROTECTED,
+                            password=GetPassCallable().getpass)
+        with self.assertRaises(ssl.SSLError):
+            ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass)
+        with self.assertRaisesRegex(ValueError, "cannot be longer"):
+            ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge)
+        with self.assertRaisesRegex(TypeError, "must return a string"):
+            ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type)
+        with self.assertRaisesRegex(Exception, "getpass error"):
+            ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception)
+        # Make sure the password function isn't called if it isn't needed
+        ctx.load_cert_chain(CERTFILE, password=getpass_exception)
 
     def test_load_verify_locations(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
@@ -443,6 +547,19 @@ class ContextTests(unittest.TestCase):
         # Issue #10989: crash if the second argument type is invalid
         self.assertRaises(TypeError, ctx.load_verify_locations, None, True)
 
+    def test_load_dh_params(self):
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+        ctx.load_dh_params(DHFILE)
+        if os.name != 'nt':
+            ctx.load_dh_params(BYTES_DHFILE)
+        self.assertRaises(TypeError, ctx.load_dh_params)
+        self.assertRaises(TypeError, ctx.load_dh_params, None)
+        with self.assertRaises(FileNotFoundError) as cm:
+            ctx.load_dh_params(WRONGCERT)
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+        with self.assertRaises(ssl.SSLError) as cm:
+            ctx.load_dh_params(CERTFILE)
+
     @skip_if_broken_ubuntu_ssl
     def test_session_stats(self):
         for proto in PROTOCOLS:
@@ -467,6 +584,57 @@ class ContextTests(unittest.TestCase):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         ctx.set_default_verify_paths()
 
+    @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build")
+    def test_set_ecdh_curve(self):
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+        ctx.set_ecdh_curve("prime256v1")
+        ctx.set_ecdh_curve(b"prime256v1")
+        self.assertRaises(TypeError, ctx.set_ecdh_curve)
+        self.assertRaises(TypeError, ctx.set_ecdh_curve, None)
+        self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo")
+        self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo")
+
+
+class SSLErrorTests(unittest.TestCase):
+
+    def test_str(self):
+        # The str() of a SSLError doesn't include the errno
+        e = ssl.SSLError(1, "foo")
+        self.assertEqual(str(e), "foo")
+        self.assertEqual(e.errno, 1)
+        # Same for a subclass
+        e = ssl.SSLZeroReturnError(1, "foo")
+        self.assertEqual(str(e), "foo")
+        self.assertEqual(e.errno, 1)
+
+    def test_lib_reason(self):
+        # Test the library and reason attributes
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+        with self.assertRaises(ssl.SSLError) as cm:
+            ctx.load_dh_params(CERTFILE)
+        self.assertEqual(cm.exception.library, 'PEM')
+        self.assertEqual(cm.exception.reason, 'NO_START_LINE')
+        s = str(cm.exception)
+        self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s)
+
+    def test_subclass(self):
+        # Check that the appropriate SSLError subclass is raised
+        # (this only tests one of them)
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+        with socket.socket() as s:
+            s.bind(("127.0.0.1", 0))
+            s.listen(5)
+            c = socket.socket()
+            c.connect(s.getsockname())
+            c.setblocking(False)
+            with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c:
+                with self.assertRaises(ssl.SSLWantReadError) as cm:
+                    c.do_handshake()
+                s = str(cm.exception)
+                self.assertTrue(s.startswith("The operation did not complete (read)"), s)
+                # For compatibility
+                self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ)
+
 
 class NetworkedTests(unittest.TestCase):
 
@@ -529,13 +697,10 @@ class NetworkedTests(unittest.TestCase):
                     try:
                         s.do_handshake()
                         break
-                    except ssl.SSLError as err:
-                        if err.args[0] == ssl.SSL_ERROR_WANT_READ:
-                            select.select([s], [], [], 5.0)
-                        elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
-                            select.select([], [s], [], 5.0)
-                        else:
-                            raise
+                    except ssl.SSLWantReadError:
+                        select.select([s], [], [], 5.0)
+                    except ssl.SSLWantWriteError:
+                        select.select([], [s], [], 5.0)
                 # SSL established
                 self.assertTrue(s.getpeercert())
             finally:
@@ -666,47 +831,49 @@ class NetworkedTests(unittest.TestCase):
                     count += 1
                     s.do_handshake()
                     break
-                except ssl.SSLError as err:
-                    if err.args[0] == ssl.SSL_ERROR_WANT_READ:
-                        select.select([s], [], [])
-                    elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
-                        select.select([], [s], [])
-                    else:
-                        raise
+                except ssl.SSLWantReadError:
+                    select.select([s], [], [])
+                except ssl.SSLWantWriteError:
+                    select.select([], [s], [])
             s.close()
             if support.verbose:
                 sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count)
 
     def test_get_server_certificate(self):
-        with support.transient_internet("svn.python.org"):
-            pem = ssl.get_server_certificate(("svn.python.org", 443))
-            if not pem:
-                self.fail("No server certificate on svn.python.org:443!")
+        def _test_get_server_certificate(host, port, cert=None):
+            with support.transient_internet(host):
+                pem = ssl.get_server_certificate((host, port))
+                if not pem:
+                    self.fail("No server certificate on %s:%s!" % (host, port))
 
-            try:
-                pem = ssl.get_server_certificate(("svn.python.org", 443), ca_certs=CERTFILE)
-            except ssl.SSLError as x:
-                #should fail
+                try:
+                    pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE)
+                except ssl.SSLError as x:
+                    #should fail
+                    if support.verbose:
+                        sys.stdout.write("%s\n" % x)
+                else:
+                    self.fail("Got server certificate %s for %s:%s!" % (pem, host, port))
+
+                pem = ssl.get_server_certificate((host, port), ca_certs=cert)
+                if not pem:
+                    self.fail("No server certificate on %s:%s!" % (host, port))
                 if support.verbose:
-                    sys.stdout.write("%s\n" % x)
-            else:
-                self.fail("Got server certificate %s for svn.python.org!" % pem)
+                    sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem))
 
-            pem = ssl.get_server_certificate(("svn.python.org", 443), ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
-            if not pem:
-                self.fail("No server certificate on svn.python.org:443!")
-            if support.verbose:
-                sys.stdout.write("\nVerified certificate for svn.python.org:443 is\n%s\n" % pem)
+        _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT)
+        if support.IPV6_ENABLED:
+            _test_get_server_certificate('ipv6.google.com', 443)
 
     def test_ciphers(self):
         remote = ("svn.python.org", 443)
         with support.transient_internet(remote[0]):
-            s = ssl.wrap_socket(socket.socket(socket.AF_INET),
-                                cert_reqs=ssl.CERT_NONE, ciphers="ALL")
-            s.connect(remote)
-            s = ssl.wrap_socket(socket.socket(socket.AF_INET),
-                                cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT")
-            s.connect(remote)
+            with ssl.wrap_socket(socket.socket(socket.AF_INET),
+                                 cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s:
+                s.connect(remote)
+            with ssl.wrap_socket(socket.socket(socket.AF_INET),
+                                 cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s:
+                s.connect(remote)
             # Error checking can happen at instantiation or when connecting
             with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"):
                 with socket.socket(socket.AF_INET) as sock:
@@ -774,13 +941,12 @@ else:
                 try:
                     self.sslconn = self.server.context.wrap_socket(
                         self.sock, server_side=True)
-                except (ssl.SSLError, socket.error) as e:
-                    # Treat ECONNRESET as though it were an SSLError - OpenSSL
-                    # on Ubuntu abruptly closes the connection when asked to use
-                    # an unsupported protocol.
-                    if (not isinstance(e, ssl.SSLError) and
-                        e.errno != errno.ECONNRESET):
-                        raise
+                    self.server.selected_protocols.append(self.sslconn.selected_npn_protocol())
+                except (ssl.SSLError, ConnectionResetError) as e:
+                    # We treat ConnectionResetError as though it were an
+                    # SSLError - OpenSSL on Ubuntu abruptly closes the
+                    # connection when asked to use an unsupported protocol.
+                    #
                     # XXX Various errors can have happened here, for example
                     # a mismatching protocol version, an invalid certificate,
                     # or a low-level bug. This should be made more discriminating.
@@ -802,6 +968,8 @@ else:
                     cipher = self.sslconn.cipher()
                     if support.verbose and self.server.chatty:
                         sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n")
+                        sys.stdout.write(" server: selected protocol is now "
+                                + str(self.sslconn.selected_npn_protocol()) + "\n")
                     return True
 
             def read(self):
@@ -856,6 +1024,11 @@ else:
                             self.sslconn = None
                             if support.verbose and self.server.connectionchatty:
                                 sys.stdout.write(" server: connection is now unencrypted...\n")
+                        elif stripped == b'CB tls-unique':
+                            if support.verbose and self.server.connectionchatty:
+                                sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n")
+                            data = self.sslconn.get_channel_binding("tls-unique")
+                            self.write(repr(data).encode("us-ascii") + b"\n")
                         else:
                             if (support.verbose and
                                 self.server.connectionchatty):
@@ -875,7 +1048,7 @@ else:
         def __init__(self, certificate=None, ssl_version=None,
                      certreqs=None, cacerts=None,
                      chatty=True, connectionchatty=False, starttls_server=False,
-                     ciphers=None, context=None):
+                     npn_protocols=None, ciphers=None, context=None):
             if context:
                 self.context = context
             else:
@@ -888,6 +1061,8 @@ else:
                     self.context.load_verify_locations(cacerts)
                 if certificate:
                     self.context.load_cert_chain(certificate)
+                if npn_protocols:
+                    self.context.set_npn_protocols(npn_protocols)
                 if ciphers:
                     self.context.set_ciphers(ciphers)
             self.chatty = chatty
@@ -897,6 +1072,7 @@ else:
             self.port = support.bind_port(self.sock)
             self.flag = None
             self.active = False
+            self.selected_protocols = []
             self.conn_errors = []
             threading.Thread.__init__(self)
             self.daemon = True
@@ -964,12 +1140,11 @@ else:
                 def _do_ssl_handshake(self):
                     try:
                         self.socket.do_handshake()
-                    except ssl.SSLError as err:
-                        if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
-                                           ssl.SSL_ERROR_WANT_WRITE):
-                            return
-                        elif err.args[0] == ssl.SSL_ERROR_EOF:
-                            return self.handle_close()
+                    except (ssl.SSLWantReadError, ssl.SSLWantWriteError):
+                        return
+                    except ssl.SSLEOFError:
+                        return self.handle_close()
+                    except ssl.SSLError:
                         raise
                     except socket.error as err:
                         if err.args[0] == errno.ECONNABORTED:
@@ -1092,6 +1267,7 @@ else:
         Launch a server, connect a client to it and try various reads
         and writes.
         """
+        stats = {}
         server = ThreadedEchoServer(context=server_context,
                                     chatty=chatty,
                                     connectionchatty=False)
@@ -1117,7 +1293,14 @@ else:
                 if connectionchatty:
                     if support.verbose:
                         sys.stdout.write(" client:  closing connection.\n")
+                stats.update({
+                    'compression': s.compression(),
+                    'cipher': s.cipher(),
+                    'client_npn_protocol': s.selected_npn_protocol()
+                })
                 s.close()
+            stats['server_npn_protocols'] = server.selected_protocols
+        return stats
 
     def try_protocol_combo(server_protocol, client_protocol, expect_success,
                            certsreqs=None, server_options=0, client_options=0):
@@ -1269,7 +1452,8 @@ else:
                 t.join()
 
         @skip_if_broken_ubuntu_ssl
-        @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), "need SSLv2")
+        @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'),
+                             "OpenSSL is compiled without SSLv2 support")
         def test_protocol_sslv2(self):
             """Connecting to an SSLv2 server with various client options"""
             if support.verbose:
@@ -1575,6 +1759,15 @@ else:
                             )
                         # consume data
                         s.read()
+
+                # Make sure sendmsg et al are disallowed to avoid
+                # inadvertent disclosure of data and/or corruption
+                # of the encrypted data stream
+                self.assertRaises(NotImplementedError, s.sendmsg, [b"data"])
+                self.assertRaises(NotImplementedError, s.recvmsg, 100)
+                self.assertRaises(NotImplementedError,
+                                  s.recvmsg_into, bytearray(100))
+
                 s.write(b"over\n")
                 s.close()
 
@@ -1659,6 +1852,8 @@ else:
             client_addr = client.getsockname()
             client.close()
             t.join()
+            remote.close()
+            server.close()
             # Sanity checks.
             self.assertIsInstance(remote, ssl.SSLSocket)
             self.assertEqual(peer, client_addr)
@@ -1673,12 +1868,140 @@ else:
             with ThreadedEchoServer(CERTFILE,
                                     ssl_version=ssl.PROTOCOL_SSLv23,
                                     chatty=False) as server:
-                with socket.socket() as sock:
-                    s = context.wrap_socket(sock)
+                with context.wrap_socket(socket.socket()) as s:
                     with self.assertRaises((OSError, ssl.SSLError)):
                         s.connect((HOST, server.port))
             self.assertIn("no shared cipher", str(server.conn_errors[0]))
 
+        @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES,
+                             "'tls-unique' channel binding not available")
+        def test_tls_unique_channel_binding(self):
+            """Test tls-unique channel binding."""
+            if support.verbose:
+                sys.stdout.write("\n")
+
+            server = ThreadedEchoServer(CERTFILE,
+                                        certreqs=ssl.CERT_NONE,
+                                        ssl_version=ssl.PROTOCOL_TLSv1,
+                                        cacerts=CERTFILE,
+                                        chatty=True,
+                                        connectionchatty=False)
+            with server:
+                s = ssl.wrap_socket(socket.socket(),
+                                    server_side=False,
+                                    certfile=CERTFILE,
+                                    ca_certs=CERTFILE,
+                                    cert_reqs=ssl.CERT_NONE,
+                                    ssl_version=ssl.PROTOCOL_TLSv1)
+                s.connect((HOST, server.port))
+                # get the data
+                cb_data = s.get_channel_binding("tls-unique")
+                if support.verbose:
+                    sys.stdout.write(" got channel binding data: {0!r}\n"
+                                     .format(cb_data))
+
+                # check if it is sane
+                self.assertIsNotNone(cb_data)
+                self.assertEqual(len(cb_data), 12) # True for TLSv1
+
+                # and compare with the peers version
+                s.write(b"CB tls-unique\n")
+                peer_data_repr = s.read().strip()
+                self.assertEqual(peer_data_repr,
+                                 repr(cb_data).encode("us-ascii"))
+                s.close()
+
+                # now, again
+                s = ssl.wrap_socket(socket.socket(),
+                                    server_side=False,
+                                    certfile=CERTFILE,
+                                    ca_certs=CERTFILE,
+                                    cert_reqs=ssl.CERT_NONE,
+                                    ssl_version=ssl.PROTOCOL_TLSv1)
+                s.connect((HOST, server.port))
+                new_cb_data = s.get_channel_binding("tls-unique")
+                if support.verbose:
+                    sys.stdout.write(" got another channel binding data: {0!r}\n"
+                                     .format(new_cb_data))
+                # is it really unique
+                self.assertNotEqual(cb_data, new_cb_data)
+                self.assertIsNotNone(cb_data)
+                self.assertEqual(len(cb_data), 12) # True for TLSv1
+                s.write(b"CB tls-unique\n")
+                peer_data_repr = s.read().strip()
+                self.assertEqual(peer_data_repr,
+                                 repr(new_cb_data).encode("us-ascii"))
+                s.close()
+
+        def test_compression(self):
+            context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+            context.load_cert_chain(CERTFILE)
+            stats = server_params_test(context, context,
+                                       chatty=True, connectionchatty=True)
+            if support.verbose:
+                sys.stdout.write(" got compression: {!r}\n".format(stats['compression']))
+            self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' })
+
+        @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'),
+                             "ssl.OP_NO_COMPRESSION needed for this test")
+        def test_compression_disabled(self):
+            context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+            context.load_cert_chain(CERTFILE)
+            context.options |= ssl.OP_NO_COMPRESSION
+            stats = server_params_test(context, context,
+                                       chatty=True, connectionchatty=True)
+            self.assertIs(stats['compression'], None)
+
+        def test_dh_params(self):
+            # Check we can get a connection with ephemeral Diffie-Hellman
+            context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+            context.load_cert_chain(CERTFILE)
+            context.load_dh_params(DHFILE)
+            context.set_ciphers("kEDH")
+            stats = server_params_test(context, context,
+                                       chatty=True, connectionchatty=True)
+            cipher = stats["cipher"][0]
+            parts = cipher.split("-")
+            if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts:
+                self.fail("Non-DH cipher: " + cipher[0])
+
+        def test_selected_npn_protocol(self):
+            # selected_npn_protocol() is None unless NPN is used
+            context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+            context.load_cert_chain(CERTFILE)
+            stats = server_params_test(context, context,
+                                       chatty=True, connectionchatty=True)
+            self.assertIs(stats['client_npn_protocol'], None)
+
+        @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test")
+        def test_npn_protocols(self):
+            server_protocols = ['http/1.1', 'spdy/2']
+            protocol_tests = [
+                (['http/1.1', 'spdy/2'], 'http/1.1'),
+                (['spdy/2', 'http/1.1'], 'http/1.1'),
+                (['spdy/2', 'test'], 'spdy/2'),
+                (['abc', 'def'], 'abc')
+            ]
+            for client_protocols, expected in protocol_tests:
+                server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+                server_context.load_cert_chain(CERTFILE)
+                server_context.set_npn_protocols(server_protocols)
+                client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+                client_context.load_cert_chain(CERTFILE)
+                client_context.set_npn_protocols(client_protocols)
+                stats = server_params_test(client_context, server_context,
+                                           chatty=True, connectionchatty=True)
+
+                msg = "failed trying %s (s) and %s (c).\n" \
+                      "was expecting %s, but got %%s from the %%s" \
+                          % (str(server_protocols), str(client_protocols),
+                             str(expected))
+                client_result = stats['client_npn_protocol']
+                self.assertEqual(client_result, expected, msg % (client_result, "client"))
+                server_result = stats['server_npn_protocols'][-1] \
+                    if len(stats['server_npn_protocols']) else 'nothing'
+                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+
 
 def test_main(verbose=False):
     if support.verbose:
@@ -1706,14 +2029,14 @@ def test_main(verbose=False):
         if not os.path.exists(filename):
             raise support.TestFailed("Can't read certificate file %r" % filename)
 
-    tests = [ContextTests, BasicSocketTests]
+    tests = [ContextTests, BasicSocketTests, SSLErrorTests]
 
     if support.is_resource_enabled('network'):
         tests.append(NetworkedTests)
 
     if _have_threads:
         thread_info = support.threading_setup()
-        if thread_info and support.is_resource_enabled('network'):
+        if thread_info:
             tests.append(ThreadedTests)
 
     try:
diff --git a/Lib/test/test_startfile.py b/Lib/test/test_startfile.py
index ae9aeb9b6c..5a9c2def24 100644
--- a/Lib/test/test_startfile.py
+++ b/Lib/test/test_startfile.py
@@ -5,7 +5,7 @@
 #
 # A possible improvement would be to have empty.vbs do something that
 # we can detect here, to make sure that not only the os.startfile()
-# call succeeded, but also the the script actually has run.
+# call succeeded, but also the script actually has run.
 
 import unittest
 from test import support
diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py
new file mode 100644
index 0000000000..48d5e3801e
--- /dev/null
+++ b/Lib/test/test_stat.py
@@ -0,0 +1,66 @@
+import unittest
+import os
+import stat
+from test.support import TESTFN, run_unittest
+
+
+def get_mode(fname=TESTFN):
+    return stat.filemode(os.lstat(fname).st_mode)
+
+
+class TestFilemode(unittest.TestCase):
+
+    def setUp(self):
+        try:
+            os.remove(TESTFN)
+        except OSError:
+            try:
+                os.rmdir(TESTFN)
+            except OSError:
+                pass
+    tearDown = setUp
+
+    def test_mode(self):
+        with open(TESTFN, 'w'):
+            pass
+        if os.name == 'posix':
+            os.chmod(TESTFN, 0o700)
+            self.assertEqual(get_mode(), '-rwx------')
+            os.chmod(TESTFN, 0o070)
+            self.assertEqual(get_mode(), '----rwx---')
+            os.chmod(TESTFN, 0o007)
+            self.assertEqual(get_mode(), '-------rwx')
+            os.chmod(TESTFN, 0o444)
+            self.assertEqual(get_mode(), '-r--r--r--')
+        else:
+            os.chmod(TESTFN, 0o700)
+            self.assertEqual(get_mode()[:3], '-rw')
+
+    def test_directory(self):
+        os.mkdir(TESTFN)
+        os.chmod(TESTFN, 0o700)
+        if os.name == 'posix':
+            self.assertEqual(get_mode(), 'drwx------')
+        else:
+            self.assertEqual(get_mode()[0], 'd')
+
+    @unittest.skipUnless(hasattr(os, 'symlink'), 'os.symlink not available')
+    def test_link(self):
+        try:
+            os.symlink(os.getcwd(), TESTFN)
+        except (OSError, NotImplementedError) as err:
+            raise unittest.SkipTest(str(err))
+        else:
+            self.assertEqual(get_mode()[0], 'l')
+
+    @unittest.skipUnless(hasattr(os, 'mkfifo'), 'os.mkfifo not available')
+    def test_fifo(self):
+        os.mkfifo(TESTFN, 0o700)
+        self.assertEqual(get_mode(), 'prwx------')
+
+
+def test_main():
+    run_unittest(TestFilemode)
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py
index 34d1dcf3fe..c2bdfdb0b0 100644
--- a/Lib/test/test_string.py
+++ b/Lib/test/test_string.py
@@ -26,6 +26,12 @@ class ModuleTest(unittest.TestCase):
         self.assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def')
         self.assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t')
 
+    def test_basic_formatter(self):
+        fmt = string.Formatter()
+        self.assertEqual(fmt.format("foo"), "foo")
+        self.assertEqual(fmt.format("foo{0}", "bar"), "foobar")
+        self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6")
+
     def test_conversion_specifiers(self):
         fmt = string.Formatter()
         self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-")
@@ -38,15 +44,26 @@ class ModuleTest(unittest.TestCase):
         self.assertEqual(fmt.format("{0!a}",  chr(255)), "'\\xff'")
         self.assertEqual(fmt.format("{0!a}",  chr(256)), "'\\u0100'")
 
-    def test_formatter(self):
+    def test_name_lookup(self):
         fmt = string.Formatter()
-        self.assertEqual(fmt.format("foo"), "foo")
-
-        self.assertEqual(fmt.format("foo{0}", "bar"), "foobar")
-        self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6")
-        self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-")
+        class AnyAttr:
+            def __getattr__(self, attr):
+                return attr
+        x = AnyAttr()
+        self.assertEqual(fmt.format("{0.lumber}{0.jack}", x), 'lumberjack')
+        with self.assertRaises(AttributeError):
+            fmt.format("{0.lumber}{0.jack}", '')
+
+    def test_index_lookup(self):
+        fmt = string.Formatter()
+        lookup = ["eggs", "and", "spam"]
+        self.assertEqual(fmt.format("{0[2]}{0[0]}", lookup), 'spameggs')
+        with self.assertRaises(IndexError):
+            fmt.format("{0[2]}{0[0]}", [])
+        with self.assertRaises(KeyError):
+            fmt.format("{0[2]}{0[0]}", {})
 
-        # override get_value ############################################
+    def test_override_get_value(self):
         class NamespaceFormatter(string.Formatter):
             def __init__(self, namespace={}):
                 string.Formatter.__init__(self)
@@ -66,7 +83,7 @@ class ModuleTest(unittest.TestCase):
         self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!')
 
 
-        # override format_field #########################################
+    def test_override_format_field(self):
         class CallFormatter(string.Formatter):
             def format_field(self, value, format_spec):
                 return format(value(), format_spec)
@@ -75,18 +92,18 @@ class ModuleTest(unittest.TestCase):
         self.assertEqual(fmt.format('*{0}*', lambda : 'result'), '*result*')
 
 
-        # override convert_field ########################################
+    def test_override_convert_field(self):
         class XFormatter(string.Formatter):
             def convert_field(self, value, conversion):
                 if conversion == 'x':
                     return None
-                return super(XFormatter, self).convert_field(value, conversion)
+                return super().convert_field(value, conversion)
 
         fmt = XFormatter()
         self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None")
 
 
-        # override parse ################################################
+    def test_override_parse(self):
         class BarFormatter(string.Formatter):
             # returns an iterable that contains tuples of the form:
             # (literal_text, field_name, format_spec, conversion)
@@ -102,7 +119,7 @@ class ModuleTest(unittest.TestCase):
         fmt = BarFormatter()
         self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '*   foo    *')
 
-        # test all parameters used
+    def test_check_unused_args(self):
         class CheckAllUsedFormatter(string.Formatter):
             def check_unused_args(self, used_args, args, kwargs):
                 # Track which arguments actually got used
@@ -124,28 +141,13 @@ class ModuleTest(unittest.TestCase):
         self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100)
         self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100)
 
-    def test_vformat_assert(self):
-        cls = string.Formatter()
-        kwargs = {
-            "i": 100
-        }
-        self.assertRaises(ValueError, cls._vformat,
-                cls.format, "{0}", kwargs, set(), -2)
-
-    def test_convert_field(self):
-        cls = string.Formatter()
-        self.assertEqual(cls.format("{0!s}", 'foo'), 'foo')
-        self.assertRaises(ValueError, cls.format, "{0!h}", 'foo')
-
-    def test_get_field(self):
-        cls = string.Formatter()
-        class MyClass:
-            name = 'lumberjack'
-        x = MyClass()
-        self.assertEqual(cls.format("{0.name}", x), 'lumberjack')
-
-        lookup = ["eggs", "and", "spam"]
-        self.assertEqual(cls.format("{0[2]}", lookup), 'spam')
+    def test_vformat_recursion_limit(self):
+        fmt = string.Formatter()
+        args = ()
+        kwargs = dict(i=100)
+        with self.assertRaises(ValueError) as err:
+            fmt._vformat("{i}", args, kwargs, set(), -1)
+        self.assertIn("recursion", str(err.exception))
 
 
 def test_main():
diff --git a/Lib/test/test_strlit.py b/Lib/test/test_strlit.py
index a4ae19803c..d01322faa6 100644
--- a/Lib/test/test_strlit.py
+++ b/Lib/test/test_strlit.py
@@ -2,10 +2,10 @@ r"""Test correct treatment of various string literals by the parser.
 
 There are four types of string literals:
 
-    'abc'   -- normal str
-    r'abc'  -- raw str
-    b'xyz'  -- normal bytes
-    br'xyz' -- raw bytes
+    'abc'             -- normal str
+    r'abc'            -- raw str
+    b'xyz'            -- normal bytes
+    br'xyz' | rb'xyz' -- raw bytes
 
 The difference between normal and raw strings is of course that in a
 raw string, \ escapes (while still used to determine the end of the
@@ -133,18 +133,38 @@ class TestLiterals(unittest.TestCase):
 
     def test_eval_bytes_raw(self):
         self.assertEqual(eval(""" br'x' """), b'x')
+        self.assertEqual(eval(""" rb'x' """), b'x')
         self.assertEqual(eval(r""" br'\x01' """), b'\\' + b'x01')
+        self.assertEqual(eval(r""" rb'\x01' """), b'\\' + b'x01')
         self.assertEqual(eval(""" br'\x01' """), byte(1))
+        self.assertEqual(eval(""" rb'\x01' """), byte(1))
         self.assertEqual(eval(r""" br'\x81' """), b"\\" + b"x81")
+        self.assertEqual(eval(r""" rb'\x81' """), b"\\" + b"x81")
         self.assertRaises(SyntaxError, eval, """ br'\x81' """)
+        self.assertRaises(SyntaxError, eval, """ rb'\x81' """)
         self.assertEqual(eval(r""" br'\u1881' """), b"\\" + b"u1881")
+        self.assertEqual(eval(r""" rb'\u1881' """), b"\\" + b"u1881")
         self.assertRaises(SyntaxError, eval, """ br'\u1881' """)
+        self.assertRaises(SyntaxError, eval, """ rb'\u1881' """)
         self.assertEqual(eval(r""" br'\U0001d120' """), b"\\" + b"U0001d120")
+        self.assertEqual(eval(r""" rb'\U0001d120' """), b"\\" + b"U0001d120")
         self.assertRaises(SyntaxError, eval, """ br'\U0001d120' """)
-        self.assertRaises(SyntaxError, eval, """ rb'' """)
+        self.assertRaises(SyntaxError, eval, """ rb'\U0001d120' """)
         self.assertRaises(SyntaxError, eval, """ bb'' """)
         self.assertRaises(SyntaxError, eval, """ rr'' """)
         self.assertRaises(SyntaxError, eval, """ brr'' """)
+        self.assertRaises(SyntaxError, eval, """ bbr'' """)
+        self.assertRaises(SyntaxError, eval, """ rrb'' """)
+        self.assertRaises(SyntaxError, eval, """ rbb'' """)
+
+    def test_eval_str_u(self):
+        self.assertEqual(eval(""" u'x' """), 'x')
+        self.assertEqual(eval(""" U'\u00e4' """), 'ä')
+        self.assertEqual(eval(""" u'\N{LATIN SMALL LETTER A WITH DIAERESIS}' """), 'ä')
+        self.assertRaises(SyntaxError, eval, """ ur'' """)
+        self.assertRaises(SyntaxError, eval, """ ru'' """)
+        self.assertRaises(SyntaxError, eval, """ bu'' """)
+        self.assertRaises(SyntaxError, eval, """ ub'' """)
 
     def check_encoding(self, encoding, extra=""):
         modname = "xx_" + encoding.replace("-", "_")
@@ -167,7 +187,7 @@ class TestLiterals(unittest.TestCase):
         self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra)
 
     def test_file_utf8(self):
-        self.check_encoding("utf8")
+        self.check_encoding("utf-8")
 
     def test_file_iso_8859_1(self):
         self.check_encoding("iso-8859-1")
diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py
index dcc73ab898..22374d244d 100644
--- a/Lib/test/test_struct.py
+++ b/Lib/test/test_struct.py
@@ -8,9 +8,19 @@ from test import support
 ISBIGENDIAN = sys.byteorder == "big"
 IS32BIT = sys.maxsize == 0x7fffffff
 
-integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q'
+integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N'
 byteorders = '', '@', '=', '<', '>', '!'
 
+def iter_integer_formats(byteorders=byteorders):
+    for code in integer_codes:
+        for byteorder in byteorders:
+            if (byteorder in ('', '@') and code in ('q', 'Q') and
+                not HAVE_LONG_LONG):
+                continue
+            if (byteorder not in ('', '@') and code in ('n', 'N')):
+                continue
+            yield code, byteorder
+
 # Native 'q' packing isn't available on systems that don't have the C
 # long long type.
 try:
@@ -30,7 +40,6 @@ def bigendian_to_native(value):
         return string_reverse(value)
 
 class StructTest(unittest.TestCase):
-
     def test_isbigendian(self):
         self.assertEqual((struct.pack('=i', 1)[0] == 0), ISBIGENDIAN)
 
@@ -142,14 +151,13 @@ class StructTest(unittest.TestCase):
             }
 
         # standard integer sizes
-        for code in integer_codes:
-            for byteorder in '=', '<', '>', '!':
-                format = byteorder+code
-                size = struct.calcsize(format)
-                self.assertEqual(size, expected_size[code])
+        for code, byteorder in iter_integer_formats(('=', '<', '>', '!')):
+            format = byteorder+code
+            size = struct.calcsize(format)
+            self.assertEqual(size, expected_size[code])
 
         # native integer sizes
-        native_pairs = 'bB', 'hH', 'iI', 'lL'
+        native_pairs = 'bB', 'hH', 'iI', 'lL', 'nN'
         if HAVE_LONG_LONG:
             native_pairs += 'qQ',
         for format_pair in native_pairs:
@@ -167,9 +175,11 @@ class StructTest(unittest.TestCase):
         if HAVE_LONG_LONG:
             self.assertLessEqual(8, struct.calcsize('q'))
             self.assertLessEqual(struct.calcsize('l'), struct.calcsize('q'))
+        self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i'))
+        self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P'))
 
     def test_integers(self):
-        # Integer tests (bBhHiIlLqQ).
+        # Integer tests (bBhHiIlLqQnN).
         import binascii
 
         class IntTester(unittest.TestCase):
@@ -183,11 +193,11 @@ class StructTest(unittest.TestCase):
                                      self.byteorder)
                 self.bytesize = struct.calcsize(format)
                 self.bitsize = self.bytesize * 8
-                if self.code in tuple('bhilq'):
+                if self.code in tuple('bhilqn'):
                     self.signed = True
                     self.min_value = -(2**(self.bitsize-1))
                     self.max_value = 2**(self.bitsize-1) - 1
-                elif self.code in tuple('BHILQ'):
+                elif self.code in tuple('BHILQN'):
                     self.signed = False
                     self.min_value = 0
                     self.max_value = 2**self.bitsize - 1
@@ -317,14 +327,23 @@ class StructTest(unittest.TestCase):
                                       struct.pack, self.format,
                                       obj)
 
-        for code in integer_codes:
-            for byteorder in byteorders:
-                if (byteorder in ('', '@') and code in ('q', 'Q') and
-                    not HAVE_LONG_LONG):
-                    continue
+        for code, byteorder in iter_integer_formats():
+            format = byteorder+code
+            t = IntTester(format)
+            t.run()
+
+    def test_nN_code(self):
+        # n and N don't exist in standard sizes
+        def assertStructError(func, *args, **kwargs):
+            with self.assertRaises(struct.error) as cm:
+                func(*args, **kwargs)
+            self.assertIn("bad char in struct format", str(cm.exception))
+        for code in 'nN':
+            for byteorder in ('=', '<', '>', '!'):
                 format = byteorder+code
-                t = IntTester(format)
-                t.run()
+                assertStructError(struct.calcsize, format)
+                assertStructError(struct.pack, format, 0)
+                assertStructError(struct.unpack, format, b"")
 
     def test_p_code(self):
         # Test p ("Pascal string") code.
@@ -378,14 +397,10 @@ class StructTest(unittest.TestCase):
         self.assertRaises(OverflowError, struct.pack, ">f", big)
 
     def test_1530559(self):
-        for byteorder in '', '@', '=', '<', '>', '!':
-            for code in integer_codes:
-                if (byteorder in ('', '@') and code in ('q', 'Q') and
-                    not HAVE_LONG_LONG):
-                    continue
-                format = byteorder + code
-                self.assertRaises(struct.error, struct.pack, format, 1.0)
-                self.assertRaises(struct.error, struct.pack, format, 1.5)
+        for code, byteorder in iter_integer_formats():
+            format = byteorder + code
+            self.assertRaises(struct.error, struct.pack, format, 1.0)
+            self.assertRaises(struct.error, struct.pack, format, 1.5)
         self.assertRaises(struct.error, struct.pack, 'P', 1.0)
         self.assertRaises(struct.error, struct.pack, 'P', 1.5)
 
@@ -559,9 +574,9 @@ class StructTest(unittest.TestCase):
 
     def check_sizeof(self, format_str, number_of_codes):
         # The size of 'PyStructObject'
-        totalsize = support.calcobjsize('5P')
+        totalsize = support.calcobjsize('2n3P')
         # The size taken up by the 'formatcode' dynamic array
-        totalsize += struct.calcsize('3P') * (number_of_codes + 1)
+        totalsize += struct.calcsize('P2n0P') * (number_of_codes + 1)
         support.check_sizeof(self, struct.Struct(format_str), totalsize)
 
     @support.cpython_only
diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py
index d6c63b792f..a89e9556c1 100644
--- a/Lib/test/test_structseq.py
+++ b/Lib/test/test_structseq.py
@@ -78,8 +78,9 @@ class StructSeqTest(unittest.TestCase):
 
     def test_fields(self):
         t = time.gmtime()
-        self.assertEqual(len(t), t.n_fields)
-        self.assertEqual(t.n_fields, t.n_sequence_fields+t.n_unnamed_fields)
+        self.assertEqual(len(t), t.n_sequence_fields)
+        self.assertEqual(t.n_unnamed_fields, 0)
+        self.assertEqual(t.n_fields, time._STRUCT_TM_ITEMS)
 
     def test_constructor(self):
         t = time.struct_time
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 45ba37352d..ff74e87433 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -16,6 +16,7 @@ import warnings
 import select
 import shutil
 import gc
+import textwrap
 
 try:
     import resource
@@ -62,6 +63,8 @@ class BaseTestCase(unittest.TestCase):
         # shutdown time.  That frustrates tests trying to check stderr produced
         # from a spawned Python process.
         actual = support.strip_python_stderr(stderr)
+        # strip_python_stderr also strips whitespace, so we do too.
+        expected = expected.strip()
         self.assertEqual(actual, expected, msg)
 
 
@@ -85,6 +88,15 @@ class ProcessTestCase(BaseTestCase):
                               "import sys; sys.exit(47)"])
         self.assertEqual(rc, 47)
 
+    def test_call_timeout(self):
+        # call() function with timeout argument; we want to test that the child
+        # process gets killed when the timeout expires.  If the child isn't
+        # killed, this call will deadlock since subprocess.call waits for the
+        # child.
+        self.assertRaises(subprocess.TimeoutExpired, subprocess.call,
+                          [sys.executable, "-c", "while True: pass"],
+                          timeout=0.1)
+
     def test_check_call_zero(self):
         # check_call() function with zero return code
         rc = subprocess.check_call([sys.executable, "-c",
@@ -127,6 +139,21 @@ class ProcessTestCase(BaseTestCase):
             self.fail("Expected ValueError when stdout arg supplied.")
         self.assertIn('stdout', c.exception.args[0])
 
+    def test_check_output_timeout(self):
+        # check_output() function with timeout arg
+        with self.assertRaises(subprocess.TimeoutExpired) as c:
+            output = subprocess.check_output(
+                    [sys.executable, "-c",
+                     "import sys, time\n"
+                     "sys.stdout.write('BDFL')\n"
+                     "sys.stdout.flush()\n"
+                     "time.sleep(3600)"],
+                    # Some heavily loaded buildbots (sparc Debian 3.x) require
+                    # this much time to start and print.
+                    timeout=3)
+            self.fail("Expected TimeoutExpired.")
+        self.assertEqual(c.exception.output, b'BDFL')
+
     def test_call_kwargs(self):
         # call() function with keyword args
         newenv = os.environ.copy()
@@ -177,6 +204,40 @@ class ProcessTestCase(BaseTestCase):
         p.wait()
         self.assertEqual(p.stderr, None)
 
+    def _assert_python(self, pre_args, **kwargs):
+        # We include sys.exit() to prevent the test runner from hanging
+        # whenever python is found.
+        args = pre_args + ["import sys; sys.exit(47)"]
+        p = subprocess.Popen(args, **kwargs)
+        p.wait()
+        self.assertEqual(47, p.returncode)
+
+    def test_executable(self):
+        # Check that the executable argument works.
+        #
+        # On Unix (non-Mac and non-Windows), Python looks at args[0] to
+        # determine where its standard library is, so we need the directory
+        # of args[0] to be valid for the Popen() call to Python to succeed.
+        # See also issue #16170 and issue #7774.
+        doesnotexist = os.path.join(os.path.dirname(sys.executable),
+                                    "doesnotexist")
+        self._assert_python([doesnotexist, "-c"], executable=sys.executable)
+
+    def test_executable_takes_precedence(self):
+        # Check that the executable argument takes precedence over args[0].
+        #
+        # Verify first that the call succeeds without the executable arg.
+        pre_args = [sys.executable, "-c"]
+        self._assert_python(pre_args)
+        self.assertRaises(FileNotFoundError, self._assert_python, pre_args,
+                          executable="doesnotexist")
+
+    @unittest.skipIf(mswindows, "executable argument replaces shell")
+    def test_executable_replaces_shell(self):
+        # Check that the executable argument replaces the default shell
+        # when shell=True.
+        self._assert_python([], executable=sys.executable, shell=True)
+
     # For use in the test_cwd* tests below.
     def _normalize_cwd(self, cwd):
         # Normalize an expected cwd (for Tru64 support).
@@ -227,9 +288,9 @@ class ProcessTestCase(BaseTestCase):
         with support.temp_cwd() as wrong_dir:
             # Before calling with the correct cwd, confirm that the call fails
             # without cwd and with the wrong cwd.
-            self.assertRaises(OSError, subprocess.Popen,
+            self.assertRaises(FileNotFoundError, subprocess.Popen,
                               [rel_python])
-            self.assertRaises(OSError, subprocess.Popen,
+            self.assertRaises(FileNotFoundError, subprocess.Popen,
                               [rel_python], cwd=wrong_dir)
             python_dir = self._normalize_cwd(python_dir)
             self._assert_cwd(python_dir, rel_python, cwd=python_dir)
@@ -244,9 +305,9 @@ class ProcessTestCase(BaseTestCase):
         with support.temp_cwd() as wrong_dir:
             # Before calling with the correct cwd, confirm that the call fails
             # without cwd and with the wrong cwd.
-            self.assertRaises(OSError, subprocess.Popen,
+            self.assertRaises(FileNotFoundError, subprocess.Popen,
                               [doesntexist], executable=rel_python)
-            self.assertRaises(OSError, subprocess.Popen,
+            self.assertRaises(FileNotFoundError, subprocess.Popen,
                               [doesntexist], executable=rel_python,
                               cwd=wrong_dir)
             python_dir = self._normalize_cwd(python_dir)
@@ -262,17 +323,21 @@ class ProcessTestCase(BaseTestCase):
         with script_helper.temp_dir() as wrong_dir:
             # Before calling with an absolute path, confirm that using a
             # relative path fails.
-            self.assertRaises(OSError, subprocess.Popen,
+            self.assertRaises(FileNotFoundError, subprocess.Popen,
                               [rel_python], cwd=wrong_dir)
             wrong_dir = self._normalize_cwd(wrong_dir)
             self._assert_cwd(wrong_dir, abs_python, cwd=wrong_dir)
 
+    @unittest.skipIf(sys.base_prefix != sys.prefix,
+                     'Test is not venv-compatible')
     def test_executable_with_cwd(self):
         python_dir, python_base = self._split_python_path()
         python_dir = self._normalize_cwd(python_dir)
         self._assert_cwd(python_dir, "somethingyoudonthave",
                          executable=sys.executable, cwd=python_dir)
 
+    @unittest.skipIf(sys.base_prefix != sys.prefix,
+                     'Test is not venv-compatible')
     @unittest.skipIf(sysconfig.is_python_build(),
                      "need an installed Python. See #7774")
     def test_executable_without_cwd(self):
@@ -410,6 +475,31 @@ class ProcessTestCase(BaseTestCase):
         rc = subprocess.call([sys.executable, "-c", cmd], stdout=1)
         self.assertEqual(rc, 2)
 
+    def test_stdout_devnull(self):
+        p = subprocess.Popen([sys.executable, "-c",
+                              'for i in range(10240):'
+                              'print("x" * 1024)'],
+                              stdout=subprocess.DEVNULL)
+        p.wait()
+        self.assertEqual(p.stdout, None)
+
+    def test_stderr_devnull(self):
+        p = subprocess.Popen([sys.executable, "-c",
+                              'import sys\n'
+                              'for i in range(10240):'
+                              'sys.stderr.write("x" * 1024)'],
+                              stderr=subprocess.DEVNULL)
+        p.wait()
+        self.assertEqual(p.stderr, None)
+
+    def test_stdin_devnull(self):
+        p = subprocess.Popen([sys.executable, "-c",
+                              'import sys;'
+                              'sys.stdin.read(1)'],
+                              stdin=subprocess.DEVNULL)
+        p.wait()
+        self.assertEqual(p.stdin, None)
+
     def test_env(self):
         newenv = os.environ.copy()
         newenv["FRUIT"] = "orange"
@@ -480,6 +570,41 @@ class ProcessTestCase(BaseTestCase):
         self.assertEqual(stdout, b"banana")
         self.assertStderrEqual(stderr, b"pineapple")
 
+    def test_communicate_timeout(self):
+        p = subprocess.Popen([sys.executable, "-c",
+                              'import sys,os,time;'
+                              'sys.stderr.write("pineapple\\n");'
+                              'time.sleep(1);'
+                              'sys.stderr.write("pear\\n");'
+                              'sys.stdout.write(sys.stdin.read())'],
+                             universal_newlines=True,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana",
+                          timeout=0.3)
+        # Make sure we can keep waiting for it, and that we get the whole output
+        # after it completes.
+        (stdout, stderr) = p.communicate()
+        self.assertEqual(stdout, "banana")
+        self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n")
+
+    def test_communicate_timeout_large_ouput(self):
+        # Test an expiring timeout while the child is outputting lots of data.
+        p = subprocess.Popen([sys.executable, "-c",
+                              'import sys,os,time;'
+                              'sys.stdout.write("a" * (64 * 1024));'
+                              'time.sleep(0.2);'
+                              'sys.stdout.write("a" * (64 * 1024));'
+                              'time.sleep(0.2);'
+                              'sys.stdout.write("a" * (64 * 1024));'
+                              'time.sleep(0.2);'
+                              'sys.stdout.write("a" * (64 * 1024));'],
+                             stdout=subprocess.PIPE)
+        self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4)
+        (stdout, _) = p.communicate()
+        self.assertEqual(len(stdout), 4 * 64 * 1024)
+
     # Test for the fd leak reported in http://bugs.python.org/issue2791.
     def test_communicate_pipe_fd_leak(self):
         for stdin_pipe in (False, True):
@@ -516,24 +641,21 @@ class ProcessTestCase(BaseTestCase):
         # This test will probably deadlock rather than fail, if
         # communicate() does not work properly.
         x, y = os.pipe()
-        if mswindows:
-            pipe_buf = 512
-        else:
-            pipe_buf = os.fpathconf(x, "PC_PIPE_BUF")
         os.close(x)
         os.close(y)
         p = subprocess.Popen([sys.executable, "-c",
                               'import sys,os;'
                               'sys.stdout.write(sys.stdin.read(47));'
-                              'sys.stderr.write("xyz"*%d);'
-                              'sys.stdout.write(sys.stdin.read())' % pipe_buf],
+                              'sys.stderr.write("x" * %d);'
+                              'sys.stdout.write(sys.stdin.read())' %
+                              support.PIPE_MAX_SIZE],
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
         self.addCleanup(p.stdout.close)
         self.addCleanup(p.stderr.close)
         self.addCleanup(p.stdin.close)
-        string_to_write = b"abc"*pipe_buf
+        string_to_write = b"a" * support.PIPE_MAX_SIZE
         (stdout, stderr) = p.communicate(string_to_write)
         self.assertEqual(stdout, string_to_write)
 
@@ -615,12 +737,12 @@ class ProcessTestCase(BaseTestCase):
     def test_universal_newlines_communicate_stdin(self):
         # universal newlines through communicate(), with only stdin
         p = subprocess.Popen([sys.executable, "-c",
-                              'import sys,os;' + SETBINARY + '''\nif True:
-                                  s = sys.stdin.readline()
-                                  assert s == "line1\\n", repr(s)
-                                  s = sys.stdin.read()
-                                  assert s == "line3\\n", repr(s)
-                              '''],
+                              'import sys,os;' + SETBINARY + textwrap.dedent('''
+                               s = sys.stdin.readline()
+                               assert s == "line1\\n", repr(s)
+                               s = sys.stdin.read()
+                               assert s == "line3\\n", repr(s)
+                              ''')],
                              stdin=subprocess.PIPE,
                              universal_newlines=1)
         (stdout, stderr) = p.communicate("line1\nline3\n")
@@ -641,18 +763,18 @@ class ProcessTestCase(BaseTestCase):
     def test_universal_newlines_communicate_stdin_stdout_stderr(self):
         # universal newlines through communicate(), with stdin, stdout, stderr
         p = subprocess.Popen([sys.executable, "-c",
-                              'import sys,os;' + SETBINARY + '''\nif True:
-                                  s = sys.stdin.buffer.readline()
-                                  sys.stdout.buffer.write(s)
-                                  sys.stdout.buffer.write(b"line2\\r")
-                                  sys.stderr.buffer.write(b"eline2\\n")
-                                  s = sys.stdin.buffer.read()
-                                  sys.stdout.buffer.write(s)
-                                  sys.stdout.buffer.write(b"line4\\n")
-                                  sys.stdout.buffer.write(b"line5\\r\\n")
-                                  sys.stderr.buffer.write(b"eline6\\r")
-                                  sys.stderr.buffer.write(b"eline7\\r\\nz")
-                              '''],
+                              'import sys,os;' + SETBINARY + textwrap.dedent('''
+                               s = sys.stdin.buffer.readline()
+                               sys.stdout.buffer.write(s)
+                               sys.stdout.buffer.write(b"line2\\r")
+                               sys.stderr.buffer.write(b"eline2\\n")
+                               s = sys.stdin.buffer.read()
+                               sys.stdout.buffer.write(s)
+                               sys.stdout.buffer.write(b"line4\\n")
+                               sys.stdout.buffer.write(b"line5\\r\\n")
+                               sys.stderr.buffer.write(b"eline6\\r")
+                               sys.stderr.buffer.write(b"eline7\\r\\nz")
+                              ''')],
                              stdin=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              stdout=subprocess.PIPE,
@@ -696,7 +818,6 @@ class ProcessTestCase(BaseTestCase):
                 stdout, stderr = popen.communicate(input='')
             finally:
                 locale.getpreferredencoding = old_getpreferredencoding
-
             self.assertEqual(stdout, '1\n2\n3\n4')
 
     def test_no_leaking(self):
@@ -756,30 +877,32 @@ class ProcessTestCase(BaseTestCase):
         self.assertEqual(subprocess.list2cmdline(['ab', '']),
                          'ab ""')
 
-
     def test_poll(self):
-        p = subprocess.Popen([sys.executable,
-                          "-c", "import time; time.sleep(1)"])
-        count = 0
-        while p.poll() is None:
-            time.sleep(0.1)
-            count += 1
-        # We expect that the poll loop probably went around about 10 times,
-        # but, based on system scheduling we can't control, it's possible
-        # poll() never returned None.  It "should be" very rare that it
-        # didn't go around at least twice.
-        self.assertGreaterEqual(count, 2)
+        p = subprocess.Popen([sys.executable, "-c",
+                              "import os; os.read(0, 1)"],
+                             stdin=subprocess.PIPE)
+        self.addCleanup(p.stdin.close)
+        self.assertIsNone(p.poll())
+        os.write(p.stdin.fileno(), b'A')
+        p.wait()
         # Subsequent invocations should just return the returncode
         self.assertEqual(p.poll(), 0)
 
-
     def test_wait(self):
-        p = subprocess.Popen([sys.executable,
-                          "-c", "import time; time.sleep(2)"])
+        p = subprocess.Popen([sys.executable, "-c", "pass"])
         self.assertEqual(p.wait(), 0)
         # Subsequent invocations should just return the returncode
         self.assertEqual(p.wait(), 0)
 
+    def test_wait_timeout(self):
+        p = subprocess.Popen([sys.executable,
+                              "-c", "import time; time.sleep(0.1)"])
+        with self.assertRaises(subprocess.TimeoutExpired) as c:
+            p.wait(timeout=0.01)
+        self.assertIn("0.01", str(c.exception))  # For coverage of __str__.
+        # Some heavily loaded buildbots (sparc Debian 3.x) require this much
+        # time to start.
+        self.assertEqual(p.wait(timeout=3), 0)
 
     def test_invalid_bufsize(self):
         # an invalid type of the bufsize argument should raise
@@ -858,25 +981,29 @@ class ProcessTestCase(BaseTestCase):
         p = subprocess.Popen([sys.executable, "-c", 'pass'],
                              stdin=subprocess.PIPE)
         self.addCleanup(p.stdin.close)
-        time.sleep(2)
+        p.wait()
         p.communicate(b"x" * 2**20)
 
-    @unittest.skipUnless(hasattr(signal, 'SIGALRM'),
-                         "Requires signal.SIGALRM")
+    @unittest.skipUnless(hasattr(signal, 'SIGUSR1'),
+                         "Requires signal.SIGUSR1")
+    @unittest.skipUnless(hasattr(os, 'kill'),
+                         "Requires os.kill")
+    @unittest.skipUnless(hasattr(os, 'getppid'),
+                         "Requires os.getppid")
     def test_communicate_eintr(self):
         # Issue #12493: communicate() should handle EINTR
         def handler(signum, frame):
             pass
-        old_handler = signal.signal(signal.SIGALRM, handler)
-        self.addCleanup(signal.signal, signal.SIGALRM, old_handler)
+        old_handler = signal.signal(signal.SIGUSR1, handler)
+        self.addCleanup(signal.signal, signal.SIGUSR1, old_handler)
 
-        # the process is running for 2 seconds
-        args = [sys.executable, "-c", 'import time; time.sleep(2)']
+        args = [sys.executable, "-c",
+                'import os, signal;'
+                'os.kill(os.getppid(), signal.SIGUSR1)']
         for stream in ('stdout', 'stderr'):
             kw = {stream: subprocess.PIPE}
             with subprocess.Popen(args, **kw) as process:
-                signal.alarm(1)
-                # communicate() will be interrupted by SIGALRM
+                # communicate() will be interrupted by SIGUSR1
                 process.communicate()
 
 
@@ -1500,6 +1627,11 @@ class POSIXProcessTestCase(BaseTestCase):
         exitcode = subprocess.call([abs_program, "-c", "pass"])
         self.assertEqual(exitcode, 0)
 
+        # absolute bytes path as a string
+        cmd = b"'" + abs_program + b"' -c pass"
+        exitcode = subprocess.call(cmd, shell=True)
+        self.assertEqual(exitcode, 0)
+
         # bytes program, unicode PATH
         env = os.environ.copy()
         env["PATH"] = path
@@ -1683,7 +1815,7 @@ class POSIXProcessTestCase(BaseTestCase):
         stdout, stderr = p.communicate()
         self.assertEqual(0, p.returncode, "sigchild_ignore.py exited"
                          " non-zero with this error:\n%s" %
-                         stderr.decode('utf8'))
+                         stderr.decode('utf-8'))
 
     def test_select_unbuffered(self):
         # Issue #11459: bufsize=0 should really set the pipes as
@@ -1930,28 +2062,6 @@ class ProcessTestCaseNoPoll(ProcessTestCase):
         ProcessTestCase.tearDown(self)
 
 
-@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False),
-                     "_posixsubprocess extension module not found.")
-class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase):
-    @classmethod
-    def setUpClass(cls):
-        global subprocess
-        assert subprocess._posixsubprocess
-        # Reimport subprocess while forcing _posixsubprocess to not exist.
-        with support.check_warnings(('.*_posixsubprocess .* not being used.*',
-                                     RuntimeWarning)):
-            subprocess = support.import_fresh_module(
-                    'subprocess', blocked=['_posixsubprocess'])
-        assert not subprocess._posixsubprocess
-
-    @classmethod
-    def tearDownClass(cls):
-        global subprocess
-        # Reimport subprocess as it should be, restoring order to the universe.
-        subprocess = support.import_fresh_module('subprocess')
-        assert subprocess._posixsubprocess
-
-
 class HelperFunctionTests(unittest.TestCase):
     @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows")
     def test_eintr_retry_call(self):
@@ -2046,20 +2156,17 @@ class ContextManagerTests(BaseTestCase):
             self.assertEqual(proc.returncode, 1)
 
     def test_invalid_args(self):
-        with self.assertRaises(EnvironmentError) as c:
+        with self.assertRaises(FileNotFoundError) as c:
             with subprocess.Popen(['nonexisting_i_hope'],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE) as proc:
                 pass
 
-        self.assertEqual(c.exception.errno, errno.ENOENT)
-
 
 def test_main():
     unit_tests = (ProcessTestCase,
                   POSIXProcessTestCase,
                   Win32ProcessTestCase,
-                  ProcessTestCasePOSIXPurePython,
                   CommandTests,
                   ProcessTestCaseNoPoll,
                   HelperFunctionTests,
diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py
index 07802d6263..fcccdf701a 100644
--- a/Lib/test/test_sundry.py
+++ b/Lib/test/test_sundry.py
@@ -9,7 +9,6 @@ class TestUntestedModules(unittest.TestCase):
         with support.check_warnings(quiet=True):
             import bdb
             import cgitb
-            import code
 
             import distutils.bcppcompiler
             import distutils.ccompiler
diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py
index 914216d2da..f6469cf263 100644
--- a/Lib/test/test_super.py
+++ b/Lib/test/test_super.py
@@ -81,6 +81,55 @@ class TestSuper(unittest.TestCase):
 
         self.assertEqual(E().f(), 'AE')
 
+    @unittest.expectedFailure
+    def test___class___set(self):
+        # See issue #12370
+        class X(A):
+            def f(self):
+                return super().f()
+            __class__ = 413
+        x = X()
+        self.assertEqual(x.f(), 'A')
+        self.assertEqual(x.__class__, 413)
+
+    def test___class___instancemethod(self):
+        # See issue #14857
+        class X:
+            def f(self):
+                return __class__
+        self.assertIs(X().f(), X)
+
+    def test___class___classmethod(self):
+        # See issue #14857
+        class X:
+            @classmethod
+            def f(cls):
+                return __class__
+        self.assertIs(X.f(), X)
+
+    def test___class___staticmethod(self):
+        # See issue #14857
+        class X:
+            @staticmethod
+            def f():
+                return __class__
+        self.assertIs(X.f(), X)
+
+    def test_obscure_super_errors(self):
+        def f():
+            super()
+        self.assertRaises(RuntimeError, f)
+        def f(x):
+            del x
+            super()
+        self.assertRaises(RuntimeError, f, None)
+        class X:
+            def f(x):
+                nonlocal __class__
+                del __class__
+                super()
+        self.assertRaises(RuntimeError, X().f)
+
 
 def test_main():
     support.run_unittest(TestSuper)
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
new file mode 100644
index 0000000000..f6ef5f6147
--- /dev/null
+++ b/Lib/test/test_support.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+
+import importlib
+import sys
+import os
+import unittest
+import socket
+import tempfile
+import errno
+from test import support
+
+TESTFN = support.TESTFN
+TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.'))
+
+
+class TestSupport(unittest.TestCase):
+    def setUp(self):
+        support.unlink(TESTFN)
+        support.rmtree(TESTDIRN)
+    tearDown = setUp
+
+    def test_import_module(self):
+        support.import_module("ftplib")
+        self.assertRaises(unittest.SkipTest, support.import_module, "foo")
+
+    def test_import_fresh_module(self):
+        support.import_fresh_module("ftplib")
+
+    def test_get_attribute(self):
+        self.assertEqual(support.get_attribute(self, "test_get_attribute"),
+                        self.test_get_attribute)
+        self.assertRaises(unittest.SkipTest, support.get_attribute, self, "foo")
+
+    @unittest.skip("failing buildbots")
+    def test_get_original_stdout(self):
+        self.assertEqual(support.get_original_stdout(), sys.stdout)
+
+    def test_unload(self):
+        import sched
+        self.assertIn("sched", sys.modules)
+        support.unload("sched")
+        self.assertNotIn("sched", sys.modules)
+
+    def test_unlink(self):
+        with open(TESTFN, "w") as f:
+            pass
+        support.unlink(TESTFN)
+        self.assertFalse(os.path.exists(TESTFN))
+        support.unlink(TESTFN)
+
+    def test_rmtree(self):
+        os.mkdir(TESTDIRN)
+        os.mkdir(os.path.join(TESTDIRN, TESTDIRN))
+        support.rmtree(TESTDIRN)
+        self.assertFalse(os.path.exists(TESTDIRN))
+        support.rmtree(TESTDIRN)
+
+    def test_forget(self):
+        mod_filename = TESTFN + '.py'
+        with open(mod_filename, 'w') as f:
+            print('foo = 1', file=f)
+        sys.path.insert(0, os.curdir)
+        importlib.invalidate_caches()
+        try:
+            mod = __import__(TESTFN)
+            self.assertIn(TESTFN, sys.modules)
+
+            support.forget(TESTFN)
+            self.assertNotIn(TESTFN, sys.modules)
+        finally:
+            del sys.path[0]
+            support.unlink(mod_filename)
+
+    def test_HOST(self):
+        s = socket.socket()
+        s.bind((support.HOST, 0))
+        s.close()
+
+    def test_find_unused_port(self):
+        port = support.find_unused_port()
+        s = socket.socket()
+        s.bind((support.HOST, port))
+        s.close()
+
+    def test_bind_port(self):
+        s = socket.socket()
+        support.bind_port(s)
+        s.listen(1)
+        s.close()
+
+    def test_temp_cwd(self):
+        here = os.getcwd()
+        with support.temp_cwd(name=TESTFN):
+            self.assertEqual(os.path.basename(os.getcwd()), TESTFN)
+        self.assertFalse(os.path.exists(TESTFN))
+        self.assertTrue(os.path.basename(os.getcwd()), here)
+
+    def test_temp_cwd__chdir_warning(self):
+        """Check the warning message when os.chdir() fails."""
+        path = TESTFN + '_does_not_exist'
+        with support.check_warnings() as recorder:
+            with support.temp_cwd(path=path, quiet=True):
+                pass
+            messages = [str(w.message) for w in recorder.warnings]
+        self.assertEqual(messages, ['tests may fail, unable to change the CWD to ' + path])
+
+    def test_sortdict(self):
+        self.assertEqual(support.sortdict({3:3, 2:2, 1:1}), "{1: 1, 2: 2, 3: 3}")
+
+    def test_make_bad_fd(self):
+        fd = support.make_bad_fd()
+        with self.assertRaises(OSError) as cm:
+            os.write(fd, b"foo")
+        self.assertEqual(cm.exception.errno, errno.EBADF)
+
+    def test_check_syntax_error(self):
+        support.check_syntax_error(self, "def class")
+        self.assertRaises(AssertionError, support.check_syntax_error, self, "1")
+
+    def test_CleanImport(self):
+        import importlib
+        with support.CleanImport("asyncore"):
+            importlib.import_module("asyncore")
+
+    def test_DirsOnSysPath(self):
+        with support.DirsOnSysPath('foo', 'bar'):
+            self.assertIn("foo", sys.path)
+            self.assertIn("bar", sys.path)
+        self.assertNotIn("foo", sys.path)
+        self.assertNotIn("bar", sys.path)
+
+    def test_captured_stdout(self):
+        with support.captured_stdout() as s:
+            print("hello")
+        self.assertEqual(s.getvalue(), "hello\n")
+
+    def test_captured_stderr(self):
+        with support.captured_stderr() as s:
+            print("hello", file=sys.stderr)
+        self.assertEqual(s.getvalue(), "hello\n")
+
+    def test_captured_stdin(self):
+        with support.captured_stdin() as s:
+            print("hello", file=sys.stdin)
+        self.assertEqual(s.getvalue(), "hello\n")
+
+    def test_gc_collect(self):
+        support.gc_collect()
+
+    def test_python_is_optimized(self):
+        self.assertIsInstance(support.python_is_optimized(), bool)
+
+    def test_swap_attr(self):
+        class Obj:
+            x = 1
+        obj = Obj()
+        with support.swap_attr(obj, "x", 5):
+            self.assertEqual(obj.x, 5)
+        self.assertEqual(obj.x, 1)
+
+    def test_swap_item(self):
+        D = {"item":1}
+        with support.swap_item(D, "item", 5):
+            self.assertEqual(D["item"], 5)
+        self.assertEqual(D["item"], 1)
+
+    # XXX -follows a list of untested API
+    # make_legacy_pyc
+    # is_resource_enabled
+    # requires
+    # fcmp
+    # umaks
+    # findfile
+    # check_warnings
+    # EnvironmentVarGuard
+    # TransientResource
+    # transient_internet
+    # run_with_locale
+    # set_memlimit
+    # bigmemtest
+    # precisionbigmemtest
+    # bigaddrspacetest
+    # requires_resource
+    # run_doctest
+    # threading_cleanup
+    # reap_threads
+    # reap_children
+    # strip_python_stderr
+    # args_from_interpreter_flags
+    # can_symlink
+    # skip_unless_symlink
+
+
+def test_main():
+    tests = [TestSupport]
+    support.run_unittest(*tests)
+
+if __name__ == '__main__':
+    test_main()
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index a52767a110..e5ec85c488 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -303,6 +303,7 @@ class SysModuleTest(unittest.TestCase):
             self.assertEqual(sys.getdlopenflags(), oldflags+1)
             sys.setdlopenflags(oldflags)
 
+    @test.support.refcount_test
     def test_refcount(self):
         # n here must be a global in order for this test to pass while
         # tracing with a python function.  Tracing calls PyFrame_FastToLocals
@@ -342,7 +343,7 @@ class SysModuleTest(unittest.TestCase):
     # Test sys._current_frames() in a WITH_THREADS build.
     @test.support.reap_threads
     def current_frames_with_threads(self):
-        import threading, _thread
+        import threading
         import traceback
 
         # Spawn a thread that blocks at a known place.  Then the main
@@ -356,7 +357,7 @@ class SysModuleTest(unittest.TestCase):
             g456()
 
         def g456():
-            thread_info.append(_thread.get_ident())
+            thread_info.append(threading.get_ident())
             entered_g.set()
             leave_g.wait()
 
@@ -372,7 +373,7 @@ class SysModuleTest(unittest.TestCase):
 
         d = sys._current_frames()
 
-        main_id = _thread.get_ident()
+        main_id = threading.get_ident()
         self.assertIn(main_id, d)
         self.assertIn(thread_id, d)
 
@@ -418,6 +419,7 @@ class SysModuleTest(unittest.TestCase):
         self.assertIsInstance(sys.builtin_module_names, tuple)
         self.assertIsInstance(sys.copyright, str)
         self.assertIsInstance(sys.exec_prefix, str)
+        self.assertIsInstance(sys.base_exec_prefix, str)
         self.assertIsInstance(sys.executable, str)
         self.assertEqual(len(sys.float_info), 11)
         self.assertEqual(sys.float_info.radix, 2)
@@ -446,8 +448,10 @@ class SysModuleTest(unittest.TestCase):
 
         self.assertIsInstance(sys.maxsize, int)
         self.assertIsInstance(sys.maxunicode, int)
+        self.assertEqual(sys.maxunicode, 0x10FFFF)
         self.assertIsInstance(sys.platform, str)
         self.assertIsInstance(sys.prefix, str)
+        self.assertIsInstance(sys.base_prefix, str)
         self.assertIsInstance(sys.version, str)
         vi = sys.version_info
         self.assertIsInstance(vi[:], tuple)
@@ -473,6 +477,14 @@ class SysModuleTest(unittest.TestCase):
         if not sys.platform.startswith('win'):
             self.assertIsInstance(sys.abiflags, str)
 
+    @unittest.skipUnless(hasattr(sys, 'thread_info'),
+                         'Threading required for this test.')
+    def test_thread_info(self):
+        info = sys.thread_info
+        self.assertEqual(len(info), 3)
+        self.assertIn(info.name, ('nt', 'os2', 'pthread', 'solaris', None))
+        self.assertIn(info.lock, ('semaphore', 'mutex+cond', None))
+
     def test_43581(self):
         # Can't use sys.stdout, as this is a StringIO object when
         # the test runs under regrtest.
@@ -500,7 +512,7 @@ class SysModuleTest(unittest.TestCase):
 
     def test_sys_flags(self):
         self.assertTrue(sys.flags)
-        attrs = ("debug", "division_warning",
+        attrs = ("debug",
                  "inspect", "interactive", "optimize", "dont_write_bytecode",
                  "no_user_site", "no_site", "ignore_environment", "verbose",
                  "bytes_warning", "quiet", "hash_randomization")
@@ -532,6 +544,8 @@ class SysModuleTest(unittest.TestCase):
         out = p.communicate()[0].strip()
         self.assertEqual(out, b'?')
 
+    @unittest.skipIf(sys.base_prefix != sys.prefix,
+                     'Test is not venv-compatible')
     def test_executable(self):
         # sys.executable should be absolute
         self.assertEqual(os.path.abspath(sys.executable), sys.executable)
@@ -568,6 +582,34 @@ class SysModuleTest(unittest.TestCase):
             expected = None
         self.check_fsencoding(fs_encoding, expected)
 
+    def test_implementation(self):
+        # This test applies to all implementations equally.
+
+        levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF}
+
+        self.assertTrue(hasattr(sys.implementation, 'name'))
+        self.assertTrue(hasattr(sys.implementation, 'version'))
+        self.assertTrue(hasattr(sys.implementation, 'hexversion'))
+        self.assertTrue(hasattr(sys.implementation, 'cache_tag'))
+
+        version = sys.implementation.version
+        self.assertEqual(version[:2], (version.major, version.minor))
+
+        hexversion = (version.major << 24 | version.minor << 16 |
+                      version.micro << 8 | levels[version.releaselevel] << 4 |
+                      version.serial << 0)
+        self.assertEqual(sys.implementation.hexversion, hexversion)
+
+        # PEP 421 requires that .name be lower case.
+        self.assertEqual(sys.implementation.name,
+                         sys.implementation.name.lower())
+
+    def test_debugmallocstats(self):
+        # Test sys._debugmallocstats()
+        from test.script_helper import assert_python_ok
+        args = ['-c', 'import sys; sys._debugmallocstats()']
+        ret, out, err = assert_python_ok(*args)
+        self.assertIn(b"free PyDictObjects", err)
 
 class SizeofTest(unittest.TestCase):
 
@@ -591,12 +633,12 @@ class SizeofTest(unittest.TestCase):
         # bool objects are not gc tracked
         self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit)
         # but lists are
-        self.assertEqual(sys.getsizeof([]), vsize('PP') + gc_header_size)
+        self.assertEqual(sys.getsizeof([]), vsize('Pn') + gc_header_size)
 
     def test_default(self):
-        vsize = test.support.calcvobjsize
-        self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit)
-        self.assertEqual(sys.getsizeof(True, -1), vsize('') + self.longdigit)
+        size = test.support.calcvobjsize
+        self.assertEqual(sys.getsizeof(True), size('') + self.longdigit)
+        self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit)
 
     def test_objecttypes(self):
         # check all types defined in Objects/
@@ -613,9 +655,9 @@ class SizeofTest(unittest.TestCase):
         samples = [b'', b'u'*100000]
         for sample in samples:
             x = bytearray(sample)
-            check(x, vsize('iPP') + x.__alloc__())
+            check(x, vsize('inP') + x.__alloc__())
         # bytearray_iterator
-        check(iter(bytearray()), size('PP'))
+        check(iter(bytearray()), size('nP'))
         # cell
         def get_cell():
             x = 42
@@ -624,27 +666,33 @@ class SizeofTest(unittest.TestCase):
             return inner
         check(get_cell().__closure__[0], size('P'))
         # code
-        check(get_cell().__code__, size('5i8Pi3P'))
+        check(get_cell().__code__, size('5i9Pi3P'))
+        check(get_cell.__code__, size('5i9Pi3P'))
+        def get_cell2(x):
+            def inner():
+                return x
+            return inner
+        check(get_cell2.__code__, size('5i9Pi3P') + 1)
         # complex
         check(complex(0,1), size('2d'))
         # method_descriptor (descriptor object)
-        check(str.lower, size('2PP'))
+        check(str.lower, size('3PP'))
         # classmethod_descriptor (descriptor object)
         # XXX
         # member_descriptor (descriptor object)
         import datetime
-        check(datetime.timedelta.days, size('2PP'))
+        check(datetime.timedelta.days, size('3PP'))
         # getset_descriptor (descriptor object)
         import collections
-        check(collections.defaultdict.default_factory, size('2PP'))
+        check(collections.defaultdict.default_factory, size('3PP'))
         # wrapper_descriptor (descriptor object)
-        check(int.__add__, size('2P2P'))
+        check(int.__add__, size('3P2P'))
         # method-wrapper (descriptor object)
         check({}.__iter__, size('2P'))
         # dict
-        check({}, size('3P2P' + 8*'P2P'))
+        check({}, size('n2P' + '2nPn' + 8*'n2P'))
         longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8}
-        check(longdict, size('3P2P' + 8*'P2P') + 16*struct.calcsize('P2P'))
+        check(longdict, size('n2P' + '2nPn') + 16*struct.calcsize('n2P'))
         # dictionary-keyiterator
         check({}.keys(), size('P'))
         # dictionary-valueiterator
@@ -652,18 +700,18 @@ class SizeofTest(unittest.TestCase):
         # dictionary-itemiterator
         check({}.items(), size('P'))
         # dictionary iterator
-        check(iter({}), size('P2PPP'))
+        check(iter({}), size('P2nPn'))
         # dictproxy
         class C(object): pass
         check(C.__dict__, size('P'))
         # BaseException
-        check(BaseException(), size('5P'))
+        check(BaseException(), size('5Pi'))
         # UnicodeEncodeError
-        check(UnicodeEncodeError("", "", 0, 0, ""), size('5P 2P2PP'))
+        check(UnicodeEncodeError("", "", 0, 0, ""), size('5Pi 2P2nP'))
         # UnicodeDecodeError
-        check(UnicodeDecodeError("", b"", 0, 0, ""), size('5P 2P2PP'))
+        check(UnicodeDecodeError("", b"", 0, 0, ""), size('5Pi 2P2nP'))
         # UnicodeTranslateError
-        check(UnicodeTranslateError("", 0, 1, ""), size('5P 2P2PP'))
+        check(UnicodeTranslateError("", 0, 1, ""), size('5Pi 2P2nP'))
         # ellipses
         check(Ellipsis, size(''))
         # EncodingMap
@@ -671,9 +719,9 @@ class SizeofTest(unittest.TestCase):
         x = codecs.charmap_build(encodings.iso8859_3.decoding_table)
         check(x, size('32B2iB'))
         # enumerate
-        check(enumerate([]), size('l3P'))
+        check(enumerate([]), size('n3P'))
         # reverse
-        check(reversed(''), size('PP'))
+        check(reversed(''), size('nP'))
         # float
         check(float(0), size('d'))
         # sys.floatinfo
@@ -689,7 +737,7 @@ class SizeofTest(unittest.TestCase):
         check(x, vsize('12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
         # function
         def func(): pass
-        check(func, size('11P'))
+        check(func, size('12P'))
         class c():
             @staticmethod
             def foo():
@@ -698,12 +746,12 @@ class SizeofTest(unittest.TestCase):
             def bar(cls):
                 pass
             # staticmethod
-            check(foo, size('P'))
+            check(foo, size('PP'))
             # classmethod
-            check(bar, size('P'))
+            check(bar, size('PP'))
         # generator
         def get_gen(): yield 1
-        check(get_gen(), size('Pi2P'))
+        check(get_gen(), size('Pb2P'))
         # iterator
         check(iter('abc'), size('lP'))
         # callable-iterator
@@ -712,7 +760,7 @@ class SizeofTest(unittest.TestCase):
         # list
         samples = [[], [1,2,3], ['1', '2', '3']]
         for sample in samples:
-            check(sample, vsize('PP') + len(sample)*self.P)
+            check(sample, vsize('Pn') + len(sample)*self.P)
         # sortwrapper (list)
         # XXX
         # cmpwrapper (list)
@@ -720,7 +768,7 @@ class SizeofTest(unittest.TestCase):
         # listiterator (list)
         check(iter([]), size('lP'))
         # listreverseiterator (list)
-        check(reversed([]), size('lP'))
+        check(reversed([]), size('nP'))
         # long
         check(0, vsize(''))
         check(1, vsize('') + self.longdigit)
@@ -730,9 +778,9 @@ class SizeofTest(unittest.TestCase):
         check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit)
         check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit)
         # memoryview
-        check(memoryview(b''), size('PP2P2i7P'))
+        check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn'))
         # module
-        check(unittest, size('3P'))
+        check(unittest, size('PnP'))
         # None
         check(None, size(''))
         # NotImplementedType
@@ -751,7 +799,7 @@ class SizeofTest(unittest.TestCase):
         # rangeiterator
         check(iter(range(1)), size('4l'))
         # reverse
-        check(reversed(''), size('PP'))
+        check(reversed(''), size('nP'))
         # range
         check(range(1), size('4P'))
         check(range(66000), size('4P'))
@@ -759,7 +807,7 @@ class SizeofTest(unittest.TestCase):
         # frozenset
         PySet_MINSIZE = 8
         samples = [[], range(10), range(50)]
-        s = size('3P2P' + PySet_MINSIZE*'PP' + 'PP')
+        s = size('3n2P' + PySet_MINSIZE*'nP' + 'nP')
         for sample in samples:
             minused = len(sample)
             if minused == 0: tmp = 1
@@ -773,10 +821,10 @@ class SizeofTest(unittest.TestCase):
                 check(set(sample), s)
                 check(frozenset(sample), s)
             else:
-                check(set(sample), s + newsize*struct.calcsize('lP'))
-                check(frozenset(sample), s + newsize*struct.calcsize('lP'))
+                check(set(sample), s + newsize*struct.calcsize('nP'))
+                check(frozenset(sample), s + newsize*struct.calcsize('nP'))
         # setiterator
-        check(iter(set()), size('P3P'))
+        check(iter(set()), size('P3n'))
         # slice
         check(slice(0), size('3P'))
         # super
@@ -785,28 +833,56 @@ class SizeofTest(unittest.TestCase):
         check((), vsize(''))
         check((1,2,3), vsize('') + 3*self.P)
         # type
-        # (PyTypeObject + PyNumberMethods + PyMappingMethods +
-        #  PySequenceMethods + PyBufferProcs)
-        s = vsize('P2P15Pl4PP9PP11PI') + struct.calcsize('16Pi17P 3P 10P 2P 2P')
+        # static type: PyTypeObject
+        s = vsize('P2n15Pl4Pn9Pn11PI')
         check(int, s)
+        # (PyTypeObject + PyNumberMethods + PyMappingMethods +
+        #  PySequenceMethods + PyBufferProcs + 4P)
+        s = vsize('P2n15Pl4Pn9Pn11PI') + struct.calcsize('34P 3P 10P 2P 4P')
+        # Separate block for PyDictKeysObject with 4 entries
+        s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P")
         # class
         class newstyleclass(object): pass
         check(newstyleclass, s)
+        # dict with shared keys
+        check(newstyleclass().__dict__, size('n2P' + '2nPn'))
         # unicode
-        usize = len('\0'.encode('unicode-internal'))
-        samples = ['', '1'*100]
-        # we need to test for both sizes, because we don't know if the string
-        # has been cached
+        # each tuple contains a string and its expected character size
+        # don't put any static strings here, as they may contain
+        # wchar_t or UTF-8 representations
+        samples = ['1'*100, '\xff'*50,
+                   '\u0100'*40, '\uffff'*100,
+                   '\U00010000'*30, '\U0010ffff'*100]
+        asciifields = "nniP"
+        compactfields = asciifields + "nPn"
+        unicodefields = compactfields + "P"
         for s in samples:
-            basicsize =  size('PPPiP') + usize * (len(s) + 1)
-            check(s, basicsize)
+            maxchar = ord(max(s))
+            if maxchar < 128:
+                L = size(asciifields) + len(s) + 1
+            elif maxchar < 256:
+                L = size(compactfields) + len(s) + 1
+            elif maxchar < 65536:
+                L = size(compactfields) + 2*(len(s) + 1)
+            else:
+                L = size(compactfields) + 4*(len(s) + 1)
+            check(s, L)
+        # verify that the UTF-8 size is accounted for
+        s = chr(0x4000)   # 4 bytes canonical representation
+        check(s, size(compactfields) + 4)
+        # compile() will trigger the generation of the UTF-8
+        # representation as a side effect
+        compile(s, "<stdin>", "eval")
+        check(s, size(compactfields) + 4 + 4)
+        # TODO: add check that forces the presence of wchar_t representation
+        # TODO: add check that forces layout of unicodefields
         # weakref
         import weakref
-        check(weakref.ref(int), size('2Pl2P'))
+        check(weakref.ref(int), size('2Pn2P'))
         # weakproxy
         # XXX
         # weakcallableproxy
-        check(weakref.proxy(int), size('2Pl2P'))
+        check(weakref.proxy(int), size('2Pn2P'))
 
     def test_pythontypes(self):
         # check all types defined in Python/
@@ -815,16 +891,13 @@ class SizeofTest(unittest.TestCase):
         check = self.check_sizeof
         # _ast.AST
         import _ast
-        check(_ast.AST(), size(''))
-        # imp.NullImporter
-        import imp
-        check(imp.NullImporter(self.file.name), size(''))
+        check(_ast.AST(), size('P'))
         try:
             raise TypeError
         except TypeError:
             tb = sys.exc_info()[2]
             # traceback
-            if tb != None:
+            if tb is not None:
                 check(tb, size('2P2i'))
         # symtable entry
         # XXX
diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index ba3bc2e911..63ae1b7180 100644
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -251,6 +251,7 @@ class TraceTestCase(unittest.TestCase):
     def setUp(self):
         self.using_gc = gc.isenabled()
         gc.disable()
+        self.addCleanup(sys.settrace, sys.gettrace())
 
     def tearDown(self):
         if self.using_gc:
@@ -389,6 +390,9 @@ class TraceTestCase(unittest.TestCase):
 
 
 class RaisingTraceFuncTestCase(unittest.TestCase):
+    def setUp(self):
+        self.addCleanup(sys.settrace, sys.gettrace())
+
     def trace(self, frame, event, arg):
         """A trace function that raises an exception in response to a
         specific trace event."""
@@ -696,6 +700,10 @@ def no_jump_without_trace_function():
 
 
 class JumpTestCase(unittest.TestCase):
+    def setUp(self):
+        self.addCleanup(sys.settrace, sys.gettrace())
+        sys.settrace(None)
+
     def compare_jump_output(self, expected, received):
         if received != expected:
             self.fail( "Outputs don't match:\n" +
@@ -747,6 +755,8 @@ class JumpTestCase(unittest.TestCase):
     def test_18_no_jump_to_non_integers(self):
         self.run_test(no_jump_to_non_integers)
     def test_19_no_jump_without_trace_function(self):
+        # Must set sys.settrace(None) in setUp(), else condition is not
+        # triggered.
         no_jump_without_trace_function()
     def test_jump_across_with(self):
         self.addCleanup(support.unlink, support.TESTFN)
diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py
index 1b6897d34d..92193602f9 100644
--- a/Lib/test/test_sysconfig.py
+++ b/Lib/test/test_sysconfig.py
@@ -1,11 +1,9 @@
-"""Tests for sysconfig."""
-
 import unittest
 import sys
 import os
 import subprocess
 import shutil
-from copy import copy, deepcopy
+from copy import copy
 
 from test.support import (run_unittest, TESTFN, unlink,
                           captured_stdout, skip_unless_symlink)
@@ -20,7 +18,6 @@ import _osx_support
 class TestSysConfig(unittest.TestCase):
 
     def setUp(self):
-        """Make a copy of sys.path"""
         super(TestSysConfig, self).setUp()
         self.sys_path = sys.path[:]
         # patching os.uname
@@ -29,7 +26,7 @@ class TestSysConfig(unittest.TestCase):
             self._uname = os.uname()
         else:
             self.uname = None
-            self._uname = None
+            self._set_uname(('',)*5)
         os.uname = self._get_uname
         # saving the environment
         self.name = os.name
@@ -39,11 +36,16 @@ class TestSysConfig(unittest.TestCase):
         self.join = os.path.join
         self.isabs = os.path.isabs
         self.splitdrive = os.path.splitdrive
-        self._config_vars = copy(sysconfig._CONFIG_VARS)
-        self.old_environ = deepcopy(os.environ)
+        self._config_vars = sysconfig._CONFIG_VARS, copy(sysconfig._CONFIG_VARS)
+        self._added_envvars = []
+        self._changed_envvars = []
+        for var in ('MACOSX_DEPLOYMENT_TARGET', 'PATH'):
+            if var in os.environ:
+                self._changed_envvars.append((var, os.environ[var]))
+            else:
+                self._added_envvars.append(var)
 
     def tearDown(self):
-        """Restore sys.path"""
         sys.path[:] = self.sys_path
         self._cleanup_testfn()
         if self.uname is not None:
@@ -57,19 +59,18 @@ class TestSysConfig(unittest.TestCase):
         os.path.join = self.join
         os.path.isabs = self.isabs
         os.path.splitdrive = self.splitdrive
-        sysconfig._CONFIG_VARS = copy(self._config_vars)
-        for key, value in self.old_environ.items():
-            if os.environ.get(key) != value:
-                os.environ[key] = value
-
-        for key in list(os.environ.keys()):
-            if key not in self.old_environ:
-                del os.environ[key]
+        sysconfig._CONFIG_VARS = self._config_vars[0]
+        sysconfig._CONFIG_VARS.clear()
+        sysconfig._CONFIG_VARS.update(self._config_vars[1])
+        for var, value in self._changed_envvars:
+            os.environ[var] = value
+        for var in self._added_envvars:
+            os.environ.pop(var, None)
 
         super(TestSysConfig, self).tearDown()
 
     def _set_uname(self, uname):
-        self._uname = uname
+        self._uname = os.uname_result(uname)
 
     def _get_uname(self):
         return self._uname
@@ -88,21 +89,19 @@ class TestSysConfig(unittest.TestCase):
         scheme = get_paths()
         default_scheme = _get_default_scheme()
         wanted = _expand_vars(default_scheme, None)
-        wanted = list(wanted.items())
-        wanted.sort()
-        scheme = list(scheme.items())
-        scheme.sort()
+        wanted = sorted(wanted.items())
+        scheme = sorted(scheme.items())
         self.assertEqual(scheme, wanted)
 
     def test_get_path(self):
-        # xxx make real tests here
+        # XXX make real tests here
         for scheme in _INSTALL_SCHEMES:
             for name in _INSTALL_SCHEMES[scheme]:
                 res = get_path(name, scheme)
 
     def test_get_config_vars(self):
         cvars = get_config_vars()
-        self.assertTrue(isinstance(cvars, dict))
+        self.assertIsInstance(cvars, dict)
         self.assertTrue(cvars)
 
     def test_get_platform(self):
@@ -244,8 +243,8 @@ class TestSysConfig(unittest.TestCase):
         # On Windows, the EXE needs to know where pythonXY.dll is at so we have
         # to add the directory to the path.
         if sys.platform == "win32":
-            os.environ["Path"] = "{};{}".format(
-                os.path.dirname(sys.executable), os.environ["Path"])
+            os.environ["PATH"] = "{};{}".format(
+                os.path.dirname(sys.executable), os.environ["PATH"])
 
         # Issue 7880
         def get(python):
@@ -269,12 +268,17 @@ class TestSysConfig(unittest.TestCase):
         # the global scheme mirrors the distinction between prefix and
         # exec-prefix but not the user scheme, so we have to adapt the paths
         # before comparing (issue #9100)
-        adapt = sys.prefix != sys.exec_prefix
+        adapt = sys.base_prefix != sys.base_exec_prefix
         for name in ('stdlib', 'platstdlib', 'purelib', 'platlib'):
             global_path = get_path(name, 'posix_prefix')
             if adapt:
-                global_path = global_path.replace(sys.exec_prefix, sys.prefix)
-                base = base.replace(sys.exec_prefix, sys.prefix)
+                global_path = global_path.replace(sys.exec_prefix, sys.base_prefix)
+                base = base.replace(sys.exec_prefix, sys.base_prefix)
+            elif sys.base_prefix != sys.prefix:
+                # virtual environment? Likewise, we have to adapt the paths
+                # before comparing
+                global_path = global_path.replace(sys.base_prefix, sys.prefix)
+                base = base.replace(sys.base_prefix, sys.prefix)
             user_path = get_path(name, 'posix_user')
             self.assertEqual(user_path, global_path.replace(base, user, 1))
 
@@ -291,7 +295,6 @@ class TestSysConfig(unittest.TestCase):
 
         self.assertIn(ldflags, ldshared)
 
-
     @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
     def test_platform_in_subprocess(self):
         my_platform = sysconfig.get_platform()
@@ -317,28 +320,58 @@ class TestSysConfig(unittest.TestCase):
         self.assertEqual(status, 0)
         self.assertEqual(my_platform, test_platform)
 
-
         # Test with MACOSX_DEPLOYMENT_TARGET in the environment, and
         # using a value that is unlikely to be the default one.
         env = os.environ.copy()
         env['MACOSX_DEPLOYMENT_TARGET'] = '10.1'
 
-        p = subprocess.Popen([
-                sys.executable, '-c',
-                'import sysconfig; print(sysconfig.get_platform())',
-            ],
-            stdout=subprocess.PIPE,
-            stderr=open('/dev/null'),
-            env=env)
-        test_platform = p.communicate()[0].strip()
-        test_platform = test_platform.decode('utf-8')
-        status = p.wait()
-
-        self.assertEqual(status, 0)
-        self.assertEqual(my_platform, test_platform)
+        with open('/dev/null') as dev_null:
+            p = subprocess.Popen([
+                    sys.executable, '-c',
+                    'import sysconfig; print(sysconfig.get_platform())',
+                ],
+                stdout=subprocess.PIPE,
+                stderr=dev_null,
+                env=env)
+            test_platform = p.communicate()[0].strip()
+            test_platform = test_platform.decode('utf-8')
+            status = p.wait()
+
+            self.assertEqual(status, 0)
+            self.assertEqual(my_platform, test_platform)
+
+    def test_srcdir(self):
+        # See Issues #15322, #15364.
+        srcdir = sysconfig.get_config_var('srcdir')
+
+        self.assertTrue(os.path.isabs(srcdir), srcdir)
+        self.assertTrue(os.path.isdir(srcdir), srcdir)
+
+        if sysconfig._PYTHON_BUILD:
+            # The python executable has not been installed so srcdir
+            # should be a full source checkout.
+            Python_h = os.path.join(srcdir, 'Include', 'Python.h')
+            self.assertTrue(os.path.exists(Python_h), Python_h)
+            self.assertTrue(sysconfig._is_python_source_dir(srcdir))
+        elif os.name == 'posix':
+            self.assertEqual(os.path.dirname(sysconfig.get_makefile_filename()),
+                                srcdir)
+
+    def test_srcdir_independent_of_cwd(self):
+        # srcdir should be independent of the current working directory
+        # See Issues #15322, #15364.
+        srcdir = sysconfig.get_config_var('srcdir')
+        cwd = os.getcwd()
+        try:
+            os.chdir('..')
+            srcdir2 = sysconfig.get_config_var('srcdir')
+        finally:
+            os.chdir(cwd)
+        self.assertEqual(srcdir, srcdir2)
 
 
 class MakefileTests(unittest.TestCase):
+
     @unittest.skipIf(sys.platform.startswith('win'),
                      'Test is not Windows compatible')
     def test_get_makefile_filename(self):
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index a53181e56c..b224bf093f 100644
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -21,6 +21,10 @@ try:
     import bz2
 except ImportError:
     bz2 = None
+try:
+    import lzma
+except ImportError:
+    lzma = None
 
 def md5sum(data):
     return md5(data).hexdigest()
@@ -29,6 +33,7 @@ TEMPDIR = os.path.abspath(support.TESTFN) + "-tardir"
 tarname = support.findfile("testtar.tar")
 gzipname = os.path.join(TEMPDIR, "testtar.tar.gz")
 bz2name = os.path.join(TEMPDIR, "testtar.tar.bz2")
+xzname = os.path.join(TEMPDIR, "testtar.tar.xz")
 tmpname = os.path.join(TEMPDIR, "tmp.tar")
 
 md5_regtype = "65f477c818ad9e15f7feab0c6d37742f"
@@ -51,13 +56,10 @@ class UstarReadTest(ReadTest):
 
     def test_fileobj_regular_file(self):
         tarinfo = self.tar.getmember("ustar/regtype")
-        fobj = self.tar.extractfile(tarinfo)
-        try:
+        with self.tar.extractfile(tarinfo) as fobj:
             data = fobj.read()
             self.assertTrue((len(data), md5sum(data)) == (tarinfo.size, md5_regtype),
                     "regular file extraction failed")
-        finally:
-            fobj.close()
 
     def test_fileobj_readlines(self):
         self.tar.extract("ustar/regtype", TEMPDIR)
@@ -65,8 +67,7 @@ class UstarReadTest(ReadTest):
         with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1:
             lines1 = fobj1.readlines()
 
-        fobj = self.tar.extractfile(tarinfo)
-        try:
+        with self.tar.extractfile(tarinfo) as fobj:
             fobj2 = io.TextIOWrapper(fobj)
             lines2 = fobj2.readlines()
             self.assertTrue(lines1 == lines2,
@@ -76,21 +77,16 @@ class UstarReadTest(ReadTest):
             self.assertTrue(lines2[83] ==
                     "I will gladly admit that Python is not the fastest running scripting language.\n",
                     "fileobj.readlines() failed")
-        finally:
-            fobj.close()
 
     def test_fileobj_iter(self):
         self.tar.extract("ustar/regtype", TEMPDIR)
         tarinfo = self.tar.getmember("ustar/regtype")
-        with open(os.path.join(TEMPDIR, "ustar/regtype"), "rU") as fobj1:
+        with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1:
             lines1 = fobj1.readlines()
-        fobj2 = self.tar.extractfile(tarinfo)
-        try:
+        with self.tar.extractfile(tarinfo) as fobj2:
             lines2 = list(io.TextIOWrapper(fobj2))
             self.assertTrue(lines1 == lines2,
                          "fileobj.__iter__() failed")
-        finally:
-            fobj2.close()
 
     def test_fileobj_seek(self):
         self.tar.extract("ustar/regtype", TEMPDIR)
@@ -142,17 +138,24 @@ class UstarReadTest(ReadTest):
                      "read() after readline() failed")
         fobj.close()
 
+    def test_fileobj_text(self):
+        with self.tar.extractfile("ustar/regtype") as fobj:
+            fobj = io.TextIOWrapper(fobj)
+            data = fobj.read().encode("iso8859-1")
+            self.assertEqual(md5sum(data), md5_regtype)
+            try:
+                fobj.seek(100)
+            except AttributeError:
+                # Issue #13815: seek() complained about a missing
+                # flush() method.
+                self.fail("seeking failed in text mode")
+
     # Test if symbolic and hard links are resolved by extractfile().  The
     # test link members each point to a regular member whose data is
     # supposed to be exported.
     def _test_fileobj_link(self, lnktype, regtype):
-        a = self.tar.extractfile(lnktype)
-        b = self.tar.extractfile(regtype)
-        try:
+        with self.tar.extractfile(lnktype) as a, self.tar.extractfile(regtype) as b:
             self.assertEqual(a.name, b.name)
-        finally:
-            a.close()
-            b.close()
 
     def test_fileobj_link1(self):
         self._test_fileobj_link("ustar/lnktype", "ustar/regtype")
@@ -204,13 +207,15 @@ class CommonReadTest(ReadTest):
             _open = gzip.GzipFile
         elif self.mode.endswith(":bz2"):
             _open = bz2.BZ2File
+        elif self.mode.endswith(":xz"):
+            _open = lzma.LZMAFile
         else:
-            _open = open
+            _open = io.FileIO
 
         for char in (b'\0', b'a'):
             # Test if EOFHeaderError ('\0') and InvalidHeaderError ('a')
             # are ignored correctly.
-            with _open(tmpname, "wb") as fobj:
+            with _open(tmpname, "w") as fobj:
                 fobj.write(char * 1024)
                 fobj.write(tarfile.TarInfo("foo").tobuf())
 
@@ -225,6 +230,10 @@ class CommonReadTest(ReadTest):
 class MiscReadTest(CommonReadTest):
 
     def test_no_name_argument(self):
+        if self.mode.endswith(("bz2", "xz")):
+            # BZ2File and LZMAFile have no name attribute.
+            self.skipTest("no name attribute")
+
         with open(self.tarname, "rb") as fobj:
             tar = tarfile.open(fileobj=fobj, mode=self.mode)
             self.assertEqual(tar.name, os.path.abspath(fobj.name))
@@ -254,9 +263,8 @@ class MiscReadTest(CommonReadTest):
             t = tar.next()
             name = t.name
             offset = t.offset
-            f = tar.extractfile(t)
-            data = f.read()
-            f.close()
+            with tar.extractfile(t) as f:
+                data = f.read()
         finally:
             tar.close()
 
@@ -265,10 +273,12 @@ class MiscReadTest(CommonReadTest):
             _open = gzip.GzipFile
         elif self.mode.endswith(":bz2"):
             _open = bz2.BZ2File
+        elif self.mode.endswith(":xz"):
+            _open = lzma.LZMAFile
         else:
-            _open = open
-        fobj = _open(self.tarname, "rb")
-        try:
+            _open = io.FileIO
+
+        with _open(self.tarname) as fobj:
             fobj.seek(offset)
 
             # Test if the tarfile starts with the second member.
@@ -281,8 +291,6 @@ class MiscReadTest(CommonReadTest):
             self.assertEqual(tar.extractfile(t).read(), data,
                     "seek back did not work")
             tar.close()
-        finally:
-            fobj.close()
 
     def test_fail_comp(self):
         # For Gzip and Bz2 Tests: fail with a ReadError on an uncompressed file.
@@ -418,27 +426,26 @@ class StreamReadTest(CommonReadTest):
         for tarinfo in self.tar:
             if not tarinfo.isreg():
                 continue
-            fobj = self.tar.extractfile(tarinfo)
-            while True:
-                try:
-                    buf = fobj.read(512)
-                except tarfile.StreamError:
-                    self.fail("simple read-through using TarFile.extractfile() failed")
-                if not buf:
-                    break
-            fobj.close()
+            with self.tar.extractfile(tarinfo) as fobj:
+                while True:
+                    try:
+                        buf = fobj.read(512)
+                    except tarfile.StreamError:
+                        self.fail("simple read-through using TarFile.extractfile() failed")
+                    if not buf:
+                        break
 
     def test_fileobj_regular_file(self):
         tarinfo = self.tar.next() # get "regtype" (can't use getmember)
-        fobj = self.tar.extractfile(tarinfo)
-        data = fobj.read()
+        with self.tar.extractfile(tarinfo) as fobj:
+            data = fobj.read()
         self.assertTrue((len(data), md5sum(data)) == (tarinfo.size, md5_regtype),
                 "regular file extraction failed")
 
     def test_provoke_stream_error(self):
         tarinfos = self.tar.getmembers()
-        f = self.tar.extractfile(tarinfos[0]) # read the first member
-        self.assertRaises(tarfile.StreamError, f.read)
+        with self.tar.extractfile(tarinfos[0]) as f: # read the first member
+            self.assertRaises(tarfile.StreamError, f.read)
 
     def test_compare_members(self):
         tar1 = tarfile.open(tarname, encoding="iso8859-1")
@@ -516,6 +523,18 @@ class DetectReadTest(unittest.TestCase):
             testfunc(bz2name, "r|*")
             testfunc(bz2name, "r|bz2")
 
+        if lzma:
+            self.assertRaises(tarfile.ReadError, tarfile.open, tarname, mode="r:xz")
+            self.assertRaises(tarfile.ReadError, tarfile.open, tarname, mode="r|xz")
+            self.assertRaises(tarfile.ReadError, tarfile.open, xzname, mode="r:")
+            self.assertRaises(tarfile.ReadError, tarfile.open, xzname, mode="r|")
+
+            testfunc(xzname, "r")
+            testfunc(xzname, "r:*")
+            testfunc(xzname, "r:xz")
+            testfunc(xzname, "r|*")
+            testfunc(xzname, "r|xz")
+
     def test_detect_file(self):
         self._test_modes(self._testfunc_file)
 
@@ -713,7 +732,7 @@ class GNUReadTest(LongnameTest):
         # Return True if the platform knows the st_blocks stat attribute and
         # uses st_blocks units of 512 bytes, and if the filesystem is able to
         # store holes in files.
-        if sys.platform == "linux2":
+        if sys.platform.startswith("linux"):
             # Linux evidentially has 512 byte st_blocks units.
             name = os.path.join(TEMPDIR, "sparse-test")
             with open(name, "wb") as fobj:
@@ -903,7 +922,7 @@ class WriteTest(WriteTestBase):
         try:
             for name in ("foo", "bar", "baz"):
                 name = os.path.join(tempdir, name)
-                open(name, "wb").close()
+                support.create_empty_file(name)
 
             exclude = os.path.isfile
 
@@ -930,7 +949,7 @@ class WriteTest(WriteTestBase):
         try:
             for name in ("foo", "bar", "baz"):
                 name = os.path.join(tempdir, name)
-                open(name, "wb").close()
+                support.create_empty_file(name)
 
             def filter(tarinfo):
                 if os.path.basename(tarinfo.name) == "bar":
@@ -969,7 +988,7 @@ class WriteTest(WriteTestBase):
         # and compare the stored name with the original.
         foo = os.path.join(TEMPDIR, "foo")
         if not dir:
-            open(foo, "w").close()
+            support.create_empty_file(foo)
         else:
             os.mkdir(foo)
 
@@ -1086,6 +1105,9 @@ class StreamWriteTest(WriteTestBase):
             data = dec.decompress(data)
             self.assertTrue(len(dec.unused_data) == 0,
                     "found trailing data")
+        elif self.mode.endswith("xz"):
+            with lzma.LZMAFile(tmpname) as fobj:
+                data = fobj.read()
         else:
             with open(tmpname, "rb") as fobj:
                 data = fobj.read()
@@ -1329,7 +1351,7 @@ class UstarUnicodeTest(unittest.TestCase):
         self._test_unicode_filename("utf7")
 
     def test_utf8_filename(self):
-        self._test_unicode_filename("utf8")
+        self._test_unicode_filename("utf-8")
 
     def _test_unicode_filename(self, encoding):
         tar = tarfile.open(tmpname, "w", format=self.format, encoding=encoding, errors="strict")
@@ -1408,7 +1430,7 @@ class GNUUnicodeTest(UstarUnicodeTest):
     def test_bad_pax_header(self):
         # Test for issue #8633. GNU tar <= 1.23 creates raw binary fields
         # without a hdrcharset=BINARY header.
-        for encoding, name in (("utf8", "pax/bad-pax-\udce4\udcf6\udcfc"),
+        for encoding, name in (("utf-8", "pax/bad-pax-\udce4\udcf6\udcfc"),
                 ("iso8859-1", "pax/bad-pax-\xe4\xf6\xfc"),):
             with tarfile.open(tarname, encoding=encoding, errors="surrogateescape") as tar:
                 try:
@@ -1423,7 +1445,7 @@ class PAXUnicodeTest(UstarUnicodeTest):
 
     def test_binary_header(self):
         # Test a POSIX.1-2008 compatible header with a hdrcharset=BINARY field.
-        for encoding, name in (("utf8", "pax/hdrcharset-\udce4\udcf6\udcfc"),
+        for encoding, name in (("utf-8", "pax/hdrcharset-\udce4\udcf6\udcfc"),
                 ("iso8859-1", "pax/hdrcharset-\xe4\xf6\xfc"),):
             with tarfile.open(tarname, encoding=encoding, errors="surrogateescape") as tar:
                 try:
@@ -1448,12 +1470,9 @@ class AppendTest(unittest.TestCase):
         with tarfile.open(tarname, encoding="iso8859-1") as src:
             t = src.getmember("ustar/regtype")
             t.name = "foo"
-            f = src.extractfile(t)
-            try:
+            with src.extractfile(t) as f:
                 with tarfile.open(self.tarname, mode) as tar:
                     tar.addfile(t, f)
-            finally:
-                f.close()
 
     def _test(self, names=["bar"], fileobj=None):
         with tarfile.open(self.tarname, fileobj=fileobj) as tar:
@@ -1500,6 +1519,12 @@ class AppendTest(unittest.TestCase):
         self._create_testtar("w:bz2")
         self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, "a")
 
+    def test_append_lzma(self):
+        if lzma is None:
+            self.skipTest("lzma module not available")
+        self._create_testtar("w:xz")
+        self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, "a")
+
     # Append mode is supposed to fail if the tarfile to append to
     # does not end with a zero block.
     def _test_error(self, data):
@@ -1778,6 +1803,21 @@ class Bz2PartialReadTest(unittest.TestCase):
         self._test_partial_input("r:bz2")
 
 
+class LzmaMiscReadTest(MiscReadTest):
+    tarname = xzname
+    mode = "r:xz"
+class LzmaUstarReadTest(UstarReadTest):
+    tarname = xzname
+    mode = "r:xz"
+class LzmaStreamReadTest(StreamReadTest):
+    tarname = xzname
+    mode = "r|xz"
+class LzmaWriteTest(WriteTest):
+    mode = "w:xz"
+class LzmaStreamWriteTest(StreamWriteTest):
+    mode = "w|xz"
+
+
 def test_main():
     support.unlink(TEMPDIR)
     os.makedirs(TEMPDIR)
@@ -1840,6 +1880,20 @@ def test_main():
             Bz2PartialReadTest,
         ]
 
+    if lzma:
+        # Create testtar.tar.xz and add lzma-specific tests.
+        support.unlink(xzname)
+        with lzma.LZMAFile(xzname, "w") as tar:
+            tar.write(data)
+
+        tests += [
+            LzmaMiscReadTest,
+            LzmaUstarReadTest,
+            LzmaStreamReadTest,
+            LzmaWriteTest,
+            LzmaStreamWriteTest,
+        ]
+
     try:
         support.run_unittest(*tests)
     finally:
diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py
index 95eebf3cbc..11e95e77b7 100644
--- a/Lib/test/test_telnetlib.py
+++ b/Lib/test/test_telnetlib.py
@@ -36,6 +36,7 @@ class GeneralTests(TestCase):
 
     def tearDown(self):
         self.thread.join()
+        del self.thread  # Clear out any dangling Thread objects.
 
     def testBasic(self):
         # connects
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
index cf8b35ac80..2962939ce0 100644
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -23,7 +23,7 @@ has_spawnl = hasattr(os, 'spawnl')
 
 # TEST_FILES may need to be tweaked for systems depending on the maximum
 # number of files that can be opened at one time (see ulimit -n)
-if sys.platform in ('openbsd3', 'openbsd4'):
+if sys.platform.startswith('openbsd'):
     TEST_FILES = 48
 else:
     TEST_FILES = 100
@@ -33,7 +33,7 @@ else:
 # threads is not done here.
 
 # Common functionality.
-class TC(unittest.TestCase):
+class BaseTestCase(unittest.TestCase):
 
     str_check = re.compile(r"[a-zA-Z0-9_-]{6}$")
 
@@ -47,11 +47,6 @@ class TC(unittest.TestCase):
         self._warnings_manager.__exit__(None, None, None)
 
 
-    def failOnException(self, what, ei=None):
-        if ei is None:
-            ei = sys.exc_info()
-        self.fail("%s raised %s: %s" % (what, ei[0], ei[1]))
-
     def nameCheck(self, name, dir, pre, suf):
         (ndir, nbase) = os.path.split(name)
         npre  = nbase[:len(pre)]
@@ -70,9 +65,8 @@ class TC(unittest.TestCase):
                      "random string '%s' does not match /^[a-zA-Z0-9_-]{6}$/"
                      % nbase)
 
-test_classes = []
 
-class test_exports(TC):
+class TestExports(BaseTestCase):
     def test_exports(self):
         # There are no surprising symbols in the tempfile module
         dict = tempfile.__dict__
@@ -99,10 +93,8 @@ class test_exports(TC):
         self.assertTrue(len(unexp) == 0,
                         "unexpected keys: %s" % unexp)
 
-test_classes.append(test_exports)
-
 
-class test__RandomNameSequence(TC):
+class TestRandomNameSequence(BaseTestCase):
     """Test the internal iterator object _RandomNameSequence."""
 
     def setUp(self):
@@ -130,13 +122,10 @@ class test__RandomNameSequence(TC):
 
         i = 0
         r = self.r
-        try:
-            for s in r:
-                i += 1
-                if i == 20:
-                    break
-        except:
-            self.failOnException("iteration")
+        for s in r:
+            i += 1
+            if i == 20:
+                break
 
     @unittest.skipUnless(hasattr(os, 'fork'),
         "os.fork is required for this test")
@@ -169,10 +158,8 @@ class test__RandomNameSequence(TC):
         self.assertNotEqual(child_value, parent_value)
 
 
-test_classes.append(test__RandomNameSequence)
-
 
-class test__candidate_tempdir_list(TC):
+class TestCandidateTempdirList(BaseTestCase):
     """Test the internal function _candidate_tempdir_list."""
 
     def test_nonempty_list(self):
@@ -211,11 +198,10 @@ class test__candidate_tempdir_list(TC):
             # Not practical to try to verify the presence of OS-specific
             # paths in this list.
 
-test_classes.append(test__candidate_tempdir_list)
 
 # We test _get_default_tempdir some more by testing gettempdir.
 
-class TestGetDefaultTempdir(TC):
+class TestGetDefaultTempdir(BaseTestCase):
     """Test _get_default_tempdir()."""
 
     def test_no_files_left_behind(self):
@@ -232,13 +218,12 @@ class TestGetDefaultTempdir(TC):
                 self.assertEqual(os.listdir(our_temp_directory), [])
 
                 def raise_OSError(*args, **kwargs):
-                    raise OSError(-1)
+                    raise OSError()
 
                 with support.swap_attr(io, "open", raise_OSError):
                     # test again with failing io.open()
-                    with self.assertRaises(IOError) as cm:
+                    with self.assertRaises(FileNotFoundError):
                         tempfile._get_default_tempdir()
-                    self.assertEqual(cm.exception.args[0], errno.ENOENT)
                     self.assertEqual(os.listdir(our_temp_directory), [])
 
                 open = io.open
@@ -249,15 +234,12 @@ class TestGetDefaultTempdir(TC):
 
                 with support.swap_attr(io, "open", bad_writer):
                     # test again with failing write()
-                    with self.assertRaises(IOError) as cm:
+                    with self.assertRaises(FileNotFoundError):
                         tempfile._get_default_tempdir()
-                    self.assertEqual(cm.exception.errno, errno.ENOENT)
                     self.assertEqual(os.listdir(our_temp_directory), [])
 
-test_classes.append(TestGetDefaultTempdir)
-
 
-class test__get_candidate_names(TC):
+class TestGetCandidateNames(BaseTestCase):
     """Test the internal function _get_candidate_names."""
 
     def test_retval(self):
@@ -272,10 +254,8 @@ class test__get_candidate_names(TC):
 
         self.assertTrue(a is b)
 
-test_classes.append(test__get_candidate_names)
 
-
-class test__mkstemp_inner(TC):
+class TestMkstempInner(BaseTestCase):
     """Test the internal function _mkstemp_inner."""
 
     class mkstemped:
@@ -300,10 +280,7 @@ class test__mkstemp_inner(TC):
     def do_create(self, dir=None, pre="", suf="", bin=1):
         if dir is None:
             dir = tempfile.gettempdir()
-        try:
-            file = self.mkstemped(dir, pre, suf, bin)
-        except:
-            self.failOnException("_mkstemp_inner")
+        file = self.mkstemped(dir, pre, suf, bin)
 
         self.nameCheck(file.name, dir, pre, suf)
         return file
@@ -395,10 +372,8 @@ class test__mkstemp_inner(TC):
         os.lseek(f.fd, 0, os.SEEK_SET)
         self.assertEqual(os.read(f.fd, 20), b"blat")
 
-test_classes.append(test__mkstemp_inner)
 
-
-class test_gettempprefix(TC):
+class TestGetTempPrefix(BaseTestCase):
     """Test gettempprefix()."""
 
     def test_sane_template(self):
@@ -418,19 +393,14 @@ class test_gettempprefix(TC):
         d = tempfile.mkdtemp(prefix="")
         try:
             p = os.path.join(d, p)
-            try:
-                fd = os.open(p, os.O_RDWR | os.O_CREAT)
-            except:
-                self.failOnException("os.open")
+            fd = os.open(p, os.O_RDWR | os.O_CREAT)
             os.close(fd)
             os.unlink(p)
         finally:
             os.rmdir(d)
 
-test_classes.append(test_gettempprefix)
-
 
-class test_gettempdir(TC):
+class TestGetTempDir(BaseTestCase):
     """Test gettempdir()."""
 
     def test_directory_exists(self):
@@ -448,12 +418,9 @@ class test_gettempdir(TC):
         # sneaky: just instantiate a NamedTemporaryFile, which
         # defaults to writing into the directory returned by
         # gettempdir.
-        try:
-            file = tempfile.NamedTemporaryFile()
-            file.write(b"blat")
-            file.close()
-        except:
-            self.failOnException("create file in %s" % tempfile.gettempdir())
+        file = tempfile.NamedTemporaryFile()
+        file.write(b"blat")
+        file.close()
 
     def test_same_thing(self):
         # gettempdir always returns the same object
@@ -462,23 +429,18 @@ class test_gettempdir(TC):
 
         self.assertTrue(a is b)
 
-test_classes.append(test_gettempdir)
-
 
-class test_mkstemp(TC):
+class TestMkstemp(BaseTestCase):
     """Test mkstemp()."""
 
     def do_create(self, dir=None, pre="", suf=""):
         if dir is None:
             dir = tempfile.gettempdir()
-        try:
-            (fd, name) = tempfile.mkstemp(dir=dir, prefix=pre, suffix=suf)
-            (ndir, nbase) = os.path.split(name)
-            adir = os.path.abspath(dir)
-            self.assertEqual(adir, ndir,
-                "Directory '%s' incorrectly returned as '%s'" % (adir, ndir))
-        except:
-            self.failOnException("mkstemp")
+        (fd, name) = tempfile.mkstemp(dir=dir, prefix=pre, suffix=suf)
+        (ndir, nbase) = os.path.split(name)
+        adir = os.path.abspath(dir)
+        self.assertEqual(adir, ndir,
+            "Directory '%s' incorrectly returned as '%s'" % (adir, ndir))
 
         try:
             self.nameCheck(name, dir, pre, suf)
@@ -503,19 +465,14 @@ class test_mkstemp(TC):
         finally:
             os.rmdir(dir)
 
-test_classes.append(test_mkstemp)
-
 
-class test_mkdtemp(TC):
+class TestMkdtemp(BaseTestCase):
     """Test mkdtemp()."""
 
     def do_create(self, dir=None, pre="", suf=""):
         if dir is None:
             dir = tempfile.gettempdir()
-        try:
-            name = tempfile.mkdtemp(dir=dir, prefix=pre, suffix=suf)
-        except:
-            self.failOnException("mkdtemp")
+        name = tempfile.mkdtemp(dir=dir, prefix=pre, suffix=suf)
 
         try:
             self.nameCheck(name, dir, pre, suf)
@@ -570,10 +527,8 @@ class test_mkdtemp(TC):
         finally:
             os.rmdir(dir)
 
-test_classes.append(test_mkdtemp)
 
-
-class test_mktemp(TC):
+class TestMktemp(BaseTestCase):
     """Test mktemp()."""
 
     # For safety, all use of mktemp must occur in a private directory.
@@ -602,10 +557,7 @@ class test_mktemp(TC):
             self._unlink(self.name)
 
     def do_create(self, pre="", suf=""):
-        try:
-            file = self.mktemped(self.dir, pre, suf)
-        except:
-            self.failOnException("mktemp")
+        file = self.mktemped(self.dir, pre, suf)
 
         self.nameCheck(file.name, self.dir, pre, suf)
         return file
@@ -632,23 +584,18 @@ class test_mktemp(TC):
 ##         self.assertRaises(RuntimeWarning,
 ##                           tempfile.mktemp, dir=self.dir)
 
-test_classes.append(test_mktemp)
-
 
 # We test _TemporaryFileWrapper by testing NamedTemporaryFile.
 
 
-class test_NamedTemporaryFile(TC):
+class TestNamedTemporaryFile(BaseTestCase):
     """Test NamedTemporaryFile()."""
 
     def do_create(self, dir=None, pre="", suf="", delete=True):
         if dir is None:
             dir = tempfile.gettempdir()
-        try:
-            file = tempfile.NamedTemporaryFile(dir=dir, prefix=pre, suffix=suf,
-                                               delete=delete)
-        except:
-            self.failOnException("NamedTemporaryFile")
+        file = tempfile.NamedTemporaryFile(dir=dir, prefix=pre, suffix=suf,
+                                           delete=delete)
 
         self.nameCheck(file.name, dir, pre, suf)
         return file
@@ -701,11 +648,8 @@ class test_NamedTemporaryFile(TC):
         f = tempfile.NamedTemporaryFile()
         f.write(b'abc\n')
         f.close()
-        try:
-            f.close()
-            f.close()
-        except:
-            self.failOnException("close")
+        f.close()
+        f.close()
 
     def test_context_manager(self):
         # A NamedTemporaryFile can be used as a context manager
@@ -719,18 +663,14 @@ class test_NamedTemporaryFile(TC):
 
     # How to test the mode and bufsize parameters?
 
-test_classes.append(test_NamedTemporaryFile)
 
-class test_SpooledTemporaryFile(TC):
+class TestSpooledTemporaryFile(BaseTestCase):
     """Test SpooledTemporaryFile()."""
 
     def do_create(self, max_size=0, dir=None, pre="", suf=""):
         if dir is None:
             dir = tempfile.gettempdir()
-        try:
-            file = tempfile.SpooledTemporaryFile(max_size=max_size, dir=dir, prefix=pre, suffix=suf)
-        except:
-            self.failOnException("SpooledTemporaryFile")
+        file = tempfile.SpooledTemporaryFile(max_size=max_size, dir=dir, prefix=pre, suffix=suf)
 
         return file
 
@@ -818,11 +758,8 @@ class test_SpooledTemporaryFile(TC):
         f.write(b'abc\n')
         self.assertFalse(f._rolled)
         f.close()
-        try:
-            f.close()
-            f.close()
-        except:
-            self.failOnException("close")
+        f.close()
+        f.close()
 
     def test_multiple_close_after_rollover(self):
         # A SpooledTemporaryFile can be closed many times without error
@@ -830,11 +767,8 @@ class test_SpooledTemporaryFile(TC):
         f.write(b'abc\n')
         self.assertTrue(f._rolled)
         f.close()
-        try:
-            f.close()
-            f.close()
-        except:
-            self.failOnException("close")
+        f.close()
+        f.close()
 
     def test_bound_methods(self):
         # It should be OK to steal a bound method from a SpooledTemporaryFile
@@ -959,66 +893,76 @@ class test_SpooledTemporaryFile(TC):
                 pass
         self.assertRaises(ValueError, use_closed)
 
+    def test_truncate_with_size_parameter(self):
+        # A SpooledTemporaryFile can be truncated to zero size
+        f = tempfile.SpooledTemporaryFile(max_size=10)
+        f.write(b'abcdefg\n')
+        f.seek(0)
+        f.truncate()
+        self.assertFalse(f._rolled)
+        self.assertEqual(f._file.getvalue(), b'')
+        # A SpooledTemporaryFile can be truncated to a specific size
+        f = tempfile.SpooledTemporaryFile(max_size=10)
+        f.write(b'abcdefg\n')
+        f.truncate(4)
+        self.assertFalse(f._rolled)
+        self.assertEqual(f._file.getvalue(), b'abcd')
+        # A SpooledTemporaryFile rolls over if truncated to large size
+        f = tempfile.SpooledTemporaryFile(max_size=10)
+        f.write(b'abcdefg\n')
+        f.truncate(20)
+        self.assertTrue(f._rolled)
+        if has_stat:
+            self.assertEqual(os.fstat(f.fileno()).st_size, 20)
 
-test_classes.append(test_SpooledTemporaryFile)
 
+if tempfile.NamedTemporaryFile is not tempfile.TemporaryFile:
 
-class test_TemporaryFile(TC):
-    """Test TemporaryFile()."""
+    class TestTemporaryFile(BaseTestCase):
+        """Test TemporaryFile()."""
 
-    def test_basic(self):
-        # TemporaryFile can create files
-        # No point in testing the name params - the file has no name.
-        try:
+        def test_basic(self):
+            # TemporaryFile can create files
+            # No point in testing the name params - the file has no name.
             tempfile.TemporaryFile()
-        except:
-            self.failOnException("TemporaryFile")
 
-    def test_has_no_name(self):
-        # TemporaryFile creates files with no names (on this system)
-        dir = tempfile.mkdtemp()
-        f = tempfile.TemporaryFile(dir=dir)
-        f.write(b'blat')
+        def test_has_no_name(self):
+            # TemporaryFile creates files with no names (on this system)
+            dir = tempfile.mkdtemp()
+            f = tempfile.TemporaryFile(dir=dir)
+            f.write(b'blat')
 
-        # Sneaky: because this file has no name, it should not prevent
-        # us from removing the directory it was created in.
-        try:
-            os.rmdir(dir)
-        except:
-            ei = sys.exc_info()
-            # cleanup
+            # Sneaky: because this file has no name, it should not prevent
+            # us from removing the directory it was created in.
+            try:
+                os.rmdir(dir)
+            except:
+                # cleanup
+                f.close()
+                os.rmdir(dir)
+                raise
+
+        def test_multiple_close(self):
+            # A TemporaryFile can be closed many times without error
+            f = tempfile.TemporaryFile()
+            f.write(b'abc\n')
             f.close()
-            os.rmdir(dir)
-            self.failOnException("rmdir", ei)
-
-    def test_multiple_close(self):
-        # A TemporaryFile can be closed many times without error
-        f = tempfile.TemporaryFile()
-        f.write(b'abc\n')
-        f.close()
-        try:
             f.close()
             f.close()
-        except:
-            self.failOnException("close")
 
-    # How to test the mode and bufsize parameters?
-    def test_mode_and_encoding(self):
+        # How to test the mode and bufsize parameters?
+        def test_mode_and_encoding(self):
 
-        def roundtrip(input, *args, **kwargs):
-            with tempfile.TemporaryFile(*args, **kwargs) as fileobj:
-                fileobj.write(input)
-                fileobj.seek(0)
-                self.assertEqual(input, fileobj.read())
+            def roundtrip(input, *args, **kwargs):
+                with tempfile.TemporaryFile(*args, **kwargs) as fileobj:
+                    fileobj.write(input)
+                    fileobj.seek(0)
+                    self.assertEqual(input, fileobj.read())
 
-        roundtrip(b"1234", "w+b")
-        roundtrip("abdc\n", "w+")
-        roundtrip("\u039B", "w+", encoding="utf-16")
-        roundtrip("foo\r\n", "w+", newline="")
-
-
-if tempfile.NamedTemporaryFile is not tempfile.TemporaryFile:
-    test_classes.append(test_TemporaryFile)
+            roundtrip(b"1234", "w+b")
+            roundtrip("abdc\n", "w+")
+            roundtrip("\u039B", "w+", encoding="utf-16")
+            roundtrip("foo\r\n", "w+", newline="")
 
 
 # Helper for test_del_on_shutdown
@@ -1037,16 +981,13 @@ class NulledModules:
             d.clear()
             d.update(c)
 
-class test_TemporaryDirectory(TC):
+class TestTemporaryDirectory(BaseTestCase):
     """Test TemporaryDirectory()."""
 
     def do_create(self, dir=None, pre="", suf="", recurse=1):
         if dir is None:
             dir = tempfile.gettempdir()
-        try:
-            tmp = tempfile.TemporaryDirectory(dir=dir, prefix=pre, suffix=suf)
-        except:
-            self.failOnException("TemporaryDirectory")
+        tmp = tempfile.TemporaryDirectory(dir=dir, prefix=pre, suffix=suf)
         self.nameCheck(tmp.name, dir, pre, suf)
         # Create a subdirectory and some files
         if recurse:
@@ -1061,8 +1002,9 @@ class test_TemporaryDirectory(TC):
         # (noted as part of Issue #10188)
         with tempfile.TemporaryDirectory() as nonexistent:
             pass
-        with self.assertRaises(os.error):
+        with self.assertRaises(FileNotFoundError) as cm:
             tempfile.TemporaryDirectory(dir=nonexistent)
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
 
     def test_explicit_cleanup(self):
         # A TemporaryDirectory is deleted when cleaned up
@@ -1170,11 +1112,8 @@ class test_TemporaryDirectory(TC):
         # Can be cleaned-up many times without error
         d = self.do_create()
         d.cleanup()
-        try:
-            d.cleanup()
-            d.cleanup()
-        except:
-            self.failOnException("cleanup")
+        d.cleanup()
+        d.cleanup()
 
     def test_context_manager(self):
         # Can be used as a context manager
@@ -1185,10 +1124,8 @@ class test_TemporaryDirectory(TC):
         self.assertFalse(os.path.exists(name))
 
 
-test_classes.append(test_TemporaryDirectory)
-
 def test_main():
-    support.run_unittest(*test_classes)
+    support.run_unittest(__name__)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py
index 3901120c87..c86f5cfae8 100644
--- a/Lib/test/test_textwrap.py
+++ b/Lib/test/test_textwrap.py
@@ -11,7 +11,7 @@
 import unittest
 from test import support
 
-from textwrap import TextWrapper, wrap, fill, dedent
+from textwrap import TextWrapper, wrap, fill, dedent, indent
 
 
 class BaseTestCase(unittest.TestCase):
@@ -100,6 +100,14 @@ What a mess!
         result = wrapper.fill(text)
         self.check(result, '\n'.join(expect))
 
+        text = "\tTest\tdefault\t\ttabsize."
+        expect = ["        Test    default         tabsize."]
+        self.check_wrap(text, 80, expect)
+
+        text = "\tTest\tcustom\t\ttabsize."
+        expect = ["    Test    custom      tabsize."]
+        self.check_wrap(text, 80, expect, tabsize=4)
+
     def test_fix_sentence_endings(self):
         wrapper = TextWrapper(60, fix_sentence_endings=True)
 
@@ -634,11 +642,147 @@ def foo():
         self.assertEqual(expect, dedent(text))
 
 
+# Test textwrap.indent
+class IndentTestCase(unittest.TestCase):
+    # The examples used for tests. If any of these change, the expected
+    # results in the various test cases must also be updated.
+    # The roundtrip cases are separate, because textwrap.dedent doesn't
+    # handle Windows line endings
+    ROUNDTRIP_CASES = (
+      # Basic test case
+      "Hi.\nThis is a test.\nTesting.",
+      # Include a blank line
+      "Hi.\nThis is a test.\n\nTesting.",
+      # Include leading and trailing blank lines
+      "\nHi.\nThis is a test.\nTesting.\n",
+    )
+    CASES = ROUNDTRIP_CASES + (
+      # Use Windows line endings
+      "Hi.\r\nThis is a test.\r\nTesting.\r\n",
+      # Pathological case
+      "\nHi.\r\nThis is a test.\n\r\nTesting.\r\n\n",
+    )
+
+    def test_indent_nomargin_default(self):
+        # indent should do nothing if 'prefix' is empty.
+        for text in self.CASES:
+            self.assertEqual(indent(text, ''), text)
+
+    def test_indent_nomargin_explicit_default(self):
+        # The same as test_indent_nomargin, but explicitly requesting
+        # the default behaviour by passing None as the predicate
+        for text in self.CASES:
+            self.assertEqual(indent(text, '', None), text)
+
+    def test_indent_nomargin_all_lines(self):
+        # The same as test_indent_nomargin, but using the optional
+        # predicate argument
+        predicate = lambda line: True
+        for text in self.CASES:
+            self.assertEqual(indent(text, '', predicate), text)
+
+    def test_indent_no_lines(self):
+        # Explicitly skip indenting any lines
+        predicate = lambda line: False
+        for text in self.CASES:
+            self.assertEqual(indent(text, '    ', predicate), text)
+
+    def test_roundtrip_spaces(self):
+        # A whitespace prefix should roundtrip with dedent
+        for text in self.ROUNDTRIP_CASES:
+            self.assertEqual(dedent(indent(text, '    ')), text)
+
+    def test_roundtrip_tabs(self):
+        # A whitespace prefix should roundtrip with dedent
+        for text in self.ROUNDTRIP_CASES:
+            self.assertEqual(dedent(indent(text, '\t\t')), text)
+
+    def test_roundtrip_mixed(self):
+        # A whitespace prefix should roundtrip with dedent
+        for text in self.ROUNDTRIP_CASES:
+            self.assertEqual(dedent(indent(text, ' \t  \t ')), text)
+
+    def test_indent_default(self):
+        # Test default indenting of lines that are not whitespace only
+        prefix = '  '
+        expected = (
+          # Basic test case
+          "  Hi.\n  This is a test.\n  Testing.",
+          # Include a blank line
+          "  Hi.\n  This is a test.\n\n  Testing.",
+          # Include leading and trailing blank lines
+          "\n  Hi.\n  This is a test.\n  Testing.\n",
+          # Use Windows line endings
+          "  Hi.\r\n  This is a test.\r\n  Testing.\r\n",
+          # Pathological case
+          "\n  Hi.\r\n  This is a test.\n\r\n  Testing.\r\n\n",
+        )
+        for text, expect in zip(self.CASES, expected):
+            self.assertEqual(indent(text, prefix), expect)
+
+    def test_indent_explicit_default(self):
+        # Test default indenting of lines that are not whitespace only
+        prefix = '  '
+        expected = (
+          # Basic test case
+          "  Hi.\n  This is a test.\n  Testing.",
+          # Include a blank line
+          "  Hi.\n  This is a test.\n\n  Testing.",
+          # Include leading and trailing blank lines
+          "\n  Hi.\n  This is a test.\n  Testing.\n",
+          # Use Windows line endings
+          "  Hi.\r\n  This is a test.\r\n  Testing.\r\n",
+          # Pathological case
+          "\n  Hi.\r\n  This is a test.\n\r\n  Testing.\r\n\n",
+        )
+        for text, expect in zip(self.CASES, expected):
+            self.assertEqual(indent(text, prefix, None), expect)
+
+    def test_indent_all_lines(self):
+        # Add 'prefix' to all lines, including whitespace-only ones.
+        prefix = '  '
+        expected = (
+          # Basic test case
+          "  Hi.\n  This is a test.\n  Testing.",
+          # Include a blank line
+          "  Hi.\n  This is a test.\n  \n  Testing.",
+          # Include leading and trailing blank lines
+          "  \n  Hi.\n  This is a test.\n  Testing.\n",
+          # Use Windows line endings
+          "  Hi.\r\n  This is a test.\r\n  Testing.\r\n",
+          # Pathological case
+          "  \n  Hi.\r\n  This is a test.\n  \r\n  Testing.\r\n  \n",
+        )
+        predicate = lambda line: True
+        for text, expect in zip(self.CASES, expected):
+            self.assertEqual(indent(text, prefix, predicate), expect)
+
+    def test_indent_empty_lines(self):
+        # Add 'prefix' solely to whitespace-only lines.
+        prefix = '  '
+        expected = (
+          # Basic test case
+          "Hi.\nThis is a test.\nTesting.",
+          # Include a blank line
+          "Hi.\nThis is a test.\n  \nTesting.",
+          # Include leading and trailing blank lines
+          "  \nHi.\nThis is a test.\nTesting.\n",
+          # Use Windows line endings
+          "Hi.\r\nThis is a test.\r\nTesting.\r\n",
+          # Pathological case
+          "  \nHi.\r\nThis is a test.\n  \r\nTesting.\r\n  \n",
+        )
+        predicate = lambda line: not line.strip()
+        for text, expect in zip(self.CASES, expected):
+            self.assertEqual(indent(text, prefix, predicate), expect)
+
+
 def test_main():
     support.run_unittest(WrapTestCase,
                               LongWordTestCase,
                               IndentTestCases,
-                              DedentTestCase)
+                              DedentTestCase,
+                              IndentTestCase)
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py
index 7791935fdf..6c2965ba6e 100644
--- a/Lib/test/test_threaded_import.py
+++ b/Lib/test/test_threaded_import.py
@@ -7,12 +7,13 @@
 
 import os
 import imp
+import importlib
 import sys
 import time
 import shutil
 import unittest
-from test.support import verbose, import_module, run_unittest, TESTFN
-thread = import_module('_thread')
+from test.support import (
+    verbose, import_module, run_unittest, TESTFN, reap_threads, forget, unlink)
 threading = import_module('threading')
 
 def task(N, done, done_tasks, errors):
@@ -30,7 +31,7 @@ def task(N, done, done_tasks, errors):
     except Exception as e:
         errors.append(e.with_traceback(None))
     finally:
-        done_tasks.append(thread.get_ident())
+        done_tasks.append(threading.get_ident())
         finished = len(done_tasks) == N
         if finished:
             done.set()
@@ -62,16 +63,17 @@ class Finder:
     def __init__(self):
         self.numcalls = 0
         self.x = 0
-        self.lock = thread.allocate_lock()
+        self.lock = threading.Lock()
 
     def find_module(self, name, path=None):
         # Simulate some thread-unsafe behaviour. If calls to find_module()
         # are properly serialized, `x` will end up the same as `numcalls`.
         # Otherwise not.
+        assert imp.lock_held()
         with self.lock:
             self.numcalls += 1
         x = self.x
-        time.sleep(0.1)
+        time.sleep(0.01)
         self.x = x + 1
 
 class FlushingFinder:
@@ -113,8 +115,10 @@ class ThreadedImportTests(unittest.TestCase):
             done_tasks = []
             done.clear()
             for i in range(N):
-                thread.start_new_thread(task, (N, done, done_tasks, errors,))
-            done.wait(60)
+                t = threading.Thread(target=task,
+                                     args=(N, done, done_tasks, errors,))
+                t.start()
+            self.assertTrue(done.wait(60))
             self.assertFalse(errors)
             if verbose:
                 print("OK.")
@@ -124,7 +128,7 @@ class ThreadedImportTests(unittest.TestCase):
 
     def test_parallel_meta_path(self):
         finder = Finder()
-        sys.meta_path.append(finder)
+        sys.meta_path.insert(0, finder)
         try:
             self.check_parallel_module_init()
             self.assertGreater(finder.numcalls, 0)
@@ -143,7 +147,7 @@ class ThreadedImportTests(unittest.TestCase):
         def path_hook(path):
             finder.find_module('')
             raise ImportError
-        sys.path_hooks.append(path_hook)
+        sys.path_hooks.insert(0, path_hook)
         sys.meta_path.append(flushing_finder)
         try:
             # Flush the cache a first time
@@ -185,8 +189,9 @@ class ThreadedImportTests(unittest.TestCase):
             contents = contents % {'delay': delay}
             with open(os.path.join(TESTFN, name + ".py"), "wb") as f:
                 f.write(contents.encode('utf-8'))
-            self.addCleanup(sys.modules.pop, name, None)
+            self.addCleanup(forget, name)
 
+        importlib.invalidate_caches()
         results = []
         def import_ab():
             import A
@@ -202,9 +207,38 @@ class ThreadedImportTests(unittest.TestCase):
         t2.join()
         self.assertEqual(set(results), {'a', 'b'})
 
-
+    def test_side_effect_import(self):
+        code = """if 1:
+            import threading
+            def target():
+                import random
+            t = threading.Thread(target=target)
+            t.start()
+            t.join()"""
+        sys.path.insert(0, os.curdir)
+        self.addCleanup(sys.path.remove, os.curdir)
+        filename = TESTFN + ".py"
+        with open(filename, "wb") as f:
+            f.write(code.encode('utf-8'))
+        self.addCleanup(unlink, filename)
+        self.addCleanup(forget, TESTFN)
+        importlib.invalidate_caches()
+        __import__(TESTFN)
+
+
+@reap_threads
 def test_main():
-    run_unittest(ThreadedImportTests)
+    old_switchinterval = None
+    try:
+        old_switchinterval = sys.getswitchinterval()
+        sys.setswitchinterval(1e-5)
+    except AttributeError:
+        pass
+    try:
+        run_unittest(ThreadedImportTests)
+    finally:
+        if old_switchinterval is not None:
+            sys.setswitchinterval(old_switchinterval)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
index 17be84b001..11e63d3e83 100644
--- a/Lib/test/test_threading.py
+++ b/Lib/test/test_threading.py
@@ -175,7 +175,7 @@ class ThreadTests(BaseTestCase):
         exception = ctypes.py_object(AsyncExc)
 
         # First check it works when setting the exception from the same thread.
-        tid = _thread.get_ident()
+        tid = threading.get_ident()
 
         try:
             result = set_async_exc(ctypes.c_long(tid), exception)
@@ -204,7 +204,7 @@ class ThreadTests(BaseTestCase):
 
         class Worker(threading.Thread):
             def run(self):
-                self.id = _thread.get_ident()
+                self.id = threading.get_ident()
                 self.finished = False
 
                 try:
@@ -409,6 +409,14 @@ class ThreadTests(BaseTestCase):
         t.daemon = True
         self.assertTrue('daemon' in repr(t))
 
+    def test_deamon_param(self):
+        t = threading.Thread()
+        self.assertFalse(t.daemon)
+        t = threading.Thread(daemon=False)
+        self.assertFalse(t.daemon)
+        t = threading.Thread(daemon=True)
+        self.assertTrue(t.daemon)
+
     @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()')
     def test_dummy_thread_after_fork(self):
         # Issue #14308: a dummy thread in the active list doesn't mess up
@@ -444,7 +452,7 @@ class ThreadJoinOnShutdown(BaseTestCase):
     # problems with some operating systems (issue #3863): skip problematic tests
     # on platforms known to behave badly.
     platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5',
-                         'os2emx')
+                         'os2emx', 'hp-ux11')
 
     def _run_and_join(self, script):
         script = """if 1:
@@ -742,6 +750,10 @@ class ThreadingExceptionTests(BaseTestCase):
         thread.start()
         self.assertRaises(RuntimeError, setattr, thread, "daemon", True)
 
+    def test_releasing_unacquired_lock(self):
+        lock = threading.Lock()
+        self.assertRaises(RuntimeError, lock.release)
+
     @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem')
     def test_recursion_limit(self):
         # Issue 9670
@@ -803,6 +815,7 @@ class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests):
 class BarrierTests(lock_tests.BarrierTests):
     barriertype = staticmethod(threading.Barrier)
 
+
 def test_main():
     test.support.run_unittest(LockTests, PyRLockTests, CRLockTests, EventTests,
                               ConditionAsRLockTests, ConditionTests,
@@ -810,7 +823,7 @@ def test_main():
                               ThreadTests,
                               ThreadJoinOnShutdown,
                               ThreadingExceptionTests,
-                              BarrierTests
+                              BarrierTests,
                               )
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py
index e0af31dae4..f975a75e85 100644
--- a/Lib/test/test_threadsignals.py
+++ b/Lib/test/test_threadsignals.py
@@ -14,10 +14,8 @@ if sys.platform[:3] in ('win', 'os2') or sys.platform=='riscos':
 process_pid = os.getpid()
 signalled_all=thread.allocate_lock()
 
-# Issue #11223: Locks are implemented using a mutex and a condition variable of
-# the pthread library on FreeBSD6. POSIX condition variables cannot be
-# interrupted by signals (see pthread_cond_wait manual page).
-USING_PTHREAD_COND = (sys.platform == 'freebsd6')
+USING_PTHREAD_COND = (sys.thread_info.name == 'pthread'
+                      and sys.thread_info.lock == 'mutex+cond')
 
 def registerSignals(for_usr1, for_usr2, for_alrm):
     usr1 = signal.signal(signal.SIGUSR1, for_usr1)
diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
index d3c49bb922..da0f555d91 100644
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -4,7 +4,17 @@ import unittest
 import locale
 import sysconfig
 import sys
-import warnings
+import platform
+try:
+    import threading
+except ImportError:
+    threading = None
+
+# Max year is only limited by the size of C int.
+SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4
+TIME_MAXYEAR = (1 << 8 * SIZEOF_INT - 1) - 1
+TIME_MINYEAR = -TIME_MAXYEAR - 1
+
 
 class TimeTestCase(unittest.TestCase):
 
@@ -17,9 +27,53 @@ class TimeTestCase(unittest.TestCase):
         time.timezone
         time.tzname
 
+    def test_time(self):
+        time.time()
+        info = time.get_clock_info('time')
+        self.assertFalse(info.monotonic)
+        self.assertTrue(info.adjustable)
+
     def test_clock(self):
         time.clock()
 
+        info = time.get_clock_info('clock')
+        self.assertTrue(info.monotonic)
+        self.assertFalse(info.adjustable)
+
+    @unittest.skipUnless(hasattr(time, 'clock_gettime'),
+                         'need time.clock_gettime()')
+    def test_clock_realtime(self):
+        time.clock_gettime(time.CLOCK_REALTIME)
+
+    @unittest.skipUnless(hasattr(time, 'clock_gettime'),
+                         'need time.clock_gettime()')
+    @unittest.skipUnless(hasattr(time, 'CLOCK_MONOTONIC'),
+                         'need time.CLOCK_MONOTONIC')
+    def test_clock_monotonic(self):
+        a = time.clock_gettime(time.CLOCK_MONOTONIC)
+        b = time.clock_gettime(time.CLOCK_MONOTONIC)
+        self.assertLessEqual(a, b)
+
+    @unittest.skipUnless(hasattr(time, 'clock_getres'),
+                         'need time.clock_getres()')
+    def test_clock_getres(self):
+        res = time.clock_getres(time.CLOCK_REALTIME)
+        self.assertGreater(res, 0.0)
+        self.assertLessEqual(res, 1.0)
+
+    @unittest.skipUnless(hasattr(time, 'clock_settime'),
+                         'need time.clock_settime()')
+    def test_clock_settime(self):
+        t = time.clock_gettime(time.CLOCK_REALTIME)
+        try:
+            time.clock_settime(time.CLOCK_REALTIME, t)
+        except PermissionError:
+            pass
+
+        if hasattr(time, 'CLOCK_MONOTONIC'):
+            self.assertRaises(OSError,
+                              time.clock_settime, time.CLOCK_MONOTONIC, 0)
+
     def test_conversions(self):
         self.assertEqual(time.ctime(self.t),
                          time.asctime(time.localtime(self.t)))
@@ -27,6 +81,8 @@ class TimeTestCase(unittest.TestCase):
                          int(self.t))
 
     def test_sleep(self):
+        self.assertRaises(ValueError, time.sleep, -2)
+        self.assertRaises(ValueError, time.sleep, -1)
         time.sleep(1.2)
 
     def test_strftime(self):
@@ -47,28 +103,34 @@ class TimeTestCase(unittest.TestCase):
             with self.assertRaises(ValueError):
                 time.strftime('%f')
 
-    def _bounds_checking(self, func=time.strftime):
+    def _bounds_checking(self, func):
         # Make sure that strftime() checks the bounds of the various parts
-        #of the time tuple (0 is valid for *all* values).
+        # of the time tuple (0 is valid for *all* values).
 
         # The year field is tested by other test cases above
 
         # Check month [1, 12] + zero support
+        func((1900, 0, 1, 0, 0, 0, 0, 1, -1))
+        func((1900, 12, 1, 0, 0, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, -1, 1, 0, 0, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, 13, 1, 0, 0, 0, 0, 1, -1))
         # Check day of month [1, 31] + zero support
+        func((1900, 1, 0, 0, 0, 0, 0, 1, -1))
+        func((1900, 1, 31, 0, 0, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, 1, -1, 0, 0, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, 1, 32, 0, 0, 0, 0, 1, -1))
         # Check hour [0, 23]
+        func((1900, 1, 1, 23, 0, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, 1, 1, -1, 0, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, 1, 1, 24, 0, 0, 0, 1, -1))
         # Check minute [0, 59]
+        func((1900, 1, 1, 0, 59, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, 1, 1, 0, -1, 0, 0, 1, -1))
         self.assertRaises(ValueError, func,
@@ -78,15 +140,21 @@ class TimeTestCase(unittest.TestCase):
                             (1900, 1, 1, 0, 0, -1, 0, 1, -1))
         # C99 only requires allowing for one leap second, but Python's docs say
         # allow two leap seconds (0..61)
+        func((1900, 1, 1, 0, 0, 60, 0, 1, -1))
+        func((1900, 1, 1, 0, 0, 61, 0, 1, -1))
         self.assertRaises(ValueError, func,
                             (1900, 1, 1, 0, 0, 62, 0, 1, -1))
         # No check for upper-bound day of week;
         #  value forced into range by a ``% 7`` calculation.
         # Start check at -2 since gettmarg() increments value before taking
         #  modulo.
+        self.assertEqual(func((1900, 1, 1, 0, 0, 0, -1, 1, -1)),
+                         func((1900, 1, 1, 0, 0, 0, +6, 1, -1)))
         self.assertRaises(ValueError, func,
                             (1900, 1, 1, 0, 0, 0, -2, 1, -1))
         # Check day of the year [1, 366] + zero support
+        func((1900, 1, 1, 0, 0, 0, 0, 0, -1))
+        func((1900, 1, 1, 0, 0, 0, 0, 366, -1))
         self.assertRaises(ValueError, func,
                             (1900, 1, 1, 0, 0, 0, 0, -1, -1))
         self.assertRaises(ValueError, func,
@@ -96,12 +164,13 @@ class TimeTestCase(unittest.TestCase):
         self._bounds_checking(lambda tup: time.strftime('', tup))
 
     def test_default_values_for_zero(self):
-        # Make sure that using all zeros uses the proper default values.
-        # No test for daylight savings since strftime() does not change output
-        # based on its value.
+        # Make sure that using all zeros uses the proper default
+        # values.  No test for daylight savings since strftime() does
+        # not change output based on its value and no test for year
+        # because systems vary in their support for year 0.
         expected = "2000 01 01 00 00 00 1 001"
         with support.check_warnings():
-            result = time.strftime("%Y %m %d %H %M %S %w %j", (0,)*9)
+            result = time.strftime("%Y %m %d %H %M %S %w %j", (2000,)+(0,)*8)
         self.assertEqual(expected, result)
 
     def test_strptime(self):
@@ -128,11 +197,13 @@ class TimeTestCase(unittest.TestCase):
         time.asctime(time.gmtime(self.t))
 
         # Max year is only limited by the size of C int.
-        sizeof_int = sysconfig.get_config_var('SIZEOF_INT') or 4
-        bigyear = (1 << 8 * sizeof_int - 1) - 1
-        asc = time.asctime((bigyear, 6, 1) + (0,)*6)
-        self.assertEqual(asc[-len(str(bigyear)):], str(bigyear))
-        self.assertRaises(OverflowError, time.asctime, (bigyear + 1,) + (0,)*8)
+        for bigyear in TIME_MAXYEAR, TIME_MINYEAR:
+            asc = time.asctime((bigyear, 6, 1) + (0,) * 6)
+            self.assertEqual(asc[-len(str(bigyear)):], str(bigyear))
+        self.assertRaises(OverflowError, time.asctime,
+                          (TIME_MAXYEAR + 1,) + (0,) * 8)
+        self.assertRaises(OverflowError, time.asctime,
+                          (TIME_MINYEAR - 1,) + (0,) * 8)
         self.assertRaises(TypeError, time.asctime, 0)
         self.assertRaises(TypeError, time.asctime, ())
         self.assertRaises(TypeError, time.asctime, (0,) * 10)
@@ -155,8 +226,8 @@ class TimeTestCase(unittest.TestCase):
             else:
                 self.assertEqual(time.ctime(testval)[20:], str(year))
 
-    @unittest.skipIf(not hasattr(time, "tzset"),
-        "time module has no attribute tzset")
+    @unittest.skipUnless(hasattr(time, "tzset"),
+                         "time module has no attribute tzset")
     def test_tzset(self):
 
         from os import environ
@@ -208,11 +279,13 @@ class TimeTestCase(unittest.TestCase):
             self.assertNotEqual(time.gmtime(xmas2002), time.localtime(xmas2002))
 
             # Issue #11886: Australian Eastern Standard Time (UTC+10) is called
-            # "EST" (as Eastern Standard Time, UTC-5) instead of "AEST" on some
-            # operating systems (e.g. FreeBSD), which is wrong. See for example
-            # this bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=93810
+            # "EST" (as Eastern Standard Time, UTC-5) instead of "AEST"
+            # (non-DST timezone), and "EDT" instead of "AEDT" (DST timezone),
+            # on some operating systems (e.g. FreeBSD), which is wrong. See for
+            # example this bug:
+            # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=93810
             self.assertIn(time.tzname[0], ('AEST' 'EST'), time.tzname[0])
-            self.assertTrue(time.tzname[1] == 'AEDT', str(time.tzname[1]))
+            self.assertTrue(time.tzname[1] in ('AEDT', 'EDT'), str(time.tzname[1]))
             self.assertEqual(len(time.tzname), 2)
             self.assertEqual(time.daylight, 1)
             self.assertEqual(time.timezone, -36000)
@@ -235,7 +308,7 @@ class TimeTestCase(unittest.TestCase):
         # results!).
         for func in time.ctime, time.gmtime, time.localtime:
             for unreasonable in -1e200, 1e200:
-                self.assertRaises(ValueError, func, unreasonable)
+                self.assertRaises(OverflowError, func, unreasonable)
 
     def test_ctime_without_arg(self):
         # Not sure how to check the values, since the clock could tick
@@ -258,6 +331,117 @@ class TimeTestCase(unittest.TestCase):
         t1 = time.mktime(lt1)
         self.assertAlmostEqual(t1, t0, delta=0.2)
 
+    def test_mktime(self):
+        # Issue #1726687
+        for t in (-2, -1, 0, 1):
+            try:
+                tt = time.localtime(t)
+            except (OverflowError, OSError):
+                pass
+            else:
+                self.assertEqual(time.mktime(tt), t)
+
+    # Issue #13309: passing extreme values to mktime() or localtime()
+    # borks the glibc's internal timezone data.
+    @unittest.skipUnless(platform.libc_ver()[0] != 'glibc',
+                         "disabled because of a bug in glibc. Issue #13309")
+    def test_mktime_error(self):
+        # It may not be possible to reliably make mktime return error
+        # on all platfom.  This will make sure that no other exception
+        # than OverflowError is raised for an extreme value.
+        tt = time.gmtime(self.t)
+        tzname = time.strftime('%Z', tt)
+        self.assertNotEqual(tzname, 'LMT')
+        try:
+            time.mktime((-1, 1, 1, 0, 0, 0, -1, -1, -1))
+        except OverflowError:
+            pass
+        self.assertEqual(time.strftime('%Z', tt), tzname)
+
+    @unittest.skipUnless(hasattr(time, 'monotonic'),
+                         'need time.monotonic')
+    def test_monotonic(self):
+        t1 = time.monotonic()
+        time.sleep(0.1)
+        t2 = time.monotonic()
+        dt = t2 - t1
+        self.assertGreater(t2, t1)
+        self.assertAlmostEqual(dt, 0.1, delta=0.2)
+
+        info = time.get_clock_info('monotonic')
+        self.assertTrue(info.monotonic)
+        self.assertFalse(info.adjustable)
+
+    def test_perf_counter(self):
+        time.perf_counter()
+
+    def test_process_time(self):
+        # process_time() should not include time spend during a sleep
+        start = time.process_time()
+        time.sleep(0.100)
+        stop = time.process_time()
+        # use 20 ms because process_time() has usually a resolution of 15 ms
+        # on Windows
+        self.assertLess(stop - start, 0.020)
+
+        info = time.get_clock_info('process_time')
+        self.assertTrue(info.monotonic)
+        self.assertFalse(info.adjustable)
+
+    @unittest.skipUnless(hasattr(time, 'monotonic'),
+                         'need time.monotonic')
+    @unittest.skipUnless(hasattr(time, 'clock_settime'),
+                         'need time.clock_settime')
+    def test_monotonic_settime(self):
+        t1 = time.monotonic()
+        realtime = time.clock_gettime(time.CLOCK_REALTIME)
+        # jump backward with an offset of 1 hour
+        try:
+            time.clock_settime(time.CLOCK_REALTIME, realtime - 3600)
+        except PermissionError as err:
+            self.skipTest(err)
+        t2 = time.monotonic()
+        time.clock_settime(time.CLOCK_REALTIME, realtime)
+        # monotonic must not be affected by system clock updates
+        self.assertGreaterEqual(t2, t1)
+
+    def test_localtime_failure(self):
+        # Issue #13847: check for localtime() failure
+        invalid_time_t = None
+        for time_t in (-1, 2**30, 2**33, 2**60):
+            try:
+                time.localtime(time_t)
+            except OverflowError:
+                self.skipTest("need 64-bit time_t")
+            except OSError:
+                invalid_time_t = time_t
+                break
+        if invalid_time_t is None:
+            self.skipTest("unable to find an invalid time_t value")
+
+        self.assertRaises(OSError, time.localtime, invalid_time_t)
+        self.assertRaises(OSError, time.ctime, invalid_time_t)
+
+    def test_get_clock_info(self):
+        clocks = ['clock', 'perf_counter', 'process_time', 'time']
+        if hasattr(time, 'monotonic'):
+            clocks.append('monotonic')
+
+        for name in clocks:
+            info = time.get_clock_info(name)
+            #self.assertIsInstance(info, dict)
+            self.assertIsInstance(info.implementation, str)
+            self.assertNotEqual(info.implementation, '')
+            self.assertIsInstance(info.monotonic, bool)
+            self.assertIsInstance(info.resolution, float)
+            # 0.0 < resolution <= 1.0
+            self.assertGreater(info.resolution, 0.0)
+            self.assertLessEqual(info.resolution, 1.0)
+            self.assertIsInstance(info.adjustable, bool)
+
+        self.assertRaises(ValueError, time.get_clock_info, 'xxx')
+
+
 class TestLocale(unittest.TestCase):
     def setUp(self):
         self.oldloc = locale.setlocale(locale.LC_ALL)
@@ -276,19 +460,12 @@ class TestLocale(unittest.TestCase):
 
 
 class _BaseYearTest(unittest.TestCase):
-    accept2dyear = None
-
-    def setUp(self):
-        self.saved_accept2dyear = time.accept2dyear
-        time.accept2dyear = self.accept2dyear
-
-    def tearDown(self):
-        time.accept2dyear = self.saved_accept2dyear
-
     def yearstr(self, y):
         raise NotImplementedError()
 
 class _TestAsctimeYear:
+    _format = '%d'
+
     def yearstr(self, y):
         return time.asctime((y,) + (0,) * 8).split()[-1]
 
@@ -298,114 +475,211 @@ class _TestAsctimeYear:
         self.assertEqual(self.yearstr(123456789), '123456789')
 
 class _TestStrftimeYear:
-    def yearstr(self, y):
-        return time.strftime('%Y', (y,) + (0,) * 8).split()[-1]
 
-    def test_large_year(self):
+    # Issue 13305:  For years < 1000, the value is not always
+    # padded to 4 digits across platforms.  The C standard
+    # assumes year >= 1900, so it does not specify the number
+    # of digits.
+
+    if time.strftime('%Y', (1,) + (0,) * 8) == '0001':
+        _format = '%04d'
+    else:
+        _format = '%d'
+
+    def yearstr(self, y):
+        return time.strftime('%Y', (y,) + (0,) * 8)
+
+    def test_4dyear(self):
+        # Check that we can return the zero padded value.
+        if self._format == '%04d':
+            self.test_year('%04d')
+        else:
+            def year4d(y):
+                return time.strftime('%4Y', (y,) + (0,) * 8)
+            self.test_year('%04d', func=year4d)
+
+    def skip_if_not_supported(y):
+        msg = "strftime() is limited to [1; 9999] with Visual Studio"
         # Check that it doesn't crash for year > 9999
         try:
-            text = self.yearstr(12345)
+            time.strftime('%Y', (y,) + (0,) * 8)
         except ValueError:
-            # strftime() is limited to [1; 9999] with Visual Studio
-            return
-        self.assertEqual(text, '12345')
-        self.assertEqual(self.yearstr(123456789), '123456789')
+            cond = False
+        else:
+            cond = True
+        return unittest.skipUnless(cond, msg)
 
-class _Test2dYear(_BaseYearTest):
-    accept2dyear = 1
+    @skip_if_not_supported(10000)
+    def test_large_year(self):
+        return super().test_large_year()
 
-    def test_year(self):
-        with support.check_warnings():
-            self.assertEqual(self.yearstr(0), '2000')
-            self.assertEqual(self.yearstr(69), '1969')
-            self.assertEqual(self.yearstr(68), '2068')
-            self.assertEqual(self.yearstr(99), '1999')
+    @skip_if_not_supported(0)
+    def test_negative(self):
+        return super().test_negative()
+
+    del skip_if_not_supported
 
-    def test_invalid(self):
-        self.assertRaises(ValueError, self.yearstr, -1)
-        self.assertRaises(ValueError, self.yearstr, 100)
-        self.assertRaises(ValueError, self.yearstr, 999)
 
 class _Test4dYear(_BaseYearTest):
-    accept2dyear = 0
+    _format = '%d'
+
+    def test_year(self, fmt=None, func=None):
+        fmt = fmt or self._format
+        func = func or self.yearstr
+        self.assertEqual(func(1),    fmt % 1)
+        self.assertEqual(func(68),   fmt % 68)
+        self.assertEqual(func(69),   fmt % 69)
+        self.assertEqual(func(99),   fmt % 99)
+        self.assertEqual(func(999),  fmt % 999)
+        self.assertEqual(func(9999), fmt % 9999)
 
-    def test_year(self):
-        self.assertIn(self.yearstr(1),     ('1', '0001'))
-        self.assertIn(self.yearstr(68),   ('68', '0068'))
-        self.assertIn(self.yearstr(69),   ('69', '0069'))
-        self.assertIn(self.yearstr(99),   ('99', '0099'))
-        self.assertIn(self.yearstr(999), ('999', '0999'))
-        self.assertEqual(self.yearstr(9999), '9999')
+    def test_large_year(self):
+        self.assertEqual(self.yearstr(12345), '12345')
+        self.assertEqual(self.yearstr(123456789), '123456789')
+        self.assertEqual(self.yearstr(TIME_MAXYEAR), str(TIME_MAXYEAR))
+        self.assertRaises(OverflowError, self.yearstr, TIME_MAXYEAR + 1)
 
     def test_negative(self):
-        try:
-            text = self.yearstr(-1)
-        except ValueError:
-            # strftime() is limited to [1; 9999] with Visual Studio
-            return
-        self.assertIn(text, ('-1', '-001'))
-
+        self.assertEqual(self.yearstr(-1), self._format % -1)
         self.assertEqual(self.yearstr(-1234), '-1234')
         self.assertEqual(self.yearstr(-123456), '-123456')
+        self.assertEqual(self.yearstr(-123456789), str(-123456789))
+        self.assertEqual(self.yearstr(-1234567890), str(-1234567890))
+        self.assertEqual(self.yearstr(TIME_MINYEAR + 1900), str(TIME_MINYEAR + 1900))
+        # Issue #13312: it may return wrong value for year < TIME_MINYEAR + 1900
+        # Skip the value test, but check that no error is raised
+        self.yearstr(TIME_MINYEAR)
+        # self.assertEqual(self.yearstr(TIME_MINYEAR), str(TIME_MINYEAR))
+        self.assertRaises(OverflowError, self.yearstr, TIME_MINYEAR - 1)
 
 
-    def test_mktime(self):
-        # Issue #1726687
-        for t in (-2, -1, 0, 1):
-            try:
-                tt = time.localtime(t)
-            except (OverflowError, ValueError):
-                pass
-            else:
-                self.assertEqual(time.mktime(tt), t)
-        # It may not be possible to reliably make mktime return error
-        # on all platfom.  This will make sure that no other exception
-        # than OverflowError is raised for an extreme value.
-        try:
-            time.mktime((-1, 1, 1, 0, 0, 0, -1, -1, -1))
-        except OverflowError:
-            pass
-
-class TestAsctimeAccept2dYear(_TestAsctimeYear, _Test2dYear):
-    pass
-
-class TestStrftimeAccept2dYear(_TestStrftimeYear, _Test2dYear):
-    pass
-
 class TestAsctime4dyear(_TestAsctimeYear, _Test4dYear):
     pass
 
 class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear):
     pass
 
-class Test2dyearBool(_TestAsctimeYear, _Test2dYear):
-    accept2dyear = True
-
-class Test4dyearBool(_TestAsctimeYear, _Test4dYear):
-    accept2dyear = False
-
-class TestAccept2YearBad(_TestAsctimeYear, _BaseYearTest):
-    class X:
-        def __bool__(self):
-            raise RuntimeError('boo')
-    accept2dyear = X()
-    def test_2dyear(self):
-        pass
-    def test_invalid(self):
-        self.assertRaises(RuntimeError, self.yearstr, 200)
 
+class TestPytime(unittest.TestCase):
+    def setUp(self):
+        self.invalid_values = (
+            -(2 ** 100), 2 ** 100,
+            -(2.0 ** 100.0), 2.0 ** 100.0,
+        )
+
+    def test_time_t(self):
+        from _testcapi import pytime_object_to_time_t
+        for obj, time_t in (
+            (0, 0),
+            (-1, -1),
+            (-1.0, -1),
+            (-1.9, -1),
+            (1.0, 1),
+            (1.9, 1),
+        ):
+            self.assertEqual(pytime_object_to_time_t(obj), time_t)
+
+        for invalid in self.invalid_values:
+            self.assertRaises(OverflowError, pytime_object_to_time_t, invalid)
+
+    def test_timeval(self):
+        from _testcapi import pytime_object_to_timeval
+        for obj, timeval in (
+            (0, (0, 0)),
+            (-1, (-1, 0)),
+            (-1.0, (-1, 0)),
+            (1e-6, (0, 1)),
+            (-1e-6, (-1, 999999)),
+            (-1.2, (-2, 800000)),
+            (1.1234560, (1, 123456)),
+            (1.1234569, (1, 123456)),
+            (-1.1234560, (-2, 876544)),
+            (-1.1234561, (-2, 876543)),
+        ):
+            self.assertEqual(pytime_object_to_timeval(obj), timeval)
+
+        for invalid in self.invalid_values:
+            self.assertRaises(OverflowError, pytime_object_to_timeval, invalid)
+
+    def test_timespec(self):
+        from _testcapi import pytime_object_to_timespec
+        for obj, timespec in (
+            (0, (0, 0)),
+            (-1, (-1, 0)),
+            (-1.0, (-1, 0)),
+            (1e-9, (0, 1)),
+            (-1e-9, (-1, 999999999)),
+            (-1.2, (-2, 800000000)),
+            (1.1234567890, (1, 123456789)),
+            (1.1234567899, (1, 123456789)),
+            (-1.1234567890, (-2, 876543211)),
+            (-1.1234567891, (-2, 876543210)),
+        ):
+            self.assertEqual(pytime_object_to_timespec(obj), timespec)
+
+        for invalid in self.invalid_values:
+            self.assertRaises(OverflowError, pytime_object_to_timespec, invalid)
+
+    @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support")
+    def test_localtime_timezone(self):
+
+        # Get the localtime and examine it for the offset and zone.
+        lt = time.localtime()
+        self.assertTrue(hasattr(lt, "tm_gmtoff"))
+        self.assertTrue(hasattr(lt, "tm_zone"))
+
+        # See if the offset and zone are similar to the module
+        # attributes.
+        if lt.tm_gmtoff is None:
+            self.assertTrue(not hasattr(time, "timezone"))
+        else:
+            self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst])
+        if lt.tm_zone is None:
+            self.assertTrue(not hasattr(time, "tzname"))
+        else:
+            self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst])
+
+        # Try and make UNIX times from the localtime and a 9-tuple
+        # created from the localtime. Test to see that the times are
+        # the same.
+        t = time.mktime(lt); t9 = time.mktime(lt[:9])
+        self.assertEqual(t, t9)
+
+        # Make localtimes from the UNIX times and compare them to
+        # the original localtime, thus making a round trip.
+        new_lt = time.localtime(t); new_lt9 = time.localtime(t9)
+        self.assertEqual(new_lt, lt)
+        self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff)
+        self.assertEqual(new_lt.tm_zone, lt.tm_zone)
+        self.assertEqual(new_lt9, lt)
+        self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff)
+        self.assertEqual(new_lt9.tm_zone, lt.tm_zone)
+
+    @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support")
+    def test_strptime_timezone(self):
+        t = time.strptime("UTC", "%Z")
+        self.assertEqual(t.tm_zone, 'UTC')
+        t = time.strptime("+0500", "%z")
+        self.assertEqual(t.tm_gmtoff, 5 * 3600)
+
+    @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support")
+    def test_short_times(self):
+
+        import pickle
+
+        # Load a short time structure using pickle.
+        st = b"ctime\nstruct_time\np0\n((I2007\nI8\nI11\nI1\nI24\nI49\nI5\nI223\nI1\ntp1\n(dp2\ntp3\nRp4\n."
+        lt = pickle.loads(st)
+        self.assertIs(lt.tm_gmtoff, None)
+        self.assertIs(lt.tm_zone, None)
 
 def test_main():
     support.run_unittest(
         TimeTestCase,
         TestLocale,
-        TestAsctimeAccept2dYear,
-        TestStrftimeAccept2dYear,
         TestAsctime4dyear,
         TestStrftime4dyear,
-        Test2dyearBool,
-        Test4dyearBool,
-        TestAccept2YearBad)
+        TestPytime)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index f9652ce7b2..b4a58f0db2 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -289,6 +289,64 @@ String literals
     OP         '+'           (1, 29) (1, 30)
     STRING     'R"ABC"'      (1, 31) (1, 37)
 
+    >>> dump_tokens("u'abc' + U'abc'")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     "u'abc'"      (1, 0) (1, 6)
+    OP         '+'           (1, 7) (1, 8)
+    STRING     "U'abc'"      (1, 9) (1, 15)
+    >>> dump_tokens('u"abc" + U"abc"')
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     'u"abc"'      (1, 0) (1, 6)
+    OP         '+'           (1, 7) (1, 8)
+    STRING     'U"abc"'      (1, 9) (1, 15)
+
+    >>> dump_tokens("b'abc' + B'abc'")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     "b'abc'"      (1, 0) (1, 6)
+    OP         '+'           (1, 7) (1, 8)
+    STRING     "B'abc'"      (1, 9) (1, 15)
+    >>> dump_tokens('b"abc" + B"abc"')
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     'b"abc"'      (1, 0) (1, 6)
+    OP         '+'           (1, 7) (1, 8)
+    STRING     'B"abc"'      (1, 9) (1, 15)
+    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     "br'abc'"     (1, 0) (1, 7)
+    OP         '+'           (1, 8) (1, 9)
+    STRING     "bR'abc'"     (1, 10) (1, 17)
+    OP         '+'           (1, 18) (1, 19)
+    STRING     "Br'abc'"     (1, 20) (1, 27)
+    OP         '+'           (1, 28) (1, 29)
+    STRING     "BR'abc'"     (1, 30) (1, 37)
+    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     'br"abc"'     (1, 0) (1, 7)
+    OP         '+'           (1, 8) (1, 9)
+    STRING     'bR"abc"'     (1, 10) (1, 17)
+    OP         '+'           (1, 18) (1, 19)
+    STRING     'Br"abc"'     (1, 20) (1, 27)
+    OP         '+'           (1, 28) (1, 29)
+    STRING     'BR"abc"'     (1, 30) (1, 37)
+    >>> dump_tokens("rb'abc' + rB'abc' + Rb'abc' + RB'abc'")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     "rb'abc'"     (1, 0) (1, 7)
+    OP         '+'           (1, 8) (1, 9)
+    STRING     "rB'abc'"     (1, 10) (1, 17)
+    OP         '+'           (1, 18) (1, 19)
+    STRING     "Rb'abc'"     (1, 20) (1, 27)
+    OP         '+'           (1, 28) (1, 29)
+    STRING     "RB'abc'"     (1, 30) (1, 37)
+    >>> dump_tokens('rb"abc" + rB"abc" + Rb"abc" + RB"abc"')
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    STRING     'rb"abc"'     (1, 0) (1, 7)
+    OP         '+'           (1, 8) (1, 9)
+    STRING     'rB"abc"'     (1, 10) (1, 17)
+    OP         '+'           (1, 18) (1, 19)
+    STRING     'Rb"abc"'     (1, 20) (1, 27)
+    OP         '+'           (1, 28) (1, 29)
+    STRING     'RB"abc"'     (1, 30) (1, 37)
+
 Operators
 
     >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
@@ -552,11 +610,6 @@ Evil tabs
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
 
-Pathological whitespace (http://bugs.python.org/issue16152)
-    >>> dump_tokens("@          ")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
-    OP         '@'           (1, 0) (1, 1)
-
 Non-ascii identifiers
 
     >>> dump_tokens("Örter = 'places'\\ngrün = 'green'")
@@ -568,15 +621,28 @@ Non-ascii identifiers
     NAME       'grün'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "'green'"     (2, 7) (2, 14)
+
+Legacy unicode literals:
+
+    >>> dump_tokens("Örter = u'places'\\ngrün = U'green'")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    NAME       'Örter'       (1, 0) (1, 5)
+    OP         '='           (1, 6) (1, 7)
+    STRING     "u'places'"   (1, 8) (1, 17)
+    NEWLINE    '\\n'          (1, 17) (1, 18)
+    NAME       'grün'        (2, 0) (2, 4)
+    OP         '='           (2, 5) (2, 6)
+    STRING     "U'green'"    (2, 7) (2, 15)
 """
 
 from test import support
 from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, tok_name, detect_encoding,
+                     STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
                      open as tokenize_open)
 from io import BytesIO
 from unittest import TestCase
 import os, sys, glob
+import token
 
 def dump_tokens(s):
     """Print out the tokens in s in a table format.
@@ -605,7 +671,7 @@ def roundtrip(f):
         f.close()
     tokens1 = [tok[:2] for tok in token_list]
     new_bytes = untokenize(tokens1)
-    readline = (line for line in new_bytes.splitlines(1)).__next__
+    readline = (line for line in new_bytes.splitlines(keepends=True)).__next__
     tokens2 = [tok[:2] for tok in tokenize(readline)]
     return tokens1 == tokens2
 
@@ -900,6 +966,35 @@ class TestDetectEncoding(TestCase):
             self.assertEqual(fp.encoding, 'utf-8-sig')
             self.assertEqual(fp.mode, 'r')
 
+    def test_filename_in_exception(self):
+        # When possible, include the file name in the exception.
+        path = 'some_file_path'
+        lines = (
+            b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S
+            )
+        class Bunk:
+            def __init__(self, lines, path):
+                self.name = path
+                self._lines = lines
+                self._index = 0
+
+            def readline(self):
+                if self._index == len(lines):
+                    raise StopIteration
+                line = lines[self._index]
+                self._index += 1
+                return line
+
+        with self.assertRaises(SyntaxError):
+            ins = Bunk(lines, path)
+            # Make sure lacking a name isn't an issue.
+            del ins.name
+            detect_encoding(ins.readline)
+        with self.assertRaisesRegex(SyntaxError, '.*{}'.format(path)):
+            ins = Bunk(lines, path)
+            detect_encoding(ins.readline)
+
+
 class TestTokenize(TestCase):
 
     def test_tokenize(self):
@@ -941,6 +1036,82 @@ class TestTokenize(TestCase):
 
         self.assertTrue(encoding_used, encoding)
 
+    def assertExactTypeEqual(self, opstr, *optypes):
+        tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline))
+        num_optypes = len(optypes)
+        self.assertEqual(len(tokens), 2 + num_optypes)
+        self.assertEqual(token.tok_name[tokens[0].exact_type],
+                         token.tok_name[ENCODING])
+        for i in range(num_optypes):
+            self.assertEqual(token.tok_name[tokens[i + 1].exact_type],
+                             token.tok_name[optypes[i]])
+        self.assertEqual(token.tok_name[tokens[1 + num_optypes].exact_type],
+                         token.tok_name[token.ENDMARKER])
+
+    def test_exact_type(self):
+        self.assertExactTypeEqual('()', token.LPAR, token.RPAR)
+        self.assertExactTypeEqual('[]', token.LSQB, token.RSQB)
+        self.assertExactTypeEqual(':', token.COLON)
+        self.assertExactTypeEqual(',', token.COMMA)
+        self.assertExactTypeEqual(';', token.SEMI)
+        self.assertExactTypeEqual('+', token.PLUS)
+        self.assertExactTypeEqual('-', token.MINUS)
+        self.assertExactTypeEqual('*', token.STAR)
+        self.assertExactTypeEqual('/', token.SLASH)
+        self.assertExactTypeEqual('|', token.VBAR)
+        self.assertExactTypeEqual('&', token.AMPER)
+        self.assertExactTypeEqual('<', token.LESS)
+        self.assertExactTypeEqual('>', token.GREATER)
+        self.assertExactTypeEqual('=', token.EQUAL)
+        self.assertExactTypeEqual('.', token.DOT)
+        self.assertExactTypeEqual('%', token.PERCENT)
+        self.assertExactTypeEqual('{}', token.LBRACE, token.RBRACE)
+        self.assertExactTypeEqual('==', token.EQEQUAL)
+        self.assertExactTypeEqual('!=', token.NOTEQUAL)
+        self.assertExactTypeEqual('<=', token.LESSEQUAL)
+        self.assertExactTypeEqual('>=', token.GREATEREQUAL)
+        self.assertExactTypeEqual('~', token.TILDE)
+        self.assertExactTypeEqual('^', token.CIRCUMFLEX)
+        self.assertExactTypeEqual('<<', token.LEFTSHIFT)
+        self.assertExactTypeEqual('>>', token.RIGHTSHIFT)
+        self.assertExactTypeEqual('**', token.DOUBLESTAR)
+        self.assertExactTypeEqual('+=', token.PLUSEQUAL)
+        self.assertExactTypeEqual('-=', token.MINEQUAL)
+        self.assertExactTypeEqual('*=', token.STAREQUAL)
+        self.assertExactTypeEqual('/=', token.SLASHEQUAL)
+        self.assertExactTypeEqual('%=', token.PERCENTEQUAL)
+        self.assertExactTypeEqual('&=', token.AMPEREQUAL)
+        self.assertExactTypeEqual('|=', token.VBAREQUAL)
+        self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL)
+        self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL)
+        self.assertExactTypeEqual('<<=', token.LEFTSHIFTEQUAL)
+        self.assertExactTypeEqual('>>=', token.RIGHTSHIFTEQUAL)
+        self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL)
+        self.assertExactTypeEqual('//', token.DOUBLESLASH)
+        self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL)
+        self.assertExactTypeEqual('@', token.AT)
+
+        self.assertExactTypeEqual('a**2+b**2==c**2',
+                                  NAME, token.DOUBLESTAR, NUMBER,
+                                  token.PLUS,
+                                  NAME, token.DOUBLESTAR, NUMBER,
+                                  token.EQEQUAL,
+                                  NAME, token.DOUBLESTAR, NUMBER)
+        self.assertExactTypeEqual('{1, 2, 3}',
+                                  token.LBRACE,
+                                  token.NUMBER, token.COMMA,
+                                  token.NUMBER, token.COMMA,
+                                  token.NUMBER,
+                                  token.RBRACE)
+        self.assertExactTypeEqual('^(x & 0x1)',
+                                  token.CIRCUMFLEX,
+                                  token.LPAR,
+                                  token.NAME, token.AMPER, token.NUMBER,
+                                  token.RPAR)
+
+    def test_pathological_trailing_whitespace(self):
+        # See http://bugs.python.org/issue16152
+        self.assertExactTypeEqual('@          ', token.AT)
 
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}
 
diff --git a/Lib/test/test_tools.py b/Lib/test/test_tools.py
index 006220a3d4..f971515899 100644
--- a/Lib/test/test_tools.py
+++ b/Lib/test/test_tools.py
@@ -6,8 +6,9 @@ Tools directory of a Python checkout or tarball, such as reindent.py.
 
 import os
 import sys
-import imp
+import importlib.machinery
 import unittest
+from unittest import mock
 import shutil
 import subprocess
 import sysconfig
@@ -365,7 +366,7 @@ class TestSundryScripts(unittest.TestCase):
     # added for a script it should be added to the whitelist below.
 
     # scripts that have independent tests.
-    whitelist = ['reindent.py']
+    whitelist = ['reindent.py', 'pdeps.py', 'gprof2html']
     # scripts that can't be imported without running
     blacklist = ['make_ctype.py']
     # scripts that use windows-only modules
@@ -404,7 +405,8 @@ class PdepsTests(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         path = os.path.join(scriptsdir, 'pdeps.py')
-        self.pdeps = imp.load_source('pdeps', path)
+        loader = importlib.machinery.SourceFileLoader('pdeps', path)
+        self.pdeps = loader.load_module()
 
     @classmethod
     def tearDownClass(self):
@@ -424,6 +426,35 @@ class PdepsTests(unittest.TestCase):
         self.pdeps.inverse({'a': []})
 
 
+class Gprof2htmlTests(unittest.TestCase):
+
+    def setUp(self):
+        path = os.path.join(scriptsdir, 'gprof2html.py')
+        loader = importlib.machinery.SourceFileLoader('gprof2html', path)
+        self.gprof = loader.load_module()
+        oldargv = sys.argv
+        def fixup():
+            sys.argv = oldargv
+        self.addCleanup(fixup)
+        sys.argv = []
+
+    def test_gprof(self):
+        # Issue #14508: this used to fail with an NameError.
+        with mock.patch.object(self.gprof, 'webbrowser') as wmock, \
+                tempfile.TemporaryDirectory() as tmpdir:
+            fn = os.path.join(tmpdir, 'abc')
+            open(fn, 'w').close()
+            sys.argv = ['gprof2html', fn]
+            self.gprof.main()
+        self.assertTrue(wmock.open.called)
+
+
+# Run the tests in Tools/parser/test_unparse.py
+with support.DirsOnSysPath(os.path.join(basepath, 'parser')):
+    from test_unparse import UnparseTestCase
+    from test_unparse import DirectoryTestCase
+
+
 def test_main():
     support.run_unittest(*[obj for obj in globals().values()
                                if isinstance(obj, type)])
diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py
index 461d1d8e6a..1cec710ee5 100644
--- a/Lib/test/test_trace.py
+++ b/Lib/test/test_trace.py
@@ -1,4 +1,5 @@
 import os
+import io
 import sys
 from test.support import (run_unittest, TESTFN, rmtree, unlink,
                                captured_stdout)
@@ -102,6 +103,7 @@ class TracedClass(object):
 class TestLineCounts(unittest.TestCase):
     """White-box testing of line-counting, via runfunc"""
     def setUp(self):
+        self.addCleanup(sys.settrace, sys.gettrace())
         self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0)
         self.my_py_filename = fix_ext_py(__file__)
 
@@ -192,6 +194,7 @@ class TestRunExecCounts(unittest.TestCase):
     """A simple sanity test of line-counting, via runctx (exec)"""
     def setUp(self):
         self.my_py_filename = fix_ext_py(__file__)
+        self.addCleanup(sys.settrace, sys.gettrace())
 
     def test_exec_counts(self):
         self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0)
@@ -218,6 +221,7 @@ class TestRunExecCounts(unittest.TestCase):
 class TestFuncs(unittest.TestCase):
     """White-box testing of funcs tracing"""
     def setUp(self):
+        self.addCleanup(sys.settrace, sys.gettrace())
         self.tracer = Trace(count=0, trace=0, countfuncs=1)
         self.filemod = my_file_and_modname()
 
@@ -242,6 +246,8 @@ class TestFuncs(unittest.TestCase):
         }
         self.assertEqual(self.tracer.results().calledfuncs, expected)
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'pre-existing trace function throws off measurements')
     def test_inst_method_calling(self):
         obj = TracedClass(20)
         self.tracer.runfunc(obj.inst_method_calling, 1)
@@ -257,9 +263,12 @@ class TestFuncs(unittest.TestCase):
 class TestCallers(unittest.TestCase):
     """White-box testing of callers tracing"""
     def setUp(self):
+        self.addCleanup(sys.settrace, sys.gettrace())
         self.tracer = Trace(count=0, trace=0, countcallers=1)
         self.filemod = my_file_and_modname()
 
+    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+                     'pre-existing trace function throws off measurements')
     def test_loop_caller_importing(self):
         self.tracer.runfunc(traced_func_importing_caller, 1)
 
@@ -280,6 +289,9 @@ class TestCallers(unittest.TestCase):
 
 # Created separately for issue #3821
 class TestCoverage(unittest.TestCase):
+    def setUp(self):
+        self.addCleanup(sys.settrace, sys.gettrace())
+
     def tearDown(self):
         rmtree(TESTFN)
         unlink(TESTFN)
@@ -305,13 +317,13 @@ class TestCoverage(unittest.TestCase):
         # Ignore all files, nothing should be traced nor printed
         libpath = os.path.normpath(os.path.dirname(os.__file__))
         # sys.prefix does not work when running from a checkout
-        tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix, libpath],
-                             trace=0, count=1)
+        tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,
+                             libpath], trace=0, count=1)
         with captured_stdout() as stdout:
             self._coverage(tracer)
         if os.path.exists(TESTFN):
             files = os.listdir(TESTFN)
-            self.assertEqual(files, [])
+            self.assertEqual(files, ['_importlib.cover'])  # Ignore __import__
 
     def test_issue9936(self):
         tracer = trace.Trace(trace=0, count=1)
@@ -350,6 +362,51 @@ class Test_Ignore(unittest.TestCase):
         self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz'))
 
 
+class TestDeprecatedMethods(unittest.TestCase):
+
+    def test_deprecated_usage(self):
+        sio = io.StringIO()
+        with self.assertWarns(DeprecationWarning):
+            trace.usage(sio)
+        self.assertIn('Usage:', sio.getvalue())
+
+    def test_deprecated_Ignore(self):
+        with self.assertWarns(DeprecationWarning):
+            trace.Ignore()
+
+    def test_deprecated_modname(self):
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual("spam", trace.modname("spam"))
+
+    def test_deprecated_fullmodname(self):
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual("spam", trace.fullmodname("spam"))
+
+    def test_deprecated_find_lines_from_code(self):
+        with self.assertWarns(DeprecationWarning):
+            def foo():
+                pass
+            trace.find_lines_from_code(foo.__code__, ["eggs"])
+
+    def test_deprecated_find_lines(self):
+        with self.assertWarns(DeprecationWarning):
+            def foo():
+                pass
+            trace.find_lines(foo.__code__, ["eggs"])
+
+    def test_deprecated_find_strings(self):
+        with open(TESTFN, 'w') as fd:
+            self.addCleanup(unlink, TESTFN)
+        with self.assertWarns(DeprecationWarning):
+            trace.find_strings(fd.name)
+
+    def test_deprecated_find_executable_linenos(self):
+        with open(TESTFN, 'w') as fd:
+            self.addCleanup(unlink, TESTFN)
+        with self.assertWarns(DeprecationWarning):
+            trace.find_executable_linenos(fd.name)
+
+
 def test_main():
     run_unittest(__name__)
 
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index 4752d37e5b..5bce2af68a 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -246,6 +246,21 @@ class BaseExceptionReportingTests:
         self.check_zero_div(blocks[0])
         self.assertIn('inner_raise() # Marker', blocks[2])
 
+    def test_context_suppression(self):
+        try:
+            try:
+                raise Exception
+            except:
+                raise ZeroDivisionError from None
+        except ZeroDivisionError as _:
+            e = _
+        lines = self.get_report(e).splitlines()
+        self.assertEqual(len(lines), 4)
+        self.assertTrue(lines[0].startswith('Traceback'))
+        self.assertTrue(lines[1].startswith('  File'))
+        self.assertIn('ZeroDivisionError from None', lines[2])
+        self.assertTrue(lines[3].startswith('ZeroDivisionError'))
+
     def test_cause_and_context(self):
         # When both a cause and a context are set, only the cause should be
         # displayed and the context should be muted.
diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py
index 6e934fb882..e41711c8e6 100644
--- a/Lib/test/test_tuple.py
+++ b/Lib/test/test_tuple.py
@@ -1,6 +1,7 @@
 from test import support, seq_tests
 
 import gc
+import pickle
 
 class TupleTest(seq_tests.CommonTest):
     type2test = tuple
@@ -164,6 +165,34 @@ class TupleTest(seq_tests.CommonTest):
         check(10)       # check our checking code
         check(1000000)
 
+    def test_iterator_pickle(self):
+        # Userlist iterators don't support pickling yet since
+        # they are based on generators.
+        data = self.type2test([4, 5, 6, 7])
+        itorg = iter(data)
+        d = pickle.dumps(itorg)
+        it = pickle.loads(d)
+        self.assertEqual(type(itorg), type(it))
+        self.assertEqual(self.type2test(it), self.type2test(data))
+
+        it = pickle.loads(d)
+        next(it)
+        d = pickle.dumps(it)
+        self.assertEqual(self.type2test(it), self.type2test(data)[1:])
+
+    def test_reversed_pickle(self):
+        data = self.type2test([4, 5, 6, 7])
+        itorg = reversed(data)
+        d = pickle.dumps(itorg)
+        it = pickle.loads(d)
+        self.assertEqual(type(itorg), type(it))
+        self.assertEqual(self.type2test(it), self.type2test(reversed(data)))
+
+        it = pickle.loads(d)
+        next(it)
+        d = pickle.dumps(it)
+        self.assertEqual(self.type2test(it), self.type2test(reversed(data))[1:])
+
     def test_no_comdat_folding(self):
         # Issue 8847: In the PGO build, the MSVC linker's COMDAT folding
         # optimization causes failures in code that relies on distinct
diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py
index 8a98a03567..3ee4c6bc5f 100644
--- a/Lib/test/test_types.py
+++ b/Lib/test/test_types.py
@@ -1,9 +1,11 @@
 # Python test set -- part 6, built-in types
 
 from test.support import run_unittest, run_with_locale
-import unittest
-import sys
+import collections
 import locale
+import sys
+import types
+import unittest
 
 class TypesTests(unittest.TestCase):
 
@@ -569,8 +571,583 @@ class TypesTests(unittest.TestCase):
         self.assertGreater(tuple.__itemsize__, 0)
 
 
+class MappingProxyTests(unittest.TestCase):
+    mappingproxy = types.MappingProxyType
+
+    def test_constructor(self):
+        class userdict(dict):
+            pass
+
+        mapping = {'x': 1, 'y': 2}
+        self.assertEqual(self.mappingproxy(mapping), mapping)
+        mapping = userdict(x=1, y=2)
+        self.assertEqual(self.mappingproxy(mapping), mapping)
+        mapping = collections.ChainMap({'x': 1}, {'y': 2})
+        self.assertEqual(self.mappingproxy(mapping), mapping)
+
+        self.assertRaises(TypeError, self.mappingproxy, 10)
+        self.assertRaises(TypeError, self.mappingproxy, ("a", "tuple"))
+        self.assertRaises(TypeError, self.mappingproxy, ["a", "list"])
+
+    def test_methods(self):
+        attrs = set(dir(self.mappingproxy({}))) - set(dir(object()))
+        self.assertEqual(attrs, {
+             '__contains__',
+             '__getitem__',
+             '__iter__',
+             '__len__',
+             'copy',
+             'get',
+             'items',
+             'keys',
+             'values',
+        })
+
+    def test_get(self):
+        view = self.mappingproxy({'a': 'A', 'b': 'B'})
+        self.assertEqual(view['a'], 'A')
+        self.assertEqual(view['b'], 'B')
+        self.assertRaises(KeyError, view.__getitem__, 'xxx')
+        self.assertEqual(view.get('a'), 'A')
+        self.assertIsNone(view.get('xxx'))
+        self.assertEqual(view.get('xxx', 42), 42)
+
+    def test_missing(self):
+        class dictmissing(dict):
+            def __missing__(self, key):
+                return "missing=%s" % key
+
+        view = self.mappingproxy(dictmissing(x=1))
+        self.assertEqual(view['x'], 1)
+        self.assertEqual(view['y'], 'missing=y')
+        self.assertEqual(view.get('x'), 1)
+        self.assertEqual(view.get('y'), None)
+        self.assertEqual(view.get('y', 42), 42)
+        self.assertTrue('x' in view)
+        self.assertFalse('y' in view)
+
+    def test_customdict(self):
+        class customdict(dict):
+            def __contains__(self, key):
+                if key == 'magic':
+                    return True
+                else:
+                    return dict.__contains__(self, key)
+
+            def __iter__(self):
+                return iter(('iter',))
+
+            def __len__(self):
+                return 500
+
+            def copy(self):
+                return 'copy'
+
+            def keys(self):
+                return 'keys'
+
+            def items(self):
+                return 'items'
+
+            def values(self):
+                return 'values'
+
+            def __getitem__(self, key):
+                return "getitem=%s" % dict.__getitem__(self, key)
+
+            def get(self, key, default=None):
+                return "get=%s" % dict.get(self, key, 'default=%r' % default)
+
+        custom = customdict({'key': 'value'})
+        view = self.mappingproxy(custom)
+        self.assertTrue('key' in view)
+        self.assertTrue('magic' in view)
+        self.assertFalse('xxx' in view)
+        self.assertEqual(view['key'], 'getitem=value')
+        self.assertRaises(KeyError, view.__getitem__, 'xxx')
+        self.assertEqual(tuple(view), ('iter',))
+        self.assertEqual(len(view), 500)
+        self.assertEqual(view.copy(), 'copy')
+        self.assertEqual(view.get('key'), 'get=value')
+        self.assertEqual(view.get('xxx'), 'get=default=None')
+        self.assertEqual(view.items(), 'items')
+        self.assertEqual(view.keys(), 'keys')
+        self.assertEqual(view.values(), 'values')
+
+    def test_chainmap(self):
+        d1 = {'x': 1}
+        d2 = {'y': 2}
+        mapping = collections.ChainMap(d1, d2)
+        view = self.mappingproxy(mapping)
+        self.assertTrue('x' in view)
+        self.assertTrue('y' in view)
+        self.assertFalse('z' in view)
+        self.assertEqual(view['x'], 1)
+        self.assertEqual(view['y'], 2)
+        self.assertRaises(KeyError, view.__getitem__, 'z')
+        self.assertEqual(tuple(sorted(view)), ('x', 'y'))
+        self.assertEqual(len(view), 2)
+        copy = view.copy()
+        self.assertIsNot(copy, mapping)
+        self.assertIsInstance(copy, collections.ChainMap)
+        self.assertEqual(copy, mapping)
+        self.assertEqual(view.get('x'), 1)
+        self.assertEqual(view.get('y'), 2)
+        self.assertIsNone(view.get('z'))
+        self.assertEqual(tuple(sorted(view.items())), (('x', 1), ('y', 2)))
+        self.assertEqual(tuple(sorted(view.keys())), ('x', 'y'))
+        self.assertEqual(tuple(sorted(view.values())), (1, 2))
+
+    def test_contains(self):
+        view = self.mappingproxy(dict.fromkeys('abc'))
+        self.assertTrue('a' in view)
+        self.assertTrue('b' in view)
+        self.assertTrue('c' in view)
+        self.assertFalse('xxx' in view)
+
+    def test_views(self):
+        mapping = {}
+        view = self.mappingproxy(mapping)
+        keys = view.keys()
+        values = view.values()
+        items = view.items()
+        self.assertEqual(list(keys), [])
+        self.assertEqual(list(values), [])
+        self.assertEqual(list(items), [])
+        mapping['key'] = 'value'
+        self.assertEqual(list(keys), ['key'])
+        self.assertEqual(list(values), ['value'])
+        self.assertEqual(list(items), [('key', 'value')])
+
+    def test_len(self):
+        for expected in range(6):
+            data = dict.fromkeys('abcde'[:expected])
+            self.assertEqual(len(data), expected)
+            view = self.mappingproxy(data)
+            self.assertEqual(len(view), expected)
+
+    def test_iterators(self):
+        keys = ('x', 'y')
+        values = (1, 2)
+        items = tuple(zip(keys, values))
+        view = self.mappingproxy(dict(items))
+        self.assertEqual(set(view), set(keys))
+        self.assertEqual(set(view.keys()), set(keys))
+        self.assertEqual(set(view.values()), set(values))
+        self.assertEqual(set(view.items()), set(items))
+
+    def test_copy(self):
+        original = {'key1': 27, 'key2': 51, 'key3': 93}
+        view = self.mappingproxy(original)
+        copy = view.copy()
+        self.assertEqual(type(copy), dict)
+        self.assertEqual(copy, original)
+        original['key1'] = 70
+        self.assertEqual(view['key1'], 70)
+        self.assertEqual(copy['key1'], 27)
+
+
+class ClassCreationTests(unittest.TestCase):
+
+    class Meta(type):
+        def __init__(cls, name, bases, ns, **kw):
+            super().__init__(name, bases, ns)
+        @staticmethod
+        def __new__(mcls, name, bases, ns, **kw):
+            return super().__new__(mcls, name, bases, ns)
+        @classmethod
+        def __prepare__(mcls, name, bases, **kw):
+            ns = super().__prepare__(name, bases)
+            ns["y"] = 1
+            ns.update(kw)
+            return ns
+
+    def test_new_class_basics(self):
+        C = types.new_class("C")
+        self.assertEqual(C.__name__, "C")
+        self.assertEqual(C.__bases__, (object,))
+
+    def test_new_class_subclass(self):
+        C = types.new_class("C", (int,))
+        self.assertTrue(issubclass(C, int))
+
+    def test_new_class_meta(self):
+        Meta = self.Meta
+        settings = {"metaclass": Meta, "z": 2}
+        # We do this twice to make sure the passed in dict isn't mutated
+        for i in range(2):
+            C = types.new_class("C" + str(i), (), settings)
+            self.assertIsInstance(C, Meta)
+            self.assertEqual(C.y, 1)
+            self.assertEqual(C.z, 2)
+
+    def test_new_class_exec_body(self):
+        Meta = self.Meta
+        def func(ns):
+            ns["x"] = 0
+        C = types.new_class("C", (), {"metaclass": Meta, "z": 2}, func)
+        self.assertIsInstance(C, Meta)
+        self.assertEqual(C.x, 0)
+        self.assertEqual(C.y, 1)
+        self.assertEqual(C.z, 2)
+
+    def test_new_class_metaclass_keywords(self):
+        #Test that keywords are passed to the metaclass:
+        def meta_func(name, bases, ns, **kw):
+            return name, bases, ns, kw
+        res = types.new_class("X",
+                              (int, object),
+                              dict(metaclass=meta_func, x=0))
+        self.assertEqual(res, ("X", (int, object), {}, {"x": 0}))
+
+    def test_new_class_defaults(self):
+        # Test defaults/keywords:
+        C = types.new_class("C", (), {}, None)
+        self.assertEqual(C.__name__, "C")
+        self.assertEqual(C.__bases__, (object,))
+
+    def test_new_class_meta_with_base(self):
+        Meta = self.Meta
+        def func(ns):
+            ns["x"] = 0
+        C = types.new_class(name="C",
+                            bases=(int,),
+                            kwds=dict(metaclass=Meta, z=2),
+                            exec_body=func)
+        self.assertTrue(issubclass(C, int))
+        self.assertIsInstance(C, Meta)
+        self.assertEqual(C.x, 0)
+        self.assertEqual(C.y, 1)
+        self.assertEqual(C.z, 2)
+
+    # Many of the following tests are derived from test_descr.py
+    def test_prepare_class(self):
+        # Basic test of metaclass derivation
+        expected_ns = {}
+        class A(type):
+            def __new__(*args, **kwargs):
+                return type.__new__(*args, **kwargs)
+
+            def __prepare__(*args):
+                return expected_ns
+
+        B = types.new_class("B", (object,))
+        C = types.new_class("C", (object,), {"metaclass": A})
+
+        # The most derived metaclass of D is A rather than type.
+        meta, ns, kwds = types.prepare_class("D", (B, C), {"metaclass": type})
+        self.assertIs(meta, A)
+        self.assertIs(ns, expected_ns)
+        self.assertEqual(len(kwds), 0)
+
+    def test_metaclass_derivation(self):
+        # issue1294232: correct metaclass calculation
+        new_calls = []  # to check the order of __new__ calls
+        class AMeta(type):
+            def __new__(mcls, name, bases, ns):
+                new_calls.append('AMeta')
+                return super().__new__(mcls, name, bases, ns)
+            @classmethod
+            def __prepare__(mcls, name, bases):
+                return {}
+
+        class BMeta(AMeta):
+            def __new__(mcls, name, bases, ns):
+                new_calls.append('BMeta')
+                return super().__new__(mcls, name, bases, ns)
+            @classmethod
+            def __prepare__(mcls, name, bases):
+                ns = super().__prepare__(name, bases)
+                ns['BMeta_was_here'] = True
+                return ns
+
+        A = types.new_class("A", (), {"metaclass": AMeta})
+        self.assertEqual(new_calls, ['AMeta'])
+        new_calls.clear()
+
+        B = types.new_class("B", (), {"metaclass": BMeta})
+        # BMeta.__new__ calls AMeta.__new__ with super:
+        self.assertEqual(new_calls, ['BMeta', 'AMeta'])
+        new_calls.clear()
+
+        C = types.new_class("C", (A, B))
+        # The most derived metaclass is BMeta:
+        self.assertEqual(new_calls, ['BMeta', 'AMeta'])
+        new_calls.clear()
+        # BMeta.__prepare__ should've been called:
+        self.assertIn('BMeta_was_here', C.__dict__)
+
+        # The order of the bases shouldn't matter:
+        C2 = types.new_class("C2", (B, A))
+        self.assertEqual(new_calls, ['BMeta', 'AMeta'])
+        new_calls.clear()
+        self.assertIn('BMeta_was_here', C2.__dict__)
+
+        # Check correct metaclass calculation when a metaclass is declared:
+        D = types.new_class("D", (C,), {"metaclass": type})
+        self.assertEqual(new_calls, ['BMeta', 'AMeta'])
+        new_calls.clear()
+        self.assertIn('BMeta_was_here', D.__dict__)
+
+        E = types.new_class("E", (C,), {"metaclass": AMeta})
+        self.assertEqual(new_calls, ['BMeta', 'AMeta'])
+        new_calls.clear()
+        self.assertIn('BMeta_was_here', E.__dict__)
+
+    def test_metaclass_override_function(self):
+        # Special case: the given metaclass isn't a class,
+        # so there is no metaclass calculation.
+        class A(metaclass=self.Meta):
+            pass
+
+        marker = object()
+        def func(*args, **kwargs):
+            return marker
+
+        X = types.new_class("X", (), {"metaclass": func})
+        Y = types.new_class("Y", (object,), {"metaclass": func})
+        Z = types.new_class("Z", (A,), {"metaclass": func})
+        self.assertIs(marker, X)
+        self.assertIs(marker, Y)
+        self.assertIs(marker, Z)
+
+    def test_metaclass_override_callable(self):
+        # The given metaclass is a class,
+        # but not a descendant of type.
+        new_calls = []  # to check the order of __new__ calls
+        prepare_calls = []  # to track __prepare__ calls
+        class ANotMeta:
+            def __new__(mcls, *args, **kwargs):
+                new_calls.append('ANotMeta')
+                return super().__new__(mcls)
+            @classmethod
+            def __prepare__(mcls, name, bases):
+                prepare_calls.append('ANotMeta')
+                return {}
+
+        class BNotMeta(ANotMeta):
+            def __new__(mcls, *args, **kwargs):
+                new_calls.append('BNotMeta')
+                return super().__new__(mcls)
+            @classmethod
+            def __prepare__(mcls, name, bases):
+                prepare_calls.append('BNotMeta')
+                return super().__prepare__(name, bases)
+
+        A = types.new_class("A", (), {"metaclass": ANotMeta})
+        self.assertIs(ANotMeta, type(A))
+        self.assertEqual(prepare_calls, ['ANotMeta'])
+        prepare_calls.clear()
+        self.assertEqual(new_calls, ['ANotMeta'])
+        new_calls.clear()
+
+        B = types.new_class("B", (), {"metaclass": BNotMeta})
+        self.assertIs(BNotMeta, type(B))
+        self.assertEqual(prepare_calls, ['BNotMeta', 'ANotMeta'])
+        prepare_calls.clear()
+        self.assertEqual(new_calls, ['BNotMeta', 'ANotMeta'])
+        new_calls.clear()
+
+        C = types.new_class("C", (A, B))
+        self.assertIs(BNotMeta, type(C))
+        self.assertEqual(prepare_calls, ['BNotMeta', 'ANotMeta'])
+        prepare_calls.clear()
+        self.assertEqual(new_calls, ['BNotMeta', 'ANotMeta'])
+        new_calls.clear()
+
+        C2 = types.new_class("C2", (B, A))
+        self.assertIs(BNotMeta, type(C2))
+        self.assertEqual(prepare_calls, ['BNotMeta', 'ANotMeta'])
+        prepare_calls.clear()
+        self.assertEqual(new_calls, ['BNotMeta', 'ANotMeta'])
+        new_calls.clear()
+
+        # This is a TypeError, because of a metaclass conflict:
+        # BNotMeta is neither a subclass, nor a superclass of type
+        with self.assertRaises(TypeError):
+            D = types.new_class("D", (C,), {"metaclass": type})
+
+        E = types.new_class("E", (C,), {"metaclass": ANotMeta})
+        self.assertIs(BNotMeta, type(E))
+        self.assertEqual(prepare_calls, ['BNotMeta', 'ANotMeta'])
+        prepare_calls.clear()
+        self.assertEqual(new_calls, ['BNotMeta', 'ANotMeta'])
+        new_calls.clear()
+
+        F = types.new_class("F", (object(), C))
+        self.assertIs(BNotMeta, type(F))
+        self.assertEqual(prepare_calls, ['BNotMeta', 'ANotMeta'])
+        prepare_calls.clear()
+        self.assertEqual(new_calls, ['BNotMeta', 'ANotMeta'])
+        new_calls.clear()
+
+        F2 = types.new_class("F2", (C, object()))
+        self.assertIs(BNotMeta, type(F2))
+        self.assertEqual(prepare_calls, ['BNotMeta', 'ANotMeta'])
+        prepare_calls.clear()
+        self.assertEqual(new_calls, ['BNotMeta', 'ANotMeta'])
+        new_calls.clear()
+
+        # TypeError: BNotMeta is neither a
+        # subclass, nor a superclass of int
+        with self.assertRaises(TypeError):
+            X = types.new_class("X", (C, int()))
+        with self.assertRaises(TypeError):
+            X = types.new_class("X", (int(), C))
+
+
+class SimpleNamespaceTests(unittest.TestCase):
+
+    def test_constructor(self):
+        ns1 = types.SimpleNamespace()
+        ns2 = types.SimpleNamespace(x=1, y=2)
+        ns3 = types.SimpleNamespace(**dict(x=1, y=2))
+
+        with self.assertRaises(TypeError):
+            types.SimpleNamespace(1, 2, 3)
+
+        self.assertEqual(len(ns1.__dict__), 0)
+        self.assertEqual(vars(ns1), {})
+        self.assertEqual(len(ns2.__dict__), 2)
+        self.assertEqual(vars(ns2), {'y': 2, 'x': 1})
+        self.assertEqual(len(ns3.__dict__), 2)
+        self.assertEqual(vars(ns3), {'y': 2, 'x': 1})
+
+    def test_unbound(self):
+        ns1 = vars(types.SimpleNamespace())
+        ns2 = vars(types.SimpleNamespace(x=1, y=2))
+
+        self.assertEqual(ns1, {})
+        self.assertEqual(ns2, {'y': 2, 'x': 1})
+
+    def test_underlying_dict(self):
+        ns1 = types.SimpleNamespace()
+        ns2 = types.SimpleNamespace(x=1, y=2)
+        ns3 = types.SimpleNamespace(a=True, b=False)
+        mapping = ns3.__dict__
+        del ns3
+
+        self.assertEqual(ns1.__dict__, {})
+        self.assertEqual(ns2.__dict__, {'y': 2, 'x': 1})
+        self.assertEqual(mapping, dict(a=True, b=False))
+
+    def test_attrget(self):
+        ns = types.SimpleNamespace(x=1, y=2, w=3)
+
+        self.assertEqual(ns.x, 1)
+        self.assertEqual(ns.y, 2)
+        self.assertEqual(ns.w, 3)
+        with self.assertRaises(AttributeError):
+            ns.z
+
+    def test_attrset(self):
+        ns1 = types.SimpleNamespace()
+        ns2 = types.SimpleNamespace(x=1, y=2, w=3)
+        ns1.a = 'spam'
+        ns1.b = 'ham'
+        ns2.z = 4
+        ns2.theta = None
+
+        self.assertEqual(ns1.__dict__, dict(a='spam', b='ham'))
+        self.assertEqual(ns2.__dict__, dict(x=1, y=2, w=3, z=4, theta=None))
+
+    def test_attrdel(self):
+        ns1 = types.SimpleNamespace()
+        ns2 = types.SimpleNamespace(x=1, y=2, w=3)
+
+        with self.assertRaises(AttributeError):
+            del ns1.spam
+        with self.assertRaises(AttributeError):
+            del ns2.spam
+
+        del ns2.y
+        self.assertEqual(vars(ns2), dict(w=3, x=1))
+        ns2.y = 'spam'
+        self.assertEqual(vars(ns2), dict(w=3, x=1, y='spam'))
+        del ns2.y
+        self.assertEqual(vars(ns2), dict(w=3, x=1))
+
+        ns1.spam = 5
+        self.assertEqual(vars(ns1), dict(spam=5))
+        del ns1.spam
+        self.assertEqual(vars(ns1), {})
+
+    def test_repr(self):
+        ns1 = types.SimpleNamespace(x=1, y=2, w=3)
+        ns2 = types.SimpleNamespace()
+        ns2.x = "spam"
+        ns2._y = 5
+
+        self.assertEqual(repr(ns1), "namespace(w=3, x=1, y=2)")
+        self.assertEqual(repr(ns2), "namespace(_y=5, x='spam')")
+
+    def test_nested(self):
+        ns1 = types.SimpleNamespace(a=1, b=2)
+        ns2 = types.SimpleNamespace()
+        ns3 = types.SimpleNamespace(x=ns1)
+        ns2.spam = ns1
+        ns2.ham = '?'
+        ns2.spam = ns3
+
+        self.assertEqual(vars(ns1), dict(a=1, b=2))
+        self.assertEqual(vars(ns2), dict(spam=ns3, ham='?'))
+        self.assertEqual(ns2.spam, ns3)
+        self.assertEqual(vars(ns3), dict(x=ns1))
+        self.assertEqual(ns3.x.a, 1)
+
+    def test_recursive(self):
+        ns1 = types.SimpleNamespace(c='cookie')
+        ns2 = types.SimpleNamespace()
+        ns3 = types.SimpleNamespace(x=1)
+        ns1.spam = ns1
+        ns2.spam = ns3
+        ns3.spam = ns2
+
+        self.assertEqual(ns1.spam, ns1)
+        self.assertEqual(ns1.spam.spam, ns1)
+        self.assertEqual(ns1.spam.spam, ns1.spam)
+        self.assertEqual(ns2.spam, ns3)
+        self.assertEqual(ns3.spam, ns2)
+        self.assertEqual(ns2.spam.spam, ns2)
+
+    def test_recursive_repr(self):
+        ns1 = types.SimpleNamespace(c='cookie')
+        ns2 = types.SimpleNamespace()
+        ns3 = types.SimpleNamespace(x=1)
+        ns1.spam = ns1
+        ns2.spam = ns3
+        ns3.spam = ns2
+
+        self.assertEqual(repr(ns1),
+                         "namespace(c='cookie', spam=namespace(...))")
+        self.assertEqual(repr(ns2),
+                         "namespace(spam=namespace(spam=namespace(...), x=1))")
+
+    def test_as_dict(self):
+        ns = types.SimpleNamespace(spam='spamspamspam')
+
+        with self.assertRaises(TypeError):
+            len(ns)
+        with self.assertRaises(TypeError):
+            iter(ns)
+        with self.assertRaises(TypeError):
+            'spam' in ns
+        with self.assertRaises(TypeError):
+            ns['spam']
+
+    def test_subclass(self):
+        class Spam(types.SimpleNamespace):
+            pass
+
+        spam = Spam(ham=8, eggs=9)
+
+        self.assertIs(type(spam), Spam)
+        self.assertEqual(vars(spam), {'ham': 8, 'eggs': 9})
+
+
 def test_main():
-    run_unittest(TypesTests)
+    run_unittest(TypesTests, MappingProxyTests, ClassCreationTests,
+                 SimpleNamespaceTests)
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py
index fbbe78b316..2e6374561f 100644
--- a/Lib/test/test_ucn.py
+++ b/Lib/test/test_ucn.py
@@ -8,9 +8,12 @@ Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
 """#"
 
 import unittest
+import unicodedata
 import _testcapi
 
 from test import support
+from http.client import HTTPException
+from test.test_normalization import check_version
 
 class UnicodeNamesTest(unittest.TestCase):
 
@@ -60,8 +63,6 @@ class UnicodeNamesTest(unittest.TestCase):
         )
 
     def test_ascii_letters(self):
-        import unicodedata
-
         for char in "".join(map(chr, range(ord("a"), ord("z")))):
             name = "LATIN SMALL LETTER %s" % char.upper()
             code = unicodedata.lookup(name)
@@ -82,7 +83,6 @@ class UnicodeNamesTest(unittest.TestCase):
         self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8")
         self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3")
 
-        import unicodedata
         self.assertRaises(ValueError, unicodedata.name, "\ud7a4")
 
     def test_cjk_unified_ideographs(self):
@@ -98,14 +98,11 @@ class UnicodeNamesTest(unittest.TestCase):
         self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D")
 
     def test_bmp_characters(self):
-        import unicodedata
-        count = 0
         for code in range(0x10000):
             char = chr(code)
             name = unicodedata.name(char, None)
             if name is not None:
                 self.assertEqual(unicodedata.lookup(name), char)
-                count += 1
 
     def test_misc_symbols(self):
         self.checkletter("PILCROW SIGN", "\u00b6")
@@ -113,8 +110,85 @@ class UnicodeNamesTest(unittest.TestCase):
         self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F")
         self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41")
 
+    def test_aliases(self):
+        # Check that the aliases defined in the NameAliases.txt file work.
+        # This should be updated when new aliases are added or the file
+        # should be downloaded and parsed instead.  See #12753.
+        aliases = [
+            ('LATIN CAPITAL LETTER GHA', 0x01A2),
+            ('LATIN SMALL LETTER GHA', 0x01A3),
+            ('KANNADA LETTER LLLA', 0x0CDE),
+            ('LAO LETTER FO FON', 0x0E9D),
+            ('LAO LETTER FO FAY', 0x0E9F),
+            ('LAO LETTER RO', 0x0EA3),
+            ('LAO LETTER LO', 0x0EA5),
+            ('TIBETAN MARK BKA- SHOG GI MGO RGYAN', 0x0FD0),
+            ('YI SYLLABLE ITERATION MARK', 0xA015),
+            ('PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET', 0xFE18),
+            ('BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS', 0x1D0C5)
+        ]
+        for alias, codepoint in aliases:
+            self.checkletter(alias, chr(codepoint))
+            name = unicodedata.name(chr(codepoint))
+            self.assertNotEqual(name, alias)
+            self.assertEqual(unicodedata.lookup(alias),
+                             unicodedata.lookup(name))
+            with self.assertRaises(KeyError):
+                unicodedata.ucd_3_2_0.lookup(alias)
+
+    def test_aliases_names_in_pua_range(self):
+        # We are storing aliases in the PUA 15, but their names shouldn't leak
+        for cp in range(0xf0000, 0xf0100):
+            with self.assertRaises(ValueError) as cm:
+                unicodedata.name(chr(cp))
+            self.assertEqual(str(cm.exception), 'no such name')
+
+    def test_named_sequences_names_in_pua_range(self):
+        # We are storing named seq in the PUA 15, but their names shouldn't leak
+        for cp in range(0xf0100, 0xf0fff):
+            with self.assertRaises(ValueError) as cm:
+                unicodedata.name(chr(cp))
+            self.assertEqual(str(cm.exception), 'no such name')
+
+    def test_named_sequences_sample(self):
+        # Check a few named sequences.  See #12753.
+        sequences = [
+            ('LATIN SMALL LETTER R WITH TILDE', '\u0072\u0303'),
+            ('TAMIL SYLLABLE SAI', '\u0BB8\u0BC8'),
+            ('TAMIL SYLLABLE MOO', '\u0BAE\u0BCB'),
+            ('TAMIL SYLLABLE NNOO', '\u0BA3\u0BCB'),
+            ('TAMIL CONSONANT KSS', '\u0B95\u0BCD\u0BB7\u0BCD'),
+        ]
+        for seqname, codepoints in sequences:
+            self.assertEqual(unicodedata.lookup(seqname), codepoints)
+            with self.assertRaises(SyntaxError):
+                self.checkletter(seqname, None)
+            with self.assertRaises(KeyError):
+                unicodedata.ucd_3_2_0.lookup(seqname)
+
+    def test_named_sequences_full(self):
+        # Check all the named sequences
+        url = ("http://www.unicode.org/Public/%s/ucd/NamedSequences.txt" %
+               unicodedata.unidata_version)
+        try:
+            testdata = support.open_urlresource(url, encoding="utf-8",
+                                                check=check_version)
+        except (IOError, HTTPException):
+            self.skipTest("Could not retrieve " + url)
+        self.addCleanup(testdata.close)
+        for line in testdata:
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+            seqname, codepoints = line.split(';')
+            codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split())
+            self.assertEqual(unicodedata.lookup(seqname), codepoints)
+            with self.assertRaises(SyntaxError):
+                self.checkletter(seqname, None)
+            with self.assertRaises(KeyError):
+                unicodedata.ucd_3_2_0.lookup(seqname)
+
     def test_errors(self):
-        import unicodedata
         self.assertRaises(TypeError, unicodedata.name)
         self.assertRaises(TypeError, unicodedata.name, 'xx')
         self.assertRaises(TypeError, unicodedata.lookup)
@@ -145,7 +219,7 @@ class UnicodeNamesTest(unittest.TestCase):
     @unittest.skipUnless(_testcapi.INT_MAX < _testcapi.PY_SSIZE_T_MAX,
                          "needs UINT_MAX < SIZE_MAX")
     @support.bigmemtest(size=_testcapi.UINT_MAX + 1,
-                        memuse=2 + 4 // len('\U00010000'), dry_run=False)
+                        memuse=2 + 1, dry_run=False)
     def test_issue16335(self, size):
         # very very long bogus character name
         x = b'\\N{SPACE' + b'x' * (_testcapi.UINT_MAX + 1) + b'}'
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 47af8b938f..d4e2222fa9 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -5,17 +5,13 @@ Written by Marc-Andre Lemburg (mal@lemburg.com).
 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
 
 """#"
+import _string
 import codecs
 import struct
 import sys
 import unittest
 import warnings
 from test import support, string_tests
-import _string
-
-# decorator to skip tests on narrow builds
-requires_wide_build = unittest.skipIf(sys.maxunicode == 65535,
-                                      'requires wide build')
 
 # Error handling (bad decoder return)
 def search_function(encoding):
@@ -37,7 +33,8 @@ codecs.register(search_function)
 
 class UnicodeTest(string_tests.CommonTest,
         string_tests.MixinStrUnicodeUserStringTest,
-        string_tests.MixinStrUnicodeTest):
+        string_tests.MixinStrUnicodeTest,
+        unittest.TestCase):
 
     type2test = str
 
@@ -175,6 +172,15 @@ class UnicodeTest(string_tests.CommonTest,
 
     def test_find(self):
         string_tests.CommonTest.test_find(self)
+        # test implementation details of the memchr fast path
+        self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102')
+        self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201')
+        self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0120')
+        self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0220')
+        self.checkequal(100, 'a' * 100 + '\U00100304', 'find', '\U00100304')
+        self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00100204')
+        self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00102004')
+        # check mixed argument types
         self.checkequalnofix(0,  'abcdefghiabc', 'find', 'abc')
         self.checkequalnofix(9,  'abcdefghiabc', 'find', 'abc', 1)
         self.checkequalnofix(-1, 'abcdefghiabc', 'find', 'def', 4)
@@ -184,6 +190,14 @@ class UnicodeTest(string_tests.CommonTest,
 
     def test_rfind(self):
         string_tests.CommonTest.test_rfind(self)
+        # test implementation details of the memrchr fast path
+        self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102')
+        self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201')
+        self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0120')
+        self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0220')
+        self.checkequal(0, '\U00100304' + 'a' * 100, 'rfind', '\U00100304')
+        self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00100204')
+        self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00102004')
         # check mixed argument types
         self.checkequalnofix(9,   'abcdefghiabc', 'rfind', 'abc')
         self.checkequalnofix(12,  'abcdefghiabc', 'rfind', '')
@@ -280,6 +294,12 @@ class UnicodeTest(string_tests.CommonTest,
         self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1)
         self.assertRaises(TypeError, 'replace'.replace, "r", 42)
 
+    @support.cpython_only
+    def test_replace_id(self):
+        pattern = 'abc'
+        text = 'abc def'
+        self.assertIs(text.replace(pattern, pattern), text)
+
     def test_bytes_comparison(self):
         with support.check_warnings():
             warnings.simplefilter('ignore', BytesWarning)
@@ -350,6 +370,8 @@ class UnicodeTest(string_tests.CommonTest,
     def test_islower(self):
         string_tests.MixinStrUnicodeUserStringTest.test_islower(self)
         self.checkequalnofix(False, '\u1FFc', 'islower')
+        self.assertFalse('\u2167'.islower())
+        self.assertTrue('\u2177'.islower())
         # non-BMP, uppercase
         self.assertFalse('\U00010401'.islower())
         self.assertFalse('\U00010427'.islower())
@@ -364,6 +386,8 @@ class UnicodeTest(string_tests.CommonTest,
         string_tests.MixinStrUnicodeUserStringTest.test_isupper(self)
         if not sys.platform.startswith('java'):
             self.checkequalnofix(False, '\u1FFc', 'isupper')
+        self.assertTrue('\u2167'.isupper())
+        self.assertFalse('\u2177'.isupper())
         # non-BMP, uppercase
         self.assertTrue('\U00010401'.isupper())
         self.assertTrue('\U00010427'.isupper())
@@ -520,7 +544,6 @@ class UnicodeTest(string_tests.CommonTest,
                 self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name))
 
 
-    @requires_wide_build
     def test_lower(self):
         string_tests.CommonTest.test_lower(self)
         self.assertEqual('\U00010427'.lower(), '\U0001044F')
@@ -530,8 +553,28 @@ class UnicodeTest(string_tests.CommonTest,
                          '\U0001044F\U0001044F')
         self.assertEqual('X\U00010427x\U0001044F'.lower(),
                          'x\U0001044Fx\U0001044F')
+        self.assertEqual('�'.lower(), '�')
+        self.assertEqual('\u0130'.lower(), '\u0069\u0307')
+        # Special case for GREEK CAPITAL LETTER SIGMA U+03A3
+        self.assertEqual('\u03a3'.lower(), '\u03c3')
+        self.assertEqual('\u0345\u03a3'.lower(), '\u0345\u03c3')
+        self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2')
+        self.assertEqual('A\u0345\u03a3a'.lower(), 'a\u0345\u03c3a')
+        self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2')
+        self.assertEqual('A\u03a3\u0345'.lower(), 'a\u03c2\u0345')
+        self.assertEqual('\u03a3\u0345 '.lower(), '\u03c3\u0345 ')
+        self.assertEqual('\U0008fffe'.lower(), '\U0008fffe')
+        self.assertEqual('\u2177'.lower(), '\u2177')
+
+    def test_casefold(self):
+        self.assertEqual('hello'.casefold(), 'hello')
+        self.assertEqual('hELlo'.casefold(), 'hello')
+        self.assertEqual('ß'.casefold(), 'ss')
+        self.assertEqual('�'.casefold(), 'fi')
+        self.assertEqual('\u03a3'.casefold(), '\u03c3')
+        self.assertEqual('A\u0345\u03a3'.casefold(), 'a\u03b9\u03c3')
+        self.assertEqual('\u00b5'.casefold(), '\u03bc')
 
-    @requires_wide_build
     def test_upper(self):
         string_tests.CommonTest.test_upper(self)
         self.assertEqual('\U0001044F'.upper(), '\U00010427')
@@ -541,8 +584,14 @@ class UnicodeTest(string_tests.CommonTest,
                          '\U00010427\U00010427')
         self.assertEqual('X\U00010427x\U0001044F'.upper(),
                          'X\U00010427X\U00010427')
+        self.assertEqual('�'.upper(), 'FI')
+        self.assertEqual('\u0130'.upper(), '\u0130')
+        self.assertEqual('\u03a3'.upper(), '\u03a3')
+        self.assertEqual('ß'.upper(), 'SS')
+        self.assertEqual('\u1fd2'.upper(), '\u0399\u0308\u0300')
+        self.assertEqual('\U0008fffe'.upper(), '\U0008fffe')
+        self.assertEqual('\u2177'.upper(), '\u2167')
 
-    @requires_wide_build
     def test_capitalize(self):
         string_tests.CommonTest.test_capitalize(self)
         self.assertEqual('\U0001044F'.capitalize(), '\U00010427')
@@ -554,8 +603,12 @@ class UnicodeTest(string_tests.CommonTest,
                          '\U00010427\U0001044F')
         self.assertEqual('X\U00010427x\U0001044F'.capitalize(),
                          'X\U0001044Fx\U0001044F')
+        self.assertEqual('h\u0130'.capitalize(), 'H\u0069\u0307')
+        exp = '\u0399\u0308\u0300\u0069\u0307'
+        self.assertEqual('\u1fd2\u0130'.capitalize(), exp)
+        self.assertEqual('�nnish'.capitalize(), 'FInnish')
+        self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2')
 
-    @requires_wide_build
     def test_title(self):
         string_tests.MixinStrUnicodeUserStringTest.test_title(self)
         self.assertEqual('\U0001044F'.title(), '\U00010427')
@@ -569,8 +622,10 @@ class UnicodeTest(string_tests.CommonTest,
                          '\U00010427\U0001044F \U00010427\U0001044F')
         self.assertEqual('X\U00010427x\U0001044F X\U00010427x\U0001044F'.title(),
                          'X\U0001044Fx\U0001044F X\U0001044Fx\U0001044F')
+        self.assertEqual('�NNISH'.title(), 'Finnish')
+        self.assertEqual('A\u03a3 \u1fa1xy'.title(), 'A\u03c2 \u1fa9xy')
+        self.assertEqual('A\u03a3A'.title(), 'A\u03c3a')
 
-    @requires_wide_build
     def test_swapcase(self):
         string_tests.CommonTest.test_swapcase(self)
         self.assertEqual('\U0001044F'.swapcase(), '\U00010427')
@@ -583,6 +638,19 @@ class UnicodeTest(string_tests.CommonTest,
                          '\U00010427\U0001044F')
         self.assertEqual('X\U00010427x\U0001044F'.swapcase(),
                          'x\U0001044FX\U00010427')
+        self.assertEqual('�'.swapcase(), 'FI')
+        self.assertEqual('\u0130'.swapcase(), '\u0069\u0307')
+        # Special case for GREEK CAPITAL LETTER SIGMA U+03A3
+        self.assertEqual('\u03a3'.swapcase(), '\u03c3')
+        self.assertEqual('\u0345\u03a3'.swapcase(), '\u0399\u03c3')
+        self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2')
+        self.assertEqual('A\u0345\u03a3a'.swapcase(), 'a\u0399\u03c3A')
+        self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2')
+        self.assertEqual('A\u03a3\u0345'.swapcase(), 'a\u03c2\u0399')
+        self.assertEqual('\u03a3\u0345 '.swapcase(), '\u03c3\u0399 ')
+        self.assertEqual('\u03a3'.swapcase(), '\u03c3')
+        self.assertEqual('ß'.swapcase(), 'SS')
+        self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300')
 
     def test_contains(self):
         # Testing Unicode contains method
@@ -776,7 +844,7 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertEqual('{0!s}'.format(G('data')), 'string is data')
 
         msg = 'object.__format__ with a non-empty format string is deprecated'
-        with support.check_warnings((msg, PendingDeprecationWarning)):
+        with support.check_warnings((msg, DeprecationWarning)):
             self.assertEqual('{0:^10}'.format(E('data')), ' E(data)  ')
             self.assertEqual('{0:^10s}'.format(E('data')), ' E(data)  ')
             self.assertEqual('{0:>15s}'.format(G('data')), ' string is data')
@@ -858,6 +926,14 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertRaises(ValueError, format, '', '#')
         self.assertRaises(ValueError, format, '', '#20')
 
+        # Non-ASCII
+        self.assertEqual("{0:s}{1:s}".format("ABC", "\u0410\u0411\u0412"),
+                         'ABC\u0410\u0411\u0412')
+        self.assertEqual("{0:.3s}".format("ABC\u0410\u0411\u0412"),
+                         'ABC')
+        self.assertEqual("{0:.0s}".format("ABC\u0410\u0411\u0412"),
+                         '')
+
     def test_format_map(self):
         self.assertEqual(''.format_map({}), '')
         self.assertEqual('a'.format_map({}), 'a')
@@ -1005,6 +1081,10 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertEqual('%f' % INF, 'inf')
         self.assertEqual('%F' % INF, 'INF')
 
+        # PEP 393
+        self.assertEqual('%.1s' % "a\xe9\u20ac", 'a')
+        self.assertEqual('%.2s' % "a\xe9\u20ac", 'a\xe9')
+
     @support.cpython_only
     def test_formatting_huge_precision(self):
         from _testcapi import INT_MAX
@@ -1041,10 +1121,13 @@ class UnicodeTest(string_tests.CommonTest,
         class UnicodeSubclass(str):
             pass
 
-        self.assertEqual(
-            str(UnicodeSubclass('unicode subclass becomes unicode')),
-            'unicode subclass becomes unicode'
-        )
+        for text in ('ascii', '\xe9', '\u20ac', '\U0010FFFF'):
+            subclass = UnicodeSubclass(text)
+            self.assertEqual(str(subclass), text)
+            self.assertEqual(len(subclass), len(text))
+            if text == 'ascii':
+                self.assertEqual(subclass.encode('ascii'), b'ascii')
+                self.assertEqual(subclass.encode('utf-8'), b'ascii')
 
         self.assertEqual(
             str('strings are converted to unicode'),
@@ -1170,15 +1253,12 @@ class UnicodeTest(string_tests.CommonTest,
     def test_codecs_utf8(self):
         self.assertEqual(''.encode('utf-8'), b'')
         self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac')
-        if sys.maxunicode == 65535:
-            self.assertEqual('\ud800\udc02'.encode('utf-8'), b'\xf0\x90\x80\x82')
-            self.assertEqual('\ud84d\udc56'.encode('utf-8'), b'\xf0\xa3\x91\x96')
+        self.assertEqual('\U00010002'.encode('utf-8'), b'\xf0\x90\x80\x82')
+        self.assertEqual('\U00023456'.encode('utf-8'), b'\xf0\xa3\x91\x96')
         self.assertEqual('\ud800'.encode('utf-8', 'surrogatepass'), b'\xed\xa0\x80')
         self.assertEqual('\udc00'.encode('utf-8', 'surrogatepass'), b'\xed\xb0\x80')
-        if sys.maxunicode == 65535:
-            self.assertEqual(
-                ('\ud800\udc02'*1000).encode('utf-8'),
-                b'\xf0\x90\x80\x82'*1000)
+        self.assertEqual(('\U00010002'*10).encode('utf-8'),
+                         b'\xf0\x90\x80\x82'*10)
         self.assertEqual(
             '\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f'
             '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00'
@@ -1290,7 +1370,7 @@ class UnicodeTest(string_tests.CommonTest,
             # with start byte of a 2-byte sequence
             (b'\xc2', FFFD), # only the start byte
             (b'\xc2\xc2', FFFD*2), # 2 start bytes
-            (b'\xc2\xc2\xc2', FFFD*3), # 2 start bytes
+            (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes
             (b'\xc2\x41', FFFD+'A'), # invalid continuation byte
             # with start byte of a 3-byte sequence
             (b'\xe1', FFFD), # only the start byte
@@ -1360,6 +1440,226 @@ class UnicodeTest(string_tests.CommonTest,
             self.assertEqual(seq.decode('utf-8', 'ignore'),
                              res.replace('\uFFFD', ''))
 
+    def to_bytestring(self, seq):
+        return bytes(int(c, 16) for c in seq.split())
+
+    def assertCorrectUTF8Decoding(self, seq, res, err):
+        """
+        Check that an invalid UTF-8 sequence raises an UnicodeDecodeError when
+        'strict' is used, returns res when 'replace' is used, and that doesn't
+        return anything when 'ignore' is used.
+        """
+        with self.assertRaises(UnicodeDecodeError) as cm:
+            seq.decode('utf-8')
+        exc = cm.exception
+
+        self.assertIn(err, str(exc))
+        self.assertEqual(seq.decode('utf-8', 'replace'), res)
+        self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'replace'),
+                         'aaaa' + res + 'bbbb')
+        res = res.replace('\ufffd', '')
+        self.assertEqual(seq.decode('utf-8', 'ignore'), res)
+        self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore'),
+                          'aaaa' + res + 'bbbb')
+
+    def test_invalid_start_byte(self):
+        """
+        Test that an 'invalid start byte' error is raised when the first byte
+        is not in the ASCII range or is not a valid start byte of a 2-, 3-, or
+        4-bytes sequence. The invalid start byte is replaced with a single
+        U+FFFD when errors='replace'.
+        E.g. <80> is a continuation byte and can appear only after a start byte.
+        """
+        FFFD = '\ufffd'
+        for byte in b'\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
+            self.assertCorrectUTF8Decoding(bytes([byte]), '\ufffd',
+                                           'invalid start byte')
+
+    def test_unexpected_end_of_data(self):
+        """
+        Test that an 'unexpected end of data' error is raised when the string
+        ends after a start byte of a 2-, 3-, or 4-bytes sequence without having
+        enough continuation bytes.  The incomplete sequence is replaced with a
+        single U+FFFD when errors='replace'.
+        E.g. in the sequence <F3 80 80>, F3 is the start byte of a 4-bytes
+        sequence, but it's followed by only 2 valid continuation bytes and the
+        last continuation bytes is missing.
+        Note: the continuation bytes must be all valid, if one of them is
+        invalid another error will be raised.
+        """
+        sequences = [
+            'C2', 'DF',
+            'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF',
+            'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF',
+            'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF',
+            'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF',
+            'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF',
+            'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
+        ]
+        FFFD = '\ufffd'
+        for seq in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), '\ufffd',
+                                           'unexpected end of data')
+
+    def test_invalid_cb_for_2bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte of a 2-bytes sequence is invalid.  The start byte
+        is replaced by a single U+FFFD and the second byte is handled
+        separately when errors='replace'.
+        E.g. in the sequence <C2 41>, C2 is the start byte of a 2-bytes
+        sequence, but 41 is not a valid continuation byte because it's the
+        ASCII letter 'A'.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('C2 00', FFFD+'\x00'), ('C2 7F', FFFD+'\x7f'),
+            ('C2 C0', FFFDx2), ('C2 FF', FFFDx2),
+            ('DF 00', FFFD+'\x00'), ('DF 7F', FFFD+'\x7f'),
+            ('DF C0', FFFDx2), ('DF FF', FFFDx2),
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+                                           'invalid continuation byte')
+
+    def test_invalid_cb_for_3bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte(s) of a 3-bytes sequence are invalid.  When
+        errors='replace', if the first continuation byte is valid, the first
+        two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the
+        third byte is handled separately, otherwise only the start byte is
+        replaced with a U+FFFD and the other continuation bytes are handled
+        separately.
+        E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
+        sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+        because it's the ASCII letter 'A'.
+        Note: when the start byte is E0 or ED, the valid ranges for the first
+        continuation byte are limited to A0..BF and 80..9F respectively.
+        Python 2 used to consider all the bytes in range 80..BF valid when the
+        start byte was ED.  This is fixed in Python 3.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('E0 00', FFFD+'\x00'), ('E0 7F', FFFD+'\x7f'), ('E0 80', FFFDx2),
+            ('E0 9F', FFFDx2), ('E0 C0', FFFDx2), ('E0 FF', FFFDx2),
+            ('E0 A0 00', FFFD+'\x00'), ('E0 A0 7F', FFFD+'\x7f'),
+            ('E0 A0 C0', FFFDx2), ('E0 A0 FF', FFFDx2),
+            ('E0 BF 00', FFFD+'\x00'), ('E0 BF 7F', FFFD+'\x7f'),
+            ('E0 BF C0', FFFDx2), ('E0 BF FF', FFFDx2), ('E1 00', FFFD+'\x00'),
+            ('E1 7F', FFFD+'\x7f'), ('E1 C0', FFFDx2), ('E1 FF', FFFDx2),
+            ('E1 80 00', FFFD+'\x00'), ('E1 80 7F', FFFD+'\x7f'),
+            ('E1 80 C0', FFFDx2), ('E1 80 FF', FFFDx2),
+            ('E1 BF 00', FFFD+'\x00'), ('E1 BF 7F', FFFD+'\x7f'),
+            ('E1 BF C0', FFFDx2), ('E1 BF FF', FFFDx2), ('EC 00', FFFD+'\x00'),
+            ('EC 7F', FFFD+'\x7f'), ('EC C0', FFFDx2), ('EC FF', FFFDx2),
+            ('EC 80 00', FFFD+'\x00'), ('EC 80 7F', FFFD+'\x7f'),
+            ('EC 80 C0', FFFDx2), ('EC 80 FF', FFFDx2),
+            ('EC BF 00', FFFD+'\x00'), ('EC BF 7F', FFFD+'\x7f'),
+            ('EC BF C0', FFFDx2), ('EC BF FF', FFFDx2), ('ED 00', FFFD+'\x00'),
+            ('ED 7F', FFFD+'\x7f'),
+            ('ED A0', FFFDx2), ('ED BF', FFFDx2), # see note ^
+            ('ED C0', FFFDx2), ('ED FF', FFFDx2), ('ED 80 00', FFFD+'\x00'),
+            ('ED 80 7F', FFFD+'\x7f'), ('ED 80 C0', FFFDx2),
+            ('ED 80 FF', FFFDx2), ('ED 9F 00', FFFD+'\x00'),
+            ('ED 9F 7F', FFFD+'\x7f'), ('ED 9F C0', FFFDx2),
+            ('ED 9F FF', FFFDx2), ('EE 00', FFFD+'\x00'),
+            ('EE 7F', FFFD+'\x7f'), ('EE C0', FFFDx2), ('EE FF', FFFDx2),
+            ('EE 80 00', FFFD+'\x00'), ('EE 80 7F', FFFD+'\x7f'),
+            ('EE 80 C0', FFFDx2), ('EE 80 FF', FFFDx2),
+            ('EE BF 00', FFFD+'\x00'), ('EE BF 7F', FFFD+'\x7f'),
+            ('EE BF C0', FFFDx2), ('EE BF FF', FFFDx2), ('EF 00', FFFD+'\x00'),
+            ('EF 7F', FFFD+'\x7f'), ('EF C0', FFFDx2), ('EF FF', FFFDx2),
+            ('EF 80 00', FFFD+'\x00'), ('EF 80 7F', FFFD+'\x7f'),
+            ('EF 80 C0', FFFDx2), ('EF 80 FF', FFFDx2),
+            ('EF BF 00', FFFD+'\x00'), ('EF BF 7F', FFFD+'\x7f'),
+            ('EF BF C0', FFFDx2), ('EF BF FF', FFFDx2),
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+                                           'invalid continuation byte')
+
+    def test_invalid_cb_for_4bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte(s) of a 4-bytes sequence are invalid.  When
+        errors='replace',the start byte and all the following valid
+        continuation bytes are replaced with a single U+FFFD, and all the bytes
+        starting from the first invalid continuation bytes (included) are
+        handled separately.
+        E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
+        sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+        because it's the ASCII letter 'A'.
+        Note: when the start byte is E0 or ED, the valid ranges for the first
+        continuation byte are limited to A0..BF and 80..9F respectively.
+        However, when the start byte is ED, Python 2 considers all the bytes
+        in range 80..BF valid.  This is fixed in Python 3.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('F0 00', FFFD+'\x00'), ('F0 7F', FFFD+'\x7f'), ('F0 80', FFFDx2),
+            ('F0 8F', FFFDx2), ('F0 C0', FFFDx2), ('F0 FF', FFFDx2),
+            ('F0 90 00', FFFD+'\x00'), ('F0 90 7F', FFFD+'\x7f'),
+            ('F0 90 C0', FFFDx2), ('F0 90 FF', FFFDx2),
+            ('F0 BF 00', FFFD+'\x00'), ('F0 BF 7F', FFFD+'\x7f'),
+            ('F0 BF C0', FFFDx2), ('F0 BF FF', FFFDx2),
+            ('F0 90 80 00', FFFD+'\x00'), ('F0 90 80 7F', FFFD+'\x7f'),
+            ('F0 90 80 C0', FFFDx2), ('F0 90 80 FF', FFFDx2),
+            ('F0 90 BF 00', FFFD+'\x00'), ('F0 90 BF 7F', FFFD+'\x7f'),
+            ('F0 90 BF C0', FFFDx2), ('F0 90 BF FF', FFFDx2),
+            ('F0 BF 80 00', FFFD+'\x00'), ('F0 BF 80 7F', FFFD+'\x7f'),
+            ('F0 BF 80 C0', FFFDx2), ('F0 BF 80 FF', FFFDx2),
+            ('F0 BF BF 00', FFFD+'\x00'), ('F0 BF BF 7F', FFFD+'\x7f'),
+            ('F0 BF BF C0', FFFDx2), ('F0 BF BF FF', FFFDx2),
+            ('F1 00', FFFD+'\x00'), ('F1 7F', FFFD+'\x7f'), ('F1 C0', FFFDx2),
+            ('F1 FF', FFFDx2), ('F1 80 00', FFFD+'\x00'),
+            ('F1 80 7F', FFFD+'\x7f'), ('F1 80 C0', FFFDx2),
+            ('F1 80 FF', FFFDx2), ('F1 BF 00', FFFD+'\x00'),
+            ('F1 BF 7F', FFFD+'\x7f'), ('F1 BF C0', FFFDx2),
+            ('F1 BF FF', FFFDx2), ('F1 80 80 00', FFFD+'\x00'),
+            ('F1 80 80 7F', FFFD+'\x7f'), ('F1 80 80 C0', FFFDx2),
+            ('F1 80 80 FF', FFFDx2), ('F1 80 BF 00', FFFD+'\x00'),
+            ('F1 80 BF 7F', FFFD+'\x7f'), ('F1 80 BF C0', FFFDx2),
+            ('F1 80 BF FF', FFFDx2), ('F1 BF 80 00', FFFD+'\x00'),
+            ('F1 BF 80 7F', FFFD+'\x7f'), ('F1 BF 80 C0', FFFDx2),
+            ('F1 BF 80 FF', FFFDx2), ('F1 BF BF 00', FFFD+'\x00'),
+            ('F1 BF BF 7F', FFFD+'\x7f'), ('F1 BF BF C0', FFFDx2),
+            ('F1 BF BF FF', FFFDx2), ('F3 00', FFFD+'\x00'),
+            ('F3 7F', FFFD+'\x7f'), ('F3 C0', FFFDx2), ('F3 FF', FFFDx2),
+            ('F3 80 00', FFFD+'\x00'), ('F3 80 7F', FFFD+'\x7f'),
+            ('F3 80 C0', FFFDx2), ('F3 80 FF', FFFDx2),
+            ('F3 BF 00', FFFD+'\x00'), ('F3 BF 7F', FFFD+'\x7f'),
+            ('F3 BF C0', FFFDx2), ('F3 BF FF', FFFDx2),
+            ('F3 80 80 00', FFFD+'\x00'), ('F3 80 80 7F', FFFD+'\x7f'),
+            ('F3 80 80 C0', FFFDx2), ('F3 80 80 FF', FFFDx2),
+            ('F3 80 BF 00', FFFD+'\x00'), ('F3 80 BF 7F', FFFD+'\x7f'),
+            ('F3 80 BF C0', FFFDx2), ('F3 80 BF FF', FFFDx2),
+            ('F3 BF 80 00', FFFD+'\x00'), ('F3 BF 80 7F', FFFD+'\x7f'),
+            ('F3 BF 80 C0', FFFDx2), ('F3 BF 80 FF', FFFDx2),
+            ('F3 BF BF 00', FFFD+'\x00'), ('F3 BF BF 7F', FFFD+'\x7f'),
+            ('F3 BF BF C0', FFFDx2), ('F3 BF BF FF', FFFDx2),
+            ('F4 00', FFFD+'\x00'), ('F4 7F', FFFD+'\x7f'), ('F4 90', FFFDx2),
+            ('F4 BF', FFFDx2), ('F4 C0', FFFDx2), ('F4 FF', FFFDx2),
+            ('F4 80 00', FFFD+'\x00'), ('F4 80 7F', FFFD+'\x7f'),
+            ('F4 80 C0', FFFDx2), ('F4 80 FF', FFFDx2),
+            ('F4 8F 00', FFFD+'\x00'), ('F4 8F 7F', FFFD+'\x7f'),
+            ('F4 8F C0', FFFDx2), ('F4 8F FF', FFFDx2),
+            ('F4 80 80 00', FFFD+'\x00'), ('F4 80 80 7F', FFFD+'\x7f'),
+            ('F4 80 80 C0', FFFDx2), ('F4 80 80 FF', FFFDx2),
+            ('F4 80 BF 00', FFFD+'\x00'), ('F4 80 BF 7F', FFFD+'\x7f'),
+            ('F4 80 BF C0', FFFDx2), ('F4 80 BF FF', FFFDx2),
+            ('F4 8F 80 00', FFFD+'\x00'), ('F4 8F 80 7F', FFFD+'\x7f'),
+            ('F4 8F 80 C0', FFFDx2), ('F4 8F 80 FF', FFFDx2),
+            ('F4 8F BF 00', FFFD+'\x00'), ('F4 8F BF 7F', FFFD+'\x7f'),
+            ('F4 8F BF C0', FFFDx2), ('F4 8F BF FF', FFFDx2)
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+                                           'invalid continuation byte')
+
     def test_codecs_idna(self):
         # Test whether trailing dot is preserved
         self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.")
@@ -1391,14 +1691,6 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertRaises(TypeError, str, b"hello", "test.unicode2")
         self.assertRaises(TypeError, "hello".encode, "test.unicode1")
         self.assertRaises(TypeError, "hello".encode, "test.unicode2")
-        # executes PyUnicode_Encode()
-        import imp
-        self.assertRaises(
-            ImportError,
-            imp.find_module,
-            "non-existing module",
-            ["non-existing dir"]
-        )
 
         # Error handling (wrong arguments)
         self.assertRaises(TypeError, "hello".encode, 42, 42, 42)
@@ -1416,18 +1708,25 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertEqual('hello'.encode('ascii'), b'hello')
         self.assertEqual('hello'.encode('utf-7'), b'hello')
         self.assertEqual('hello'.encode('utf-8'), b'hello')
-        self.assertEqual('hello'.encode('utf8'), b'hello')
+        self.assertEqual('hello'.encode('utf-8'), b'hello')
         self.assertEqual('hello'.encode('utf-16-le'), b'h\000e\000l\000l\000o\000')
         self.assertEqual('hello'.encode('utf-16-be'), b'\000h\000e\000l\000l\000o')
         self.assertEqual('hello'.encode('latin-1'), b'hello')
 
+        # Default encoding is utf-8
+        self.assertEqual('\u2603'.encode(), b'\xe2\x98\x83')
+
         # Roundtrip safety for BMP (just the first 1024 chars)
         for c in range(1024):
             u = chr(c)
             for encoding in ('utf-7', 'utf-8', 'utf-16', 'utf-16-le',
                              'utf-16-be', 'raw_unicode_escape',
                              'unicode_escape', 'unicode_internal'):
-                self.assertEqual(str(u.encode(encoding),encoding), u)
+                with warnings.catch_warnings():
+                    # unicode-internal has been deprecated
+                    warnings.simplefilter("ignore", DeprecationWarning)
+
+                    self.assertEqual(str(u.encode(encoding),encoding), u)
 
         # Roundtrip safety for BMP (just the first 256 chars)
         for c in range(256):
@@ -1442,18 +1741,20 @@ class UnicodeTest(string_tests.CommonTest,
                 self.assertEqual(str(u.encode(encoding),encoding), u)
 
         # Roundtrip safety for non-BMP (just a few chars)
-        u = '\U00010001\U00020002\U00030003\U00040004\U00050005'
-        for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
-                         #'raw_unicode_escape',
-                         'unicode_escape', 'unicode_internal'):
-            self.assertEqual(str(u.encode(encoding),encoding), u)
+        with warnings.catch_warnings():
+            # unicode-internal has been deprecated
+            warnings.simplefilter("ignore", DeprecationWarning)
+
+            u = '\U00010001\U00020002\U00030003\U00040004\U00050005'
+            for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
+                             'raw_unicode_escape',
+                             'unicode_escape', 'unicode_internal'):
+                self.assertEqual(str(u.encode(encoding),encoding), u)
 
-        # UTF-8 must be roundtrip safe for all UCS-2 code points
-        # This excludes surrogates: in the full range, there would be
-        # a surrogate pair (\udbff\udc00), which gets converted back
-        # to a non-BMP character (\U0010fc00)
-        u = ''.join(map(chr, list(range(0,0xd800)) +
-                             list(range(0xe000,0x10000))))
+        # UTF-8 must be roundtrip safe for all code points
+        # (except surrogates, which are forbidden).
+        u = ''.join(map(chr, list(range(0, 0xd800)) +
+                             list(range(0xe000, 0x110000))))
         for encoding in ('utf-8',):
             self.assertEqual(str(u.encode(encoding),encoding), u)
 
@@ -1638,17 +1939,39 @@ class UnicodeTest(string_tests.CommonTest,
             return
         self.assertRaises(OverflowError, 't\tt\t'.expandtabs, sys.maxsize)
 
+    @support.cpython_only
+    def test_expandtabs_optimization(self):
+        s = 'abc'
+        self.assertIs(s.expandtabs(), s)
+
     def test_raiseMemError(self):
-        # Ensure that the freelist contains a consistent object, even
-        # when a string allocation fails with a MemoryError.
-        # This used to crash the interpreter,
-        # or leak references when the number was smaller.
-        charwidth = 4 if sys.maxunicode >= 0x10000 else 2
-        # Note: sys.maxsize is half of the actual max allocation because of
-        # the signedness of Py_ssize_t.
-        alloc = lambda: "a" * (sys.maxsize // charwidth * 2)
-        self.assertRaises(MemoryError, alloc)
-        self.assertRaises(MemoryError, alloc)
+        if struct.calcsize('P') == 8:
+            # 64 bits pointers
+            ascii_struct_size = 48
+            compact_struct_size = 72
+        else:
+            # 32 bits pointers
+            ascii_struct_size = 24
+            compact_struct_size = 36
+
+        for char in ('a', '\xe9', '\u20ac', '\U0010ffff'):
+            code = ord(char)
+            if code < 0x100:
+                char_size = 1  # sizeof(Py_UCS1)
+                struct_size = ascii_struct_size
+            elif code < 0x10000:
+                char_size = 2  # sizeof(Py_UCS2)
+                struct_size = compact_struct_size
+            else:
+                char_size = 4  # sizeof(Py_UCS4)
+                struct_size = compact_struct_size
+            # Note: sys.maxsize is half of the actual max allocation because of
+            # the signedness of Py_ssize_t. Strings of maxlen-1 should in principle
+            # be allocatable, given enough memory.
+            maxlen = ((sys.maxsize - struct_size) // char_size)
+            alloc = lambda: char * maxlen
+            self.assertRaises(MemoryError, alloc)
+            self.assertRaises(MemoryError, alloc)
 
     def test_format_subclass(self):
         class S(str):
@@ -1661,11 +1984,10 @@ class UnicodeTest(string_tests.CommonTest,
     # Test PyUnicode_FromFormat()
     def test_from_format(self):
         support.import_module('ctypes')
-        from ctypes import pythonapi, py_object, c_int
-        if sys.maxunicode == 65535:
-            name = "PyUnicodeUCS2_FromFormat"
-        else:
-            name = "PyUnicodeUCS4_FromFormat"
+        from ctypes import (pythonapi, py_object,
+            c_int, c_long, c_longlong, c_ssize_t,
+            c_uint, c_ulong, c_ulonglong, c_size_t)
+        name = "PyUnicode_FromFormat"
         _PyUnicode_FromFormat = getattr(pythonapi, name)
         _PyUnicode_FromFormat.restype = py_object
 
@@ -1686,13 +2008,40 @@ class UnicodeTest(string_tests.CommonTest,
             'string, got a non-ASCII byte: 0xe9$',
             PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii')
 
+        # test "%c"
         self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0xabcd)), '\uabcd')
         self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0x10ffff)), '\U0010ffff')
 
-        # other tests
+        # test "%"
+        self.assertEqual(PyUnicode_FromFormat(b'%'), '%')
+        self.assertEqual(PyUnicode_FromFormat(b'%%'), '%')
+        self.assertEqual(PyUnicode_FromFormat(b'%%s'), '%s')
+        self.assertEqual(PyUnicode_FromFormat(b'[%%]'), '[%]')
+        self.assertEqual(PyUnicode_FromFormat(b'%%%s', b'abc'), '%abc')
+
+        # test integer formats (%i, %d, %u)
+        self.assertEqual(PyUnicode_FromFormat(b'%03i', c_int(10)), '010')
+        self.assertEqual(PyUnicode_FromFormat(b'%0.4i', c_int(10)), '0010')
+        self.assertEqual(PyUnicode_FromFormat(b'%i', c_int(-123)), '-123')
+        self.assertEqual(PyUnicode_FromFormat(b'%li', c_long(-123)), '-123')
+        self.assertEqual(PyUnicode_FromFormat(b'%lli', c_longlong(-123)), '-123')
+        self.assertEqual(PyUnicode_FromFormat(b'%zi', c_ssize_t(-123)), '-123')
+
+        self.assertEqual(PyUnicode_FromFormat(b'%d', c_int(-123)), '-123')
+        self.assertEqual(PyUnicode_FromFormat(b'%ld', c_long(-123)), '-123')
+        self.assertEqual(PyUnicode_FromFormat(b'%lld', c_longlong(-123)), '-123')
+        self.assertEqual(PyUnicode_FromFormat(b'%zd', c_ssize_t(-123)), '-123')
+
+        self.assertEqual(PyUnicode_FromFormat(b'%u', c_uint(123)), '123')
+        self.assertEqual(PyUnicode_FromFormat(b'%lu', c_ulong(123)), '123')
+        self.assertEqual(PyUnicode_FromFormat(b'%llu', c_ulonglong(123)), '123')
+        self.assertEqual(PyUnicode_FromFormat(b'%zu', c_size_t(123)), '123')
+
+        # test %A
         text = PyUnicode_FromFormat(b'%%A:%A', 'abc\xe9\uabcd\U0010ffff')
         self.assertEqual(text, r"%A:'abc\xe9\uabcd\U0010ffff'")
 
+        # test %V
         text = PyUnicode_FromFormat(b'repr=%V', 'abc', b'xyz')
         self.assertEqual(text, 'repr=abc')
 
@@ -1706,6 +2055,13 @@ class UnicodeTest(string_tests.CommonTest,
         text = PyUnicode_FromFormat(b'repr=%V', None, b'abc\xff')
         self.assertEqual(text, 'repr=abc\ufffd')
 
+        # not supported: copy the raw format string. these tests are just here
+        # to check for crashs and should not be considered as specifications
+        self.assertEqual(PyUnicode_FromFormat(b'%1%s', b'abc'), '%s')
+        self.assertEqual(PyUnicode_FromFormat(b'%1abc'), '%1abc')
+        self.assertEqual(PyUnicode_FromFormat(b'%+i', c_int(10)), '%+i')
+        self.assertEqual(PyUnicode_FromFormat(b'%.%s', b'abc'), '%.%s')
+
     # Test PyUnicode_AsWideChar()
     def test_aswidechar(self):
         from _testcapi import unicode_aswidechar
@@ -1766,6 +2122,68 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertEqual(size, nchar)
         self.assertEqual(wchar, nonbmp + '\0')
 
+    def test_subclass_add(self):
+        class S(str):
+            def __add__(self, o):
+                return "3"
+        self.assertEqual(S("4") + S("5"), "3")
+        class S(str):
+            def __iadd__(self, o):
+                return "3"
+        s = S("1")
+        s += "4"
+        self.assertEqual(s, "3")
+
+    def test_encode_decimal(self):
+        from _testcapi import unicode_encodedecimal
+        self.assertEqual(unicode_encodedecimal('123'),
+                         b'123')
+        self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'),
+                         b'3.14')
+        self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"),
+                         b' 3.14 ')
+        self.assertRaises(UnicodeEncodeError,
+                          unicode_encodedecimal, "123\u20ac", "strict")
+        self.assertRaisesRegex(
+            ValueError,
+            "^'decimal' codec can't encode character",
+            unicode_encodedecimal, "123\u20ac", "replace")
+
+    def test_transform_decimal(self):
+        from _testcapi import unicode_transformdecimaltoascii as transform_decimal
+        self.assertEqual(transform_decimal('123'),
+                         '123')
+        self.assertEqual(transform_decimal('\u0663.\u0661\u0664'),
+                         '3.14')
+        self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"),
+                         "\N{EM SPACE}3.14\N{EN SPACE}")
+        self.assertEqual(transform_decimal('123\u20ac'),
+                         '123\u20ac')
+
+    def test_getnewargs(self):
+        text = 'abc'
+        args = text.__getnewargs__()
+        self.assertIsNot(args[0], text)
+        self.assertEqual(args[0], text)
+        self.assertEqual(len(args), 1)
+
+    def test_resize(self):
+        for length in range(1, 100, 7):
+            # generate a fresh string (refcount=1)
+            text = 'a' * length + 'b'
+
+            with support.check_warnings(('unicode_internal codec has been '
+                                         'deprecated', DeprecationWarning)):
+                # fill wstr internal field
+                abc = text.encode('unicode_internal')
+                self.assertEqual(abc.decode('unicode_internal'), text)
+
+                # resize text: wstr field must be cleared and then recomputed
+                text += 'c'
+                abcdef = text.encode('unicode_internal')
+                self.assertNotEqual(abc, abcdef)
+                self.assertEqual(abcdef.decode('unicode_internal'), text)
+
 
 class StringModuleTest(unittest.TestCase):
     def test_formatter_parser(self):
@@ -1817,45 +2235,6 @@ class StringModuleTest(unittest.TestCase):
             ]])
         self.assertRaises(TypeError, _string.formatter_field_name_split, 1)
 
-    def test_encode_decimal(self):
-        from _testcapi import unicode_encodedecimal
-        self.assertEqual(unicode_encodedecimal('123'),
-                         b'123')
-        self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'),
-                         b'3.14')
-        self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"),
-                         b' 3.14 ')
-        self.assertRaises(UnicodeEncodeError,
-                          unicode_encodedecimal, "123\u20ac", "strict")
-        self.assertEqual(unicode_encodedecimal("123\u20ac", "replace"),
-                         b'123?')
-        self.assertEqual(unicode_encodedecimal("123\u20ac", "ignore"),
-                         b'123')
-        self.assertEqual(unicode_encodedecimal("123\u20ac", "xmlcharrefreplace"),
-                         b'123&#8364;')
-        self.assertEqual(unicode_encodedecimal("123\u20ac", "backslashreplace"),
-                         b'123\\u20ac')
-        self.assertEqual(unicode_encodedecimal("123\u20ac\N{EM SPACE}", "replace"),
-                         b'123? ')
-        self.assertEqual(unicode_encodedecimal("123\u20ac\u20ac", "replace"),
-                         b'123??')
-        self.assertEqual(unicode_encodedecimal("123\u20ac\u0660", "replace"),
-                         b'123?0')
-
-    def test_transform_decimal(self):
-        from _testcapi import unicode_transformdecimaltoascii as transform_decimal
-        self.assertEqual(transform_decimal('123'),
-                         '123')
-        self.assertEqual(transform_decimal('\u0663.\u0661\u0664'),
-                         '3.14')
-        self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"),
-                         "\N{EM SPACE}3.14\N{EN SPACE}")
-        self.assertEqual(transform_decimal('123\u20ac'),
-                         '123\u20ac')
-
-
-def test_main():
-    support.run_unittest(__name__)
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_unicode_file.py b/Lib/test/test_unicode_file.py
index 6c2011ace8..faa8da3c0a 100644
--- a/Lib/test/test_unicode_file.py
+++ b/Lib/test/test_unicode_file.py
@@ -6,7 +6,7 @@ import unicodedata
 
 import unittest
 from test.support import (run_unittest, rmtree,
-    TESTFN_ENCODING, TESTFN_UNICODE, TESTFN_UNENCODABLE)
+    TESTFN_ENCODING, TESTFN_UNICODE, TESTFN_UNENCODABLE, create_empty_file)
 
 if not os.path.supports_unicode_filenames:
     try:
@@ -56,16 +56,20 @@ class TestUnicodeFiles(unittest.TestCase):
         # Should be able to rename the file using either name.
         self.assertTrue(os.path.isfile(filename1)) # must exist.
         os.rename(filename1, filename2 + ".new")
-        self.assertTrue(os.path.isfile(filename1+".new"))
+        self.assertFalse(os.path.isfile(filename2))
+        self.assertTrue(os.path.isfile(filename1 + '.new'))
         os.rename(filename1 + ".new", filename2)
+        self.assertFalse(os.path.isfile(filename1 + '.new'))
         self.assertTrue(os.path.isfile(filename2))
 
         shutil.copy(filename1, filename2 + ".new")
         os.unlink(filename1 + ".new") # remove using equiv name.
         # And a couple of moves, one using each name.
         shutil.move(filename1, filename2 + ".new")
-        self.assertTrue(not os.path.exists(filename2))
+        self.assertFalse(os.path.exists(filename2))
+        self.assertTrue(os.path.exists(filename1 + '.new'))
         shutil.move(filename1 + ".new", filename2)
+        self.assertFalse(os.path.exists(filename2 + '.new'))
         self.assertTrue(os.path.exists(filename1))
         # Note - due to the implementation of shutil.move,
         # it tries a rename first.  This only fails on Windows when on
@@ -73,10 +77,12 @@ class TestUnicodeFiles(unittest.TestCase):
         # So we test the shutil.copy2 function, which is the thing most
         # likely to fail.
         shutil.copy2(filename1, filename2 + ".new")
+        self.assertTrue(os.path.isfile(filename1 + '.new'))
         os.unlink(filename1 + ".new")
+        self.assertFalse(os.path.exists(filename2 + '.new'))
 
     def _do_directory(self, make_name, chdir_name):
-        cwd = os.getcwdb()
+        cwd = os.getcwd()
         if os.path.isdir(make_name):
             rmtree(make_name)
         os.mkdir(make_name)
@@ -99,8 +105,7 @@ class TestUnicodeFiles(unittest.TestCase):
     # top-level 'test' functions would be if they could take params
     def _test_single(self, filename):
         remove_if_exists(filename)
-        f = open(filename, "w")
-        f.close()
+        create_empty_file(filename)
         try:
             self._do_single(filename)
         finally:
diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index 97442564e7..99aa0033cd 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -21,7 +21,7 @@ errors = 'surrogatepass'
 class UnicodeMethodsTest(unittest.TestCase):
 
     # update this, if the database changes
-    expectedchecksum = '21b90f1aed00081b81ca7942b22196af090015a0'
+    expectedchecksum = 'bf7a78f1a532421b5033600102e23a92044dbba9'
 
     def test_method_checksum(self):
         h = hashlib.sha1()
@@ -80,7 +80,7 @@ class UnicodeDatabaseTest(unittest.TestCase):
 class UnicodeFunctionsTest(UnicodeDatabaseTest):
 
     # update this, if the database changes
-    expectedchecksum = 'c23dfc0b5eaf3ca2aad32d733de96bb182ccda50'
+    expectedchecksum = '17fe2f12b788e4fff5479b469c4404bb6ecf841f'
     def test_function_checksum(self):
         data = []
         h = hashlib.sha1()
@@ -108,6 +108,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
         self.assertEqual(self.db.digit('\u215b', None), None)
         self.assertEqual(self.db.digit('\u2468'), 9)
         self.assertEqual(self.db.digit('\U00020000', None), None)
+        self.assertEqual(self.db.digit('\U0001D7FD'), 7)
 
         self.assertRaises(TypeError, self.db.digit)
         self.assertRaises(TypeError, self.db.digit, 'xx')
@@ -120,6 +121,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
         self.assertEqual(self.db.numeric('\u2468'), 9.0)
         self.assertEqual(self.db.numeric('\ua627'), 7.0)
         self.assertEqual(self.db.numeric('\U00020000', None), None)
+        self.assertEqual(self.db.numeric('\U0001012A'), 9000)
 
         self.assertRaises(TypeError, self.db.numeric)
         self.assertRaises(TypeError, self.db.numeric, 'xx')
@@ -131,6 +133,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
         self.assertEqual(self.db.decimal('\u215b', None), None)
         self.assertEqual(self.db.decimal('\u2468', None), None)
         self.assertEqual(self.db.decimal('\U00020000', None), None)
+        self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
 
         self.assertRaises(TypeError, self.db.decimal)
         self.assertRaises(TypeError, self.db.decimal, 'xx')
@@ -141,6 +144,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
         self.assertEqual(self.db.category('a'), 'Ll')
         self.assertEqual(self.db.category('A'), 'Lu')
         self.assertEqual(self.db.category('\U00020000'), 'Lo')
+        self.assertEqual(self.db.category('\U0001012A'), 'No')
 
         self.assertRaises(TypeError, self.db.category)
         self.assertRaises(TypeError, self.db.category, 'xx')
@@ -308,14 +312,6 @@ class UnicodeMiscTest(UnicodeDatabaseTest):
                 self.assertEqual(len(lines), 1,
                                  r"\u%.4x should not be a linebreak" % i)
 
-    def test_UCS4(self):
-        # unicodedata should work with code points outside the BMP
-        # even on a narrow Unicode build
-        self.assertEqual(self.db.category("\U0001012A"), "No")
-        self.assertEqual(self.db.numeric("\U0001012A"), 9000)
-        self.assertEqual(self.db.decimal("\U0001D7FD"), 7)
-        self.assertEqual(self.db.digit("\U0001D7FD"), 7)
-
 def test_main():
     test.support.run_unittest(
         UnicodeMiscTest,
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 3fc499ec07..4f71b24a08 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -30,7 +30,10 @@ def urlopen(url, data=None, proxies=None):
     if proxies is not None:
         opener = urllib.request.FancyURLopener(proxies=proxies)
     elif not _urlopener:
-        opener = urllib.request.FancyURLopener()
+        with support.check_warnings(
+                ('FancyURLopener style of invoking requests is deprecated.',
+                DeprecationWarning)):
+            opener = urllib.request.FancyURLopener()
         _urlopener = opener
     else:
         opener = _urlopener
@@ -270,10 +273,9 @@ Content-Type: text/html; charset=iso-8859-1
 
     def test_missing_localfile(self):
         # Test for #10836
-        with self.assertRaises(urllib.error.URLError) as e:
+        # 3.3 - URLError is not captured, explicit IOError is raised.
+        with self.assertRaises(IOError):
             urlopen('file://localhost/a/file/which/doesnot/exists.py')
-        self.assertTrue(e.exception.filename)
-        self.assertTrue(e.exception.reason)
 
     def test_file_notexists(self):
         fd, tmp_file = tempfile.mkstemp()
@@ -286,21 +288,20 @@ Content-Type: text/html; charset=iso-8859-1
             os.close(fd)
             os.unlink(tmp_file)
         self.assertFalse(os.path.exists(tmp_file))
-        with self.assertRaises(urllib.error.URLError):
+        # 3.3 - IOError instead of URLError
+        with self.assertRaises(IOError):
             urlopen(tmp_fileurl)
 
     def test_ftp_nohost(self):
         test_ftp_url = 'ftp:///path'
-        with self.assertRaises(urllib.error.URLError) as e:
+        # 3.3 - IOError instead of URLError
+        with self.assertRaises(IOError):
             urlopen(test_ftp_url)
-        self.assertFalse(e.exception.filename)
-        self.assertTrue(e.exception.reason)
 
     def test_ftp_nonexisting(self):
-        with self.assertRaises(urllib.error.URLError) as e:
+        # 3.3 - IOError instead of URLError
+        with self.assertRaises(IOError):
             urlopen('ftp://localhost/a/file/which/doesnot/exists.py')
-        self.assertFalse(e.exception.filename)
-        self.assertTrue(e.exception.reason)
 
 
     def test_userpass_inurl(self):
@@ -333,6 +334,10 @@ Content-Type: text/html; charset=iso-8859-1
         finally:
             self.unfakehttp()
 
+    def test_URLopener_deprecation(self):
+        with support.check_warnings(('',DeprecationWarning)):
+            urllib.request.URLopener()
+
 class urlretrieve_FileTests(unittest.TestCase):
     """Test urllib.urlretrieve() on local files"""
 
@@ -366,7 +371,7 @@ class urlretrieve_FileTests(unittest.TestCase):
     def constructLocalFileUrl(self, filePath):
         filePath = os.path.abspath(filePath)
         try:
-            filePath.encode("utf8")
+            filePath.encode("utf-8")
         except UnicodeEncodeError:
             raise unittest.SkipTest("filePath is not encodable to utf8")
         return "file://%s" % urllib.request.pathname2url(filePath)
@@ -419,11 +424,11 @@ class urlretrieve_FileTests(unittest.TestCase):
 
     def test_reporthook(self):
         # Make sure that the reporthook works.
-        def hooktester(count, block_size, total_size, count_holder=[0]):
-            self.assertIsInstance(count, int)
-            self.assertIsInstance(block_size, int)
-            self.assertIsInstance(total_size, int)
-            self.assertEqual(count, count_holder[0])
+        def hooktester(block_count, block_read_size, file_size, count_holder=[0]):
+            self.assertIsInstance(block_count, int)
+            self.assertIsInstance(block_read_size, int)
+            self.assertIsInstance(file_size, int)
+            self.assertEqual(block_count, count_holder[0])
             count_holder[0] = count_holder[0] + 1
         second_temp = "%s.2" % support.TESTFN
         self.registerFileForCleanUp(second_temp)
@@ -434,8 +439,8 @@ class urlretrieve_FileTests(unittest.TestCase):
     def test_reporthook_0_bytes(self):
         # Test on zero length file. Should call reporthook only 1 time.
         report = []
-        def hooktester(count, block_size, total_size, _report=report):
-            _report.append((count, block_size, total_size))
+        def hooktester(block_count, block_read_size, file_size, _report=report):
+            _report.append((block_count, block_read_size, file_size))
         srcFileName = self.createNewTempFile()
         urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
             support.TESTFN, hooktester)
@@ -445,31 +450,32 @@ class urlretrieve_FileTests(unittest.TestCase):
     def test_reporthook_5_bytes(self):
         # Test on 5 byte file. Should call reporthook only 2 times (once when
         # the "network connection" is established and once when the block is
-        # read). Since the block size is 8192 bytes, only one block read is
-        # required to read the entire file.
+        # read).
         report = []
-        def hooktester(count, block_size, total_size, _report=report):
-            _report.append((count, block_size, total_size))
+        def hooktester(block_count, block_read_size, file_size, _report=report):
+            _report.append((block_count, block_read_size, file_size))
         srcFileName = self.createNewTempFile(b"x" * 5)
         urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
             support.TESTFN, hooktester)
         self.assertEqual(len(report), 2)
-        self.assertEqual(report[0][1], 8192)
         self.assertEqual(report[0][2], 5)
+        self.assertEqual(report[1][2], 5)
 
     def test_reporthook_8193_bytes(self):
         # Test on 8193 byte file. Should call reporthook only 3 times (once
         # when the "network connection" is established, once for the next 8192
         # bytes, and once for the last byte).
         report = []
-        def hooktester(count, block_size, total_size, _report=report):
-            _report.append((count, block_size, total_size))
+        def hooktester(block_count, block_read_size, file_size, _report=report):
+            _report.append((block_count, block_read_size, file_size))
         srcFileName = self.createNewTempFile(b"x" * 8193)
         urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
             support.TESTFN, hooktester)
         self.assertEqual(len(report), 3)
-        self.assertEqual(report[0][1], 8192)
         self.assertEqual(report[0][2], 8193)
+        self.assertEqual(report[0][1], 8192)
+        self.assertEqual(report[1][1], 8192)
+        self.assertEqual(report[2][1], 8192)
 
 
 class urlretrieve_HttpTests(unittest.TestCase, FakeHTTPMixin):
@@ -1193,14 +1199,16 @@ class URLopener_Tests(unittest.TestCase):
         class DummyURLopener(urllib.request.URLopener):
             def open_spam(self, url):
                 return url
+        with support.check_warnings(
+                ('DummyURLopener style of invoking requests is deprecated.',
+                DeprecationWarning)):
+            self.assertEqual(DummyURLopener().open(
+                'spam://example/ /'),'//example/%20/')
 
-        self.assertEqual(DummyURLopener().open(
-            'spam://example/ /'),'//example/%20/')
-
-        # test the safe characters are not quoted by urlopen
-        self.assertEqual(DummyURLopener().open(
-            "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"),
-            "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/")
+            # test the safe characters are not quoted by urlopen
+            self.assertEqual(DummyURLopener().open(
+                "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"),
+                "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/")
 
 # Just commented them out.
 # Can't really tell why keep failing in windows and sparc.
@@ -1280,6 +1288,28 @@ class URLopener_Tests(unittest.TestCase):
 #         self.assertEqual(ftp.ftp.sock.gettimeout(), 30)
 #         ftp.close()
 
+class RequestTests(unittest.TestCase):
+    """Unit tests for urllib.request.Request."""
+
+    def test_default_values(self):
+        Request = urllib.request.Request
+        request = Request("http://www.python.org")
+        self.assertEqual(request.get_method(), 'GET')
+        request = Request("http://www.python.org", {})
+        self.assertEqual(request.get_method(), 'POST')
+
+    def test_with_method_arg(self):
+        Request = urllib.request.Request
+        request = Request("http://www.python.org", method='HEAD')
+        self.assertEqual(request.method, 'HEAD')
+        self.assertEqual(request.get_method(), 'HEAD')
+        request = Request("http://www.python.org", {}, method='HEAD')
+        self.assertEqual(request.method, 'HEAD')
+        self.assertEqual(request.get_method(), 'HEAD')
+        request = Request("http://www.python.org", method='GET')
+        self.assertEqual(request.get_method(), 'GET')
+        request.method = 'HEAD'
+        self.assertEqual(request.get_method(), 'HEAD')
 
 
 def test_main():
@@ -1296,6 +1326,7 @@ def test_main():
         Utility_Tests,
         URLopener_Tests,
         #FTPWrapperTests,
+        RequestTests,
     )
 
 
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 5eab30af76..ccd5419b70 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -5,6 +5,7 @@ import os
 import io
 import socket
 import array
+import sys
 
 import urllib.request
 # The proxy bypass method imported below has logic specific to the OSX
@@ -18,6 +19,22 @@ import urllib.error
 # parse_keqv_list, parse_http_list, HTTPDigestAuthHandler
 
 class TrivialTests(unittest.TestCase):
+
+    def test___all__(self):
+        # Verify which names are exposed
+        for module in 'request', 'response', 'parse', 'error', 'robotparser':
+            context = {}
+            exec('from urllib.%s import *' % module, context)
+            del context['__builtins__']
+            if module == 'request' and os.name == 'nt':
+                u, p = context.pop('url2pathname'), context.pop('pathname2url')
+                self.assertEqual(u.__module__, 'nturl2path')
+                self.assertEqual(p.__module__, 'nturl2path')
+            for k, v in context.items():
+                self.assertEqual(v.__module__, 'urllib.%s' % module,
+                    "%r is exposed in 'urllib.%s' but defined in %r" %
+                    (k, module, v.__module__))
+
     def test_trivial(self):
         # A couple trivial tests
 
@@ -536,10 +553,6 @@ class OpenerDirectorTests(unittest.TestCase):
         self.assertRaises(urllib.error.URLError, o.open, req)
         self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})])
 
-##     def test_error(self):
-##         # XXX this doesn't actually seem to be used in standard library,
-##         #  but should really be tested anyway...
-
     def test_http_error(self):
         # XXX http_error_default
         # http errors are a special case
@@ -567,6 +580,7 @@ class OpenerDirectorTests(unittest.TestCase):
             self.assertEqual((handler, method_name), got[:2])
             self.assertEqual(args, got[2])
 
+
     def test_processors(self):
         # *_request / *_response methods get called appropriately
         o = OpenerDirector()
@@ -602,10 +616,30 @@ class OpenerDirectorTests(unittest.TestCase):
                 self.assertTrue(args[1] is None or
                              isinstance(args[1], MockResponse))
 
+    def test_method_deprecations(self):
+        req = Request("http://www.example.com")
+
+        with self.assertWarns(DeprecationWarning):
+            req.add_data("data")
+        with self.assertWarns(DeprecationWarning):
+            req.get_data()
+        with self.assertWarns(DeprecationWarning):
+            req.has_data()
+        with self.assertWarns(DeprecationWarning):
+            req.get_host()
+        with self.assertWarns(DeprecationWarning):
+            req.get_selector()
+        with self.assertWarns(DeprecationWarning):
+            req.is_unverifiable()
+        with self.assertWarns(DeprecationWarning):
+            req.get_origin_req_host()
+        with self.assertWarns(DeprecationWarning):
+            req.get_type()
+
 
 def sanepathname2url(path):
     try:
-        path.encode("utf8")
+        path.encode("utf-8")
     except UnicodeEncodeError:
         raise unittest.SkipTest("path is not encodable to utf8")
     urlpath = urllib.request.pathname2url(path)
@@ -957,8 +991,8 @@ class HandlerTests(unittest.TestCase):
         newreq = h.http_request(req)
         self.assertIs(cj.ach_req, req)
         self.assertIs(cj.ach_req, newreq)
-        self.assertEqual(req.get_origin_req_host(), "example.com")
-        self.assertFalse(req.is_unverifiable())
+        self.assertEqual(req.origin_req_host, "example.com")
+        self.assertFalse(req.unverifiable)
         newr = h.http_response(req, r)
         self.assertIs(cj.ec_req, req)
         self.assertIs(cj.ec_r, r)
@@ -990,7 +1024,7 @@ class HandlerTests(unittest.TestCase):
                 try:
                     self.assertEqual(o.req.get_method(), "GET")
                 except AttributeError:
-                    self.assertFalse(o.req.has_data())
+                    self.assertFalse(o.req.data)
 
                 # now it's a GET, there should not be headers regarding content
                 # (possibly dragged from before being a POST)
@@ -1106,9 +1140,9 @@ class HandlerTests(unittest.TestCase):
         handlers = add_ordered_mock_handlers(o, meth_spec)
 
         req = Request("http://acme.example.com/")
-        self.assertEqual(req.get_host(), "acme.example.com")
+        self.assertEqual(req.host, "acme.example.com")
         r = o.open(req)
-        self.assertEqual(req.get_host(), "proxy.example.com:3128")
+        self.assertEqual(req.host, "proxy.example.com:3128")
 
         self.assertEqual([(handlers[0], "http_open")],
                          [tup[0:2] for tup in o.calls])
@@ -1119,13 +1153,13 @@ class HandlerTests(unittest.TestCase):
         ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
         o.add_handler(ph)
         req = Request("http://www.perl.org/")
-        self.assertEqual(req.get_host(), "www.perl.org")
+        self.assertEqual(req.host, "www.perl.org")
         r = o.open(req)
-        self.assertEqual(req.get_host(), "proxy.example.com")
+        self.assertEqual(req.host, "proxy.example.com")
         req = Request("http://www.python.org")
-        self.assertEqual(req.get_host(), "www.python.org")
+        self.assertEqual(req.host, "www.python.org")
         r = o.open(req)
-        self.assertEqual(req.get_host(), "www.python.org")
+        self.assertEqual(req.host, "www.python.org")
         del os.environ['no_proxy']
 
     def test_proxy_no_proxy_all(self):
@@ -1134,9 +1168,9 @@ class HandlerTests(unittest.TestCase):
         ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
         o.add_handler(ph)
         req = Request("http://www.python.org")
-        self.assertEqual(req.get_host(), "www.python.org")
+        self.assertEqual(req.host, "www.python.org")
         r = o.open(req)
-        self.assertEqual(req.get_host(), "www.python.org")
+        self.assertEqual(req.host, "www.python.org")
         del os.environ['no_proxy']
 
 
@@ -1150,9 +1184,9 @@ class HandlerTests(unittest.TestCase):
         handlers = add_ordered_mock_handlers(o, meth_spec)
 
         req = Request("https://www.example.com/")
-        self.assertEqual(req.get_host(), "www.example.com")
+        self.assertEqual(req.host, "www.example.com")
         r = o.open(req)
-        self.assertEqual(req.get_host(), "proxy.example.com:3128")
+        self.assertEqual(req.host, "proxy.example.com:3128")
         self.assertEqual([(handlers[0], "https_open")],
                          [tup[0:2] for tup in o.calls])
 
@@ -1165,7 +1199,7 @@ class HandlerTests(unittest.TestCase):
         req = Request("https://www.example.com/")
         req.add_header("Proxy-Authorization","FooBar")
         req.add_header("User-Agent","Grail")
-        self.assertEqual(req.get_host(), "www.example.com")
+        self.assertEqual(req.host, "www.example.com")
         self.assertIsNone(req._tunnel_host)
         r = o.open(req)
         # Verify Proxy-Authorization gets tunneled to request.
@@ -1176,9 +1210,11 @@ class HandlerTests(unittest.TestCase):
         self.assertIn(("User-Agent","Grail"),
                       https_handler.httpconn.req_headers)
         self.assertIsNotNone(req._tunnel_host)
-        self.assertEqual(req.get_host(), "proxy.example.com:3128")
+        self.assertEqual(req.host, "proxy.example.com:3128")
         self.assertEqual(req.get_header("Proxy-authorization"),"FooBar")
 
+    # TODO: This should be only for OSX
+    @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX")
     def test_osx_proxy_bypass(self):
         bypass = {
             'exclude_simple': False,
@@ -1298,6 +1334,26 @@ class HandlerTests(unittest.TestCase):
         # _test_basic_auth called .open() twice)
         self.assertEqual(opener.recorded, ["digest", "basic"]*2)
 
+    def test_unsupported_auth_digest_handler(self):
+        opener = OpenerDirector()
+        # While using DigestAuthHandler
+        digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None)
+        http_handler = MockHTTPHandler(
+            401, 'WWW-Authenticate: Kerberos\r\n\r\n')
+        opener.add_handler(digest_auth_handler)
+        opener.add_handler(http_handler)
+        self.assertRaises(ValueError,opener.open,"http://www.example.com")
+
+    def test_unsupported_auth_basic_handler(self):
+        # While using BasicAuthHandler
+        opener = OpenerDirector()
+        basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None)
+        http_handler = MockHTTPHandler(
+            401, 'WWW-Authenticate: NTLM\r\n\r\n')
+        opener.add_handler(basic_auth_handler)
+        opener.add_handler(http_handler)
+        self.assertRaises(ValueError,opener.open,"http://www.example.com")
+
     def _test_basic_auth(self, opener, auth_handler, auth_header,
                          realm, http_handler, password_manager,
                          request_url, protected_url):
@@ -1335,6 +1391,7 @@ class HandlerTests(unittest.TestCase):
         self.assertEqual(len(http_handler.requests), 1)
         self.assertFalse(http_handler.requests[0].has_header(auth_header))
 
+
 class MiscTests(unittest.TestCase):
 
     def test_build_opener(self):
@@ -1390,11 +1447,11 @@ class RequestTests(unittest.TestCase):
         self.assertEqual("POST", self.post.get_method())
         self.assertEqual("GET", self.get.get_method())
 
-    def test_add_data(self):
-        self.assertFalse(self.get.has_data())
+    def test_data(self):
+        self.assertFalse(self.get.data)
         self.assertEqual("GET", self.get.get_method())
-        self.get.add_data("spam")
-        self.assertTrue(self.get.has_data())
+        self.get.data = "spam"
+        self.assertTrue(self.get.data)
         self.assertEqual("POST", self.get.get_method())
 
     def test_get_full_url(self):
@@ -1402,36 +1459,36 @@ class RequestTests(unittest.TestCase):
                          self.get.get_full_url())
 
     def test_selector(self):
-        self.assertEqual("/~jeremy/", self.get.get_selector())
+        self.assertEqual("/~jeremy/", self.get.selector)
         req = Request("http://www.python.org/")
-        self.assertEqual("/", req.get_selector())
+        self.assertEqual("/", req.selector)
 
     def test_get_type(self):
-        self.assertEqual("http", self.get.get_type())
+        self.assertEqual("http", self.get.type)
 
     def test_get_host(self):
-        self.assertEqual("www.python.org", self.get.get_host())
+        self.assertEqual("www.python.org", self.get.host)
 
     def test_get_host_unquote(self):
         req = Request("http://www.%70ython.org/")
-        self.assertEqual("www.python.org", req.get_host())
+        self.assertEqual("www.python.org", req.host)
 
     def test_proxy(self):
         self.assertFalse(self.get.has_proxy())
         self.get.set_proxy("www.perl.org", "http")
         self.assertTrue(self.get.has_proxy())
-        self.assertEqual("www.python.org", self.get.get_origin_req_host())
-        self.assertEqual("www.perl.org", self.get.get_host())
+        self.assertEqual("www.python.org", self.get.origin_req_host)
+        self.assertEqual("www.perl.org", self.get.host)
 
     def test_wrapped_url(self):
         req = Request("<URL:http://www.python.org>")
-        self.assertEqual("www.python.org", req.get_host())
+        self.assertEqual("www.python.org", req.host)
 
     def test_url_fragment(self):
         req = Request("http://www.python.org/?qs=query#fragment=true")
-        self.assertEqual("/?qs=query", req.get_selector())
+        self.assertEqual("/?qs=query", req.selector)
         req = Request("http://www.python.org/#fun=true")
-        self.assertEqual("/", req.get_selector())
+        self.assertEqual("/", req.selector)
 
         # Issue 11703: geturl() omits fragment in the original URL.
         url = 'http://docs.python.org/library/urllib2.html#OK'
@@ -1443,7 +1500,9 @@ class RequestTests(unittest.TestCase):
         Issue 13211 reveals that HTTPError didn't implement the URLError
         interface even though HTTPError is a subclass of URLError.
 
-        >>> err = urllib.error.HTTPError(msg='something bad happened', url=None, code=None, hdrs=None, fp=None)
+        >>> msg = 'something bad happened'
+        >>> url = code = hdrs = fp = None
+        >>> err = urllib.error.HTTPError(url, code, msg, hdrs, fp)
         >>> assert hasattr(err, 'reason')
         >>> err.reason
         'something bad happened'
diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py
index f07471dd0e..1eba14bacb 100644
--- a/Lib/test/test_urllib2_localnet.py
+++ b/Lib/test/test_urllib2_localnet.py
@@ -476,6 +476,13 @@ class TestUrlopen(unittest.TestCase):
             self.urlopen("https://localhost:%s/bizarre" % handler.port,
                          cafile=CERT_fakehostname)
 
+    def test_https_with_cadefault(self):
+        handler = self.start_https_server(certfile=CERT_localhost)
+        # Self-signed cert should fail verification with system certificate store
+        with self.assertRaises(urllib.error.URLError) as cm:
+            self.urlopen("https://localhost:%s/bizarre" % handler.port,
+                         cadefault=True)
+
     def test_sending_headers(self):
         handler = self.start_server()
         req = urllib.request.Request("http://localhost:%s/" % handler.port,
diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py
index e2b1d95c04..7f3c93adaa 100644
--- a/Lib/test/test_urllib2net.py
+++ b/Lib/test/test_urllib2net.py
@@ -83,12 +83,13 @@ class CloseSocketTest(unittest.TestCase):
     def test_close(self):
         # calling .close() on urllib2's response objects should close the
         # underlying socket
-
-        response = _urlopen_with_retry("http://www.python.org/")
-        sock = response.fp
-        self.assertTrue(not sock.closed)
-        response.close()
-        self.assertTrue(sock.closed)
+        url = "http://www.python.org/"
+        with support.transient_internet(url):
+            response = _urlopen_with_retry(url)
+            sock = response.fp
+            self.assertTrue(not sock.closed)
+            response.close()
+            self.assertTrue(sock.closed)
 
 class OtherNetworkTests(unittest.TestCase):
     def setUp(self):
diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py
index 383b2affd0..d3fe69dbce 100644
--- a/Lib/test/test_urllibnet.py
+++ b/Lib/test/test_urllibnet.py
@@ -137,10 +137,10 @@ class urlretrieveNetworkTests(unittest.TestCase):
     """Tests urllib.request.urlretrieve using the network."""
 
     @contextlib.contextmanager
-    def urlretrieve(self, *args):
+    def urlretrieve(self, *args, **kwargs):
         resource = args[0]
         with support.transient_internet(resource):
-            file_location, info = urllib.request.urlretrieve(*args)
+            file_location, info = urllib.request.urlretrieve(*args, **kwargs)
             try:
                 yield file_location, info
             finally:
@@ -170,9 +170,10 @@ class urlretrieveNetworkTests(unittest.TestCase):
             self.assertIsInstance(info, email.message.Message,
                                   "info is not an instance of email.message.Message")
 
+    logo = "http://www.python.org/community/logos/python-logo-master-v3-TM.png"
+
     def test_data_header(self):
-        logo = "http://www.python.org/community/logos/python-logo-master-v3-TM.png"
-        with self.urlretrieve(logo) as (file_location, fileheaders):
+        with self.urlretrieve(self.logo) as (file_location, fileheaders):
             datevalue = fileheaders.get('Date')
             dateformat = '%a, %d %b %Y %H:%M:%S GMT'
             try:
@@ -180,6 +181,31 @@ class urlretrieveNetworkTests(unittest.TestCase):
             except ValueError:
                 self.fail('Date value not in %r format', dateformat)
 
+    def test_reporthook(self):
+        records = []
+        def recording_reporthook(blocks, block_size, total_size):
+            records.append((blocks, block_size, total_size))
+
+        with self.urlretrieve(self.logo, reporthook=recording_reporthook) as (
+                file_location, fileheaders):
+            expected_size = int(fileheaders['Content-Length'])
+
+        records_repr = repr(records)  # For use in error messages.
+        self.assertGreater(len(records), 1, msg="There should always be two "
+                           "calls; the first one before the transfer starts.")
+        self.assertEqual(records[0][0], 0)
+        self.assertGreater(records[0][1], 0,
+                           msg="block size can't be 0 in %s" % records_repr)
+        self.assertEqual(records[0][2], expected_size)
+        self.assertEqual(records[-1][2], expected_size)
+
+        block_sizes = {block_size for _, block_size, _ in records}
+        self.assertEqual({records[0][1]}, block_sizes,
+                         msg="block sizes in %s must be equal" % records_repr)
+        self.assertGreaterEqual(records[-1][0]*records[0][1], expected_size,
+                                msg="number of blocks * block size must be"
+                                " >= total size in %s" % records_repr)
+
 
 def test_main():
     support.requires('network')
diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py
index 868ed24ccc..6381070f56 100644
--- a/Lib/test/test_userlist.py
+++ b/Lib/test/test_userlist.py
@@ -52,6 +52,12 @@ class UserListTest(list_tests.CommonTest):
                 return str(key) + '!!!'
         self.assertEqual(next(iter(T((1,2)))), "0!!!")
 
+    def test_userlist_copy(self):
+        u = self.type2test([6, 8, 1, 9, 1])
+        v = u.copy()
+        self.assertEqual(u, v)
+        self.assertEqual(type(u), type(v))
+
 def test_main():
     support.run_unittest(UserListTest)
 
diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py
index 7a8b9323f8..71fcac25e1 100755
--- a/Lib/test/test_userstring.py
+++ b/Lib/test/test_userstring.py
@@ -3,6 +3,7 @@
 # UserString instances should behave similar to builtin string objects.
 
 import string
+import unittest
 from test import support, string_tests
 
 from collections import UserString
@@ -10,6 +11,7 @@ from collections import UserString
 class UserStringTest(
     string_tests.CommonTest,
     string_tests.MixinStrUnicodeUserStringTest,
+    unittest.TestCase
     ):
 
     type2test = UserString
@@ -17,11 +19,11 @@ class UserStringTest(
     # Overwrite the three testing methods, because UserString
     # can't cope with arguments propagated to UserString
     # (and we don't test with subclasses)
-    def checkequal(self, result, object, methodname, *args):
+    def checkequal(self, result, object, methodname, *args, **kwargs):
         result = self.fixtype(result)
         object = self.fixtype(object)
         # we don't fix the arguments, because UserString can't cope with it
-        realresult = getattr(object, methodname)(*args)
+        realresult = getattr(object, methodname)(*args, **kwargs)
         self.assertEqual(
             result,
             realresult
@@ -42,8 +44,5 @@ class UserStringTest(
         getattr(object, methodname)(*args)
 
 
-def test_main():
-    support.run_unittest(UserStringTest)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py
index 43fa656102..7bc59ed9fd 100644
--- a/Lib/test/test_uuid.py
+++ b/Lib/test/test_uuid.py
@@ -471,14 +471,14 @@ class TestUUID(TestCase):
         if pid == 0:
             os.close(fds[0])
             value = uuid.uuid4()
-            os.write(fds[1], value.hex.encode('latin1'))
+            os.write(fds[1], value.hex.encode('latin-1'))
             os._exit(0)
 
         else:
             os.close(fds[1])
             parent_value = uuid.uuid4().hex
             os.waitpid(pid, 0)
-            child_value = os.read(fds[0], 100).decode('latin1')
+            child_value = os.read(fds[0], 100).decode('latin-1')
 
             self.assertNotEqual(parent_value, child_value)
 
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
new file mode 100644
index 0000000000..96968444c6
--- /dev/null
+++ b/Lib/test/test_venv.py
@@ -0,0 +1,203 @@
+"""
+Test harness for the venv module.
+
+Copyright (C) 2011-2012 Vinay Sajip.
+Licensed to the PSF under a contributor agreement.
+"""
+
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+from test.support import (captured_stdout, captured_stderr, run_unittest,
+                          can_symlink)
+import unittest
+import venv
+
+class BaseTest(unittest.TestCase):
+    """Base class for venv tests."""
+
+    def setUp(self):
+        self.env_dir = os.path.realpath(tempfile.mkdtemp())
+        if os.name == 'nt':
+            self.bindir = 'Scripts'
+            self.pydocname = 'pydoc.py'
+            self.lib = ('Lib',)
+            self.include = 'Include'
+        else:
+            self.bindir = 'bin'
+            self.pydocname = 'pydoc'
+            self.lib = ('lib', 'python%s' % sys.version[:3])
+            self.include = 'include'
+        if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ:
+            executable = os.environ['__PYVENV_LAUNCHER__']
+        else:
+            executable = sys.executable
+        self.exe = os.path.split(executable)[-1]
+
+    def tearDown(self):
+        shutil.rmtree(self.env_dir)
+
+    def run_with_capture(self, func, *args, **kwargs):
+        with captured_stdout() as output:
+            with captured_stderr() as error:
+                func(*args, **kwargs)
+        return output.getvalue(), error.getvalue()
+
+    def get_env_file(self, *args):
+        return os.path.join(self.env_dir, *args)
+
+    def get_text_file_contents(self, *args):
+        with open(self.get_env_file(*args), 'r') as f:
+            result = f.read()
+        return result
+
+class BasicTest(BaseTest):
+    """Test venv module functionality."""
+
+    def isdir(self, *args):
+        fn = self.get_env_file(*args)
+        self.assertTrue(os.path.isdir(fn))
+
+    def test_defaults(self):
+        """
+        Test the create function with default arguments.
+        """
+        shutil.rmtree(self.env_dir)
+        self.run_with_capture(venv.create, self.env_dir)
+        self.isdir(self.bindir)
+        self.isdir(self.include)
+        self.isdir(*self.lib)
+        data = self.get_text_file_contents('pyvenv.cfg')
+        if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__'
+                                         in os.environ):
+            executable =  os.environ['__PYVENV_LAUNCHER__']
+        else:
+            executable = sys.executable
+        path = os.path.dirname(executable)
+        self.assertIn('home = %s' % path, data)
+        data = self.get_text_file_contents(self.bindir, self.pydocname)
+        self.assertTrue(data.startswith('#!%s%s' % (self.env_dir, os.sep)))
+        fn = self.get_env_file(self.bindir, self.exe)
+        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
+            bd = self.get_env_file(self.bindir)
+            print('Contents of %r:' % bd)
+            print('    %r' % os.listdir(bd))
+        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
+
+    @unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate '
+                     'in a venv')
+    def test_prefixes(self):
+        """
+        Test that the prefix values are as expected.
+        """
+        #check our prefixes
+        self.assertEqual(sys.base_prefix, sys.prefix)
+        self.assertEqual(sys.base_exec_prefix, sys.exec_prefix)
+
+        # check a venv's prefixes
+        shutil.rmtree(self.env_dir)
+        self.run_with_capture(venv.create, self.env_dir)
+        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
+        cmd = [envpy, '-c', None]
+        for prefix, expected in (
+            ('prefix', self.env_dir),
+            ('prefix', self.env_dir),
+            ('base_prefix', sys.prefix),
+            ('base_exec_prefix', sys.exec_prefix)):
+            cmd[2] = 'import sys; print(sys.%s)' % prefix
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
+            out, err = p.communicate()
+            self.assertEqual(out.strip(), expected.encode())
+
+    def test_overwrite_existing(self):
+        """
+        Test control of overwriting an existing environment directory.
+        """
+        self.assertRaises(ValueError, venv.create, self.env_dir)
+        builder = venv.EnvBuilder(clear=True)
+        builder.create(self.env_dir)
+
+    def test_upgrade(self):
+        """
+        Test upgrading an existing environment directory.
+        """
+        builder = venv.EnvBuilder(upgrade=True)
+        self.run_with_capture(builder.create, self.env_dir)
+        self.isdir(self.bindir)
+        self.isdir(self.include)
+        self.isdir(*self.lib)
+        fn = self.get_env_file(self.bindir, self.exe)
+        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
+            bd = self.get_env_file(self.bindir)
+            print('Contents of %r:' % bd)
+            print('    %r' % os.listdir(bd))
+        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
+
+    def test_isolation(self):
+        """
+        Test isolation from system site-packages
+        """
+        for ssp, s in ((True, 'true'), (False, 'false')):
+            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
+            builder.create(self.env_dir)
+            data = self.get_text_file_contents('pyvenv.cfg')
+            self.assertIn('include-system-site-packages = %s\n' % s, data)
+
+    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
+    def test_symlinking(self):
+        """
+        Test symlinking works as expected
+        """
+        for usl in (False, True):
+            builder = venv.EnvBuilder(clear=True, symlinks=usl)
+            builder.create(self.env_dir)
+            fn = self.get_env_file(self.bindir, self.exe)
+            # Don't test when False, because e.g. 'python' is always
+            # symlinked to 'python3.3' in the env, even when symlinking in
+            # general isn't wanted.
+            if usl:
+                self.assertTrue(os.path.islink(fn))
+
+    # If a venv is created from a source build and that venv is used to
+    # run the test, the pyvenv.cfg in the venv created in the test will
+    # point to the venv being used to run the test, and we lose the link
+    # to the source build - so Python can't initialise properly.
+    @unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate '
+                     'in a venv')
+    def test_executable(self):
+        """
+        Test that the sys.executable value is as expected.
+        """
+        shutil.rmtree(self.env_dir)
+        self.run_with_capture(venv.create, self.env_dir)
+        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
+        cmd = [envpy, '-c', 'import sys; print(sys.executable)']
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        self.assertEqual(out.strip(), envpy.encode())
+
+    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
+    def test_executable_symlinks(self):
+        """
+        Test that the sys.executable value is as expected.
+        """
+        shutil.rmtree(self.env_dir)
+        builder = venv.EnvBuilder(clear=True, symlinks=True)
+        builder.create(self.env_dir)
+        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
+        cmd = [envpy, '-c', 'import sys; print(sys.executable)']
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        self.assertEqual(out.strip(), envpy.encode())
+
+def test_main():
+    run_unittest(BasicTest)
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py
index 786e60b327..bd06c8d8bd 100644
--- a/Lib/test/test_wait3.py
+++ b/Lib/test/test_wait3.py
@@ -19,13 +19,16 @@ except AttributeError:
 
 class Wait3Test(ForkWait):
     def wait_impl(self, cpid):
-        for i in range(10):
+        # This many iterations can be required, since some previously run
+        # tests (e.g. test_ctypes) could have spawned a lot of children
+        # very quickly.
+        for i in range(30):
             # wait3() shouldn't hang, but some of the buildbots seem to hang
             # in the forking tests.  This is an attempt to fix the problem.
             spid, status, rusage = os.wait3(os.WNOHANG)
             if spid == cpid:
                 break
-            time.sleep(1.0)
+            time.sleep(0.1)
 
         self.assertEqual(spid, cpid)
         self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8))
diff --git a/Lib/test/test_warnings.py b/Lib/test/test_warnings.py
index 79be835bbd..464ff40d42 100644
--- a/Lib/test/test_warnings.py
+++ b/Lib/test/test_warnings.py
@@ -40,7 +40,7 @@ def warnings_state(module):
         module.filters = original_filters
 
 
-class BaseTest(unittest.TestCase):
+class BaseTest:
 
     """Basic bookkeeping required for testing."""
 
@@ -63,7 +63,7 @@ class BaseTest(unittest.TestCase):
         super(BaseTest, self).tearDown()
 
 
-class FilterTests(object):
+class FilterTests(BaseTest):
 
     """Testing the filtering functionality."""
 
@@ -186,14 +186,14 @@ class FilterTests(object):
             self.assertEqual(str(w[-1].message), text)
             self.assertTrue(w[-1].category is UserWarning)
 
-class CFilterTests(BaseTest, FilterTests):
+class CFilterTests(FilterTests, unittest.TestCase):
     module = c_warnings
 
-class PyFilterTests(BaseTest, FilterTests):
+class PyFilterTests(FilterTests, unittest.TestCase):
     module = py_warnings
 
 
-class WarnTests(unittest.TestCase):
+class WarnTests(BaseTest):
 
     """Test warnings.warn() and warnings.warn_explicit()."""
 
@@ -360,7 +360,7 @@ class WarnTests(unittest.TestCase):
             self.module.warn(BadStrWarning())
 
 
-class CWarnTests(BaseTest, WarnTests):
+class CWarnTests(WarnTests, unittest.TestCase):
     module = c_warnings
 
     # As an early adopter, we sanity check the
@@ -369,7 +369,7 @@ class CWarnTests(BaseTest, WarnTests):
         self.assertFalse(original_warnings is self.module)
         self.assertFalse(hasattr(self.module.warn, '__code__'))
 
-class PyWarnTests(BaseTest, WarnTests):
+class PyWarnTests(WarnTests, unittest.TestCase):
     module = py_warnings
 
     # As an early adopter, we sanity check the
@@ -379,7 +379,7 @@ class PyWarnTests(BaseTest, WarnTests):
         self.assertTrue(hasattr(self.module.warn, '__code__'))
 
 
-class WCmdLineTests(unittest.TestCase):
+class WCmdLineTests(BaseTest):
 
     def test_improper_input(self):
         # Uses the private _setoption() function to test the parsing
@@ -410,14 +410,14 @@ class WCmdLineTests(unittest.TestCase):
         self.assertFalse(out.strip())
         self.assertNotIn(b'RuntimeWarning', err)
 
-class CWCmdLineTests(BaseTest, WCmdLineTests):
+class CWCmdLineTests(WCmdLineTests, unittest.TestCase):
     module = c_warnings
 
-class PyWCmdLineTests(BaseTest, WCmdLineTests):
+class PyWCmdLineTests(WCmdLineTests, unittest.TestCase):
     module = py_warnings
 
 
-class _WarningsTests(BaseTest):
+class _WarningsTests(BaseTest, unittest.TestCase):
 
     """Tests specific to the _warnings module."""
 
@@ -512,12 +512,11 @@ class _WarningsTests(BaseTest):
     def test_showwarning_not_callable(self):
         with original_warnings.catch_warnings(module=self.module):
             self.module.filterwarnings("always", category=UserWarning)
-            old_showwarning = self.module.showwarning
+            self.module.showwarning = print
+            with support.captured_output('stdout'):
+                self.module.warn('Warning!')
             self.module.showwarning = 23
-            try:
-                self.assertRaises(TypeError, self.module.warn, "Warning!")
-            finally:
-                self.module.showwarning = old_showwarning
+            self.assertRaises(TypeError, self.module.warn, "Warning!")
 
     def test_show_warning_output(self):
         # With showarning() missing, make sure that output is okay.
@@ -547,15 +546,18 @@ class _WarningsTests(BaseTest):
         globals_dict = globals()
         oldfile = globals_dict['__file__']
         try:
-            with original_warnings.catch_warnings(module=self.module) as w:
+            catch = original_warnings.catch_warnings(record=True,
+                                                     module=self.module)
+            with catch as w:
                 self.module.filterwarnings("always", category=UserWarning)
                 globals_dict['__file__'] = None
                 original_warnings.warn('test', UserWarning)
+                self.assertTrue(len(w))
         finally:
             globals_dict['__file__'] = oldfile
 
 
-class WarningsDisplayTests(unittest.TestCase):
+class WarningsDisplayTests(BaseTest):
 
     """Test the displaying of warnings and the ability to overload functions
     related to displaying warnings."""
@@ -599,10 +601,10 @@ class WarningsDisplayTests(unittest.TestCase):
                                 file_object, expected_file_line)
         self.assertEqual(expect, file_object.getvalue())
 
-class CWarningsDisplayTests(BaseTest, WarningsDisplayTests):
+class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
     module = c_warnings
 
-class PyWarningsDisplayTests(BaseTest, WarningsDisplayTests):
+class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
     module = py_warnings
 
 
@@ -708,10 +710,10 @@ class CatchWarningTests(BaseTest):
             with support.check_warnings(('foo', RuntimeWarning)):
                 wmod.warn("foo")
 
-class CCatchWarningTests(CatchWarningTests):
+class CCatchWarningTests(CatchWarningTests, unittest.TestCase):
     module = c_warnings
 
-class PyCatchWarningTests(CatchWarningTests):
+class PyCatchWarningTests(CatchWarningTests, unittest.TestCase):
     module = py_warnings
 
 
@@ -760,10 +762,10 @@ class EnvironmentVariableTests(BaseTest):
                 "['ignore:DeprecaciónWarning']".encode('utf-8'))
         self.assertEqual(p.wait(), 0)
 
-class CEnvironmentVariableTests(EnvironmentVariableTests):
+class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase):
     module = c_warnings
 
-class PyEnvironmentVariableTests(EnvironmentVariableTests):
+class PyEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase):
     module = py_warnings
 
 
@@ -786,20 +788,12 @@ class BootstrapTest(unittest.TestCase):
                 env=env)
             self.assertEqual(retcode, 0)
 
-def test_main():
+
+def setUpModule():
     py_warnings.onceregistry.clear()
     c_warnings.onceregistry.clear()
-    support.run_unittest(
-        CFilterTests, PyFilterTests,
-        CWarnTests, PyWarnTests,
-        CWCmdLineTests, PyWCmdLineTests,
-        _WarningsTests,
-        CWarningsDisplayTests, PyWarningsDisplayTests,
-        CCatchWarningTests, PyCatchWarningTests,
-        CEnvironmentVariableTests, PyEnvironmentVariableTests,
-        BootstrapTest,
-    )
 
+tearDownModule = setUpModule
 
 if __name__ == "__main__":
-    test_main()
+    unittest.main()
diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py
new file mode 100644
index 0000000000..34b9364a3f
--- /dev/null
+++ b/Lib/test/test_webbrowser.py
@@ -0,0 +1,192 @@
+import webbrowser
+import unittest
+import subprocess
+from unittest import mock
+from test import support
+
+
+URL = 'http://www.example.com'
+CMD_NAME = 'test'
+
+
+class PopenMock(mock.MagicMock):
+
+    def poll(self):
+        return 0
+
+    def wait(self, seconds=None):
+        return 0
+
+
+class CommandTestMixin:
+
+    def _test(self, meth, *, args=[URL], kw={}, options, arguments):
+        """Given a web browser instance method name along with arguments and
+        keywords for same (which defaults to the single argument URL), creates
+        a browser instance from the class pointed to by self.browser, calls the
+        indicated instance method with the indicated arguments, and compares
+        the resulting options and arguments passed to Popen by the browser
+        instance against the 'options' and 'args' lists.  Options are compared
+        in a position independent fashion, and the arguments are compared in
+        sequence order to whatever is left over after removing the options.
+
+        """
+        popen = PopenMock()
+        support.patch(self, subprocess, 'Popen', popen)
+        browser = self.browser_class(name=CMD_NAME)
+        getattr(browser, meth)(*args, **kw)
+        popen_args = subprocess.Popen.call_args[0][0]
+        self.assertEqual(popen_args[0], CMD_NAME)
+        popen_args.pop(0)
+        for option in options:
+            self.assertIn(option, popen_args)
+            popen_args.pop(popen_args.index(option))
+        self.assertEqual(popen_args, arguments)
+
+
+class GenericBrowserCommandTest(CommandTestMixin, unittest.TestCase):
+
+    browser_class = webbrowser.GenericBrowser
+
+    def test_open(self):
+        self._test('open',
+                   options=[],
+                   arguments=[URL])
+
+
+class BackgroundBrowserCommandTest(CommandTestMixin, unittest.TestCase):
+
+    browser_class = webbrowser.BackgroundBrowser
+
+    def test_open(self):
+        self._test('open',
+                   options=[],
+                   arguments=[URL])
+
+
+class ChromeCommandTest(CommandTestMixin, unittest.TestCase):
+
+    browser_class = webbrowser.Chrome
+
+    def test_open(self):
+        self._test('open',
+                   options=[],
+                   arguments=[URL])
+
+    def test_open_with_autoraise_false(self):
+        self._test('open', kw=dict(autoraise=False),
+                   options=[],
+                   arguments=[URL])
+
+    def test_open_new(self):
+        self._test('open_new',
+                   options=['--new-window'],
+                   arguments=[URL])
+
+    def test_open_new_tab(self):
+        self._test('open_new_tab',
+                   options=[],
+                   arguments=[URL])
+
+
+class MozillaCommandTest(CommandTestMixin, unittest.TestCase):
+
+    browser_class = webbrowser.Mozilla
+
+    def test_open(self):
+        self._test('open',
+                   options=['-raise', '-remote'],
+                   arguments=['openURL({})'.format(URL)])
+
+    def test_open_with_autoraise_false(self):
+        self._test('open', kw=dict(autoraise=False),
+                   options=['-noraise', '-remote'],
+                   arguments=['openURL({})'.format(URL)])
+
+    def test_open_new(self):
+        self._test('open_new',
+                   options=['-raise', '-remote'],
+                   arguments=['openURL({},new-window)'.format(URL)])
+
+    def test_open_new_tab(self):
+        self._test('open_new_tab',
+                   options=['-raise', '-remote'],
+                   arguments=['openURL({},new-tab)'.format(URL)])
+
+
+class GaleonCommandTest(CommandTestMixin, unittest.TestCase):
+
+    browser_class = webbrowser.Galeon
+
+    def test_open(self):
+        self._test('open',
+                   options=['-n'],
+                   arguments=[URL])
+
+    def test_open_with_autoraise_false(self):
+        self._test('open', kw=dict(autoraise=False),
+                   options=['-noraise', '-n'],
+                   arguments=[URL])
+
+    def test_open_new(self):
+        self._test('open_new',
+                   options=['-w'],
+                   arguments=[URL])
+
+    def test_open_new_tab(self):
+        self._test('open_new_tab',
+                   options=['-w'],
+                   arguments=[URL])
+
+
+class OperaCommandTest(CommandTestMixin, unittest.TestCase):
+
+    browser_class = webbrowser.Opera
+
+    def test_open(self):
+        self._test('open',
+                   options=['-remote'],
+                   arguments=['openURL({})'.format(URL)])
+
+    def test_open_with_autoraise_false(self):
+        self._test('open', kw=dict(autoraise=False),
+                   options=['-remote', '-noraise'],
+                   arguments=['openURL({})'.format(URL)])
+
+    def test_open_new(self):
+        self._test('open_new',
+                   options=['-remote'],
+                   arguments=['openURL({},new-window)'.format(URL)])
+
+    def test_open_new(self):
+        self._test('open_new_tab',
+                   options=['-remote'],
+                   arguments=['openURL({},new-page)'.format(URL)])
+
+
+class ELinksCommandTest(CommandTestMixin, unittest.TestCase):
+
+    browser_class = webbrowser.Elinks
+
+    def test_open(self):
+        self._test('open', options=['-remote'],
+                           arguments=['openURL({})'.format(URL)])
+
+    def test_open_with_autoraise_false(self):
+        self._test('open',
+                   options=['-remote'],
+                   arguments=['openURL({})'.format(URL)])
+
+    def test_open_new(self):
+        self._test('open_new',
+                   options=['-remote'],
+                   arguments=['openURL({},new-window)'.format(URL)])
+
+    def test_open_new_tab(self):
+        self._test('open_new_tab',
+                   options=['-remote'],
+                   arguments=['openURL({},new-tab)'.format(URL)])
+
+
+if __name__=='__main__':
+    unittest.main()
diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py
index 34c3deaf7e..eb7f75f066 100644
--- a/Lib/test/test_winsound.py
+++ b/Lib/test/test_winsound.py
@@ -16,16 +16,12 @@ def has_sound(sound):
     try:
         # Ask the mixer API for the number of devices it knows about.
         # When there are no devices, PlaySound will fail.
-        if ctypes.windll.winmm.mixerGetNumDevs() is 0:
+        if ctypes.windll.winmm.mixerGetNumDevs() == 0:
             return False
 
         key = winreg.OpenKeyEx(winreg.HKEY_CURRENT_USER,
                 "AppEvents\Schemes\Apps\.Default\{0}\.Default".format(sound))
-        value = winreg.EnumValue(key, 0)[1]
-        if value is not "":
-            return True
-        else:
-            return False
+        return winreg.EnumValue(key, 0)[1] != ""
     except WindowsError:
         return False
 
diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py
index 08f8d9a604..05c1f4fc23 100644
--- a/Lib/test/test_wsgiref.py
+++ b/Lib/test/test_wsgiref.py
@@ -9,6 +9,8 @@ from wsgiref.simple_server import WSGIServer, WSGIRequestHandler, demo_app
 from wsgiref.simple_server import make_server
 from io import StringIO, BytesIO, BufferedReader
 from socketserver import BaseServer
+from platform import python_implementation
+
 import os
 import re
 import sys
@@ -100,9 +102,11 @@ def compare_generic_iter(make_it,match):
 class IntegrationTests(TestCase):
 
     def check_hello(self, out, has_length=True):
+        pyver = (python_implementation() + "/" +
+                sys.version.split()[0])
         self.assertEqual(out,
             ("HTTP/1.0 200 OK\r\n"
-            "Server: WSGIServer/0.2 Python/"+sys.version.split()[0]+"\r\n"
+            "Server: WSGIServer/0.2 " + pyver +"\r\n"
             "Content-Type: text/plain\r\n"
             "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" +
             (has_length and  "Content-Length: 13\r\n" or "") +
@@ -156,9 +160,11 @@ class IntegrationTests(TestCase):
         out, err = run_amock(validator(app))
         self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n'))
         ver = sys.version.split()[0].encode('ascii')
+        py  = python_implementation().encode('ascii')
+        pyver = py + b"/" + ver
         self.assertEqual(
                 b"HTTP/1.0 200 OK\r\n"
-                b"Server: WSGIServer/0.2 Python/" + ver + b"\r\n"
+                b"Server: WSGIServer/0.2 "+ pyver + b"\r\n"
                 b"Content-Type: text/plain; charset=utf-8\r\n"
                 b"Date: Wed, 24 Dec 2008 13:29:32 GMT\r\n"
                 b"\r\n"
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index f61292f546..f3683f2d7e 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -1,28 +1,32 @@
-# xml.etree test.  This file contains enough tests to make sure that
-# all included components work as they should.
-# Large parts are extracted from the upstream test suite.
-
-# IMPORTANT: the same doctests are run from "test_xml_etree_c" in
-# order to ensure consistency between the C implementation and the
-# Python implementation.
+# IMPORTANT: the same tests are run from "test_xml_etree_c" in order
+# to ensure consistency between the C implementation and the Python
+# implementation.
 #
 # For this purpose, the module-level "ET" symbol is temporarily
 # monkey-patched when running the "test_xml_etree_c" test suite.
-# Don't re-import "xml.etree.ElementTree" module in the docstring,
-# except if the test is specific to the Python implementation.
 
-import sys
 import html
+import io
+import operator
+import pickle
+import sys
 import unittest
+import weakref
 
+from itertools import product
 from test import support
-from test.support import findfile
+from test.support import TESTFN, findfile, import_fresh_module, gc_collect
 
-from xml.etree import ElementTree as ET
+# pyET is the pure-Python implementation.
+#
+# ET is pyET in test_xml_etree and is the C accelerated version in
+# test_xml_etree_c.
+pyET = None
+ET = None
 
 SIMPLE_XMLFILE = findfile("simple.xml", subdir="xmltestdata")
 try:
-    SIMPLE_XMLFILE.encode("utf8")
+    SIMPLE_XMLFILE.encode("utf-8")
 except UnicodeEncodeError:
     raise unittest.SkipTest("filename is not encodable to utf8")
 SIMPLE_NS_XMLFILE = findfile("simple-ns.xml", subdir="xmltestdata")
@@ -57,22 +61,44 @@ SAMPLE_XML_NS = """
 </body>
 """
 
+SAMPLE_XML_NS_ELEMS = """
+<root>
+<h:table xmlns:h="hello">
+  <h:tr>
+    <h:td>Apples</h:td>
+    <h:td>Bananas</h:td>
+  </h:tr>
+</h:table>
+
+<f:table xmlns:f="foo">
+  <f:name>African Coffee Table</f:name>
+  <f:width>80</f:width>
+  <f:length>120</f:length>
+</f:table>
+</root>
+"""
+
+ENTITY_XML = """\
+<!DOCTYPE points [
+<!ENTITY % user-entities SYSTEM 'user-entities.xml'>
+%user-entities;
+]>
+<document>&entity;</document>
+"""
 
-def sanity():
-    """
-    Import sanity.
 
-    >>> from xml.etree import ElementTree
-    >>> from xml.etree import ElementInclude
-    >>> from xml.etree import ElementPath
-    """
+class ModuleTest(unittest.TestCase):
+    # TODO: this should be removed once we get rid of the global module vars
+
+    def test_sanity(self):
+        # Import sanity.
+
+        from xml.etree import ElementTree
+        from xml.etree import ElementInclude
+        from xml.etree import ElementPath
 
-def check_method(method):
-    if not hasattr(method, '__call__'):
-        print(method, "not callable")
 
 def serialize(elem, to_string=True, encoding='unicode', **options):
-    import io
     if encoding != 'unicode':
         file = io.BytesIO()
     else:
@@ -85,1182 +111,797 @@ def serialize(elem, to_string=True, encoding='unicode', **options):
         file.seek(0)
         return file
 
-def summarize(elem):
-    if elem.tag == ET.Comment:
-        return "<Comment>"
-    return elem.tag
-
 def summarize_list(seq):
-    return [summarize(elem) for elem in seq]
-
-def normalize_crlf(tree):
-    for elem in tree.iter():
-        if elem.text:
-            elem.text = elem.text.replace("\r\n", "\n")
-        if elem.tail:
-            elem.tail = elem.tail.replace("\r\n", "\n")
-
-def normalize_exception(func, *args, **kwargs):
-    # Ignore the exception __module__
-    try:
-        func(*args, **kwargs)
-    except Exception as err:
-        print("Traceback (most recent call last):")
-        print("{}: {}".format(err.__class__.__name__, err))
-
-def check_string(string):
-    len(string)
-    for char in string:
-        if len(char) != 1:
-            print("expected one-character string, got %r" % char)
-    new_string = string + ""
-    new_string = string + " "
-    string[:0]
-
-def check_mapping(mapping):
-    len(mapping)
-    keys = mapping.keys()
-    items = mapping.items()
-    for key in keys:
-        item = mapping[key]
-    mapping["key"] = "value"
-    if mapping["key"] != "value":
-        print("expected value string, got %r" % mapping["key"])
-
-def check_element(element):
-    if not ET.iselement(element):
-        print("not an element")
-    if not hasattr(element, "tag"):
-        print("no tag member")
-    if not hasattr(element, "attrib"):
-        print("no attrib member")
-    if not hasattr(element, "text"):
-        print("no text member")
-    if not hasattr(element, "tail"):
-        print("no tail member")
-
-    check_string(element.tag)
-    check_mapping(element.attrib)
-    if element.text is not None:
-        check_string(element.text)
-    if element.tail is not None:
-        check_string(element.tail)
-    for elem in element:
-        check_element(elem)
+    return [elem.tag for elem in seq]
+
+
+class ElementTestCase:
+    @classmethod
+    def setUpClass(cls):
+        cls.modules = {pyET, ET}
+
+    def pickleRoundTrip(self, obj, name, dumper, loader):
+        save_m = sys.modules[name]
+        try:
+            sys.modules[name] = dumper
+            temp = pickle.dumps(obj)
+            sys.modules[name] = loader
+            result = pickle.loads(temp)
+        except pickle.PicklingError as pe:
+            # pyET must be second, because pyET may be (equal to) ET.
+            human = dict([(ET, "cET"), (pyET, "pyET")])
+            raise support.TestFailed("Failed to round-trip %r from %r to %r"
+                                     % (obj,
+                                        human.get(dumper, dumper),
+                                        human.get(loader, loader))) from pe
+        finally:
+            sys.modules[name] = save_m
+        return result
+
+    def assertEqualElements(self, alice, bob):
+        self.assertIsInstance(alice, (ET.Element, pyET.Element))
+        self.assertIsInstance(bob, (ET.Element, pyET.Element))
+        self.assertEqual(len(list(alice)), len(list(bob)))
+        for x, y in zip(alice, bob):
+            self.assertEqualElements(x, y)
+        properties = operator.attrgetter('tag', 'tail', 'text', 'attrib')
+        self.assertEqual(properties(alice), properties(bob))
 
 # --------------------------------------------------------------------
 # element tree tests
 
-def interface():
-    """
-    Test element tree interface.
-
-    >>> element = ET.Element("tag")
-    >>> check_element(element)
-    >>> tree = ET.ElementTree(element)
-    >>> check_element(tree.getroot())
-
-    >>> element = ET.Element("t\\xe4g", key="value")
-    >>> tree = ET.ElementTree(element)
-    >>> repr(element)   # doctest: +ELLIPSIS
-    "<Element 't\\xe4g' at 0x...>"
-    >>> element = ET.Element("tag", key="value")
-
-    Make sure all standard element methods exist.
-
-    >>> check_method(element.append)
-    >>> check_method(element.extend)
-    >>> check_method(element.insert)
-    >>> check_method(element.remove)
-    >>> check_method(element.getchildren)
-    >>> check_method(element.find)
-    >>> check_method(element.iterfind)
-    >>> check_method(element.findall)
-    >>> check_method(element.findtext)
-    >>> check_method(element.clear)
-    >>> check_method(element.get)
-    >>> check_method(element.set)
-    >>> check_method(element.keys)
-    >>> check_method(element.items)
-    >>> check_method(element.iter)
-    >>> check_method(element.itertext)
-    >>> check_method(element.getiterator)
-
-    These methods return an iterable. See bug 6472.
-
-    >>> check_method(element.iter("tag").__next__)
-    >>> check_method(element.iterfind("tag").__next__)
-    >>> check_method(element.iterfind("*").__next__)
-    >>> check_method(tree.iter("tag").__next__)
-    >>> check_method(tree.iterfind("tag").__next__)
-    >>> check_method(tree.iterfind("*").__next__)
-
-    These aliases are provided:
-
-    >>> assert ET.XML == ET.fromstring
-    >>> assert ET.PI == ET.ProcessingInstruction
-    >>> assert ET.XMLParser == ET.XMLTreeBuilder
-    """
-
-def simpleops():
-    """
-    Basic method sanity checks.
-
-    >>> elem = ET.XML("<body><tag/></body>")
-    >>> serialize(elem)
-    '<body><tag /></body>'
-    >>> e = ET.Element("tag2")
-    >>> elem.append(e)
-    >>> serialize(elem)
-    '<body><tag /><tag2 /></body>'
-    >>> elem.remove(e)
-    >>> serialize(elem)
-    '<body><tag /></body>'
-    >>> elem.insert(0, e)
-    >>> serialize(elem)
-    '<body><tag2 /><tag /></body>'
-    >>> elem.remove(e)
-    >>> elem.extend([e])
-    >>> serialize(elem)
-    '<body><tag /><tag2 /></body>'
-    >>> elem.remove(e)
-
-    >>> element = ET.Element("tag", key="value")
-    >>> serialize(element) # 1
-    '<tag key="value" />'
-    >>> subelement = ET.Element("subtag")
-    >>> element.append(subelement)
-    >>> serialize(element) # 2
-    '<tag key="value"><subtag /></tag>'
-    >>> element.insert(0, subelement)
-    >>> serialize(element) # 3
-    '<tag key="value"><subtag /><subtag /></tag>'
-    >>> element.remove(subelement)
-    >>> serialize(element) # 4
-    '<tag key="value"><subtag /></tag>'
-    >>> element.remove(subelement)
-    >>> serialize(element) # 5
-    '<tag key="value" />'
-    >>> element.remove(subelement)
-    Traceback (most recent call last):
-    ValueError: list.remove(x): x not in list
-    >>> serialize(element) # 6
-    '<tag key="value" />'
-    >>> element[0:0] = [subelement, subelement, subelement]
-    >>> serialize(element[1])
-    '<subtag />'
-    >>> element[1:9] == [element[1], element[2]]
-    True
-    >>> element[:9:2] == [element[0], element[2]]
-    True
-    >>> del element[1:2]
-    >>> serialize(element)
-    '<tag key="value"><subtag /><subtag /></tag>'
-    """
-
-def cdata():
-    """
-    Test CDATA handling (etc).
-
-    >>> serialize(ET.XML("<tag>hello</tag>"))
-    '<tag>hello</tag>'
-    >>> serialize(ET.XML("<tag>&#104;&#101;&#108;&#108;&#111;</tag>"))
-    '<tag>hello</tag>'
-    >>> serialize(ET.XML("<tag><![CDATA[hello]]></tag>"))
-    '<tag>hello</tag>'
-    """
-
-# Only with Python implementation
-def simplefind():
-    """
-    Test find methods using the elementpath fallback.
-
-    >>> from xml.etree import ElementTree
-
-    >>> CurrentElementPath = ElementTree.ElementPath
-    >>> ElementTree.ElementPath = ElementTree._SimpleElementPath()
-    >>> elem = ElementTree.XML(SAMPLE_XML)
-    >>> elem.find("tag").tag
-    'tag'
-    >>> ElementTree.ElementTree(elem).find("tag").tag
-    'tag'
-    >>> elem.findtext("tag")
-    'text'
-    >>> elem.findtext("tog")
-    >>> elem.findtext("tog", "default")
-    'default'
-    >>> ElementTree.ElementTree(elem).findtext("tag")
-    'text'
-    >>> summarize_list(elem.findall("tag"))
-    ['tag', 'tag']
-    >>> summarize_list(elem.findall(".//tag"))
-    ['tag', 'tag', 'tag']
-
-    Path syntax doesn't work in this case.
-
-    >>> elem.find("section/tag")
-    >>> elem.findtext("section/tag")
-    >>> summarize_list(elem.findall("section/tag"))
-    []
-
-    >>> ElementTree.ElementPath = CurrentElementPath
-    """
-
-def find():
-    """
-    Test find methods (including xpath syntax).
-
-    >>> elem = ET.XML(SAMPLE_XML)
-    >>> elem.find("tag").tag
-    'tag'
-    >>> ET.ElementTree(elem).find("tag").tag
-    'tag'
-    >>> elem.find("section/tag").tag
-    'tag'
-    >>> elem.find("./tag").tag
-    'tag'
-    >>> ET.ElementTree(elem).find("./tag").tag
-    'tag'
-    >>> ET.ElementTree(elem).find("/tag").tag
-    'tag'
-    >>> elem[2] = ET.XML(SAMPLE_SECTION)
-    >>> elem.find("section/nexttag").tag
-    'nexttag'
-    >>> ET.ElementTree(elem).find("section/tag").tag
-    'tag'
-    >>> ET.ElementTree(elem).find("tog")
-    >>> ET.ElementTree(elem).find("tog/foo")
-    >>> elem.findtext("tag")
-    'text'
-    >>> elem.findtext("section/nexttag")
-    ''
-    >>> elem.findtext("section/nexttag", "default")
-    ''
-    >>> elem.findtext("tog")
-    >>> elem.findtext("tog", "default")
-    'default'
-    >>> ET.ElementTree(elem).findtext("tag")
-    'text'
-    >>> ET.ElementTree(elem).findtext("tog/foo")
-    >>> ET.ElementTree(elem).findtext("tog/foo", "default")
-    'default'
-    >>> ET.ElementTree(elem).findtext("./tag")
-    'text'
-    >>> ET.ElementTree(elem).findtext("/tag")
-    'text'
-    >>> elem.findtext("section/tag")
-    'subtext'
-    >>> ET.ElementTree(elem).findtext("section/tag")
-    'subtext'
-    >>> ET.XML('<root><empty /></root>').findtext('empty')
-    ''
-    >>> summarize_list(elem.findall("."))
-    ['body']
-    >>> summarize_list(elem.findall("tag"))
-    ['tag', 'tag']
-    >>> summarize_list(elem.findall("tog"))
-    []
-    >>> summarize_list(elem.findall("tog/foo"))
-    []
-    >>> summarize_list(elem.findall("*"))
-    ['tag', 'tag', 'section']
-    >>> summarize_list(elem.findall(".//tag"))
-    ['tag', 'tag', 'tag', 'tag']
-    >>> summarize_list(elem.findall("section/tag"))
-    ['tag']
-    >>> summarize_list(elem.findall("section//tag"))
-    ['tag', 'tag']
-    >>> summarize_list(elem.findall("section/*"))
-    ['tag', 'nexttag', 'nextsection']
-    >>> summarize_list(elem.findall("section//*"))
-    ['tag', 'nexttag', 'nextsection', 'tag']
-    >>> summarize_list(elem.findall("section/.//*"))
-    ['tag', 'nexttag', 'nextsection', 'tag']
-    >>> summarize_list(elem.findall("*/*"))
-    ['tag', 'nexttag', 'nextsection']
-    >>> summarize_list(elem.findall("*//*"))
-    ['tag', 'nexttag', 'nextsection', 'tag']
-    >>> summarize_list(elem.findall("*/tag"))
-    ['tag']
-    >>> summarize_list(elem.findall("*/./tag"))
-    ['tag']
-    >>> summarize_list(elem.findall("./tag"))
-    ['tag', 'tag']
-    >>> summarize_list(elem.findall(".//tag"))
-    ['tag', 'tag', 'tag', 'tag']
-    >>> summarize_list(elem.findall("././tag"))
-    ['tag', 'tag']
-    >>> summarize_list(elem.findall(".//tag[@class]"))
-    ['tag', 'tag', 'tag']
-    >>> summarize_list(elem.findall(".//tag[@class='a']"))
-    ['tag']
-    >>> summarize_list(elem.findall(".//tag[@class='b']"))
-    ['tag', 'tag']
-    >>> summarize_list(elem.findall(".//tag[@id]"))
-    ['tag']
-    >>> summarize_list(elem.findall(".//section[tag]"))
-    ['section']
-    >>> summarize_list(elem.findall(".//section[element]"))
-    []
-    >>> summarize_list(elem.findall("../tag"))
-    []
-    >>> summarize_list(elem.findall("section/../tag"))
-    ['tag', 'tag']
-    >>> summarize_list(ET.ElementTree(elem).findall("./tag"))
-    ['tag', 'tag']
-
-    Following example is invalid in 1.2.
-    A leading '*' is assumed in 1.3.
-
-    >>> elem.findall("section//") == elem.findall("section//*")
-    True
-
-    ET's Path module handles this case incorrectly; this gives
-    a warning in 1.3, and the behaviour will be modified in 1.4.
-
-    >>> summarize_list(ET.ElementTree(elem).findall("/tag"))
-    ['tag', 'tag']
-
-    >>> elem = ET.XML(SAMPLE_XML_NS)
-    >>> summarize_list(elem.findall("tag"))
-    []
-    >>> summarize_list(elem.findall("{http://effbot.org/ns}tag"))
-    ['{http://effbot.org/ns}tag', '{http://effbot.org/ns}tag']
-    >>> summarize_list(elem.findall(".//{http://effbot.org/ns}tag"))
-    ['{http://effbot.org/ns}tag', '{http://effbot.org/ns}tag', '{http://effbot.org/ns}tag']
-    """
-
-def file_init():
-    """
-    >>> import io
-
-    >>> stringfile = io.BytesIO(SAMPLE_XML.encode("utf-8"))
-    >>> tree = ET.ElementTree(file=stringfile)
-    >>> tree.find("tag").tag
-    'tag'
-    >>> tree.find("section/tag").tag
-    'tag'
-
-    >>> tree = ET.ElementTree(file=SIMPLE_XMLFILE)
-    >>> tree.find("element").tag
-    'element'
-    >>> tree.find("element/../empty-element").tag
-    'empty-element'
-    """
-
-def bad_find():
-    """
-    Check bad or unsupported path expressions.
-
-    >>> elem = ET.XML(SAMPLE_XML)
-    >>> elem.findall("/tag")
-    Traceback (most recent call last):
-    SyntaxError: cannot use absolute path on element
-    """
-
-def path_cache():
-    """
-    Check that the path cache behaves sanely.
-
-    >>> elem = ET.XML(SAMPLE_XML)
-    >>> for i in range(10): ET.ElementTree(elem).find('./'+str(i))
-    >>> cache_len_10 = len(ET.ElementPath._cache)
-    >>> for i in range(10): ET.ElementTree(elem).find('./'+str(i))
-    >>> len(ET.ElementPath._cache) == cache_len_10
-    True
-    >>> for i in range(20): ET.ElementTree(elem).find('./'+str(i))
-    >>> len(ET.ElementPath._cache) > cache_len_10
-    True
-    >>> for i in range(600): ET.ElementTree(elem).find('./'+str(i))
-    >>> len(ET.ElementPath._cache) < 500
-    True
-    """
-
-def copy():
-    """
-    Test copy handling (etc).
-
-    >>> import copy
-    >>> e1 = ET.XML("<tag>hello<foo/></tag>")
-    >>> e2 = copy.copy(e1)
-    >>> e3 = copy.deepcopy(e1)
-    >>> e1.find("foo").tag = "bar"
-    >>> serialize(e1)
-    '<tag>hello<bar /></tag>'
-    >>> serialize(e2)
-    '<tag>hello<bar /></tag>'
-    >>> serialize(e3)
-    '<tag>hello<foo /></tag>'
-
-    """
-
-def attrib():
-    """
-    Test attribute handling.
-
-    >>> elem = ET.Element("tag")
-    >>> elem.get("key") # 1.1
-    >>> elem.get("key", "default") # 1.2
-    'default'
-    >>> elem.set("key", "value")
-    >>> elem.get("key") # 1.3
-    'value'
-
-    >>> elem = ET.Element("tag", key="value")
-    >>> elem.get("key") # 2.1
-    'value'
-    >>> elem.attrib # 2.2
-    {'key': 'value'}
-
-    >>> attrib = {"key": "value"}
-    >>> elem = ET.Element("tag", attrib)
-    >>> attrib.clear() # check for aliasing issues
-    >>> elem.get("key") # 3.1
-    'value'
-    >>> elem.attrib # 3.2
-    {'key': 'value'}
-
-    >>> attrib = {"key": "value"}
-    >>> elem = ET.Element("tag", **attrib)
-    >>> attrib.clear() # check for aliasing issues
-    >>> elem.get("key") # 4.1
-    'value'
-    >>> elem.attrib # 4.2
-    {'key': 'value'}
-
-    >>> elem = ET.Element("tag", {"key": "other"}, key="value")
-    >>> elem.get("key") # 5.1
-    'value'
-    >>> elem.attrib # 5.2
-    {'key': 'value'}
-
-    >>> elem = ET.Element('test')
-    >>> elem.text = "aa"
-    >>> elem.set('testa', 'testval')
-    >>> elem.set('testb', 'test2')
-    >>> ET.tostring(elem)
-    b'<test testa="testval" testb="test2">aa</test>'
-    >>> sorted(elem.keys())
-    ['testa', 'testb']
-    >>> sorted(elem.items())
-    [('testa', 'testval'), ('testb', 'test2')]
-    >>> elem.attrib['testb']
-    'test2'
-    >>> elem.attrib['testb'] = 'test1'
-    >>> elem.attrib['testc'] = 'test2'
-    >>> ET.tostring(elem)
-    b'<test testa="testval" testb="test1" testc="test2">aa</test>'
-    """
-
-def makeelement():
-    """
-    Test makeelement handling.
-
-    >>> elem = ET.Element("tag")
-    >>> attrib = {"key": "value"}
-    >>> subelem = elem.makeelement("subtag", attrib)
-    >>> if subelem.attrib is attrib:
-    ...     print("attrib aliasing")
-    >>> elem.append(subelem)
-    >>> serialize(elem)
-    '<tag><subtag key="value" /></tag>'
-
-    >>> elem.clear()
-    >>> serialize(elem)
-    '<tag />'
-    >>> elem.append(subelem)
-    >>> serialize(elem)
-    '<tag><subtag key="value" /></tag>'
-    >>> elem.extend([subelem, subelem])
-    >>> serialize(elem)
-    '<tag><subtag key="value" /><subtag key="value" /><subtag key="value" /></tag>'
-    >>> elem[:] = [subelem]
-    >>> serialize(elem)
-    '<tag><subtag key="value" /></tag>'
-    >>> elem[:] = tuple([subelem])
-    >>> serialize(elem)
-    '<tag><subtag key="value" /></tag>'
-
-    """
-
-def parsefile():
-    """
-    Test parsing from file.
-
-    >>> tree = ET.parse(SIMPLE_XMLFILE)
-    >>> normalize_crlf(tree)
-    >>> tree.write(sys.stdout, encoding='unicode')
-    <root>
-       <element key="value">text</element>
-       <element>text</element>tail
-       <empty-element />
-    </root>
-    >>> tree = ET.parse(SIMPLE_NS_XMLFILE)
-    >>> normalize_crlf(tree)
-    >>> tree.write(sys.stdout, encoding='unicode')
-    <ns0:root xmlns:ns0="namespace">
-       <ns0:element key="value">text</ns0:element>
-       <ns0:element>text</ns0:element>tail
-       <ns0:empty-element />
-    </ns0:root>
-
-    >>> with open(SIMPLE_XMLFILE) as f:
-    ...     data = f.read()
-
-    >>> parser = ET.XMLParser()
-    >>> parser.version  # doctest: +ELLIPSIS
-    'Expat ...'
-    >>> parser.feed(data)
-    >>> print(serialize(parser.close()))
-    <root>
-       <element key="value">text</element>
-       <element>text</element>tail
-       <empty-element />
-    </root>
-
-    >>> parser = ET.XMLTreeBuilder() # 1.2 compatibility
-    >>> parser.feed(data)
-    >>> print(serialize(parser.close()))
-    <root>
-       <element key="value">text</element>
-       <element>text</element>tail
-       <empty-element />
-    </root>
-
-    >>> target = ET.TreeBuilder()
-    >>> parser = ET.XMLParser(target=target)
-    >>> parser.feed(data)
-    >>> print(serialize(parser.close()))
-    <root>
-       <element key="value">text</element>
-       <element>text</element>tail
-       <empty-element />
-    </root>
-    """
-
-def parseliteral():
-    """
-    >>> element = ET.XML("<html><body>text</body></html>")
-    >>> ET.ElementTree(element).write(sys.stdout, encoding='unicode')
-    <html><body>text</body></html>
-    >>> element = ET.fromstring("<html><body>text</body></html>")
-    >>> ET.ElementTree(element).write(sys.stdout, encoding='unicode')
-    <html><body>text</body></html>
-    >>> sequence = ["<html><body>", "text</bo", "dy></html>"]
-    >>> element = ET.fromstringlist(sequence)
-    >>> ET.tostring(element)
-    b'<html><body>text</body></html>'
-    >>> b"".join(ET.tostringlist(element))
-    b'<html><body>text</body></html>'
-    >>> ET.tostring(element, "ascii")
-    b"<?xml version='1.0' encoding='ascii'?>\\n<html><body>text</body></html>"
-    >>> _, ids = ET.XMLID("<html><body>text</body></html>")
-    >>> len(ids)
-    0
-    >>> _, ids = ET.XMLID("<html><body id='body'>text</body></html>")
-    >>> len(ids)
-    1
-    >>> ids["body"].tag
-    'body'
-    """
-
-def iterparse():
-    """
-    Test iterparse interface.
-
-    >>> iterparse = ET.iterparse
-
-    >>> context = iterparse(SIMPLE_XMLFILE)
-    >>> action, elem = next(context)
-    >>> print(action, elem.tag)
-    end element
-    >>> for action, elem in context:
-    ...   print(action, elem.tag)
-    end element
-    end empty-element
-    end root
-    >>> context.root.tag
-    'root'
-
-    >>> context = iterparse(SIMPLE_NS_XMLFILE)
-    >>> for action, elem in context:
-    ...   print(action, elem.tag)
-    end {namespace}element
-    end {namespace}element
-    end {namespace}empty-element
-    end {namespace}root
-
-    >>> events = ()
-    >>> context = iterparse(SIMPLE_XMLFILE, events)
-    >>> for action, elem in context:
-    ...   print(action, elem.tag)
-
-    >>> events = ()
-    >>> context = iterparse(SIMPLE_XMLFILE, events=events)
-    >>> for action, elem in context:
-    ...   print(action, elem.tag)
-
-    >>> events = ("start", "end")
-    >>> context = iterparse(SIMPLE_XMLFILE, events)
-    >>> for action, elem in context:
-    ...   print(action, elem.tag)
-    start root
-    start element
-    end element
-    start element
-    end element
-    start empty-element
-    end empty-element
-    end root
-
-    >>> events = ("start", "end", "start-ns", "end-ns")
-    >>> context = iterparse(SIMPLE_NS_XMLFILE, events)
-    >>> for action, elem in context:
-    ...   if action in ("start", "end"):
-    ...     print(action, elem.tag)
-    ...   else:
-    ...     print(action, elem)
-    start-ns ('', 'namespace')
-    start {namespace}root
-    start {namespace}element
-    end {namespace}element
-    start {namespace}element
-    end {namespace}element
-    start {namespace}empty-element
-    end {namespace}empty-element
-    end {namespace}root
-    end-ns None
-
-    >>> events = ("start", "end", "bogus")
-    >>> with open(SIMPLE_XMLFILE, "rb") as f:
-    ...     iterparse(f, events)
-    Traceback (most recent call last):
-    ValueError: unknown event 'bogus'
-
-    >>> import io
-
-    >>> source = io.BytesIO(
-    ...     b"<?xml version='1.0' encoding='iso-8859-1'?>\\n"
-    ...     b"<body xmlns='http://&#233;ffbot.org/ns'\\n"
-    ...     b"      xmlns:cl\\xe9='http://effbot.org/ns'>text</body>\\n")
-    >>> events = ("start-ns",)
-    >>> context = iterparse(source, events)
-    >>> for action, elem in context:
-    ...     print(action, elem)
-    start-ns ('', 'http://\\xe9ffbot.org/ns')
-    start-ns ('cl\\xe9', 'http://effbot.org/ns')
-
-    >>> source = io.StringIO("<document />junk")
-    >>> try:
-    ...   for action, elem in iterparse(source):
-    ...     print(action, elem.tag)
-    ... except ET.ParseError as v:
-    ...   print(v)
-    end document
-    junk after document element: line 1, column 12
-    """
-
-def writefile():
-    """
-    >>> elem = ET.Element("tag")
-    >>> elem.text = "text"
-    >>> serialize(elem)
-    '<tag>text</tag>'
-    >>> ET.SubElement(elem, "subtag").text = "subtext"
-    >>> serialize(elem)
-    '<tag>text<subtag>subtext</subtag></tag>'
-
-    Test tag suppression
-    >>> elem.tag = None
-    >>> serialize(elem)
-    'text<subtag>subtext</subtag>'
-    >>> elem.insert(0, ET.Comment("comment"))
-    >>> serialize(elem)     # assumes 1.3
-    'text<!--comment--><subtag>subtext</subtag>'
-    >>> elem[0] = ET.PI("key", "value")
-    >>> serialize(elem)
-    'text<?key value?><subtag>subtext</subtag>'
-    """
-
-def custom_builder():
-    """
-    Test parser w. custom builder.
-
-    >>> with open(SIMPLE_XMLFILE) as f:
-    ...     data = f.read()
-    >>> class Builder:
-    ...     def start(self, tag, attrib):
-    ...         print("start", tag)
-    ...     def end(self, tag):
-    ...         print("end", tag)
-    ...     def data(self, text):
-    ...         pass
-    >>> builder = Builder()
-    >>> parser = ET.XMLParser(target=builder)
-    >>> parser.feed(data)
-    start root
-    start element
-    end element
-    start element
-    end element
-    start empty-element
-    end empty-element
-    end root
-
-    >>> with open(SIMPLE_NS_XMLFILE) as f:
-    ...     data = f.read()
-    >>> class Builder:
-    ...     def start(self, tag, attrib):
-    ...         print("start", tag)
-    ...     def end(self, tag):
-    ...         print("end", tag)
-    ...     def data(self, text):
-    ...         pass
-    ...     def pi(self, target, data):
-    ...         print("pi", target, repr(data))
-    ...     def comment(self, data):
-    ...         print("comment", repr(data))
-    >>> builder = Builder()
-    >>> parser = ET.XMLParser(target=builder)
-    >>> parser.feed(data)
-    pi pi 'data'
-    comment ' comment '
-    start {namespace}root
-    start {namespace}element
-    end {namespace}element
-    start {namespace}element
-    end {namespace}element
-    start {namespace}empty-element
-    end {namespace}empty-element
-    end {namespace}root
-
-    """
-
-def getchildren():
-    """
-    Test Element.getchildren()
-
-    >>> with open(SIMPLE_XMLFILE, "rb") as f:
-    ...     tree = ET.parse(f)
-    >>> for elem in tree.getroot().iter():
-    ...     summarize_list(elem.getchildren())
-    ['element', 'element', 'empty-element']
-    []
-    []
-    []
-    >>> for elem in tree.getiterator():
-    ...     summarize_list(elem.getchildren())
-    ['element', 'element', 'empty-element']
-    []
-    []
-    []
-
-    >>> elem = ET.XML(SAMPLE_XML)
-    >>> len(elem.getchildren())
-    3
-    >>> len(elem[2].getchildren())
-    1
-    >>> elem[:] == elem.getchildren()
-    True
-    >>> child1 = elem[0]
-    >>> child2 = elem[2]
-    >>> del elem[1:2]
-    >>> len(elem.getchildren())
-    2
-    >>> child1 == elem[0]
-    True
-    >>> child2 == elem[1]
-    True
-    >>> elem[0:2] = [child2, child1]
-    >>> child2 == elem[0]
-    True
-    >>> child1 == elem[1]
-    True
-    >>> child1 == elem[0]
-    False
-    >>> elem.clear()
-    >>> elem.getchildren()
-    []
-    """
-
-def writestring():
-    """
-    >>> elem = ET.XML("<html><body>text</body></html>")
-    >>> ET.tostring(elem)
-    b'<html><body>text</body></html>'
-    >>> elem = ET.fromstring("<html><body>text</body></html>")
-    >>> ET.tostring(elem)
-    b'<html><body>text</body></html>'
-    """
-
-def check_encoding(encoding):
-    """
-    >>> check_encoding("ascii")
-    >>> check_encoding("us-ascii")
-    >>> check_encoding("iso-8859-1")
-    >>> check_encoding("iso-8859-15")
-    >>> check_encoding("cp437")
-    >>> check_encoding("mac-roman")
-    """
-    ET.XML("<?xml version='1.0' encoding='%s'?><xml />" % encoding)
-
-def encoding():
-    r"""
-    Test encoding issues.
-
-    >>> elem = ET.Element("tag")
-    >>> elem.text = "abc"
-    >>> serialize(elem)
-    '<tag>abc</tag>'
-    >>> serialize(elem, encoding="utf-8")
-    b'<tag>abc</tag>'
-    >>> serialize(elem, encoding="us-ascii")
-    b'<tag>abc</tag>'
-    >>> serialize(elem, encoding="iso-8859-1")
-    b"<?xml version='1.0' encoding='iso-8859-1'?>\n<tag>abc</tag>"
-
-    >>> elem.text = "<&\"\'>"
-    >>> serialize(elem)
-    '<tag>&lt;&amp;"\'&gt;</tag>'
-    >>> serialize(elem, encoding="utf-8")
-    b'<tag>&lt;&amp;"\'&gt;</tag>'
-    >>> serialize(elem, encoding="us-ascii") # cdata characters
-    b'<tag>&lt;&amp;"\'&gt;</tag>'
-    >>> serialize(elem, encoding="iso-8859-1")
-    b'<?xml version=\'1.0\' encoding=\'iso-8859-1\'?>\n<tag>&lt;&amp;"\'&gt;</tag>'
-
-    >>> elem.attrib["key"] = "<&\"\'>"
-    >>> elem.text = None
-    >>> serialize(elem)
-    '<tag key="&lt;&amp;&quot;\'&gt;" />'
-    >>> serialize(elem, encoding="utf-8")
-    b'<tag key="&lt;&amp;&quot;\'&gt;" />'
-    >>> serialize(elem, encoding="us-ascii")
-    b'<tag key="&lt;&amp;&quot;\'&gt;" />'
-    >>> serialize(elem, encoding="iso-8859-1")
-    b'<?xml version=\'1.0\' encoding=\'iso-8859-1\'?>\n<tag key="&lt;&amp;&quot;\'&gt;" />'
-
-    >>> elem.text = '\xe5\xf6\xf6<>'
-    >>> elem.attrib.clear()
-    >>> serialize(elem)
-    '<tag>\xe5\xf6\xf6&lt;&gt;</tag>'
-    >>> serialize(elem, encoding="utf-8")
-    b'<tag>\xc3\xa5\xc3\xb6\xc3\xb6&lt;&gt;</tag>'
-    >>> serialize(elem, encoding="us-ascii")
-    b'<tag>&#229;&#246;&#246;&lt;&gt;</tag>'
-    >>> serialize(elem, encoding="iso-8859-1")
-    b"<?xml version='1.0' encoding='iso-8859-1'?>\n<tag>\xe5\xf6\xf6&lt;&gt;</tag>"
-
-    >>> elem.attrib["key"] = '\xe5\xf6\xf6<>'
-    >>> elem.text = None
-    >>> serialize(elem)
-    '<tag key="\xe5\xf6\xf6&lt;&gt;" />'
-    >>> serialize(elem, encoding="utf-8")
-    b'<tag key="\xc3\xa5\xc3\xb6\xc3\xb6&lt;&gt;" />'
-    >>> serialize(elem, encoding="us-ascii")
-    b'<tag key="&#229;&#246;&#246;&lt;&gt;" />'
-    >>> serialize(elem, encoding="iso-8859-1")
-    b'<?xml version=\'1.0\' encoding=\'iso-8859-1\'?>\n<tag key="\xe5\xf6\xf6&lt;&gt;" />'
-    """
-
-def methods():
-    r"""
-    Test serialization methods.
-
-    >>> e = ET.XML("<html><link/><script>1 &lt; 2</script></html>")
-    >>> e.tail = "\n"
-    >>> serialize(e)
-    '<html><link /><script>1 &lt; 2</script></html>\n'
-    >>> serialize(e, method=None)
-    '<html><link /><script>1 &lt; 2</script></html>\n'
-    >>> serialize(e, method="xml")
-    '<html><link /><script>1 &lt; 2</script></html>\n'
-    >>> serialize(e, method="html")
-    '<html><link><script>1 < 2</script></html>\n'
-    >>> serialize(e, method="text")
-    '1 < 2\n'
-    """
-
-def iterators():
-    """
-    Test iterators.
-
-    >>> e = ET.XML("<html><body>this is a <i>paragraph</i>.</body>..</html>")
-    >>> summarize_list(e.iter())
-    ['html', 'body', 'i']
-    >>> summarize_list(e.find("body").iter())
-    ['body', 'i']
-    >>> summarize(next(e.iter()))
-    'html'
-    >>> "".join(e.itertext())
-    'this is a paragraph...'
-    >>> "".join(e.find("body").itertext())
-    'this is a paragraph.'
-    >>> next(e.itertext())
-    'this is a '
-
-    Method iterparse should return an iterator. See bug 6472.
-
-    >>> sourcefile = serialize(e, to_string=False)
-    >>> next(ET.iterparse(sourcefile))  # doctest: +ELLIPSIS
-    ('end', <Element 'i' at 0x...>)
-
-    >>> tree = ET.ElementTree(None)
-    >>> tree.iter()
-    Traceback (most recent call last):
-    AttributeError: 'NoneType' object has no attribute 'iter'
-    """
+class ElementTreeTest(unittest.TestCase):
+
+    def serialize_check(self, elem, expected):
+        self.assertEqual(serialize(elem), expected)
+
+    def test_interface(self):
+        # Test element tree interface.
+
+        def check_string(string):
+            len(string)
+            for char in string:
+                self.assertEqual(len(char), 1,
+                        msg="expected one-character string, got %r" % char)
+            new_string = string + ""
+            new_string = string + " "
+            string[:0]
+
+        def check_mapping(mapping):
+            len(mapping)
+            keys = mapping.keys()
+            items = mapping.items()
+            for key in keys:
+                item = mapping[key]
+            mapping["key"] = "value"
+            self.assertEqual(mapping["key"], "value",
+                    msg="expected value string, got %r" % mapping["key"])
+
+        def check_element(element):
+            self.assertTrue(ET.iselement(element), msg="not an element")
+            self.assertTrue(hasattr(element, "tag"), msg="no tag member")
+            self.assertTrue(hasattr(element, "attrib"), msg="no attrib member")
+            self.assertTrue(hasattr(element, "text"), msg="no text member")
+            self.assertTrue(hasattr(element, "tail"), msg="no tail member")
+
+            check_string(element.tag)
+            check_mapping(element.attrib)
+            if element.text is not None:
+                check_string(element.text)
+            if element.tail is not None:
+                check_string(element.tail)
+            for elem in element:
+                check_element(elem)
+
+        element = ET.Element("tag")
+        check_element(element)
+        tree = ET.ElementTree(element)
+        check_element(tree.getroot())
+        element = ET.Element("t\xe4g", key="value")
+        tree = ET.ElementTree(element)
+        self.assertRegex(repr(element), r"^<Element 't\xe4g' at 0x.*>$")
+        element = ET.Element("tag", key="value")
+
+        # Make sure all standard element methods exist.
+
+        def check_method(method):
+            self.assertTrue(hasattr(method, '__call__'),
+                    msg="%s not callable" % method)
+
+        check_method(element.append)
+        check_method(element.extend)
+        check_method(element.insert)
+        check_method(element.remove)
+        check_method(element.getchildren)
+        check_method(element.find)
+        check_method(element.iterfind)
+        check_method(element.findall)
+        check_method(element.findtext)
+        check_method(element.clear)
+        check_method(element.get)
+        check_method(element.set)
+        check_method(element.keys)
+        check_method(element.items)
+        check_method(element.iter)
+        check_method(element.itertext)
+        check_method(element.getiterator)
+
+        # These methods return an iterable. See bug 6472.
+
+        def check_iter(it):
+            check_method(it.__next__)
+
+        check_iter(element.iterfind("tag"))
+        check_iter(element.iterfind("*"))
+        check_iter(tree.iterfind("tag"))
+        check_iter(tree.iterfind("*"))
+
+        # These aliases are provided:
+
+        self.assertEqual(ET.XML, ET.fromstring)
+        self.assertEqual(ET.PI, ET.ProcessingInstruction)
+        self.assertEqual(ET.XMLParser, ET.XMLTreeBuilder)
+
+    def test_simpleops(self):
+        # Basic method sanity checks.
+
+        elem = ET.XML("<body><tag/></body>")
+        self.serialize_check(elem, '<body><tag /></body>')
+        e = ET.Element("tag2")
+        elem.append(e)
+        self.serialize_check(elem, '<body><tag /><tag2 /></body>')
+        elem.remove(e)
+        self.serialize_check(elem, '<body><tag /></body>')
+        elem.insert(0, e)
+        self.serialize_check(elem, '<body><tag2 /><tag /></body>')
+        elem.remove(e)
+        elem.extend([e])
+        self.serialize_check(elem, '<body><tag /><tag2 /></body>')
+        elem.remove(e)
+
+        element = ET.Element("tag", key="value")
+        self.serialize_check(element, '<tag key="value" />') # 1
+        subelement = ET.Element("subtag")
+        element.append(subelement)
+        self.serialize_check(element, '<tag key="value"><subtag /></tag>') # 2
+        element.insert(0, subelement)
+        self.serialize_check(element,
+                '<tag key="value"><subtag /><subtag /></tag>') # 3
+        element.remove(subelement)
+        self.serialize_check(element, '<tag key="value"><subtag /></tag>') # 4
+        element.remove(subelement)
+        self.serialize_check(element, '<tag key="value" />') # 5
+        with self.assertRaises(ValueError) as cm:
+            element.remove(subelement)
+        self.assertEqual(str(cm.exception), 'list.remove(x): x not in list')
+        self.serialize_check(element, '<tag key="value" />') # 6
+        element[0:0] = [subelement, subelement, subelement]
+        self.serialize_check(element[1], '<subtag />')
+        self.assertEqual(element[1:9], [element[1], element[2]])
+        self.assertEqual(element[:9:2], [element[0], element[2]])
+        del element[1:2]
+        self.serialize_check(element,
+                '<tag key="value"><subtag /><subtag /></tag>')
+
+    def test_cdata(self):
+        # Test CDATA handling (etc).
+
+        self.serialize_check(ET.XML("<tag>hello</tag>"),
+                '<tag>hello</tag>')
+        self.serialize_check(ET.XML("<tag>&#104;&#101;&#108;&#108;&#111;</tag>"),
+                '<tag>hello</tag>')
+        self.serialize_check(ET.XML("<tag><![CDATA[hello]]></tag>"),
+                '<tag>hello</tag>')
+
+    def test_file_init(self):
+        stringfile = io.BytesIO(SAMPLE_XML.encode("utf-8"))
+        tree = ET.ElementTree(file=stringfile)
+        self.assertEqual(tree.find("tag").tag, 'tag')
+        self.assertEqual(tree.find("section/tag").tag, 'tag')
+
+        tree = ET.ElementTree(file=SIMPLE_XMLFILE)
+        self.assertEqual(tree.find("element").tag, 'element')
+        self.assertEqual(tree.find("element/../empty-element").tag,
+                'empty-element')
+
+    def test_path_cache(self):
+        # Check that the path cache behaves sanely.
+
+        from xml.etree import ElementPath
+
+        elem = ET.XML(SAMPLE_XML)
+        for i in range(10): ET.ElementTree(elem).find('./'+str(i))
+        cache_len_10 = len(ElementPath._cache)
+        for i in range(10): ET.ElementTree(elem).find('./'+str(i))
+        self.assertEqual(len(ElementPath._cache), cache_len_10)
+        for i in range(20): ET.ElementTree(elem).find('./'+str(i))
+        self.assertGreater(len(ElementPath._cache), cache_len_10)
+        for i in range(600): ET.ElementTree(elem).find('./'+str(i))
+        self.assertLess(len(ElementPath._cache), 500)
+
+    def test_copy(self):
+        # Test copy handling (etc).
+
+        import copy
+        e1 = ET.XML("<tag>hello<foo/></tag>")
+        e2 = copy.copy(e1)
+        e3 = copy.deepcopy(e1)
+        e1.find("foo").tag = "bar"
+        self.serialize_check(e1, '<tag>hello<bar /></tag>')
+        self.serialize_check(e2, '<tag>hello<bar /></tag>')
+        self.serialize_check(e3, '<tag>hello<foo /></tag>')
+
+    def test_attrib(self):
+        # Test attribute handling.
+
+        elem = ET.Element("tag")
+        elem.get("key") # 1.1
+        self.assertEqual(elem.get("key", "default"), 'default') # 1.2
+
+        elem.set("key", "value")
+        self.assertEqual(elem.get("key"), 'value') # 1.3
+
+        elem = ET.Element("tag", key="value")
+        self.assertEqual(elem.get("key"), 'value') # 2.1
+        self.assertEqual(elem.attrib, {'key': 'value'}) # 2.2
+
+        attrib = {"key": "value"}
+        elem = ET.Element("tag", attrib)
+        attrib.clear() # check for aliasing issues
+        self.assertEqual(elem.get("key"), 'value') # 3.1
+        self.assertEqual(elem.attrib, {'key': 'value'}) # 3.2
+
+        attrib = {"key": "value"}
+        elem = ET.Element("tag", **attrib)
+        attrib.clear() # check for aliasing issues
+        self.assertEqual(elem.get("key"), 'value') # 4.1
+        self.assertEqual(elem.attrib, {'key': 'value'}) # 4.2
+
+        elem = ET.Element("tag", {"key": "other"}, key="value")
+        self.assertEqual(elem.get("key"), 'value') # 5.1
+        self.assertEqual(elem.attrib, {'key': 'value'}) # 5.2
+
+        elem = ET.Element('test')
+        elem.text = "aa"
+        elem.set('testa', 'testval')
+        elem.set('testb', 'test2')
+        self.assertEqual(ET.tostring(elem),
+                b'<test testa="testval" testb="test2">aa</test>')
+        self.assertEqual(sorted(elem.keys()), ['testa', 'testb'])
+        self.assertEqual(sorted(elem.items()),
+                [('testa', 'testval'), ('testb', 'test2')])
+        self.assertEqual(elem.attrib['testb'], 'test2')
+        elem.attrib['testb'] = 'test1'
+        elem.attrib['testc'] = 'test2'
+        self.assertEqual(ET.tostring(elem),
+                b'<test testa="testval" testb="test1" testc="test2">aa</test>')
+
+    def test_makeelement(self):
+        # Test makeelement handling.
+
+        elem = ET.Element("tag")
+        attrib = {"key": "value"}
+        subelem = elem.makeelement("subtag", attrib)
+        self.assertIsNot(subelem.attrib, attrib, msg="attrib aliasing")
+        elem.append(subelem)
+        self.serialize_check(elem, '<tag><subtag key="value" /></tag>')
+
+        elem.clear()
+        self.serialize_check(elem, '<tag />')
+        elem.append(subelem)
+        self.serialize_check(elem, '<tag><subtag key="value" /></tag>')
+        elem.extend([subelem, subelem])
+        self.serialize_check(elem,
+            '<tag><subtag key="value" /><subtag key="value" /><subtag key="value" /></tag>')
+        elem[:] = [subelem]
+        self.serialize_check(elem, '<tag><subtag key="value" /></tag>')
+        elem[:] = tuple([subelem])
+        self.serialize_check(elem, '<tag><subtag key="value" /></tag>')
+
+    def test_parsefile(self):
+        # Test parsing from file.
+
+        tree = ET.parse(SIMPLE_XMLFILE)
+        stream = io.StringIO()
+        tree.write(stream, encoding='unicode')
+        self.assertEqual(stream.getvalue(),
+                '<root>\n'
+                '   <element key="value">text</element>\n'
+                '   <element>text</element>tail\n'
+                '   <empty-element />\n'
+                '</root>')
+        tree = ET.parse(SIMPLE_NS_XMLFILE)
+        stream = io.StringIO()
+        tree.write(stream, encoding='unicode')
+        self.assertEqual(stream.getvalue(),
+                '<ns0:root xmlns:ns0="namespace">\n'
+                '   <ns0:element key="value">text</ns0:element>\n'
+                '   <ns0:element>text</ns0:element>tail\n'
+                '   <ns0:empty-element />\n'
+                '</ns0:root>')
+
+        with open(SIMPLE_XMLFILE) as f:
+            data = f.read()
+
+        parser = ET.XMLParser()
+        self.assertRegex(parser.version, r'^Expat ')
+        parser.feed(data)
+        self.serialize_check(parser.close(),
+                '<root>\n'
+                '   <element key="value">text</element>\n'
+                '   <element>text</element>tail\n'
+                '   <empty-element />\n'
+                '</root>')
+
+        parser = ET.XMLTreeBuilder() # 1.2 compatibility
+        parser.feed(data)
+        self.serialize_check(parser.close(),
+                '<root>\n'
+                '   <element key="value">text</element>\n'
+                '   <element>text</element>tail\n'
+                '   <empty-element />\n'
+                '</root>')
+
+        target = ET.TreeBuilder()
+        parser = ET.XMLParser(target=target)
+        parser.feed(data)
+        self.serialize_check(parser.close(),
+                '<root>\n'
+                '   <element key="value">text</element>\n'
+                '   <element>text</element>tail\n'
+                '   <empty-element />\n'
+                '</root>')
+
+    def test_parseliteral(self):
+        element = ET.XML("<html><body>text</body></html>")
+        self.assertEqual(ET.tostring(element, encoding='unicode'),
+                '<html><body>text</body></html>')
+        element = ET.fromstring("<html><body>text</body></html>")
+        self.assertEqual(ET.tostring(element, encoding='unicode'),
+                '<html><body>text</body></html>')
+        sequence = ["<html><body>", "text</bo", "dy></html>"]
+        element = ET.fromstringlist(sequence)
+        self.assertEqual(ET.tostring(element),
+                b'<html><body>text</body></html>')
+        self.assertEqual(b"".join(ET.tostringlist(element)),
+                b'<html><body>text</body></html>')
+        self.assertEqual(ET.tostring(element, "ascii"),
+                b"<?xml version='1.0' encoding='ascii'?>\n"
+                b"<html><body>text</body></html>")
+        _, ids = ET.XMLID("<html><body>text</body></html>")
+        self.assertEqual(len(ids), 0)
+        _, ids = ET.XMLID("<html><body id='body'>text</body></html>")
+        self.assertEqual(len(ids), 1)
+        self.assertEqual(ids["body"].tag, 'body')
+
+    def test_iterparse(self):
+        # Test iterparse interface.
+
+        iterparse = ET.iterparse
+
+        context = iterparse(SIMPLE_XMLFILE)
+        action, elem = next(context)
+        self.assertEqual((action, elem.tag), ('end', 'element'))
+        self.assertEqual([(action, elem.tag) for action, elem in context], [
+                ('end', 'element'),
+                ('end', 'empty-element'),
+                ('end', 'root'),
+            ])
+        self.assertEqual(context.root.tag, 'root')
+
+        context = iterparse(SIMPLE_NS_XMLFILE)
+        self.assertEqual([(action, elem.tag) for action, elem in context], [
+                ('end', '{namespace}element'),
+                ('end', '{namespace}element'),
+                ('end', '{namespace}empty-element'),
+                ('end', '{namespace}root'),
+            ])
+
+        events = ()
+        context = iterparse(SIMPLE_XMLFILE, events)
+        self.assertEqual([(action, elem.tag) for action, elem in context], [])
+
+        events = ()
+        context = iterparse(SIMPLE_XMLFILE, events=events)
+        self.assertEqual([(action, elem.tag) for action, elem in context], [])
+
+        events = ("start", "end")
+        context = iterparse(SIMPLE_XMLFILE, events)
+        self.assertEqual([(action, elem.tag) for action, elem in context], [
+                ('start', 'root'),
+                ('start', 'element'),
+                ('end', 'element'),
+                ('start', 'element'),
+                ('end', 'element'),
+                ('start', 'empty-element'),
+                ('end', 'empty-element'),
+                ('end', 'root'),
+            ])
+
+        events = ("start", "end", "start-ns", "end-ns")
+        context = iterparse(SIMPLE_NS_XMLFILE, events)
+        self.assertEqual([(action, elem.tag) if action in ("start", "end")
+                                             else (action, elem)
+                          for action, elem in context], [
+                ('start-ns', ('', 'namespace')),
+                ('start', '{namespace}root'),
+                ('start', '{namespace}element'),
+                ('end', '{namespace}element'),
+                ('start', '{namespace}element'),
+                ('end', '{namespace}element'),
+                ('start', '{namespace}empty-element'),
+                ('end', '{namespace}empty-element'),
+                ('end', '{namespace}root'),
+                ('end-ns', None),
+            ])
+
+        events = ("start", "end", "bogus")
+        with self.assertRaises(ValueError) as cm:
+            with open(SIMPLE_XMLFILE, "rb") as f:
+                iterparse(f, events)
+        self.assertEqual(str(cm.exception), "unknown event 'bogus'")
+
+        source = io.BytesIO(
+            b"<?xml version='1.0' encoding='iso-8859-1'?>\n"
+            b"<body xmlns='http://&#233;ffbot.org/ns'\n"
+            b"      xmlns:cl\xe9='http://effbot.org/ns'>text</body>\n")
+        events = ("start-ns",)
+        context = iterparse(source, events)
+        self.assertEqual([(action, elem) for action, elem in context], [
+                ('start-ns', ('', 'http://\xe9ffbot.org/ns')),
+                ('start-ns', ('cl\xe9', 'http://effbot.org/ns')),
+            ])
+
+        source = io.StringIO("<document />junk")
+        it = iterparse(source)
+        action, elem = next(it)
+        self.assertEqual((action, elem.tag), ('end', 'document'))
+        with self.assertRaises(ET.ParseError) as cm:
+            next(it)
+        self.assertEqual(str(cm.exception),
+                'junk after document element: line 1, column 12')
+
+    def test_writefile(self):
+        elem = ET.Element("tag")
+        elem.text = "text"
+        self.serialize_check(elem, '<tag>text</tag>')
+        ET.SubElement(elem, "subtag").text = "subtext"
+        self.serialize_check(elem, '<tag>text<subtag>subtext</subtag></tag>')
+
+        # Test tag suppression
+        elem.tag = None
+        self.serialize_check(elem, 'text<subtag>subtext</subtag>')
+        elem.insert(0, ET.Comment("comment"))
+        self.serialize_check(elem,
+                'text<!--comment--><subtag>subtext</subtag>')     # assumes 1.3
+
+        elem[0] = ET.PI("key", "value")
+        self.serialize_check(elem, 'text<?key value?><subtag>subtext</subtag>')
+
+    def test_custom_builder(self):
+        # Test parser w. custom builder.
+
+        with open(SIMPLE_XMLFILE) as f:
+            data = f.read()
+        class Builder(list):
+            def start(self, tag, attrib):
+                self.append(("start", tag))
+            def end(self, tag):
+                self.append(("end", tag))
+            def data(self, text):
+                pass
+        builder = Builder()
+        parser = ET.XMLParser(target=builder)
+        parser.feed(data)
+        self.assertEqual(builder, [
+                ('start', 'root'),
+                ('start', 'element'),
+                ('end', 'element'),
+                ('start', 'element'),
+                ('end', 'element'),
+                ('start', 'empty-element'),
+                ('end', 'empty-element'),
+                ('end', 'root'),
+            ])
+
+        with open(SIMPLE_NS_XMLFILE) as f:
+            data = f.read()
+        class Builder(list):
+            def start(self, tag, attrib):
+                self.append(("start", tag))
+            def end(self, tag):
+                self.append(("end", tag))
+            def data(self, text):
+                pass
+            def pi(self, target, data):
+                self.append(("pi", target, data))
+            def comment(self, data):
+                self.append(("comment", data))
+        builder = Builder()
+        parser = ET.XMLParser(target=builder)
+        parser.feed(data)
+        self.assertEqual(builder, [
+                ('pi', 'pi', 'data'),
+                ('comment', ' comment '),
+                ('start', '{namespace}root'),
+                ('start', '{namespace}element'),
+                ('end', '{namespace}element'),
+                ('start', '{namespace}element'),
+                ('end', '{namespace}element'),
+                ('start', '{namespace}empty-element'),
+                ('end', '{namespace}empty-element'),
+                ('end', '{namespace}root'),
+            ])
+
+
+    def test_getchildren(self):
+        # Test Element.getchildren()
+
+        with open(SIMPLE_XMLFILE, "rb") as f:
+            tree = ET.parse(f)
+        self.assertEqual([summarize_list(elem.getchildren())
+                          for elem in tree.getroot().iter()], [
+                ['element', 'element', 'empty-element'],
+                [],
+                [],
+                [],
+            ])
+        self.assertEqual([summarize_list(elem.getchildren())
+                          for elem in tree.getiterator()], [
+                ['element', 'element', 'empty-element'],
+                [],
+                [],
+                [],
+            ])
+
+        elem = ET.XML(SAMPLE_XML)
+        self.assertEqual(len(elem.getchildren()), 3)
+        self.assertEqual(len(elem[2].getchildren()), 1)
+        self.assertEqual(elem[:], elem.getchildren())
+        child1 = elem[0]
+        child2 = elem[2]
+        del elem[1:2]
+        self.assertEqual(len(elem.getchildren()), 2)
+        self.assertEqual(child1, elem[0])
+        self.assertEqual(child2, elem[1])
+        elem[0:2] = [child2, child1]
+        self.assertEqual(child2, elem[0])
+        self.assertEqual(child1, elem[1])
+        self.assertNotEqual(child1, elem[0])
+        elem.clear()
+        self.assertEqual(elem.getchildren(), [])
+
+    def test_writestring(self):
+        elem = ET.XML("<html><body>text</body></html>")
+        self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>')
+        elem = ET.fromstring("<html><body>text</body></html>")
+        self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>')
+
+    def test_encoding(encoding):
+        def check(encoding):
+            ET.XML("<?xml version='1.0' encoding='%s'?><xml />" % encoding)
+        check("ascii")
+        check("us-ascii")
+        check("iso-8859-1")
+        check("iso-8859-15")
+        check("cp437")
+        check("mac-roman")
+
+    def test_methods(self):
+        # Test serialization methods.
+
+        e = ET.XML("<html><link/><script>1 &lt; 2</script></html>")
+        e.tail = "\n"
+        self.assertEqual(serialize(e),
+                '<html><link /><script>1 &lt; 2</script></html>\n')
+        self.assertEqual(serialize(e, method=None),
+                '<html><link /><script>1 &lt; 2</script></html>\n')
+        self.assertEqual(serialize(e, method="xml"),
+                '<html><link /><script>1 &lt; 2</script></html>\n')
+        self.assertEqual(serialize(e, method="html"),
+                '<html><link><script>1 < 2</script></html>\n')
+        self.assertEqual(serialize(e, method="text"), '1 < 2\n')
+
+    def test_entity(self):
+        # Test entity handling.
+
+        # 1) good entities
+
+        e = ET.XML("<document title='&#x8230;'>test</document>")
+        self.assertEqual(serialize(e, encoding="us-ascii"),
+                b'<document title="&#33328;">test</document>')
+        self.serialize_check(e, '<document title="\u8230">test</document>')
+
+        # 2) bad entities
+
+        with self.assertRaises(ET.ParseError) as cm:
+            ET.XML("<document>&entity;</document>")
+        self.assertEqual(str(cm.exception),
+                'undefined entity: line 1, column 10')
+
+        with self.assertRaises(ET.ParseError) as cm:
+            ET.XML(ENTITY_XML)
+        self.assertEqual(str(cm.exception),
+                'undefined entity &entity;: line 5, column 10')
+
+        # 3) custom entity
+
+        parser = ET.XMLParser()
+        parser.entity["entity"] = "text"
+        parser.feed(ENTITY_XML)
+        root = parser.close()
+        self.serialize_check(root, '<document>text</document>')
+
+    def test_namespace(self):
+        # Test namespace issues.
+
+        # 1) xml namespace
+
+        elem = ET.XML("<tag xml:lang='en' />")
+        self.serialize_check(elem, '<tag xml:lang="en" />') # 1.1
+
+        # 2) other "well-known" namespaces
+
+        elem = ET.XML("<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' />")
+        self.serialize_check(elem,
+            '<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" />') # 2.1
+
+        elem = ET.XML("<html:html xmlns:html='http://www.w3.org/1999/xhtml' />")
+        self.serialize_check(elem,
+            '<html:html xmlns:html="http://www.w3.org/1999/xhtml" />') # 2.2
+
+        elem = ET.XML("<soap:Envelope xmlns:soap='http://schemas.xmlsoap.org/soap/envelope' />")
+        self.serialize_check(elem,
+            '<ns0:Envelope xmlns:ns0="http://schemas.xmlsoap.org/soap/envelope" />') # 2.3
+
+        # 3) unknown namespaces
+        elem = ET.XML(SAMPLE_XML_NS)
+        self.serialize_check(elem,
+            '<ns0:body xmlns:ns0="http://effbot.org/ns">\n'
+            '  <ns0:tag>text</ns0:tag>\n'
+            '  <ns0:tag />\n'
+            '  <ns0:section>\n'
+            '    <ns0:tag>subtext</ns0:tag>\n'
+            '  </ns0:section>\n'
+            '</ns0:body>')
+
+    def test_qname(self):
+        # Test QName handling.
+
+        # 1) decorated tags
+
+        elem = ET.Element("{uri}tag")
+        self.serialize_check(elem, '<ns0:tag xmlns:ns0="uri" />') # 1.1
+        elem = ET.Element(ET.QName("{uri}tag"))
+        self.serialize_check(elem, '<ns0:tag xmlns:ns0="uri" />') # 1.2
+        elem = ET.Element(ET.QName("uri", "tag"))
+        self.serialize_check(elem, '<ns0:tag xmlns:ns0="uri" />') # 1.3
+        elem = ET.Element(ET.QName("uri", "tag"))
+        subelem = ET.SubElement(elem, ET.QName("uri", "tag1"))
+        subelem = ET.SubElement(elem, ET.QName("uri", "tag2"))
+        self.serialize_check(elem,
+            '<ns0:tag xmlns:ns0="uri"><ns0:tag1 /><ns0:tag2 /></ns0:tag>') # 1.4
+
+        # 2) decorated attributes
+
+        elem.clear()
+        elem.attrib["{uri}key"] = "value"
+        self.serialize_check(elem,
+            '<ns0:tag xmlns:ns0="uri" ns0:key="value" />') # 2.1
+
+        elem.clear()
+        elem.attrib[ET.QName("{uri}key")] = "value"
+        self.serialize_check(elem,
+            '<ns0:tag xmlns:ns0="uri" ns0:key="value" />') # 2.2
+
+        # 3) decorated values are not converted by default, but the
+        # QName wrapper can be used for values
+
+        elem.clear()
+        elem.attrib["{uri}key"] = "{uri}value"
+        self.serialize_check(elem,
+            '<ns0:tag xmlns:ns0="uri" ns0:key="{uri}value" />') # 3.1
+
+        elem.clear()
+        elem.attrib["{uri}key"] = ET.QName("{uri}value")
+        self.serialize_check(elem,
+            '<ns0:tag xmlns:ns0="uri" ns0:key="ns0:value" />') # 3.2
+
+        elem.clear()
+        subelem = ET.Element("tag")
+        subelem.attrib["{uri1}key"] = ET.QName("{uri2}value")
+        elem.append(subelem)
+        elem.append(subelem)
+        self.serialize_check(elem,
+            '<ns0:tag xmlns:ns0="uri" xmlns:ns1="uri1" xmlns:ns2="uri2">'
+            '<tag ns1:key="ns2:value" />'
+            '<tag ns1:key="ns2:value" />'
+            '</ns0:tag>') # 3.3
+
+        # 4) Direct QName tests
+
+        self.assertEqual(str(ET.QName('ns', 'tag')), '{ns}tag')
+        self.assertEqual(str(ET.QName('{ns}tag')), '{ns}tag')
+        q1 = ET.QName('ns', 'tag')
+        q2 = ET.QName('ns', 'tag')
+        self.assertEqual(q1, q2)
+        q2 = ET.QName('ns', 'other-tag')
+        self.assertNotEqual(q1, q2)
+        self.assertNotEqual(q1, 'ns:tag')
+        self.assertEqual(q1, '{ns}tag')
+
+    def test_doctype_public(self):
+        # Test PUBLIC doctype.
+
+        elem = ET.XML('<!DOCTYPE html PUBLIC'
+                ' "-//W3C//DTD XHTML 1.0 Transitional//EN"'
+                ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'
+                '<html>text</html>')
+
+    def test_xpath_tokenizer(self):
+        # Test the XPath tokenizer.
+        from xml.etree import ElementPath
+        def check(p, expected):
+            self.assertEqual([op or tag
+                              for op, tag in ElementPath.xpath_tokenizer(p)],
+                             expected)
+
+        # tests from the xml specification
+        check("*", ['*'])
+        check("text()", ['text', '()'])
+        check("@name", ['@', 'name'])
+        check("@*", ['@', '*'])
+        check("para[1]", ['para', '[', '1', ']'])
+        check("para[last()]", ['para', '[', 'last', '()', ']'])
+        check("*/para", ['*', '/', 'para'])
+        check("/doc/chapter[5]/section[2]",
+              ['/', 'doc', '/', 'chapter', '[', '5', ']',
+               '/', 'section', '[', '2', ']'])
+        check("chapter//para", ['chapter', '//', 'para'])
+        check("//para", ['//', 'para'])
+        check("//olist/item", ['//', 'olist', '/', 'item'])
+        check(".", ['.'])
+        check(".//para", ['.', '//', 'para'])
+        check("..", ['..'])
+        check("../@lang", ['..', '/', '@', 'lang'])
+        check("chapter[title]", ['chapter', '[', 'title', ']'])
+        check("employee[@secretary and @assistant]", ['employee',
+              '[', '@', 'secretary', '', 'and', '', '@', 'assistant', ']'])
+
+        # additional tests
+        check("{http://spam}egg", ['{http://spam}egg'])
+        check("./spam.egg", ['.', '/', 'spam.egg'])
+        check(".//{http://spam}egg", ['.', '//', '{http://spam}egg'])
+
+    def test_processinginstruction(self):
+        # Test ProcessingInstruction directly
+
+        self.assertEqual(ET.tostring(ET.ProcessingInstruction('test', 'instruction')),
+                b'<?test instruction?>')
+        self.assertEqual(ET.tostring(ET.PI('test', 'instruction')),
+                b'<?test instruction?>')
+
+        # Issue #2746
+
+        self.assertEqual(ET.tostring(ET.PI('test', '<testing&>')),
+                b'<?test <testing&>?>')
+        self.assertEqual(ET.tostring(ET.PI('test', '<testing&>\xe3'), 'latin-1'),
+                b"<?xml version='1.0' encoding='latin-1'?>\n"
+                b"<?test <testing&>\xe3?>")
+
+    def test_html_empty_elems_serialization(self):
+        # issue 15970
+        # from http://www.w3.org/TR/html401/index/elements.html
+        for element in ['AREA', 'BASE', 'BASEFONT', 'BR', 'COL', 'FRAME', 'HR',
+                        'IMG', 'INPUT', 'ISINDEX', 'LINK', 'META', 'PARAM']:
+            for elem in [element, element.lower()]:
+                expected = '<%s>' % elem
+                serialized = serialize(ET.XML('<%s />' % elem), method='html')
+                self.assertEqual(serialized, expected)
+                serialized = serialize(ET.XML('<%s></%s>' % (elem,elem)),
+                                       method='html')
+                self.assertEqual(serialized, expected)
 
-ENTITY_XML = """\
-<!DOCTYPE points [
-<!ENTITY % user-entities SYSTEM 'user-entities.xml'>
-%user-entities;
-]>
-<document>&entity;</document>
-"""
-
-def entity():
-    """
-    Test entity handling.
-
-    1) good entities
-
-    >>> e = ET.XML("<document title='&#x8230;'>test</document>")
-    >>> serialize(e, encoding="us-ascii")
-    b'<document title="&#33328;">test</document>'
-    >>> serialize(e)
-    '<document title="\u8230">test</document>'
-
-    2) bad entities
-
-    >>> normalize_exception(ET.XML, "<document>&entity;</document>")
-    Traceback (most recent call last):
-    ParseError: undefined entity: line 1, column 10
-
-    >>> normalize_exception(ET.XML, ENTITY_XML)
-    Traceback (most recent call last):
-    ParseError: undefined entity &entity;: line 5, column 10
-
-    3) custom entity
-
-    >>> parser = ET.XMLParser()
-    >>> parser.entity["entity"] = "text"
-    >>> parser.feed(ENTITY_XML)
-    >>> root = parser.close()
-    >>> serialize(root)
-    '<document>text</document>'
-    """
-
-def error(xml):
-    """
-
-    Test error handling.
-
-    >>> issubclass(ET.ParseError, SyntaxError)
-    True
-    >>> error("foo").position
-    (1, 0)
-    >>> error("<tag>&foo;</tag>").position
-    (1, 5)
-    >>> error("foobar<").position
-    (1, 6)
-
-    """
-    try:
-        ET.XML(xml)
-    except ET.ParseError:
-        return sys.exc_info()[1]
-
-def namespace():
-    """
-    Test namespace issues.
-
-    1) xml namespace
-
-    >>> elem = ET.XML("<tag xml:lang='en' />")
-    >>> serialize(elem) # 1.1
-    '<tag xml:lang="en" />'
-
-    2) other "well-known" namespaces
-
-    >>> elem = ET.XML("<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' />")
-    >>> serialize(elem) # 2.1
-    '<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" />'
-
-    >>> elem = ET.XML("<html:html xmlns:html='http://www.w3.org/1999/xhtml' />")
-    >>> serialize(elem) # 2.2
-    '<html:html xmlns:html="http://www.w3.org/1999/xhtml" />'
-
-    >>> elem = ET.XML("<soap:Envelope xmlns:soap='http://schemas.xmlsoap.org/soap/envelope' />")
-    >>> serialize(elem) # 2.3
-    '<ns0:Envelope xmlns:ns0="http://schemas.xmlsoap.org/soap/envelope" />'
-
-    3) unknown namespaces
-    >>> elem = ET.XML(SAMPLE_XML_NS)
-    >>> print(serialize(elem))
-    <ns0:body xmlns:ns0="http://effbot.org/ns">
-      <ns0:tag>text</ns0:tag>
-      <ns0:tag />
-      <ns0:section>
-        <ns0:tag>subtext</ns0:tag>
-      </ns0:section>
-    </ns0:body>
-    """
-
-def qname():
-    """
-    Test QName handling.
-
-    1) decorated tags
-
-    >>> elem = ET.Element("{uri}tag")
-    >>> serialize(elem) # 1.1
-    '<ns0:tag xmlns:ns0="uri" />'
-    >>> elem = ET.Element(ET.QName("{uri}tag"))
-    >>> serialize(elem) # 1.2
-    '<ns0:tag xmlns:ns0="uri" />'
-    >>> elem = ET.Element(ET.QName("uri", "tag"))
-    >>> serialize(elem) # 1.3
-    '<ns0:tag xmlns:ns0="uri" />'
-    >>> elem = ET.Element(ET.QName("uri", "tag"))
-    >>> subelem = ET.SubElement(elem, ET.QName("uri", "tag1"))
-    >>> subelem = ET.SubElement(elem, ET.QName("uri", "tag2"))
-    >>> serialize(elem) # 1.4
-    '<ns0:tag xmlns:ns0="uri"><ns0:tag1 /><ns0:tag2 /></ns0:tag>'
-
-    2) decorated attributes
-
-    >>> elem.clear()
-    >>> elem.attrib["{uri}key"] = "value"
-    >>> serialize(elem) # 2.1
-    '<ns0:tag xmlns:ns0="uri" ns0:key="value" />'
-
-    >>> elem.clear()
-    >>> elem.attrib[ET.QName("{uri}key")] = "value"
-    >>> serialize(elem) # 2.2
-    '<ns0:tag xmlns:ns0="uri" ns0:key="value" />'
-
-    3) decorated values are not converted by default, but the
-       QName wrapper can be used for values
-
-    >>> elem.clear()
-    >>> elem.attrib["{uri}key"] = "{uri}value"
-    >>> serialize(elem) # 3.1
-    '<ns0:tag xmlns:ns0="uri" ns0:key="{uri}value" />'
-
-    >>> elem.clear()
-    >>> elem.attrib["{uri}key"] = ET.QName("{uri}value")
-    >>> serialize(elem) # 3.2
-    '<ns0:tag xmlns:ns0="uri" ns0:key="ns0:value" />'
-
-    >>> elem.clear()
-    >>> subelem = ET.Element("tag")
-    >>> subelem.attrib["{uri1}key"] = ET.QName("{uri2}value")
-    >>> elem.append(subelem)
-    >>> elem.append(subelem)
-    >>> serialize(elem) # 3.3
-    '<ns0:tag xmlns:ns0="uri" xmlns:ns1="uri1" xmlns:ns2="uri2"><tag ns1:key="ns2:value" /><tag ns1:key="ns2:value" /></ns0:tag>'
-
-    4) Direct QName tests
-
-    >>> str(ET.QName('ns', 'tag'))
-    '{ns}tag'
-    >>> str(ET.QName('{ns}tag'))
-    '{ns}tag'
-    >>> q1 = ET.QName('ns', 'tag')
-    >>> q2 = ET.QName('ns', 'tag')
-    >>> q1 == q2
-    True
-    >>> q2 = ET.QName('ns', 'other-tag')
-    >>> q1 == q2
-    False
-    >>> q1 == 'ns:tag'
-    False
-    >>> q1 == '{ns}tag'
-    True
-    """
-
-def doctype_public():
-    """
-    Test PUBLIC doctype.
-
-    >>> elem = ET.XML('<!DOCTYPE html PUBLIC'
-    ...   ' "-//W3C//DTD XHTML 1.0 Transitional//EN"'
-    ...   ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'
-    ...   '<html>text</html>')
-
-    """
-
-def xpath_tokenizer(p):
-    """
-    Test the XPath tokenizer.
-
-    >>> # tests from the xml specification
-    >>> xpath_tokenizer("*")
-    ['*']
-    >>> xpath_tokenizer("text()")
-    ['text', '()']
-    >>> xpath_tokenizer("@name")
-    ['@', 'name']
-    >>> xpath_tokenizer("@*")
-    ['@', '*']
-    >>> xpath_tokenizer("para[1]")
-    ['para', '[', '1', ']']
-    >>> xpath_tokenizer("para[last()]")
-    ['para', '[', 'last', '()', ']']
-    >>> xpath_tokenizer("*/para")
-    ['*', '/', 'para']
-    >>> xpath_tokenizer("/doc/chapter[5]/section[2]")
-    ['/', 'doc', '/', 'chapter', '[', '5', ']', '/', 'section', '[', '2', ']']
-    >>> xpath_tokenizer("chapter//para")
-    ['chapter', '//', 'para']
-    >>> xpath_tokenizer("//para")
-    ['//', 'para']
-    >>> xpath_tokenizer("//olist/item")
-    ['//', 'olist', '/', 'item']
-    >>> xpath_tokenizer(".")
-    ['.']
-    >>> xpath_tokenizer(".//para")
-    ['.', '//', 'para']
-    >>> xpath_tokenizer("..")
-    ['..']
-    >>> xpath_tokenizer("../@lang")
-    ['..', '/', '@', 'lang']
-    >>> xpath_tokenizer("chapter[title]")
-    ['chapter', '[', 'title', ']']
-    >>> xpath_tokenizer("employee[@secretary and @assistant]")
-    ['employee', '[', '@', 'secretary', '', 'and', '', '@', 'assistant', ']']
-
-    >>> # additional tests
-    >>> xpath_tokenizer("{http://spam}egg")
-    ['{http://spam}egg']
-    >>> xpath_tokenizer("./spam.egg")
-    ['.', '/', 'spam.egg']
-    >>> xpath_tokenizer(".//{http://spam}egg")
-    ['.', '//', '{http://spam}egg']
-    """
-    from xml.etree import ElementPath
-    out = []
-    for op, tag in ElementPath.xpath_tokenizer(p):
-        out.append(op or tag)
-    return out
-
-def processinginstruction():
-    """
-    Test ProcessingInstruction directly
-
-    >>> ET.tostring(ET.ProcessingInstruction('test', 'instruction'))
-    b'<?test instruction?>'
-    >>> ET.tostring(ET.PI('test', 'instruction'))
-    b'<?test instruction?>'
-
-    Issue #2746
-
-    >>> ET.tostring(ET.PI('test', '<testing&>'))
-    b'<?test <testing&>?>'
-    >>> ET.tostring(ET.PI('test', '<testing&>\xe3'), 'latin1')
-    b"<?xml version='1.0' encoding='latin1'?>\\n<?test <testing&>\\xe3?>"
-    """
 
 #
 # xinclude tests (samples from appendix C of the xinclude specification)
@@ -1338,96 +979,6 @@ XINCLUDE["default.xml"] = """\
 </document>
 """.format(html.escape(SIMPLE_XMLFILE, True))
 
-def xinclude_loader(href, parse="xml", encoding=None):
-    try:
-        data = XINCLUDE[href]
-    except KeyError:
-        raise IOError("resource not found")
-    if parse == "xml":
-        from xml.etree.ElementTree import XML
-        return XML(data)
-    return data
-
-def xinclude():
-    r"""
-    Basic inclusion example (XInclude C.1)
-
-    >>> from xml.etree import ElementTree as ET
-    >>> from xml.etree import ElementInclude
-
-    >>> document = xinclude_loader("C1.xml")
-    >>> ElementInclude.include(document, xinclude_loader)
-    >>> print(serialize(document)) # C1
-    <document>
-      <p>120 Mz is adequate for an average home user.</p>
-      <disclaimer>
-      <p>The opinions represented herein represent those of the individual
-      and should not be interpreted as official policy endorsed by this
-      organization.</p>
-    </disclaimer>
-    </document>
-
-    Textual inclusion example (XInclude C.2)
-
-    >>> document = xinclude_loader("C2.xml")
-    >>> ElementInclude.include(document, xinclude_loader)
-    >>> print(serialize(document)) # C2
-    <document>
-      <p>This document has been accessed
-      324387 times.</p>
-    </document>
-
-    Textual inclusion after sibling element (based on modified XInclude C.2)
-
-    >>> document = xinclude_loader("C2b.xml")
-    >>> ElementInclude.include(document, xinclude_loader)
-    >>> print(serialize(document)) # C2b
-    <document>
-      <p>This document has been <em>accessed</em>
-      324387 times.</p>
-    </document>
-
-    Textual inclusion of XML example (XInclude C.3)
-
-    >>> document = xinclude_loader("C3.xml")
-    >>> ElementInclude.include(document, xinclude_loader)
-    >>> print(serialize(document)) # C3
-    <document>
-      <p>The following is the source of the "data.xml" resource:</p>
-      <example>&lt;?xml version='1.0'?&gt;
-    &lt;data&gt;
-      &lt;item&gt;&lt;![CDATA[Brooks &amp; Shields]]&gt;&lt;/item&gt;
-    &lt;/data&gt;
-    </example>
-    </document>
-
-    Fallback example (XInclude C.5)
-    Note! Fallback support is not yet implemented
-
-    >>> document = xinclude_loader("C5.xml")
-    >>> ElementInclude.include(document, xinclude_loader)
-    Traceback (most recent call last):
-    IOError: resource not found
-    >>> # print(serialize(document)) # C5
-    """
-
-def xinclude_default():
-    """
-    >>> from xml.etree import ElementInclude
-
-    >>> document = xinclude_loader("default.xml")
-    >>> ElementInclude.include(document)
-    >>> print(serialize(document)) # default
-    <document>
-      <p>Example.</p>
-      <root>
-       <element key="value">text</element>
-       <element>text</element>tail
-       <empty-element />
-    </root>
-    </document>
-    """
-
 #
 # badly formatted xi:include tags
 
@@ -1448,430 +999,1230 @@ XINCLUDE_BAD["B2.xml"] = """\
 </div>
 """
 
-def xinclude_failures():
-    r"""
-    Test failure to locate included XML file.
-
-    >>> from xml.etree import ElementInclude
-
-    >>> def none_loader(href, parser, encoding=None):
-    ...     return None
-
-    >>> document = ET.XML(XINCLUDE["C1.xml"])
-    >>> ElementInclude.include(document, loader=none_loader)
-    Traceback (most recent call last):
-    xml.etree.ElementInclude.FatalIncludeError: cannot load 'disclaimer.xml' as 'xml'
-
-    Test failure to locate included text file.
-
-    >>> document = ET.XML(XINCLUDE["C2.xml"])
-    >>> ElementInclude.include(document, loader=none_loader)
-    Traceback (most recent call last):
-    xml.etree.ElementInclude.FatalIncludeError: cannot load 'count.txt' as 'text'
-
-    Test bad parse type.
-
-    >>> document = ET.XML(XINCLUDE_BAD["B1.xml"])
-    >>> ElementInclude.include(document, loader=none_loader)
-    Traceback (most recent call last):
-    xml.etree.ElementInclude.FatalIncludeError: unknown parse type in xi:include tag ('BAD_TYPE')
-
-    Test xi:fallback outside xi:include.
-
-    >>> document = ET.XML(XINCLUDE_BAD["B2.xml"])
-    >>> ElementInclude.include(document, loader=none_loader)
-    Traceback (most recent call last):
-    xml.etree.ElementInclude.FatalIncludeError: xi:fallback tag must be child of xi:include ('{http://www.w3.org/2001/XInclude}fallback')
-    """
+class XIncludeTest(unittest.TestCase):
+
+    def xinclude_loader(self, href, parse="xml", encoding=None):
+        try:
+            data = XINCLUDE[href]
+        except KeyError:
+            raise OSError("resource not found")
+        if parse == "xml":
+            data = ET.XML(data)
+        return data
+
+    def none_loader(self, href, parser, encoding=None):
+        return None
+
+    def _my_loader(self, href, parse):
+        # Used to avoid a test-dependency problem where the default loader
+        # of ElementInclude uses the pyET parser for cET tests.
+        if parse == 'xml':
+            with open(href, 'rb') as f:
+                return ET.parse(f).getroot()
+        else:
+            return None
+
+    def test_xinclude_default(self):
+        from xml.etree import ElementInclude
+        doc = self.xinclude_loader('default.xml')
+        ElementInclude.include(doc, self._my_loader)
+        self.assertEqual(serialize(doc),
+            '<document>\n'
+            '  <p>Example.</p>\n'
+            '  <root>\n'
+            '   <element key="value">text</element>\n'
+            '   <element>text</element>tail\n'
+            '   <empty-element />\n'
+            '</root>\n'
+            '</document>')
+
+    def test_xinclude(self):
+        from xml.etree import ElementInclude
+
+        # Basic inclusion example (XInclude C.1)
+        document = self.xinclude_loader("C1.xml")
+        ElementInclude.include(document, self.xinclude_loader)
+        self.assertEqual(serialize(document),
+            '<document>\n'
+            '  <p>120 Mz is adequate for an average home user.</p>\n'
+            '  <disclaimer>\n'
+            '  <p>The opinions represented herein represent those of the individual\n'
+            '  and should not be interpreted as official policy endorsed by this\n'
+            '  organization.</p>\n'
+            '</disclaimer>\n'
+            '</document>') # C1
+
+        # Textual inclusion example (XInclude C.2)
+        document = self.xinclude_loader("C2.xml")
+        ElementInclude.include(document, self.xinclude_loader)
+        self.assertEqual(serialize(document),
+            '<document>\n'
+            '  <p>This document has been accessed\n'
+            '  324387 times.</p>\n'
+            '</document>') # C2
+
+        # Textual inclusion after sibling element (based on modified XInclude C.2)
+        document = self.xinclude_loader("C2b.xml")
+        ElementInclude.include(document, self.xinclude_loader)
+        self.assertEqual(serialize(document),
+            '<document>\n'
+            '  <p>This document has been <em>accessed</em>\n'
+            '  324387 times.</p>\n'
+            '</document>') # C2b
+
+        # Textual inclusion of XML example (XInclude C.3)
+        document = self.xinclude_loader("C3.xml")
+        ElementInclude.include(document, self.xinclude_loader)
+        self.assertEqual(serialize(document),
+            '<document>\n'
+            '  <p>The following is the source of the "data.xml" resource:</p>\n'
+            "  <example>&lt;?xml version='1.0'?&gt;\n"
+            '&lt;data&gt;\n'
+            '  &lt;item&gt;&lt;![CDATA[Brooks &amp; Shields]]&gt;&lt;/item&gt;\n'
+            '&lt;/data&gt;\n'
+            '</example>\n'
+            '</document>') # C3
+
+        # Fallback example (XInclude C.5)
+        # Note! Fallback support is not yet implemented
+        document = self.xinclude_loader("C5.xml")
+        with self.assertRaises(OSError) as cm:
+            ElementInclude.include(document, self.xinclude_loader)
+        self.assertEqual(str(cm.exception), 'resource not found')
+        self.assertEqual(serialize(document),
+            '<div xmlns:ns0="http://www.w3.org/2001/XInclude">\n'
+            '  <ns0:include href="example.txt" parse="text">\n'
+            '    <ns0:fallback>\n'
+            '      <ns0:include href="fallback-example.txt" parse="text">\n'
+            '        <ns0:fallback><a href="mailto:bob@example.org">Report error</a></ns0:fallback>\n'
+            '      </ns0:include>\n'
+            '    </ns0:fallback>\n'
+            '  </ns0:include>\n'
+            '</div>') # C5
+
+    def test_xinclude_failures(self):
+        from xml.etree import ElementInclude
+
+        # Test failure to locate included XML file.
+        document = ET.XML(XINCLUDE["C1.xml"])
+        with self.assertRaises(ElementInclude.FatalIncludeError) as cm:
+            ElementInclude.include(document, loader=self.none_loader)
+        self.assertEqual(str(cm.exception),
+                "cannot load 'disclaimer.xml' as 'xml'")
+
+        # Test failure to locate included text file.
+        document = ET.XML(XINCLUDE["C2.xml"])
+        with self.assertRaises(ElementInclude.FatalIncludeError) as cm:
+            ElementInclude.include(document, loader=self.none_loader)
+        self.assertEqual(str(cm.exception),
+                "cannot load 'count.txt' as 'text'")
+
+        # Test bad parse type.
+        document = ET.XML(XINCLUDE_BAD["B1.xml"])
+        with self.assertRaises(ElementInclude.FatalIncludeError) as cm:
+            ElementInclude.include(document, loader=self.none_loader)
+        self.assertEqual(str(cm.exception),
+                "unknown parse type in xi:include tag ('BAD_TYPE')")
+
+        # Test xi:fallback outside xi:include.
+        document = ET.XML(XINCLUDE_BAD["B2.xml"])
+        with self.assertRaises(ElementInclude.FatalIncludeError) as cm:
+            ElementInclude.include(document, loader=self.none_loader)
+        self.assertEqual(str(cm.exception),
+                "xi:fallback tag must be child of xi:include "
+                "('{http://www.w3.org/2001/XInclude}fallback')")
 
 # --------------------------------------------------------------------
 # reported bugs
 
-def bug_xmltoolkit21():
-    """
-
-    marshaller gives obscure errors for non-string values
-
-    >>> elem = ET.Element(123)
-    >>> serialize(elem) # tag
-    Traceback (most recent call last):
-    TypeError: cannot serialize 123 (type int)
-    >>> elem = ET.Element("elem")
-    >>> elem.text = 123
-    >>> serialize(elem) # text
-    Traceback (most recent call last):
-    TypeError: cannot serialize 123 (type int)
-    >>> elem = ET.Element("elem")
-    >>> elem.tail = 123
-    >>> serialize(elem) # tail
-    Traceback (most recent call last):
-    TypeError: cannot serialize 123 (type int)
-    >>> elem = ET.Element("elem")
-    >>> elem.set(123, "123")
-    >>> serialize(elem) # attribute key
-    Traceback (most recent call last):
-    TypeError: cannot serialize 123 (type int)
-    >>> elem = ET.Element("elem")
-    >>> elem.set("123", 123)
-    >>> serialize(elem) # attribute value
-    Traceback (most recent call last):
-    TypeError: cannot serialize 123 (type int)
-
-    """
-
-def bug_xmltoolkit25():
-    """
-
-    typo in ElementTree.findtext
+class BugsTest(unittest.TestCase):
 
-    >>> elem = ET.XML(SAMPLE_XML)
-    >>> tree = ET.ElementTree(elem)
-    >>> tree.findtext("tag")
-    'text'
-    >>> tree.findtext("section/tag")
-    'subtext'
+    def test_bug_xmltoolkit21(self):
+        # marshaller gives obscure errors for non-string values
 
-    """
+        def check(elem):
+            with self.assertRaises(TypeError) as cm:
+                serialize(elem)
+            self.assertEqual(str(cm.exception),
+                    'cannot serialize 123 (type int)')
 
-def bug_xmltoolkit28():
-    """
+        elem = ET.Element(123)
+        check(elem) # tag
 
-    .//tag causes exceptions
+        elem = ET.Element("elem")
+        elem.text = 123
+        check(elem) # text
 
-    >>> tree = ET.XML("<doc><table><tbody/></table></doc>")
-    >>> summarize_list(tree.findall(".//thead"))
-    []
-    >>> summarize_list(tree.findall(".//tbody"))
-    ['tbody']
+        elem = ET.Element("elem")
+        elem.tail = 123
+        check(elem) # tail
 
-    """
+        elem = ET.Element("elem")
+        elem.set(123, "123")
+        check(elem) # attribute key
 
-def bug_xmltoolkitX1():
-    """
+        elem = ET.Element("elem")
+        elem.set("123", 123)
+        check(elem) # attribute value
 
-    dump() doesn't flush the output buffer
+    def test_bug_xmltoolkit25(self):
+        # typo in ElementTree.findtext
 
-    >>> tree = ET.XML("<doc><table><tbody/></table></doc>")
-    >>> ET.dump(tree); print("tail")
-    <doc><table><tbody /></table></doc>
-    tail
+        elem = ET.XML(SAMPLE_XML)
+        tree = ET.ElementTree(elem)
+        self.assertEqual(tree.findtext("tag"), 'text')
+        self.assertEqual(tree.findtext("section/tag"), 'subtext')
 
-    """
+    def test_bug_xmltoolkit28(self):
+        # .//tag causes exceptions
 
-def bug_xmltoolkit39():
-    """
+        tree = ET.XML("<doc><table><tbody/></table></doc>")
+        self.assertEqual(summarize_list(tree.findall(".//thead")), [])
+        self.assertEqual(summarize_list(tree.findall(".//tbody")), ['tbody'])
 
-    non-ascii element and attribute names doesn't work
+    def test_bug_xmltoolkitX1(self):
+        # dump() doesn't flush the output buffer
 
-    >>> tree = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?><t\\xe4g />")
-    >>> ET.tostring(tree, "utf-8")
-    b'<t\\xc3\\xa4g />'
+        tree = ET.XML("<doc><table><tbody/></table></doc>")
+        with support.captured_stdout() as stdout:
+            ET.dump(tree)
+            self.assertEqual(stdout.getvalue(), '<doc><table><tbody /></table></doc>\n')
 
-    >>> tree = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?><tag \\xe4ttr='v&#228;lue' />")
-    >>> tree.attrib
-    {'\\xe4ttr': 'v\\xe4lue'}
-    >>> ET.tostring(tree, "utf-8")
-    b'<tag \\xc3\\xa4ttr="v\\xc3\\xa4lue" />'
+    def test_bug_xmltoolkit39(self):
+        # non-ascii element and attribute names doesn't work
 
-    >>> tree = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?><t\\xe4g>text</t\\xe4g>")
-    >>> ET.tostring(tree, "utf-8")
-    b'<t\\xc3\\xa4g>text</t\\xc3\\xa4g>'
+        tree = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?><t\xe4g />")
+        self.assertEqual(ET.tostring(tree, "utf-8"), b'<t\xc3\xa4g />')
 
-    >>> tree = ET.Element("t\u00e4g")
-    >>> ET.tostring(tree, "utf-8")
-    b'<t\\xc3\\xa4g />'
+        tree = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?>"
+                      b"<tag \xe4ttr='v&#228;lue' />")
+        self.assertEqual(tree.attrib, {'\xe4ttr': 'v\xe4lue'})
+        self.assertEqual(ET.tostring(tree, "utf-8"),
+                b'<tag \xc3\xa4ttr="v\xc3\xa4lue" />')
 
-    >>> tree = ET.Element("tag")
-    >>> tree.set("\u00e4ttr", "v\u00e4lue")
-    >>> ET.tostring(tree, "utf-8")
-    b'<tag \\xc3\\xa4ttr="v\\xc3\\xa4lue" />'
+        tree = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?>"
+                      b'<t\xe4g>text</t\xe4g>')
+        self.assertEqual(ET.tostring(tree, "utf-8"),
+                b'<t\xc3\xa4g>text</t\xc3\xa4g>')
 
-    """
+        tree = ET.Element("t\u00e4g")
+        self.assertEqual(ET.tostring(tree, "utf-8"), b'<t\xc3\xa4g />')
 
-def bug_xmltoolkit54():
-    """
+        tree = ET.Element("tag")
+        tree.set("\u00e4ttr", "v\u00e4lue")
+        self.assertEqual(ET.tostring(tree, "utf-8"),
+                b'<tag \xc3\xa4ttr="v\xc3\xa4lue" />')
 
-    problems handling internally defined entities
+    def test_bug_xmltoolkit54(self):
+        # problems handling internally defined entities
 
-    >>> e = ET.XML("<!DOCTYPE doc [<!ENTITY ldots '&#x8230;'>]><doc>&ldots;</doc>")
-    >>> serialize(e, encoding="us-ascii")
-    b'<doc>&#33328;</doc>'
-    >>> serialize(e)
-    '<doc>\u8230</doc>'
+        e = ET.XML("<!DOCTYPE doc [<!ENTITY ldots '&#x8230;'>]>"
+                   '<doc>&ldots;</doc>')
+        self.assertEqual(serialize(e, encoding="us-ascii"),
+                b'<doc>&#33328;</doc>')
+        self.assertEqual(serialize(e), '<doc>\u8230</doc>')
 
-    """
+    def test_bug_xmltoolkit55(self):
+        # make sure we're reporting the first error, not the last
 
-def bug_xmltoolkit55():
-    """
+        with self.assertRaises(ET.ParseError) as cm:
+            ET.XML(b"<!DOCTYPE doc SYSTEM 'doc.dtd'>"
+                   b'<doc>&ldots;&ndots;&rdots;</doc>')
+        self.assertEqual(str(cm.exception),
+                'undefined entity &ldots;: line 1, column 36')
 
-    make sure we're reporting the first error, not the last
+    def test_bug_xmltoolkit60(self):
+        # Handle crash in stream source.
 
-    >>> normalize_exception(ET.XML, b"<!DOCTYPE doc SYSTEM 'doc.dtd'><doc>&ldots;&ndots;&rdots;</doc>")
-    Traceback (most recent call last):
-    ParseError: undefined entity &ldots;: line 1, column 36
+        class ExceptionFile:
+            def read(self, x):
+                raise OSError
 
-    """
+        self.assertRaises(OSError, ET.parse, ExceptionFile())
 
-class ExceptionFile:
-    def read(self, x):
-        raise IOError
+    def test_bug_xmltoolkit62(self):
+        # Don't crash when using custom entities.
 
-def xmltoolkit60():
-    """
-
-    Handle crash in stream source.
-    >>> tree = ET.parse(ExceptionFile())
-    Traceback (most recent call last):
-    IOError
-
-    """
-
-XMLTOOLKIT62_DOC = """<?xml version="1.0" encoding="UTF-8"?>
+        ENTITIES = {'rsquo': '\u2019', 'lsquo': '\u2018'}
+        parser = ET.XMLTreeBuilder()
+        parser.entity.update(ENTITIES)
+        parser.feed("""<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE patent-application-publication SYSTEM "pap-v15-2001-01-31.dtd" []>
 <patent-application-publication>
 <subdoc-abstract>
 <paragraph id="A-0001" lvl="0">A new cultivar of Begonia plant named &lsquo;BCT9801BEG&rsquo;.</paragraph>
 </subdoc-abstract>
-</patent-application-publication>"""
-
-
-def xmltoolkit62():
-    """
-
-    Don't crash when using custom entities.
-
-    >>> xmltoolkit62()
-    'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.'
-
-    """
-    ENTITIES = {'rsquo': '\u2019', 'lsquo': '\u2018'}
-    parser = ET.XMLTreeBuilder()
-    parser.entity.update(ENTITIES)
-    parser.feed(XMLTOOLKIT62_DOC)
-    t = parser.close()
-    return t.find('.//paragraph').text
-
-def xmltoolkit63():
-    """
-
-    Check reference leak.
-    >>> xmltoolkit63()
-    >>> count = sys.getrefcount(None)
-    >>> for i in range(1000):
-    ...     xmltoolkit63()
-    >>> sys.getrefcount(None) - count
-    0
-
-    """
-    tree = ET.TreeBuilder()
-    tree.start("tag", {})
-    tree.data("text")
-    tree.end("tag")
+</patent-application-publication>""")
+        t = parser.close()
+        self.assertEqual(t.find('.//paragraph').text,
+            'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.')
+
+    def test_bug_xmltoolkit63(self):
+        # Check reference leak.
+        def xmltoolkit63():
+            tree = ET.TreeBuilder()
+            tree.start("tag", {})
+            tree.data("text")
+            tree.end("tag")
+
+        xmltoolkit63()
+        count = sys.getrefcount(None)
+        for i in range(1000):
+            xmltoolkit63()
+        self.assertEqual(sys.getrefcount(None), count)
+
+    def test_bug_200708_newline(self):
+        # Preserve newlines in attributes.
+
+        e = ET.Element('SomeTag', text="def _f():\n  return 3\n")
+        self.assertEqual(ET.tostring(e),
+                b'<SomeTag text="def _f():&#10;  return 3&#10;" />')
+        self.assertEqual(ET.XML(ET.tostring(e)).get("text"),
+                'def _f():\n  return 3\n')
+        self.assertEqual(ET.tostring(ET.XML(ET.tostring(e))),
+                b'<SomeTag text="def _f():&#10;  return 3&#10;" />')
+
+    def test_bug_200708_close(self):
+        # Test default builder.
+        parser = ET.XMLParser() # default
+        parser.feed("<element>some text</element>")
+        self.assertEqual(parser.close().tag, 'element')
+
+        # Test custom builder.
+        class EchoTarget:
+            def close(self):
+                return ET.Element("element") # simulate root
+        parser = ET.XMLParser(EchoTarget())
+        parser.feed("<element>some text</element>")
+        self.assertEqual(parser.close().tag, 'element')
+
+    def test_bug_200709_default_namespace(self):
+        e = ET.Element("{default}elem")
+        s = ET.SubElement(e, "{default}elem")
+        self.assertEqual(serialize(e, default_namespace="default"), # 1
+                '<elem xmlns="default"><elem /></elem>')
+
+        e = ET.Element("{default}elem")
+        s = ET.SubElement(e, "{default}elem")
+        s = ET.SubElement(e, "{not-default}elem")
+        self.assertEqual(serialize(e, default_namespace="default"), # 2
+            '<elem xmlns="default" xmlns:ns1="not-default">'
+            '<elem />'
+            '<ns1:elem />'
+            '</elem>')
+
+        e = ET.Element("{default}elem")
+        s = ET.SubElement(e, "{default}elem")
+        s = ET.SubElement(e, "elem") # unprefixed name
+        with self.assertRaises(ValueError) as cm:
+            serialize(e, default_namespace="default") # 3
+        self.assertEqual(str(cm.exception),
+                'cannot use non-qualified names with default_namespace option')
+
+    def test_bug_200709_register_namespace(self):
+        e = ET.Element("{http://namespace.invalid/does/not/exist/}title")
+        self.assertEqual(ET.tostring(e),
+            b'<ns0:title xmlns:ns0="http://namespace.invalid/does/not/exist/" />')
+        ET.register_namespace("foo", "http://namespace.invalid/does/not/exist/")
+        e = ET.Element("{http://namespace.invalid/does/not/exist/}title")
+        self.assertEqual(ET.tostring(e),
+            b'<foo:title xmlns:foo="http://namespace.invalid/does/not/exist/" />')
+
+        # And the Dublin Core namespace is in the default list:
+
+        e = ET.Element("{http://purl.org/dc/elements/1.1/}title")
+        self.assertEqual(ET.tostring(e),
+            b'<dc:title xmlns:dc="http://purl.org/dc/elements/1.1/" />')
+
+    def test_bug_200709_element_comment(self):
+        # Not sure if this can be fixed, really (since the serializer needs
+        # ET.Comment, not cET.comment).
+
+        a = ET.Element('a')
+        a.append(ET.Comment('foo'))
+        self.assertEqual(a[0].tag, ET.Comment)
+
+        a = ET.Element('a')
+        a.append(ET.PI('foo'))
+        self.assertEqual(a[0].tag, ET.PI)
+
+    def test_bug_200709_element_insert(self):
+        a = ET.Element('a')
+        b = ET.SubElement(a, 'b')
+        c = ET.SubElement(a, 'c')
+        d = ET.Element('d')
+        a.insert(0, d)
+        self.assertEqual(summarize_list(a), ['d', 'b', 'c'])
+        a.insert(-1, d)
+        self.assertEqual(summarize_list(a), ['d', 'b', 'd', 'c'])
+
+    def test_bug_200709_iter_comment(self):
+        a = ET.Element('a')
+        b = ET.SubElement(a, 'b')
+        comment_b = ET.Comment("TEST-b")
+        b.append(comment_b)
+        self.assertEqual(summarize_list(a.iter(ET.Comment)), [ET.Comment])
+
+    # --------------------------------------------------------------------
+    # reported on bugs.python.org
+
+    def test_bug_1534630(self):
+        bob = ET.TreeBuilder()
+        e = bob.data("data")
+        e = bob.start("tag", {})
+        e = bob.end("tag")
+        e = bob.close()
+        self.assertEqual(serialize(e), '<tag />')
+
+    def test_issue6233(self):
+        e = ET.XML(b"<?xml version='1.0' encoding='utf-8'?>"
+                   b'<body>t\xc3\xa3g</body>')
+        self.assertEqual(ET.tostring(e, 'ascii'),
+                b"<?xml version='1.0' encoding='ascii'?>\n"
+                b'<body>t&#227;g</body>')
+        e = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?>"
+                   b'<body>t\xe3g</body>')
+        self.assertEqual(ET.tostring(e, 'ascii'),
+                b"<?xml version='1.0' encoding='ascii'?>\n"
+                b'<body>t&#227;g</body>')
+
+    def test_issue3151(self):
+        e = ET.XML('<prefix:localname xmlns:prefix="${stuff}"/>')
+        self.assertEqual(e.tag, '{${stuff}}localname')
+        t = ET.ElementTree(e)
+        self.assertEqual(ET.tostring(e), b'<ns0:localname xmlns:ns0="${stuff}" />')
+
+    def test_issue6565(self):
+        elem = ET.XML("<body><tag/></body>")
+        self.assertEqual(summarize_list(elem), ['tag'])
+        newelem = ET.XML(SAMPLE_XML)
+        elem[:] = newelem[:]
+        self.assertEqual(summarize_list(elem), ['tag', 'tag', 'section'])
+
+    def test_issue10777(self):
+        # Registering a namespace twice caused a "dictionary changed size during
+        # iteration" bug.
+
+        ET.register_namespace('test10777', 'http://myuri/')
+        ET.register_namespace('test10777', 'http://myuri/')
 
 # --------------------------------------------------------------------
 
 
-def bug_200708_newline():
-    r"""
-
-    Preserve newlines in attributes.
-
-    >>> e = ET.Element('SomeTag', text="def _f():\n  return 3\n")
-    >>> ET.tostring(e)
-    b'<SomeTag text="def _f():&#10;  return 3&#10;" />'
-    >>> ET.XML(ET.tostring(e)).get("text")
-    'def _f():\n  return 3\n'
-    >>> ET.tostring(ET.XML(ET.tostring(e)))
-    b'<SomeTag text="def _f():&#10;  return 3&#10;" />'
-
-    """
-
-def bug_200708_close():
-    """
-
-    Test default builder.
-    >>> parser = ET.XMLParser() # default
-    >>> parser.feed("<element>some text</element>")
-    >>> summarize(parser.close())
-    'element'
-
-    Test custom builder.
-    >>> class EchoTarget:
-    ...     def close(self):
-    ...         return ET.Element("element") # simulate root
-    >>> parser = ET.XMLParser(EchoTarget())
-    >>> parser.feed("<element>some text</element>")
-    >>> summarize(parser.close())
-    'element'
-
-    """
+class BasicElementTest(ElementTestCase, unittest.TestCase):
+    def test_augmentation_type_errors(self):
+        e = ET.Element('joe')
+        self.assertRaises(TypeError, e.append, 'b')
+        self.assertRaises(TypeError, e.extend, [ET.Element('bar'), 'foo'])
+        self.assertRaises(TypeError, e.insert, 0, 'foo')
+
+    def test_cyclic_gc(self):
+        class Dummy:
+            pass
+
+        # Test the shortest cycle: d->element->d
+        d = Dummy()
+        d.dummyref = ET.Element('joe', attr=d)
+        wref = weakref.ref(d)
+        del d
+        gc_collect()
+        self.assertIsNone(wref())
+
+        # A longer cycle: d->e->e2->d
+        e = ET.Element('joe')
+        d = Dummy()
+        d.dummyref = e
+        wref = weakref.ref(d)
+        e2 = ET.SubElement(e, 'foo', attr=d)
+        del d, e, e2
+        gc_collect()
+        self.assertIsNone(wref())
+
+        # A cycle between Element objects as children of one another
+        # e1->e2->e3->e1
+        e1 = ET.Element('e1')
+        e2 = ET.Element('e2')
+        e3 = ET.Element('e3')
+        e1.append(e2)
+        e2.append(e2)
+        e3.append(e1)
+        wref = weakref.ref(e1)
+        del e1, e2, e3
+        gc_collect()
+        self.assertIsNone(wref())
+
+    def test_weakref(self):
+        flag = False
+        def wref_cb(w):
+            nonlocal flag
+            flag = True
+        e = ET.Element('e')
+        wref = weakref.ref(e, wref_cb)
+        self.assertEqual(wref().tag, 'e')
+        del e
+        self.assertEqual(flag, True)
+        self.assertEqual(wref(), None)
+
+    def test_get_keyword_args(self):
+        e1 = ET.Element('foo' , x=1, y=2, z=3)
+        self.assertEqual(e1.get('x', default=7), 1)
+        self.assertEqual(e1.get('w', default=7), 7)
+
+    def test_pickle(self):
+        # issue #16076: the C implementation wasn't pickleable.
+        for dumper, loader in product(self.modules, repeat=2):
+            e = dumper.Element('foo', bar=42)
+            e.text = "text goes here"
+            e.tail = "opposite of head"
+            dumper.SubElement(e, 'child').append(dumper.Element('grandchild'))
+            e.append(dumper.Element('child'))
+            e.findall('.//grandchild')[0].set('attr', 'other value')
+
+            e2 = self.pickleRoundTrip(e, 'xml.etree.ElementTree',
+                                      dumper, loader)
+
+            self.assertEqual(e2.tag, 'foo')
+            self.assertEqual(e2.attrib['bar'], 42)
+            self.assertEqual(len(e2), 2)
+            self.assertEqualElements(e, e2)
+
+
+class ElementTreeTypeTest(unittest.TestCase):
+    def test_istype(self):
+        self.assertIsInstance(ET.ParseError, type)
+        self.assertIsInstance(ET.QName, type)
+        self.assertIsInstance(ET.ElementTree, type)
+        self.assertIsInstance(ET.Element, type)
+        self.assertIsInstance(ET.TreeBuilder, type)
+        self.assertIsInstance(ET.XMLParser, type)
+
+    def test_Element_subclass_trivial(self):
+        class MyElement(ET.Element):
+            pass
+
+        mye = MyElement('foo')
+        self.assertIsInstance(mye, ET.Element)
+        self.assertIsInstance(mye, MyElement)
+        self.assertEqual(mye.tag, 'foo')
+
+        # test that attribute assignment works (issue 14849)
+        mye.text = "joe"
+        self.assertEqual(mye.text, "joe")
+
+    def test_Element_subclass_constructor(self):
+        class MyElement(ET.Element):
+            def __init__(self, tag, attrib={}, **extra):
+                super(MyElement, self).__init__(tag + '__', attrib, **extra)
+
+        mye = MyElement('foo', {'a': 1, 'b': 2}, c=3, d=4)
+        self.assertEqual(mye.tag, 'foo__')
+        self.assertEqual(sorted(mye.items()),
+            [('a', 1), ('b', 2), ('c', 3), ('d', 4)])
+
+    def test_Element_subclass_new_method(self):
+        class MyElement(ET.Element):
+            def newmethod(self):
+                return self.tag
+
+        mye = MyElement('joe')
+        self.assertEqual(mye.newmethod(), 'joe')
+
+
+class ElementFindTest(unittest.TestCase):
+    def test_find_simple(self):
+        e = ET.XML(SAMPLE_XML)
+        self.assertEqual(e.find('tag').tag, 'tag')
+        self.assertEqual(e.find('section/tag').tag, 'tag')
+        self.assertEqual(e.find('./tag').tag, 'tag')
+
+        e[2] = ET.XML(SAMPLE_SECTION)
+        self.assertEqual(e.find('section/nexttag').tag, 'nexttag')
+
+        self.assertEqual(e.findtext('./tag'), 'text')
+        self.assertEqual(e.findtext('section/tag'), 'subtext')
+
+        # section/nexttag is found but has no text
+        self.assertEqual(e.findtext('section/nexttag'), '')
+        self.assertEqual(e.findtext('section/nexttag', 'default'), '')
+
+        # tog doesn't exist and 'default' kicks in
+        self.assertIsNone(e.findtext('tog'))
+        self.assertEqual(e.findtext('tog', 'default'), 'default')
+
+        # Issue #16922
+        self.assertEqual(ET.XML('<tag><empty /></tag>').findtext('empty'), '')
+
+    def test_find_xpath(self):
+        LINEAR_XML = '''
+        <body>
+            <tag class='a'/>
+            <tag class='b'/>
+            <tag class='c'/>
+            <tag class='d'/>
+        </body>'''
+        e = ET.XML(LINEAR_XML)
+
+        # Test for numeric indexing and last()
+        self.assertEqual(e.find('./tag[1]').attrib['class'], 'a')
+        self.assertEqual(e.find('./tag[2]').attrib['class'], 'b')
+        self.assertEqual(e.find('./tag[last()]').attrib['class'], 'd')
+        self.assertEqual(e.find('./tag[last()-1]').attrib['class'], 'c')
+        self.assertEqual(e.find('./tag[last()-2]').attrib['class'], 'b')
+
+    def test_findall(self):
+        e = ET.XML(SAMPLE_XML)
+        e[2] = ET.XML(SAMPLE_SECTION)
+        self.assertEqual(summarize_list(e.findall('.')), ['body'])
+        self.assertEqual(summarize_list(e.findall('tag')), ['tag', 'tag'])
+        self.assertEqual(summarize_list(e.findall('tog')), [])
+        self.assertEqual(summarize_list(e.findall('tog/foo')), [])
+        self.assertEqual(summarize_list(e.findall('*')),
+            ['tag', 'tag', 'section'])
+        self.assertEqual(summarize_list(e.findall('.//tag')),
+            ['tag'] * 4)
+        self.assertEqual(summarize_list(e.findall('section/tag')), ['tag'])
+        self.assertEqual(summarize_list(e.findall('section//tag')), ['tag'] * 2)
+        self.assertEqual(summarize_list(e.findall('section/*')),
+            ['tag', 'nexttag', 'nextsection'])
+        self.assertEqual(summarize_list(e.findall('section//*')),
+            ['tag', 'nexttag', 'nextsection', 'tag'])
+        self.assertEqual(summarize_list(e.findall('section/.//*')),
+            ['tag', 'nexttag', 'nextsection', 'tag'])
+        self.assertEqual(summarize_list(e.findall('*/*')),
+            ['tag', 'nexttag', 'nextsection'])
+        self.assertEqual(summarize_list(e.findall('*//*')),
+            ['tag', 'nexttag', 'nextsection', 'tag'])
+        self.assertEqual(summarize_list(e.findall('*/tag')), ['tag'])
+        self.assertEqual(summarize_list(e.findall('*/./tag')), ['tag'])
+        self.assertEqual(summarize_list(e.findall('./tag')), ['tag'] * 2)
+        self.assertEqual(summarize_list(e.findall('././tag')), ['tag'] * 2)
+
+        self.assertEqual(summarize_list(e.findall('.//tag[@class]')),
+            ['tag'] * 3)
+        self.assertEqual(summarize_list(e.findall('.//tag[@class="a"]')),
+            ['tag'])
+        self.assertEqual(summarize_list(e.findall('.//tag[@class="b"]')),
+            ['tag'] * 2)
+        self.assertEqual(summarize_list(e.findall('.//tag[@id]')),
+            ['tag'])
+        self.assertEqual(summarize_list(e.findall('.//section[tag]')),
+            ['section'])
+        self.assertEqual(summarize_list(e.findall('.//section[element]')), [])
+        self.assertEqual(summarize_list(e.findall('../tag')), [])
+        self.assertEqual(summarize_list(e.findall('section/../tag')),
+            ['tag'] * 2)
+        self.assertEqual(e.findall('section//'), e.findall('section//*'))
+
+    def test_test_find_with_ns(self):
+        e = ET.XML(SAMPLE_XML_NS)
+        self.assertEqual(summarize_list(e.findall('tag')), [])
+        self.assertEqual(
+            summarize_list(e.findall("{http://effbot.org/ns}tag")),
+            ['{http://effbot.org/ns}tag'] * 2)
+        self.assertEqual(
+            summarize_list(e.findall(".//{http://effbot.org/ns}tag")),
+            ['{http://effbot.org/ns}tag'] * 3)
+
+    def test_bad_find(self):
+        e = ET.XML(SAMPLE_XML)
+        with self.assertRaisesRegex(SyntaxError, 'cannot use absolute path'):
+            e.findall('/tag')
+
+    def test_find_through_ElementTree(self):
+        e = ET.XML(SAMPLE_XML)
+        self.assertEqual(ET.ElementTree(e).find('tag').tag, 'tag')
+        self.assertEqual(ET.ElementTree(e).findtext('tag'), 'text')
+        self.assertEqual(summarize_list(ET.ElementTree(e).findall('tag')),
+            ['tag'] * 2)
+        # this produces a warning
+        self.assertEqual(summarize_list(ET.ElementTree(e).findall('//tag')),
+            ['tag'] * 3)
+
+
+class ElementIterTest(unittest.TestCase):
+    def _ilist(self, elem, tag=None):
+        return summarize_list(elem.iter(tag))
+
+    def test_basic(self):
+        doc = ET.XML("<html><body>this is a <i>paragraph</i>.</body>..</html>")
+        self.assertEqual(self._ilist(doc), ['html', 'body', 'i'])
+        self.assertEqual(self._ilist(doc.find('body')), ['body', 'i'])
+        self.assertEqual(next(doc.iter()).tag, 'html')
+        self.assertEqual(''.join(doc.itertext()), 'this is a paragraph...')
+        self.assertEqual(''.join(doc.find('body').itertext()),
+            'this is a paragraph.')
+        self.assertEqual(next(doc.itertext()), 'this is a ')
+
+        # iterparse should return an iterator
+        sourcefile = serialize(doc, to_string=False)
+        self.assertEqual(next(ET.iterparse(sourcefile))[0], 'end')
+
+        # With an explitit parser too (issue #9708)
+        sourcefile = serialize(doc, to_string=False)
+        parser = ET.XMLParser(target=ET.TreeBuilder())
+        self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0],
+                         'end')
+
+        tree = ET.ElementTree(None)
+        self.assertRaises(AttributeError, tree.iter)
+
+        # Issue #16913
+        doc = ET.XML("<root>a&amp;<sub>b&amp;</sub>c&amp;</root>")
+        self.assertEqual(''.join(doc.itertext()), 'a&b&c&')
+
+    def test_corners(self):
+        # single root, no subelements
+        a = ET.Element('a')
+        self.assertEqual(self._ilist(a), ['a'])
+
+        # one child
+        b = ET.SubElement(a, 'b')
+        self.assertEqual(self._ilist(a), ['a', 'b'])
+
+        # one child and one grandchild
+        c = ET.SubElement(b, 'c')
+        self.assertEqual(self._ilist(a), ['a', 'b', 'c'])
+
+        # two children, only first with grandchild
+        d = ET.SubElement(a, 'd')
+        self.assertEqual(self._ilist(a), ['a', 'b', 'c', 'd'])
+
+        # replace first child by second
+        a[0] = a[1]
+        del a[1]
+        self.assertEqual(self._ilist(a), ['a', 'd'])
+
+    def test_iter_by_tag(self):
+        doc = ET.XML('''
+            <document>
+                <house>
+                    <room>bedroom1</room>
+                    <room>bedroom2</room>
+                </house>
+                <shed>nothing here
+                </shed>
+                <house>
+                    <room>bedroom8</room>
+                </house>
+            </document>''')
+
+        self.assertEqual(self._ilist(doc, 'room'), ['room'] * 3)
+        self.assertEqual(self._ilist(doc, 'house'), ['house'] * 2)
+
+        # test that iter also accepts 'tag' as a keyword arg
+        self.assertEqual(
+            summarize_list(doc.iter(tag='room')),
+            ['room'] * 3)
+
+        # make sure both tag=None and tag='*' return all tags
+        all_tags = ['document', 'house', 'room', 'room',
+                    'shed', 'house', 'room']
+        self.assertEqual(self._ilist(doc), all_tags)
+        self.assertEqual(self._ilist(doc, '*'), all_tags)
+
+
+class TreeBuilderTest(unittest.TestCase):
+    sample1 = ('<!DOCTYPE html PUBLIC'
+        ' "-//W3C//DTD XHTML 1.0 Transitional//EN"'
+        ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'
+        '<html>text<div>subtext</div>tail</html>')
+
+    sample2 = '''<toplevel>sometext</toplevel>'''
+
+    def _check_sample1_element(self, e):
+        self.assertEqual(e.tag, 'html')
+        self.assertEqual(e.text, 'text')
+        self.assertEqual(e.tail, None)
+        self.assertEqual(e.attrib, {})
+        children = list(e)
+        self.assertEqual(len(children), 1)
+        child = children[0]
+        self.assertEqual(child.tag, 'div')
+        self.assertEqual(child.text, 'subtext')
+        self.assertEqual(child.tail, 'tail')
+        self.assertEqual(child.attrib, {})
+
+    def test_dummy_builder(self):
+        class BaseDummyBuilder:
+            def close(self):
+                return 42
+
+        class DummyBuilder(BaseDummyBuilder):
+            data = start = end = lambda *a: None
+
+        parser = ET.XMLParser(target=DummyBuilder())
+        parser.feed(self.sample1)
+        self.assertEqual(parser.close(), 42)
+
+        parser = ET.XMLParser(target=BaseDummyBuilder())
+        parser.feed(self.sample1)
+        self.assertEqual(parser.close(), 42)
+
+        parser = ET.XMLParser(target=object())
+        parser.feed(self.sample1)
+        self.assertIsNone(parser.close())
+
+    def test_subclass(self):
+        class MyTreeBuilder(ET.TreeBuilder):
+            def foobar(self, x):
+                return x * 2
+
+        tb = MyTreeBuilder()
+        self.assertEqual(tb.foobar(10), 20)
+
+        parser = ET.XMLParser(target=tb)
+        parser.feed(self.sample1)
+
+        e = parser.close()
+        self._check_sample1_element(e)
+
+    def test_element_factory(self):
+        lst = []
+        def myfactory(tag, attrib):
+            nonlocal lst
+            lst.append(tag)
+            return ET.Element(tag, attrib)
+
+        tb = ET.TreeBuilder(element_factory=myfactory)
+        parser = ET.XMLParser(target=tb)
+        parser.feed(self.sample2)
+        parser.close()
+
+        self.assertEqual(lst, ['toplevel'])
+
+    def _check_element_factory_class(self, cls):
+        tb = ET.TreeBuilder(element_factory=cls)
+
+        parser = ET.XMLParser(target=tb)
+        parser.feed(self.sample1)
+        e = parser.close()
+        self.assertIsInstance(e, cls)
+        self._check_sample1_element(e)
+
+    def test_element_factory_subclass(self):
+        class MyElement(ET.Element):
+            pass
+        self._check_element_factory_class(MyElement)
+
+    def test_element_factory_pure_python_subclass(self):
+        # Mimick SimpleTAL's behaviour (issue #16089): both versions of
+        # TreeBuilder should be able to cope with a subclass of the
+        # pure Python Element class.
+        base = ET._Element
+        # Not from a C extension
+        self.assertEqual(base.__module__, 'xml.etree.ElementTree')
+        # Force some multiple inheritance with a C class to make things
+        # more interesting.
+        class MyElement(base, ValueError):
+            pass
+        self._check_element_factory_class(MyElement)
+
+    def test_doctype(self):
+        class DoctypeParser:
+            _doctype = None
+
+            def doctype(self, name, pubid, system):
+                self._doctype = (name, pubid, system)
+
+            def close(self):
+                return self._doctype
+
+        parser = ET.XMLParser(target=DoctypeParser())
+        parser.feed(self.sample1)
+
+        self.assertEqual(parser.close(),
+            ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN',
+             'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'))
+
+
+class XMLParserTest(unittest.TestCase):
+    sample1 = '<file><line>22</line></file>'
+    sample2 = ('<!DOCTYPE html PUBLIC'
+        ' "-//W3C//DTD XHTML 1.0 Transitional//EN"'
+        ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">'
+        '<html>text</html>')
+
+    def _check_sample_element(self, e):
+        self.assertEqual(e.tag, 'file')
+        self.assertEqual(e[0].tag, 'line')
+        self.assertEqual(e[0].text, '22')
+
+    def test_constructor_args(self):
+        # Positional args. The first (html) is not supported, but should be
+        # nevertheless correctly accepted.
+        parser = ET.XMLParser(None, ET.TreeBuilder(), 'utf-8')
+        parser.feed(self.sample1)
+        self._check_sample_element(parser.close())
+
+        # Now as keyword args.
+        parser2 = ET.XMLParser(encoding='utf-8',
+                               html=[{}],
+                               target=ET.TreeBuilder())
+        parser2.feed(self.sample1)
+        self._check_sample_element(parser2.close())
+
+    def test_subclass(self):
+        class MyParser(ET.XMLParser):
+            pass
+        parser = MyParser()
+        parser.feed(self.sample1)
+        self._check_sample_element(parser.close())
+
+    def test_subclass_doctype(self):
+        _doctype = None
+        class MyParserWithDoctype(ET.XMLParser):
+            def doctype(self, name, pubid, system):
+                nonlocal _doctype
+                _doctype = (name, pubid, system)
+
+        parser = MyParserWithDoctype()
+        parser.feed(self.sample2)
+        parser.close()
+        self.assertEqual(_doctype,
+            ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN',
+             'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'))
+
+
+class NamespaceParseTest(unittest.TestCase):
+    def test_find_with_namespace(self):
+        nsmap = {'h': 'hello', 'f': 'foo'}
+        doc = ET.fromstring(SAMPLE_XML_NS_ELEMS)
+
+        self.assertEqual(len(doc.findall('{hello}table', nsmap)), 1)
+        self.assertEqual(len(doc.findall('.//{hello}td', nsmap)), 2)
+        self.assertEqual(len(doc.findall('.//{foo}name', nsmap)), 1)
+
+
+class ElementSlicingTest(unittest.TestCase):
+    def _elem_tags(self, elemlist):
+        return [e.tag for e in elemlist]
+
+    def _subelem_tags(self, elem):
+        return self._elem_tags(list(elem))
+
+    def _make_elem_with_children(self, numchildren):
+        """Create an Element with a tag 'a', with the given amount of children
+           named 'a0', 'a1' ... and so on.
+
+        """
+        e = ET.Element('a')
+        for i in range(numchildren):
+            ET.SubElement(e, 'a%s' % i)
+        return e
+
+    def test_getslice_single_index(self):
+        e = self._make_elem_with_children(10)
+
+        self.assertEqual(e[1].tag, 'a1')
+        self.assertEqual(e[-2].tag, 'a8')
+
+        self.assertRaises(IndexError, lambda: e[12])
+
+    def test_getslice_range(self):
+        e = self._make_elem_with_children(6)
+
+        self.assertEqual(self._elem_tags(e[3:]), ['a3', 'a4', 'a5'])
+        self.assertEqual(self._elem_tags(e[3:6]), ['a3', 'a4', 'a5'])
+        self.assertEqual(self._elem_tags(e[3:16]), ['a3', 'a4', 'a5'])
+        self.assertEqual(self._elem_tags(e[3:5]), ['a3', 'a4'])
+        self.assertEqual(self._elem_tags(e[3:-1]), ['a3', 'a4'])
+        self.assertEqual(self._elem_tags(e[:2]), ['a0', 'a1'])
+
+    def test_getslice_steps(self):
+        e = self._make_elem_with_children(10)
+
+        self.assertEqual(self._elem_tags(e[8:10:1]), ['a8', 'a9'])
+        self.assertEqual(self._elem_tags(e[::3]), ['a0', 'a3', 'a6', 'a9'])
+        self.assertEqual(self._elem_tags(e[::8]), ['a0', 'a8'])
+        self.assertEqual(self._elem_tags(e[1::8]), ['a1', 'a9'])
+
+    def test_getslice_negative_steps(self):
+        e = self._make_elem_with_children(4)
+
+        self.assertEqual(self._elem_tags(e[::-1]), ['a3', 'a2', 'a1', 'a0'])
+        self.assertEqual(self._elem_tags(e[::-2]), ['a3', 'a1'])
+
+    def test_delslice(self):
+        e = self._make_elem_with_children(4)
+        del e[0:2]
+        self.assertEqual(self._subelem_tags(e), ['a2', 'a3'])
+
+        e = self._make_elem_with_children(4)
+        del e[0:]
+        self.assertEqual(self._subelem_tags(e), [])
+
+        e = self._make_elem_with_children(4)
+        del e[::-1]
+        self.assertEqual(self._subelem_tags(e), [])
+
+        e = self._make_elem_with_children(4)
+        del e[::-2]
+        self.assertEqual(self._subelem_tags(e), ['a0', 'a2'])
+
+        e = self._make_elem_with_children(4)
+        del e[1::2]
+        self.assertEqual(self._subelem_tags(e), ['a0', 'a2'])
+
+        e = self._make_elem_with_children(2)
+        del e[::2]
+        self.assertEqual(self._subelem_tags(e), ['a1'])
+
+
+class IOTest(unittest.TestCase):
+    def tearDown(self):
+        support.unlink(TESTFN)
+
+    def test_encoding(self):
+        # Test encoding issues.
+        elem = ET.Element("tag")
+        elem.text = "abc"
+        self.assertEqual(serialize(elem), '<tag>abc</tag>')
+        self.assertEqual(serialize(elem, encoding="utf-8"),
+                b'<tag>abc</tag>')
+        self.assertEqual(serialize(elem, encoding="us-ascii"),
+                b'<tag>abc</tag>')
+        for enc in ("iso-8859-1", "utf-16", "utf-32"):
+            self.assertEqual(serialize(elem, encoding=enc),
+                    ("<?xml version='1.0' encoding='%s'?>\n"
+                     "<tag>abc</tag>" % enc).encode(enc))
+
+        elem = ET.Element("tag")
+        elem.text = "<&\"\'>"
+        self.assertEqual(serialize(elem), '<tag>&lt;&amp;"\'&gt;</tag>')
+        self.assertEqual(serialize(elem, encoding="utf-8"),
+                b'<tag>&lt;&amp;"\'&gt;</tag>')
+        self.assertEqual(serialize(elem, encoding="us-ascii"),
+                b'<tag>&lt;&amp;"\'&gt;</tag>')
+        for enc in ("iso-8859-1", "utf-16", "utf-32"):
+            self.assertEqual(serialize(elem, encoding=enc),
+                    ("<?xml version='1.0' encoding='%s'?>\n"
+                     "<tag>&lt;&amp;\"'&gt;</tag>" % enc).encode(enc))
+
+        elem = ET.Element("tag")
+        elem.attrib["key"] = "<&\"\'>"
+        self.assertEqual(serialize(elem), '<tag key="&lt;&amp;&quot;\'&gt;" />')
+        self.assertEqual(serialize(elem, encoding="utf-8"),
+                b'<tag key="&lt;&amp;&quot;\'&gt;" />')
+        self.assertEqual(serialize(elem, encoding="us-ascii"),
+                b'<tag key="&lt;&amp;&quot;\'&gt;" />')
+        for enc in ("iso-8859-1", "utf-16", "utf-32"):
+            self.assertEqual(serialize(elem, encoding=enc),
+                    ("<?xml version='1.0' encoding='%s'?>\n"
+                     "<tag key=\"&lt;&amp;&quot;'&gt;\" />" % enc).encode(enc))
+
+        elem = ET.Element("tag")
+        elem.text = '\xe5\xf6\xf6<>'
+        self.assertEqual(serialize(elem), '<tag>\xe5\xf6\xf6&lt;&gt;</tag>')
+        self.assertEqual(serialize(elem, encoding="utf-8"),
+                b'<tag>\xc3\xa5\xc3\xb6\xc3\xb6&lt;&gt;</tag>')
+        self.assertEqual(serialize(elem, encoding="us-ascii"),
+                b'<tag>&#229;&#246;&#246;&lt;&gt;</tag>')
+        for enc in ("iso-8859-1", "utf-16", "utf-32"):
+            self.assertEqual(serialize(elem, encoding=enc),
+                    ("<?xml version='1.0' encoding='%s'?>\n"
+                     "<tag>åöö&lt;&gt;</tag>" % enc).encode(enc))
+
+        elem = ET.Element("tag")
+        elem.attrib["key"] = '\xe5\xf6\xf6<>'
+        self.assertEqual(serialize(elem), '<tag key="\xe5\xf6\xf6&lt;&gt;" />')
+        self.assertEqual(serialize(elem, encoding="utf-8"),
+                b'<tag key="\xc3\xa5\xc3\xb6\xc3\xb6&lt;&gt;" />')
+        self.assertEqual(serialize(elem, encoding="us-ascii"),
+                b'<tag key="&#229;&#246;&#246;&lt;&gt;" />')
+        for enc in ("iso-8859-1", "utf-16", "utf-16le", "utf-16be", "utf-32"):
+            self.assertEqual(serialize(elem, encoding=enc),
+                    ("<?xml version='1.0' encoding='%s'?>\n"
+                     "<tag key=\"åöö&lt;&gt;\" />" % enc).encode(enc))
+
+    def test_write_to_filename(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        tree.write(TESTFN)
+        with open(TESTFN, 'rb') as f:
+            self.assertEqual(f.read(), b'''<site />''')
+
+    def test_write_to_text_file(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        with open(TESTFN, 'w', encoding='utf-8') as f:
+            tree.write(f, encoding='unicode')
+            self.assertFalse(f.closed)
+        with open(TESTFN, 'rb') as f:
+            self.assertEqual(f.read(), b'''<site />''')
+
+    def test_write_to_binary_file(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        with open(TESTFN, 'wb') as f:
+            tree.write(f)
+            self.assertFalse(f.closed)
+        with open(TESTFN, 'rb') as f:
+            self.assertEqual(f.read(), b'''<site />''')
+
+    def test_write_to_binary_file_with_bom(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        # test BOM writing to buffered file
+        with open(TESTFN, 'wb') as f:
+            tree.write(f, encoding='utf-16')
+            self.assertFalse(f.closed)
+        with open(TESTFN, 'rb') as f:
+            self.assertEqual(f.read(),
+                    '''<?xml version='1.0' encoding='utf-16'?>\n'''
+                    '''<site />'''.encode("utf-16"))
+        # test BOM writing to non-buffered file
+        with open(TESTFN, 'wb', buffering=0) as f:
+            tree.write(f, encoding='utf-16')
+            self.assertFalse(f.closed)
+        with open(TESTFN, 'rb') as f:
+            self.assertEqual(f.read(),
+                    '''<?xml version='1.0' encoding='utf-16'?>\n'''
+                    '''<site />'''.encode("utf-16"))
+
+    def test_read_from_stringio(self):
+        tree = ET.ElementTree()
+        stream = io.StringIO('''<?xml version="1.0"?><site></site>''')
+        tree.parse(stream)
+        self.assertEqual(tree.getroot().tag, 'site')
+
+    def test_write_to_stringio(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        stream = io.StringIO()
+        tree.write(stream, encoding='unicode')
+        self.assertEqual(stream.getvalue(), '''<site />''')
+
+    def test_read_from_bytesio(self):
+        tree = ET.ElementTree()
+        raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''')
+        tree.parse(raw)
+        self.assertEqual(tree.getroot().tag, 'site')
+
+    def test_write_to_bytesio(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        raw = io.BytesIO()
+        tree.write(raw)
+        self.assertEqual(raw.getvalue(), b'''<site />''')
+
+    class dummy:
+        pass
+
+    def test_read_from_user_text_reader(self):
+        stream = io.StringIO('''<?xml version="1.0"?><site></site>''')
+        reader = self.dummy()
+        reader.read = stream.read
+        tree = ET.ElementTree()
+        tree.parse(reader)
+        self.assertEqual(tree.getroot().tag, 'site')
+
+    def test_write_to_user_text_writer(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        stream = io.StringIO()
+        writer = self.dummy()
+        writer.write = stream.write
+        tree.write(writer, encoding='unicode')
+        self.assertEqual(stream.getvalue(), '''<site />''')
+
+    def test_read_from_user_binary_reader(self):
+        raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''')
+        reader = self.dummy()
+        reader.read = raw.read
+        tree = ET.ElementTree()
+        tree.parse(reader)
+        self.assertEqual(tree.getroot().tag, 'site')
+        tree = ET.ElementTree()
+
+    def test_write_to_user_binary_writer(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        raw = io.BytesIO()
+        writer = self.dummy()
+        writer.write = raw.write
+        tree.write(writer)
+        self.assertEqual(raw.getvalue(), b'''<site />''')
+
+    def test_write_to_user_binary_writer_with_bom(self):
+        tree = ET.ElementTree(ET.XML('''<site />'''))
+        raw = io.BytesIO()
+        writer = self.dummy()
+        writer.write = raw.write
+        writer.seekable = lambda: True
+        writer.tell = raw.tell
+        tree.write(writer, encoding="utf-16")
+        self.assertEqual(raw.getvalue(),
+                '''<?xml version='1.0' encoding='utf-16'?>\n'''
+                '''<site />'''.encode("utf-16"))
+
+    def test_tostringlist_invariant(self):
+        root = ET.fromstring('<tag>foo</tag>')
+        self.assertEqual(
+            ET.tostring(root, 'unicode'),
+            ''.join(ET.tostringlist(root, 'unicode')))
+        self.assertEqual(
+            ET.tostring(root, 'utf-16'),
+            b''.join(ET.tostringlist(root, 'utf-16')))
+
+
+class ParseErrorTest(unittest.TestCase):
+    def test_subclass(self):
+        self.assertIsInstance(ET.ParseError(), SyntaxError)
+
+    def _get_error(self, s):
+        try:
+            ET.fromstring(s)
+        except ET.ParseError as e:
+            return e
+
+    def test_error_position(self):
+        self.assertEqual(self._get_error('foo').position, (1, 0))
+        self.assertEqual(self._get_error('<tag>&foo;</tag>').position, (1, 5))
+        self.assertEqual(self._get_error('foobar<').position, (1, 6))
+
+    def test_error_code(self):
+        import xml.parsers.expat.errors as ERRORS
+        self.assertEqual(self._get_error('foo').code,
+                ERRORS.codes[ERRORS.XML_ERROR_SYNTAX])
+
+
+class KeywordArgsTest(unittest.TestCase):
+    # Test various issues with keyword arguments passed to ET.Element
+    # constructor and methods
+    def test_issue14818(self):
+        x = ET.XML("<a>foo</a>")
+        self.assertEqual(x.find('a', None),
+                         x.find(path='a', namespaces=None))
+        self.assertEqual(x.findtext('a', None, None),
+                         x.findtext(path='a', default=None, namespaces=None))
+        self.assertEqual(x.findall('a', None),
+                         x.findall(path='a', namespaces=None))
+        self.assertEqual(list(x.iterfind('a', None)),
+                         list(x.iterfind(path='a', namespaces=None)))
+
+        self.assertEqual(ET.Element('a').attrib, {})
+        elements = [
+            ET.Element('a', dict(href="#", id="foo")),
+            ET.Element('a', attrib=dict(href="#", id="foo")),
+            ET.Element('a', dict(href="#"), id="foo"),
+            ET.Element('a', href="#", id="foo"),
+            ET.Element('a', dict(href="#", id="foo"), href="#", id="foo"),
+        ]
+        for e in elements:
+            self.assertEqual(e.tag, 'a')
+            self.assertEqual(e.attrib, dict(href="#", id="foo"))
+
+        e2 = ET.SubElement(elements[0], 'foobar', attrib={'key1': 'value1'})
+        self.assertEqual(e2.attrib['key1'], 'value1')
+
+        with self.assertRaisesRegex(TypeError, 'must be dict, not str'):
+            ET.Element('a', "I'm not a dict")
+        with self.assertRaisesRegex(TypeError, 'must be dict, not str'):
+            ET.Element('a', attrib="I'm not a dict")
 
-def bug_200709_default_namespace():
-    """
-
-    >>> e = ET.Element("{default}elem")
-    >>> s = ET.SubElement(e, "{default}elem")
-    >>> serialize(e, default_namespace="default") # 1
-    '<elem xmlns="default"><elem /></elem>'
-
-    >>> e = ET.Element("{default}elem")
-    >>> s = ET.SubElement(e, "{default}elem")
-    >>> s = ET.SubElement(e, "{not-default}elem")
-    >>> serialize(e, default_namespace="default") # 2
-    '<elem xmlns="default" xmlns:ns1="not-default"><elem /><ns1:elem /></elem>'
-
-    >>> e = ET.Element("{default}elem")
-    >>> s = ET.SubElement(e, "{default}elem")
-    >>> s = ET.SubElement(e, "elem") # unprefixed name
-    >>> serialize(e, default_namespace="default") # 3
-    Traceback (most recent call last):
-    ValueError: cannot use non-qualified names with default_namespace option
-
-    """
-
-def bug_200709_register_namespace():
-    """
-
-    >>> ET.tostring(ET.Element("{http://namespace.invalid/does/not/exist/}title"))
-    b'<ns0:title xmlns:ns0="http://namespace.invalid/does/not/exist/" />'
-    >>> ET.register_namespace("foo", "http://namespace.invalid/does/not/exist/")
-    >>> ET.tostring(ET.Element("{http://namespace.invalid/does/not/exist/}title"))
-    b'<foo:title xmlns:foo="http://namespace.invalid/does/not/exist/" />'
-
-    And the Dublin Core namespace is in the default list:
-
-    >>> ET.tostring(ET.Element("{http://purl.org/dc/elements/1.1/}title"))
-    b'<dc:title xmlns:dc="http://purl.org/dc/elements/1.1/" />'
-
-    """
-
-def bug_200709_element_comment():
-    """
-
-    Not sure if this can be fixed, really (since the serializer needs
-    ET.Comment, not cET.comment).
-
-    >>> a = ET.Element('a')
-    >>> a.append(ET.Comment('foo'))
-    >>> a[0].tag == ET.Comment
-    True
-
-    >>> a = ET.Element('a')
-    >>> a.append(ET.PI('foo'))
-    >>> a[0].tag == ET.PI
-    True
-
-    """
-
-def bug_200709_element_insert():
-    """
-
-    >>> a = ET.Element('a')
-    >>> b = ET.SubElement(a, 'b')
-    >>> c = ET.SubElement(a, 'c')
-    >>> d = ET.Element('d')
-    >>> a.insert(0, d)
-    >>> summarize_list(a)
-    ['d', 'b', 'c']
-    >>> a.insert(-1, d)
-    >>> summarize_list(a)
-    ['d', 'b', 'd', 'c']
-
-    """
-
-def bug_200709_iter_comment():
-    """
-
-    >>> a = ET.Element('a')
-    >>> b = ET.SubElement(a, 'b')
-    >>> comment_b = ET.Comment("TEST-b")
-    >>> b.append(comment_b)
-    >>> summarize_list(a.iter(ET.Comment))
-    ['<Comment>']
+# --------------------------------------------------------------------
 
-    """
+class NoAcceleratorTest(unittest.TestCase):
+    def setUp(self):
+        if not pyET:
+            raise unittest.SkipTest('only for the Python version')
 
-# --------------------------------------------------------------------
-# reported on bugs.python.org
-
-def bug_1534630():
-    """
-
-    >>> bob = ET.TreeBuilder()
-    >>> e = bob.data("data")
-    >>> e = bob.start("tag", {})
-    >>> e = bob.end("tag")
-    >>> e = bob.close()
-    >>> serialize(e)
-    '<tag />'
-
-    """
-
-def check_issue6233():
-    """
-
-    >>> e = ET.XML(b"<?xml version='1.0' encoding='utf-8'?><body>t\\xc3\\xa3g</body>")
-    >>> ET.tostring(e, 'ascii')
-    b"<?xml version='1.0' encoding='ascii'?>\\n<body>t&#227;g</body>"
-    >>> e = ET.XML(b"<?xml version='1.0' encoding='iso-8859-1'?><body>t\\xe3g</body>")
-    >>> ET.tostring(e, 'ascii')
-    b"<?xml version='1.0' encoding='ascii'?>\\n<body>t&#227;g</body>"
-
-    """
-
-def check_issue3151():
-    """
-
-    >>> e = ET.XML('<prefix:localname xmlns:prefix="${stuff}"/>')
-    >>> e.tag
-    '{${stuff}}localname'
-    >>> t = ET.ElementTree(e)
-    >>> ET.tostring(e)
-    b'<ns0:localname xmlns:ns0="${stuff}" />'
-
-    """
-
-def check_issue6565():
-    """
-
-    >>> elem = ET.XML("<body><tag/></body>")
-    >>> summarize_list(elem)
-    ['tag']
-    >>> newelem = ET.XML(SAMPLE_XML)
-    >>> elem[:] = newelem[:]
-    >>> summarize_list(elem)
-    ['tag', 'tag', 'section']
-
-    """
-
-def check_issue10777():
-    """
-    Registering a namespace twice caused a "dictionary changed size during
-    iteration" bug.
-
-    >>> ET.register_namespace('test10777', 'http://myuri/')
-    >>> ET.register_namespace('test10777', 'http://myuri/')
-    """
-
-def check_html_empty_elems_serialization(self):
-    # issue 15970
-    # from http://www.w3.org/TR/html401/index/elements.html
-    """
-
-    >>> empty_elems = ['AREA', 'BASE', 'BASEFONT', 'BR', 'COL', 'FRAME', 'HR',
-    ...                'IMG', 'INPUT', 'ISINDEX', 'LINK', 'META', 'PARAM']
-    >>> elems = ''.join('<%s />' % elem for elem in empty_elems)
-    >>> serialize(ET.XML('<html>%s</html>' % elems), method='html')
-    '<html><AREA><BASE><BASEFONT><BR><COL><FRAME><HR><IMG><INPUT><ISINDEX><LINK><META><PARAM></html>'
-    >>> serialize(ET.XML('<html>%s</html>' % elems.lower()), method='html')
-    '<html><area><base><basefont><br><col><frame><hr><img><input><isindex><link><meta><param></html>'
-    >>> elems = ''.join('<%s></%s>' % (elem, elem) for elem in empty_elems)
-    >>> serialize(ET.XML('<html>%s</html>' % elems), method='html')
-    '<html><AREA><BASE><BASEFONT><BR><COL><FRAME><HR><IMG><INPUT><ISINDEX><LINK><META><PARAM></html>'
-    >>> serialize(ET.XML('<html>%s</html>' % elems.lower()), method='html')
-    '<html><area><base><basefont><br><col><frame><hr><img><input><isindex><link><meta><param></html>'
-
-    """
+    # Test that the C accelerator was not imported for pyET
+    def test_correct_import_pyET(self):
+        self.assertEqual(pyET.Element.__module__, 'xml.etree.ElementTree')
+        self.assertEqual(pyET.SubElement.__module__, 'xml.etree.ElementTree')
 
 # --------------------------------------------------------------------
 
@@ -1894,44 +2245,71 @@ class CleanContext(object):
             ("This method will be removed in future versions.  "
              "Use .+ instead.", DeprecationWarning),
             ("This method will be removed in future versions.  "
-             "Use .+ instead.", PendingDeprecationWarning),
-            # XMLParser.doctype() is deprecated.
-            ("This method of XMLParser is deprecated.  Define doctype.. "
-             "method on the TreeBuilder target.", DeprecationWarning))
+             "Use .+ instead.", PendingDeprecationWarning))
         self.checkwarnings = support.check_warnings(*deprecations, quiet=quiet)
 
     def __enter__(self):
-        from xml.etree import ElementTree
-        self._nsmap = ElementTree._namespace_map
-        self._path_cache = ElementTree.ElementPath._cache
+        from xml.etree import ElementPath
+        self._nsmap = ET.register_namespace._namespace_map
         # Copy the default namespace mapping
-        ElementTree._namespace_map = self._nsmap.copy()
+        self._nsmap_copy = self._nsmap.copy()
         # Copy the path cache (should be empty)
-        ElementTree.ElementPath._cache = self._path_cache.copy()
+        self._path_cache = ElementPath._cache
+        ElementPath._cache = self._path_cache.copy()
         self.checkwarnings.__enter__()
 
     def __exit__(self, *args):
-        from xml.etree import ElementTree
+        from xml.etree import ElementPath
         # Restore mapping and path cache
-        ElementTree._namespace_map = self._nsmap
-        ElementTree.ElementPath._cache = self._path_cache
+        self._nsmap.clear()
+        self._nsmap.update(self._nsmap_copy)
+        ElementPath._cache = self._path_cache
         self.checkwarnings.__exit__(*args)
 
 
-def test_main(module_name='xml.etree.ElementTree'):
-    from test import test_xml_etree
+def test_main(module=None):
+    # When invoked without a module, runs the Python ET tests by loading pyET.
+    # Otherwise, uses the given module as the ET.
+    global pyET
+    pyET = import_fresh_module('xml.etree.ElementTree',
+                               blocked=['_elementtree'])
+    if module is None:
+        module = pyET
+
+    global ET
+    ET = module
+
+    test_classes = [
+        ModuleTest,
+        ElementSlicingTest,
+        BasicElementTest,
+        ElementTreeTest,
+        IOTest,
+        ParseErrorTest,
+        XIncludeTest,
+        ElementTreeTypeTest,
+        ElementFindTest,
+        ElementIterTest,
+        TreeBuilderTest,
+        BugsTest,
+        ]
+
+    # These tests will only run for the pure-Python version that doesn't import
+    # _elementtree. We can't use skipUnless here, because pyET is filled in only
+    # after the module is loaded.
+    if pyET is not ET:
+        test_classes.extend([
+            NoAcceleratorTest,
+            ])
 
-    use_py_module = (module_name == 'xml.etree.ElementTree')
-
-    # The same doctests are used for both the Python and the C implementations
-    assert test_xml_etree.ET.__name__ == module_name
-
-    # XXX the C module should give the same warnings as the Python module
-    with CleanContext(quiet=not use_py_module):
-        support.run_doctest(test_xml_etree, verbosity=True)
+    try:
+        # XXX the C module should give the same warnings as the Python module
+        with CleanContext(quiet=(pyET is not ET)):
+            support.run_unittest(*test_classes)
+    finally:
+        # don't interfere with subsequent tests
+        ET = pyET = None
 
-    # The module should not be changed by the tests
-    assert test_xml_etree.ET.__name__ == module_name
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
index 2ff118fae8..bcaa724344 100644
--- a/Lib/test/test_xml_etree_c.py
+++ b/Lib/test/test_xml_etree_c.py
@@ -1,36 +1,13 @@
 # xml.etree test for cElementTree
-
+import sys, struct
 from test import support
-from test.support import bigmemtest, _2G
+from test.support import import_fresh_module
 import unittest
 
-cET = support.import_module('xml.etree.cElementTree')
-
-
-# cElementTree specific tests
-
-def sanity():
-    r"""
-    Import sanity.
-
-    >>> from xml.etree import cElementTree
-
-    Issue #6697.
-
-    >>> e = cElementTree.Element('a')
-    >>> getattr(e, '\uD800')           # doctest: +ELLIPSIS
-    Traceback (most recent call last):
-      ...
-    UnicodeEncodeError: ...
-
-    >>> p = cElementTree.XMLParser()
-    >>> p.version.split()[0]
-    'Expat'
-    >>> getattr(p, '\uD800')
-    Traceback (most recent call last):
-     ...
-    AttributeError: 'XMLParser' object has no attribute '\ud800'
-    """
+cET = import_fresh_module('xml.etree.ElementTree',
+                          fresh=['_elementtree'])
+cET_alias = import_fresh_module('xml.etree.cElementTree',
+                                fresh=['_elementtree', 'xml.etree'])
 
 
 class MiscTests(unittest.TestCase):
@@ -47,27 +24,64 @@ class MiscTests(unittest.TestCase):
             data = None
 
 
+@unittest.skipUnless(cET, 'requires _elementtree')
+class TestAliasWorking(unittest.TestCase):
+    # Test that the cET alias module is alive
+    def test_alias_working(self):
+        e = cET_alias.Element('foo')
+        self.assertEqual(e.tag, 'foo')
+
+
+@unittest.skipUnless(cET, 'requires _elementtree')
+class TestAcceleratorImported(unittest.TestCase):
+    # Test that the C accelerator was imported, as expected
+    def test_correct_import_cET(self):
+        self.assertEqual(cET.SubElement.__module__, '_elementtree')
+
+    def test_correct_import_cET_alias(self):
+        self.assertEqual(cET_alias.SubElement.__module__, '_elementtree')
+
+
+@unittest.skipUnless(cET, 'requires _elementtree')
+@support.cpython_only
+class SizeofTest(unittest.TestCase):
+    def setUp(self):
+        self.elementsize = support.calcobjsize('5P')
+        # extra
+        self.extra = struct.calcsize('PiiP4P')
+
+    check_sizeof = support.check_sizeof
+
+    def test_element(self):
+        e = cET.Element('a')
+        self.check_sizeof(e, self.elementsize)
+
+    def test_element_with_attrib(self):
+        e = cET.Element('a', href='about:')
+        self.check_sizeof(e, self.elementsize + self.extra)
+
+    def test_element_with_children(self):
+        e = cET.Element('a')
+        for i in range(5):
+            cET.SubElement(e, 'span')
+        # should have space for 8 children now
+        self.check_sizeof(e, self.elementsize + self.extra +
+                             struct.calcsize('8P'))
+
 def test_main():
     from test import test_xml_etree, test_xml_etree_c
 
     # Run the tests specific to the C implementation
-    support.run_doctest(test_xml_etree_c, verbosity=True)
-
-    support.run_unittest(MiscTests)
-
-    # Assign the C implementation before running the doctests
-    # Patch the __name__, to prevent confusion with the pure Python test
-    pyET = test_xml_etree.ET
-    py__name__ = test_xml_etree.__name__
-    test_xml_etree.ET = cET
-    if __name__ != '__main__':
-        test_xml_etree.__name__ = __name__
-    try:
-        # Run the same test suite as xml.etree.ElementTree
-        test_xml_etree.test_main(module_name='xml.etree.cElementTree')
-    finally:
-        test_xml_etree.ET = pyET
-        test_xml_etree.__name__ = py__name__
+    support.run_unittest(
+        MiscTests,
+        TestAliasWorking,
+        TestAcceleratorImported,
+        SizeofTest,
+        )
+
+    # Run the same test suite as the Python module
+    test_xml_etree.test_main(module=cET)
+
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py
index 38141912a1..16f85c5555 100644
--- a/Lib/test/test_xmlrpc.py
+++ b/Lib/test/test_xmlrpc.py
@@ -24,6 +24,8 @@ alist = [{'astring': 'foo@bar.baz.spam',
           'ashortlong': 2,
           'anotherlist': ['.zyx.41'],
           'abase64': xmlrpclib.Binary(b"my dog has fleas"),
+          'b64bytes': b"my dog has fleas",
+          'b64bytearray': bytearray(b"my dog has fleas"),
           'boolean': False,
           'unicode': '\u4000\u6000\u8000',
           'ukey\u4000': 'regular value',
@@ -44,27 +46,54 @@ class XMLRPCTestCase(unittest.TestCase):
     def test_dump_bare_datetime(self):
         # This checks that an unwrapped datetime.date object can be handled
         # by the marshalling code.  This can't be done via test_dump_load()
-        # since with use_datetime set to 1 the unmarshaller would create
+        # since with use_builtin_types set to 1 the unmarshaller would create
         # datetime objects for the 'datetime[123]' keys as well
         dt = datetime.datetime(2005, 2, 10, 11, 41, 23)
+        self.assertEqual(dt, xmlrpclib.DateTime('20050210T11:41:23'))
         s = xmlrpclib.dumps((dt,))
-        (newdt,), m = xmlrpclib.loads(s, use_datetime=1)
+
+        result, m = xmlrpclib.loads(s, use_builtin_types=True)
+        (newdt,) = result
         self.assertEqual(newdt, dt)
-        self.assertEqual(m, None)
+        self.assertIs(type(newdt), datetime.datetime)
+        self.assertIsNone(m)
+
+        result, m = xmlrpclib.loads(s, use_builtin_types=False)
+        (newdt,) = result
+        self.assertEqual(newdt, dt)
+        self.assertIs(type(newdt), xmlrpclib.DateTime)
+        self.assertIsNone(m)
+
+        result, m = xmlrpclib.loads(s, use_datetime=True)
+        (newdt,) = result
+        self.assertEqual(newdt, dt)
+        self.assertIs(type(newdt), datetime.datetime)
+        self.assertIsNone(m)
+
+        result, m = xmlrpclib.loads(s, use_datetime=False)
+        (newdt,) = result
+        self.assertEqual(newdt, dt)
+        self.assertIs(type(newdt), xmlrpclib.DateTime)
+        self.assertIsNone(m)
 
-        (newdt,), m = xmlrpclib.loads(s, use_datetime=0)
-        self.assertEqual(newdt, xmlrpclib.DateTime('20050210T11:41:23'))
 
     def test_datetime_before_1900(self):
         # same as before but with a date before 1900
         dt = datetime.datetime(1,  2, 10, 11, 41, 23)
+        self.assertEqual(dt, xmlrpclib.DateTime('00010210T11:41:23'))
         s = xmlrpclib.dumps((dt,))
-        (newdt,), m = xmlrpclib.loads(s, use_datetime=1)
+
+        result, m = xmlrpclib.loads(s, use_builtin_types=True)
+        (newdt,) = result
         self.assertEqual(newdt, dt)
-        self.assertEqual(m, None)
+        self.assertIs(type(newdt), datetime.datetime)
+        self.assertIsNone(m)
 
-        (newdt,), m = xmlrpclib.loads(s, use_datetime=0)
-        self.assertEqual(newdt, xmlrpclib.DateTime('00010210T11:41:23'))
+        result, m = xmlrpclib.loads(s, use_builtin_types=False)
+        (newdt,) = result
+        self.assertEqual(newdt, dt)
+        self.assertIs(type(newdt), xmlrpclib.DateTime)
+        self.assertIsNone(m)
 
     def test_bug_1164912 (self):
         d = xmlrpclib.DateTime()
@@ -125,6 +154,22 @@ class XMLRPCTestCase(unittest.TestCase):
         self.assertRaises(OverflowError, m.dump_int,
                           xmlrpclib.MININT-1, dummy_write)
 
+    def test_dump_double(self):
+        xmlrpclib.dumps((float(2 ** 34),))
+        xmlrpclib.dumps((float(xmlrpclib.MAXINT),
+                         float(xmlrpclib.MININT)))
+        xmlrpclib.dumps((float(xmlrpclib.MAXINT + 42),
+                         float(xmlrpclib.MININT - 42)))
+
+        def dummy_write(s):
+            pass
+
+        m = xmlrpclib.Marshaller()
+        m.dump_double(xmlrpclib.MAXINT, dummy_write)
+        m.dump_double(xmlrpclib.MININT, dummy_write)
+        m.dump_double(xmlrpclib.MAXINT + 42, dummy_write)
+        m.dump_double(xmlrpclib.MININT - 42, dummy_write)
+
     def test_dump_none(self):
         value = alist + [None]
         arg1 = (alist + [None],)
@@ -133,6 +178,25 @@ class XMLRPCTestCase(unittest.TestCase):
                           xmlrpclib.loads(strg)[0][0])
         self.assertRaises(TypeError, xmlrpclib.dumps, (arg1,))
 
+    def test_dump_bytes(self):
+        sample = b"my dog has fleas"
+        self.assertEqual(sample, xmlrpclib.Binary(sample))
+        for type_ in bytes, bytearray, xmlrpclib.Binary:
+            value = type_(sample)
+            s = xmlrpclib.dumps((value,))
+
+            result, m = xmlrpclib.loads(s, use_builtin_types=True)
+            (newvalue,) = result
+            self.assertEqual(newvalue, sample)
+            self.assertIs(type(newvalue), bytes)
+            self.assertIsNone(m)
+
+            result, m = xmlrpclib.loads(s, use_builtin_types=False)
+            (newvalue,) = result
+            self.assertEqual(newvalue, sample)
+            self.assertIs(type(newvalue), xmlrpclib.Binary)
+            self.assertIsNone(m)
+
     def test_get_host_info(self):
         # see bug #3613, this raised a TypeError
         transp = xmlrpc.client.Transport()
@@ -140,9 +204,6 @@ class XMLRPCTestCase(unittest.TestCase):
                           ('host.tld',
                            [('Authorization', 'Basic dXNlcg==')], {}))
 
-    def test_dump_bytes(self):
-        self.assertRaises(TypeError, xmlrpclib.dumps, (b"my dog has fleas",))
-
     def test_ssl_presence(self):
         try:
             import ssl
@@ -980,10 +1041,44 @@ class CGIHandlerTestCase(unittest.TestCase):
             len(content))
 
 
+class UseBuiltinTypesTestCase(unittest.TestCase):
+
+    def test_use_builtin_types(self):
+        # SimpleXMLRPCDispatcher.__init__ accepts use_builtin_types, which
+        # makes all dispatch of binary data as bytes instances, and all
+        # dispatch of datetime argument as datetime.datetime instances.
+        self.log = []
+        expected_bytes = b"my dog has fleas"
+        expected_date = datetime.datetime(2008, 5, 26, 18, 25, 12)
+        marshaled = xmlrpclib.dumps((expected_bytes, expected_date), 'foobar')
+        def foobar(*args):
+            self.log.extend(args)
+        handler = xmlrpc.server.SimpleXMLRPCDispatcher(
+            allow_none=True, encoding=None, use_builtin_types=True)
+        handler.register_function(foobar)
+        handler._marshaled_dispatch(marshaled)
+        self.assertEqual(len(self.log), 2)
+        mybytes, mydate = self.log
+        self.assertEqual(self.log, [expected_bytes, expected_date])
+        self.assertIs(type(mydate), datetime.datetime)
+        self.assertIs(type(mybytes), bytes)
+
+    def test_cgihandler_has_use_builtin_types_flag(self):
+        handler = xmlrpc.server.CGIXMLRPCRequestHandler(use_builtin_types=True)
+        self.assertTrue(handler.use_builtin_types)
+
+    def test_xmlrpcserver_has_use_builtin_types_flag(self):
+        server = xmlrpc.server.SimpleXMLRPCServer(("localhost", 0),
+            use_builtin_types=True)
+        server.server_close()
+        self.assertTrue(server.use_builtin_types)
+
+
 @support.reap_threads
 def test_main():
     xmlrpc_tests = [XMLRPCTestCase, HelperTestCase, DateTimeTestCase,
          BinaryTestCase, FaultTestCase]
+    xmlrpc_tests.append(UseBuiltinTypesTestCase)
     xmlrpc_tests.append(SimpleServerTestCase)
     xmlrpc_tests.append(KeepaliveServerTestCase1)
     xmlrpc_tests.append(KeepaliveServerTestCase2)
diff --git a/Lib/test/test_xmlrpc_net.py b/Lib/test/test_xmlrpc_net.py
index d72f8ac895..dfb5f9aa3d 100644
--- a/Lib/test/test_xmlrpc_net.py
+++ b/Lib/test/test_xmlrpc_net.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import collections
+import collections.abc
 import errno
 import socket
 import sys
@@ -49,7 +49,7 @@ class CurrentTimeTest(unittest.TestCase):
 
         # Perform a minimal sanity check on the result, just to be sure
         # the request means what we think it means.
-        self.assertIsInstance(builders, collections.Sequence)
+        self.assertIsInstance(builders, collections.abc.Sequence)
         self.assertTrue([x for x in builders if "3.x" in x], builders)
 
 
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index c00c83ef05..5e837cd6de 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -1,9 +1,3 @@
-# We can test part of the module without zlib.
-try:
-    import zlib
-except ImportError:
-    zlib = None
-
 import io
 import os
 import sys
@@ -20,7 +14,8 @@ from random import randint, random
 from unittest import skipUnless
 
 from test.support import (TESTFN, run_unittest, findfile, unlink,
-                            captured_stdout)
+                          requires_zlib, requires_bz2, requires_lzma,
+                          captured_stdout)
 
 TESTFN2 = TESTFN + "2"
 TESTFNDIR = TESTFN + "d"
@@ -270,44 +265,44 @@ class TestsWithSourceFile(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_iterlines_test(f, zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_test(f, zipfile.ZIP_DEFLATED)
 
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_open_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_open_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_random_open_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_random_open_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_readline_read_deflated(self):
         # Issue #7610: calls to readline() interleaved with calls to read().
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_readline_read_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_readline_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_readline_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_readlines_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_readlines_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_iterlines_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_iterlines_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_low_compression(self):
         """Check for cases where compressed data is larger than original."""
         # Create the ZIP archive
@@ -320,6 +315,103 @@ class TestsWithSourceFile(unittest.TestCase):
                 self.assertEqual(openobj.read(1), b'1')
                 self.assertEqual(openobj.read(1), b'2')
 
+    @requires_bz2
+    def test_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_open_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_open_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_random_open_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_random_open_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_readline_read_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readline_read_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_readline_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readline_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_readlines_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readlines_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_iterlines_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_iterlines_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_low_compression_bzip2(self):
+        """Check for cases where compressed data is larger than original."""
+        # Create the ZIP archive
+        with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_BZIP2) as zipfp:
+            zipfp.writestr("strfile", '12')
+
+        # Get an open object for strfile
+        with zipfile.ZipFile(TESTFN2, "r", zipfile.ZIP_BZIP2) as zipfp:
+            with zipfp.open("strfile") as openobj:
+                self.assertEqual(openobj.read(1), b'1')
+                self.assertEqual(openobj.read(1), b'2')
+
+    @requires_lzma
+    def test_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_open_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_open_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_random_open_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_random_open_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_readline_read_lzma(self):
+        # Issue #7610: calls to readline() interleaved with calls to read().
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readline_read_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_readline_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readline_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_readlines_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readlines_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_iterlines_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_iterlines_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_low_compression_lzma(self):
+        """Check for cases where compressed data is larger than original."""
+        # Create the ZIP archive
+        with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_LZMA) as zipfp:
+            zipfp.writestr("strfile", '12')
+
+        # Get an open object for strfile
+        with zipfile.ZipFile(TESTFN2, "r", zipfile.ZIP_LZMA) as zipfp:
+            with zipfp.open("strfile") as openobj:
+                self.assertEqual(openobj.read(1), b'1')
+                self.assertEqual(openobj.read(1), b'2')
+
     def test_absolute_arcnames(self):
         with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp:
             zipfp.write(TESTFN, "/absolute")
@@ -378,7 +470,7 @@ class TestsWithSourceFile(unittest.TestCase):
             with open(TESTFN, "rb") as f:
                 self.assertEqual(zipfp.read(TESTFN), f.read())
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_per_file_compression(self):
         """Check that files within a Zip archive can have different
         compression options."""
@@ -446,8 +538,15 @@ class TestsWithSourceFile(unittest.TestCase):
         with open(filename, 'rb') as f:
             self.assertEqual(f.read(), content)
 
-    def test_extract_hackers_arcnames(self):
-        hacknames = [
+    def test_sanitize_windows_name(self):
+        san = zipfile.ZipFile._sanitize_windows_name
+        # Passing pathsep in allows this test to work regardless of platform.
+        self.assertEqual(san(r',,?,C:,foo,bar/z', ','), r'_,C_,foo,bar/z')
+        self.assertEqual(san(r'a\b,c<d>e|f"g?h*i', ','), r'a\b,c_d_e_f_g_h_i')
+        self.assertEqual(san('../../foo../../ba..r', '/'), r'foo/ba..r')
+
+    def test_extract_hackers_arcnames_common_cases(self):
+        common_hacknames = [
             ('../foo/bar', 'foo/bar'),
             ('foo/../bar', 'foo/bar'),
             ('foo/../../bar', 'foo/bar'),
@@ -457,8 +556,12 @@ class TestsWithSourceFile(unittest.TestCase):
             ('/foo/../bar', 'foo/bar'),
             ('/foo/../../bar', 'foo/bar'),
         ]
-        if os.path.sep == '\\':  # Windows.
-            hacknames.extend([
+        self._test_extract_hackers_arcnames(common_hacknames)
+
+    @unittest.skipIf(os.path.sep != '\\', 'Requires \\ as path separator.')
+    def test_extract_hackers_arcnames_windows_only(self):
+        """Test combination of path fixing and windows name sanitization."""
+        windows_hacknames = [
                 (r'..\foo\bar', 'foo/bar'),
                 (r'..\/foo\/bar', 'foo/bar'),
                 (r'foo/\..\/bar', 'foo/bar'),
@@ -478,14 +581,19 @@ class TestsWithSourceFile(unittest.TestCase):
                 (r'C:/../C:/foo/bar', 'C_/foo/bar'),
                 (r'a:b\c<d>e|f"g?h*i', 'b/c_d_e_f_g_h_i'),
                 ('../../foo../../ba..r', 'foo/ba..r'),
-            ])
-        else:  # Unix
-            hacknames.extend([
-                ('//foo/bar', 'foo/bar'),
-                ('../../foo../../ba..r', 'foo../ba..r'),
-                (r'foo/..\bar', r'foo/..\bar'),
-            ])
+        ]
+        self._test_extract_hackers_arcnames(windows_hacknames)
+
+    @unittest.skipIf(os.path.sep != '/', r'Requires / as path separator.')
+    def test_extract_hackers_arcnames_posix_only(self):
+        posix_hacknames = [
+            ('//foo/bar', 'foo/bar'),
+            ('../../foo../../ba..r', 'foo../ba..r'),
+            (r'foo/..\bar', r'foo/..\bar'),
+        ]
+        self._test_extract_hackers_arcnames(posix_hacknames)
 
+    def _test_extract_hackers_arcnames(self, hacknames):
         for arcname, fixedname in hacknames:
             content = b'foobar' + arcname.encode()
             with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_STORED) as zipfp:
@@ -502,7 +610,8 @@ class TestsWithSourceFile(unittest.TestCase):
             with zipfile.ZipFile(TESTFN2, 'r') as zipfp:
                 writtenfile = zipfp.extract(arcname, targetpath)
                 self.assertEqual(writtenfile, correctfile,
-                                 msg="extract %r" % arcname)
+                                 msg='extract %r: %r != %r' %
+                                 (arcname, writtenfile, correctfile))
             self.check_file(correctfile, content)
             shutil.rmtree('target')
 
@@ -527,19 +636,32 @@ class TestsWithSourceFile(unittest.TestCase):
 
             os.remove(TESTFN2)
 
-    def test_writestr_compression(self):
+    def test_writestr_compression_stored(self):
         zipfp = zipfile.ZipFile(TESTFN2, "w")
         zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED)
-        if zlib:
-            zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
-
         info = zipfp.getinfo('a.txt')
         self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
 
-        if zlib:
-            info = zipfp.getinfo('b.txt')
-            self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
+    @requires_zlib
+    def test_writestr_compression_deflated(self):
+        zipfp = zipfile.ZipFile(TESTFN2, "w")
+        zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
+        info = zipfp.getinfo('b.txt')
+        self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_writestr_compression_bzip2(self):
+        zipfp = zipfile.ZipFile(TESTFN2, "w")
+        zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_BZIP2)
+        info = zipfp.getinfo('b.txt')
+        self.assertEqual(info.compress_type, zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_writestr_compression_lzma(self):
+        zipfp = zipfile.ZipFile(TESTFN2, "w")
+        zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_LZMA)
+        info = zipfp.getinfo('b.txt')
+        self.assertEqual(info.compress_type, zipfile.ZIP_LZMA)
 
     def zip_test_writestr_permissions(self, f, compression):
         # Make sure that writestr creates files with mode 0600,
@@ -595,7 +717,12 @@ class TestsWithSourceFile(unittest.TestCase):
             self.assertRaises(ValueError, zipfp.write, TESTFN)
 
 
-    @skipUnless(zlib, "requires zlib")
+
+
+
+
+
+    @requires_zlib
     def test_unicode_filenames(self):
         # bug #10801
         fname = findfile('zip_cp437_header.zip')
@@ -704,11 +831,21 @@ class TestZip64InSmallFiles(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_test(f, zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_test(f, zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_test(f, zipfile.ZIP_LZMA)
+
     def test_absolute_arcnames(self):
         with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED,
                              allowZip64=True) as zipfp:
@@ -859,8 +996,40 @@ class OtherTests(unittest.TestCase):
             b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00'
             b'\x00afilePK\x05\x06\x00\x00\x00\x00\x01\x00'
             b'\x01\x003\x00\x00\x003\x00\x00\x00\x00\x00'),
+        zipfile.ZIP_BZIP2: (
+            b'PK\x03\x04\x14\x03\x00\x00\x0c\x00nu\x0c=FA'
+            b'KE8\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af'
+            b'ileBZh91AY&SY\xd4\xa8\xca'
+            b'\x7f\x00\x00\x0f\x11\x80@\x00\x06D\x90\x80 \x00 \xa5'
+            b'P\xd9!\x03\x03\x13\x13\x13\x89\xa9\xa9\xc2u5:\x9f'
+            b'\x8b\xb9"\x9c(HjTe?\x80PK\x01\x02\x14'
+            b'\x03\x14\x03\x00\x00\x0c\x00nu\x0c=FAKE8'
+            b'\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00'
+            b'\x00 \x80\x80\x81\x00\x00\x00\x00afilePK'
+            b'\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00\x00[\x00'
+            b'\x00\x00\x00\x00'),
+        zipfile.ZIP_LZMA: (
+            b'PK\x03\x04\x14\x03\x00\x00\x0e\x00nu\x0c=FA'
+            b'KE\x1b\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af'
+            b'ile\t\x04\x05\x00]\x00\x00\x00\x04\x004\x19I'
+            b'\xee\x8d\xe9\x17\x89:3`\tq!.8\x00PK'
+            b'\x01\x02\x14\x03\x14\x03\x00\x00\x0e\x00nu\x0c=FA'
+            b'KE\x1b\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00\x00\x00'
+            b'\x00\x00\x00\x00 \x80\x80\x81\x00\x00\x00\x00afil'
+            b'ePK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00'
+            b'\x00>\x00\x00\x00\x00\x00'),
     }
 
+    def test_unsupported_version(self):
+        # File has an extract_version of 120
+        data = (b'PK\x03\x04x\x00\x00\x00\x00\x00!p\xa1@\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00xPK\x01\x02x\x03x\x00\x00\x00\x00'
+        b'\x00!p\xa1@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00\x00xPK\x05\x06'
+        b'\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x1f\x00\x00\x00\x00\x00')
+        self.assertRaises(NotImplementedError, zipfile.ZipFile,
+                          io.BytesIO(data), 'r')
+
     def test_unicode_filenames(self):
         with zipfile.ZipFile(TESTFN, "w") as zf:
             zf.writestr("foo.txt", "Test for unicode filename")
@@ -1151,10 +1320,18 @@ class OtherTests(unittest.TestCase):
     def test_testzip_with_bad_crc_stored(self):
         self.check_testzip_with_bad_crc(zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_testzip_with_bad_crc_deflated(self):
         self.check_testzip_with_bad_crc(zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_testzip_with_bad_crc_bzip2(self):
+        self.check_testzip_with_bad_crc(zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_testzip_with_bad_crc_lzma(self):
+        self.check_testzip_with_bad_crc(zipfile.ZIP_LZMA)
+
     def check_read_with_bad_crc(self, compression):
         """Tests that files with bad CRCs raise a BadZipFile exception when read."""
         zipdata = self.zips_with_bad_crc[compression]
@@ -1179,10 +1356,18 @@ class OtherTests(unittest.TestCase):
     def test_read_with_bad_crc_stored(self):
         self.check_read_with_bad_crc(zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_read_with_bad_crc_deflated(self):
         self.check_read_with_bad_crc(zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_read_with_bad_crc_bzip2(self):
+        self.check_read_with_bad_crc(zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_read_with_bad_crc_lzma(self):
+        self.check_read_with_bad_crc(zipfile.ZIP_LZMA)
+
     def check_read_return_size(self, compression):
         # Issue #9837: ZipExtFile.read() shouldn't return more bytes
         # than requested.
@@ -1199,10 +1384,18 @@ class OtherTests(unittest.TestCase):
     def test_read_return_size_stored(self):
         self.check_read_return_size(zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_read_return_size_deflated(self):
         self.check_read_return_size(zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_read_return_size_bzip2(self):
+        self.check_read_return_size(zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_read_return_size_lzma(self):
+        self.check_read_return_size(zipfile.ZIP_LZMA)
+
     def test_empty_zipfile(self):
         # Check that creating a file in 'w' or 'a' mode and closing without
         # adding any files to the archives creates a valid empty ZIP file
@@ -1289,7 +1482,7 @@ class DecryptionTests(unittest.TestCase):
         self.zip2.setpassword(b"perl")
         self.assertRaises(RuntimeError, self.zip2.read, "zero")
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_good_password(self):
         self.zip.setpassword(b"python")
         self.assertEqual(self.zip.read("test.txt"), self.plain)
@@ -1339,11 +1532,21 @@ class TestsWithRandomBinaryFiles(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_test(f, zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_test(f, zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_test(f, zipfile.ZIP_LZMA)
+
     def zip_open_test(self, f, compression):
         self.make_test_archive(f, compression)
 
@@ -1379,11 +1582,21 @@ class TestsWithRandomBinaryFiles(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_open_test(f, zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_open_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_open_test(f, zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_open_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_open_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_open_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_open_test(f, zipfile.ZIP_LZMA)
+
     def zip_random_open_test(self, f, compression):
         self.make_test_archive(f, compression)
 
@@ -1407,13 +1620,23 @@ class TestsWithRandomBinaryFiles(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_random_open_test(f, zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_random_open_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_random_open_test(f, zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_random_open_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_random_open_test(f, zipfile.ZIP_BZIP2)
 
-@skipUnless(zlib, "requires zlib")
+    @requires_lzma
+    def test_random_open_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_random_open_test(f, zipfile.ZIP_LZMA)
+
+
+@requires_zlib
 class TestsWithMultipleOpens(unittest.TestCase):
     def setUp(self):
         # Create the ZIP archive
@@ -1605,32 +1828,82 @@ class UniversalNewlineTests(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.iterlines_test(f, zipfile.ZIP_STORED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_read_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.read_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_readline_read_deflated(self):
         # Issue #7610: calls to readline() interleaved with calls to read().
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.readline_read_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_readline_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.readline_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_readlines_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.readlines_test(f, zipfile.ZIP_DEFLATED)
 
-    @skipUnless(zlib, "requires zlib")
+    @requires_zlib
     def test_iterlines_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.iterlines_test(f, zipfile.ZIP_DEFLATED)
 
+    @requires_bz2
+    def test_read_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.read_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_readline_read_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readline_read_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_readline_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readline_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_readlines_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readlines_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_bz2
+    def test_iterlines_bzip2(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.iterlines_test(f, zipfile.ZIP_BZIP2)
+
+    @requires_lzma
+    def test_read_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.read_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_readline_read_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readline_read_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_readline_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readline_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_readlines_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readlines_test(f, zipfile.ZIP_LZMA)
+
+    @requires_lzma
+    def test_iterlines_lzma(self):
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.iterlines_test(f, zipfile.ZIP_LZMA)
+
     def tearDown(self):
         for sep, fn in self.arcfiles.items():
             os.remove(fn)
diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py
index 0e7d73f1ff..a8fb7aba54 100644
--- a/Lib/test/test_zipfile64.py
+++ b/Lib/test/test_zipfile64.py
@@ -11,12 +11,6 @@ support.requires(
         'test requires loads of disk-space bytes and a long time to run'
     )
 
-# We can test part of the module without zlib.
-try:
-    import zlib
-except ImportError:
-    zlib = None
-
 import zipfile, os, unittest
 import time
 import sys
@@ -24,7 +18,7 @@ import sys
 from io import StringIO
 from tempfile import TemporaryFile
 
-from test.support import TESTFN, run_unittest
+from test.support import TESTFN, run_unittest, requires_zlib
 
 TESTFN2 = TESTFN + "2"
 
@@ -81,12 +75,12 @@ class TestsWithSourceFile(unittest.TestCase):
         for f in TemporaryFile(), TESTFN2:
             self.zipTest(f, zipfile.ZIP_STORED)
 
-    if zlib:
-        def testDeflated(self):
-            # Try the temp file first.  If we do TESTFN2 first, then it hogs
-            # gigabytes of disk space for the duration of the test.
-            for f in TemporaryFile(), TESTFN2:
-                self.zipTest(f, zipfile.ZIP_DEFLATED)
+    @requires_zlib
+    def testDeflated(self):
+        # Try the temp file first.  If we do TESTFN2 first, then it hogs
+        # gigabytes of disk space for the duration of the test.
+        for f in TemporaryFile(), TESTFN2:
+            self.zipTest(f, zipfile.ZIP_DEFLATED)
 
     def tearDown(self):
         for fname in TESTFN, TESTFN2:
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index df5ff9d6a6..f7cb8b977f 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -9,12 +9,6 @@ import unittest
 from test import support
 from test.test_importhooks import ImportHooksBaseTestCase, test_src, test_co
 
-# some tests can be ran even without zlib
-try:
-    import zlib
-except ImportError:
-    zlib = None
-
 from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED
 
 import zipimport
@@ -25,7 +19,7 @@ import io
 from traceback import extract_tb, extract_stack, print_tb
 raise_src = 'def do_raise(): raise TypeError\n'
 
-def make_pyc(co, mtime):
+def make_pyc(co, mtime, size):
     data = marshal.dumps(co)
     if type(mtime) is type(0.0):
         # Mac mtimes need a bit of special casing
@@ -33,14 +27,14 @@ def make_pyc(co, mtime):
             mtime = int(mtime)
         else:
             mtime = int(-0x100000000 + int(mtime))
-    pyc = imp.get_magic() + struct.pack("<i", int(mtime)) + data
+    pyc = imp.get_magic() + struct.pack("<ii", int(mtime), size & 0xFFFFFFFF) + data
     return pyc
 
 def module_path_to_dotted_name(path):
     return path.replace(os.sep, '.')
 
 NOW = time.time()
-test_pyc = make_pyc(test_co, NOW)
+test_pyc = make_pyc(test_co, NOW, len(test_src))
 
 
 TESTMOD = "ziptestmodule"
@@ -211,6 +205,10 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             mod = zi.load_module(TESTPACK)
             self.assertEqual(zi.get_filename(TESTPACK), mod.__file__)
 
+            existing_pack_path = __import__(TESTPACK).__path__[0]
+            expected_path_path = os.path.join(TEMP_ZIP, TESTPACK)
+            self.assertEqual(existing_pack_path, expected_path_path)
+
             self.assertEqual(zi.is_package(packdir + '__init__'), False)
             self.assertEqual(zi.is_package(packdir + TESTPACK2), True)
             self.assertEqual(zi.is_package(packdir2 + TESTMOD), False)
@@ -299,7 +297,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             return __file__
         if __loader__.get_data("some.data") != b"some data":
             raise AssertionError("bad data")\n"""
-        pyc = make_pyc(compile(src, "<???>", "exec"), NOW)
+        pyc = make_pyc(compile(src, "<???>", "exec"), NOW, len(src))
         files = {TESTMOD + pyc_ext: (NOW, pyc),
                  "some.data": (NOW, "some data")}
         self.doTest(pyc_ext, files, TESTMOD)
@@ -319,7 +317,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
         self.doTest(".py", files, TESTMOD, call=self.assertModuleSource)
 
     def testGetCompiledSource(self):
-        pyc = make_pyc(compile(test_src, "<???>", "exec"), NOW)
+        pyc = make_pyc(compile(test_src, "<???>", "exec"), NOW, len(test_src))
         files = {TESTMOD + ".py": (NOW, test_src),
                  TESTMOD + pyc_ext: (NOW, pyc)}
         self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource)
@@ -392,7 +390,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             os.remove(filename)
 
 
-@unittest.skipUnless(zlib, "requires zlib")
+@support.requires_zlib
 class CompressedZipImportTestCase(UncompressedZipImportTestCase):
     compression = ZIP_DEFLATED
 
@@ -417,7 +415,7 @@ class BadFileZipImportTestCase(unittest.TestCase):
 
     def testEmptyFile(self):
         support.unlink(TESTMOD)
-        open(TESTMOD, 'w+').close()
+        support.create_empty_file(TESTMOD)
         self.assertZipFailure(TESTMOD)
 
     def testFileUnreadable(self):
diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py
index 060108f2d0..f7f3398015 100644
--- a/Lib/test/test_zipimport_support.py
+++ b/Lib/test/test_zipimport_support.py
@@ -167,20 +167,19 @@ class ZipSupportTests(unittest.TestCase):
                     test_zipped_doctest.test_DocTestRunner.verbose_flag,
                     test_zipped_doctest.test_Example,
                     test_zipped_doctest.test_debug,
-                    test_zipped_doctest.test_pdb_set_trace,
-                    test_zipped_doctest.test_pdb_set_trace_nested,
                     test_zipped_doctest.test_testsource,
                     test_zipped_doctest.test_trailing_space_in_test,
                     test_zipped_doctest.test_DocTestSuite,
                     test_zipped_doctest.test_DocTestFinder,
                 ]
-                # These remaining tests are the ones which need access
+                # These tests are the ones which need access
                 # to the data files, so we don't run them
                 fail_due_to_missing_data_files = [
                     test_zipped_doctest.test_DocFileSuite,
                     test_zipped_doctest.test_testfile,
                     test_zipped_doctest.test_unittest_reportflags,
                 ]
+
                 for obj in known_good_tests:
                     _run_object_doctest(obj, test_zipped_doctest)
             finally:
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
index 4661c1d613..2f6f84042f 100644
--- a/Lib/test/test_zlib.py
+++ b/Lib/test/test_zlib.py
@@ -7,10 +7,16 @@ from test.support import bigmemtest, _1G, _4G
 
 zlib = support.import_module('zlib')
 
-try:
-    import mmap
-except ImportError:
-    mmap = None
+
+class VersionTestCase(unittest.TestCase):
+
+    def test_library_version(self):
+        # Test that the major version of the actual library in use matches the
+        # major version that we were compiled against. We can't guarantee that
+        # the minor versions will match (even on the machine on which the module
+        # was compiled), and the API is stable between minor versions, so
+        # testing only the major versions avoids spurious failures.
+        self.assertEqual(zlib.ZLIB_RUNTIME_VERSION[0], zlib.ZLIB_VERSION[0])
 
 
 class ChecksumTestCase(unittest.TestCase):
@@ -173,10 +179,8 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
     def test_big_decompress_buffer(self, size):
         self.check_big_decompress_buffer(size, zlib.decompress)
 
-    @bigmemtest(size=_4G + 100, memuse=1)
+    @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
     def test_length_overflow(self, size):
-        if size < _4G + 100:
-            self.skipTest("not enough free memory, need at least 4 GB")
         data = b'x' * size
         try:
             self.assertRaises(OverflowError, zlib.compress, data, 1)
@@ -421,6 +425,35 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
         dco = zlib.decompressobj()
         self.assertEqual(dco.flush(), b"") # Returns nothing
 
+    def test_dictionary(self):
+        h = HAMLET_SCENE
+        # Build a simulated dictionary out of the words in HAMLET.
+        words = h.split()
+        random.shuffle(words)
+        zdict = b''.join(words)
+        # Use it to compress HAMLET.
+        co = zlib.compressobj(zdict=zdict)
+        cd = co.compress(h) + co.flush()
+        # Verify that it will decompress with the dictionary.
+        dco = zlib.decompressobj(zdict=zdict)
+        self.assertEqual(dco.decompress(cd) + dco.flush(), h)
+        # Verify that it fails when not given the dictionary.
+        dco = zlib.decompressobj()
+        self.assertRaises(zlib.error, dco.decompress, cd)
+
+    def test_dictionary_streaming(self):
+        # This simulates the reuse of a compressor object for compressing
+        # several separate data streams.
+        co = zlib.compressobj(zdict=HAMLET_SCENE)
+        do = zlib.decompressobj(zdict=HAMLET_SCENE)
+        piece = HAMLET_SCENE[1000:1500]
+        d0 = co.compress(piece) + co.flush(zlib.Z_SYNC_FLUSH)
+        d1 = co.compress(piece[100:]) + co.flush(zlib.Z_SYNC_FLUSH)
+        d2 = co.compress(piece[:-100]) + co.flush(zlib.Z_SYNC_FLUSH)
+        self.assertEqual(do.decompress(d0), piece)
+        self.assertEqual(do.decompress(d1), piece[100:])
+        self.assertEqual(do.decompress(d2), piece[:-100])
+
     def test_decompress_incomplete_stream(self):
         # This is 'foo', deflated
         x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'
@@ -434,6 +467,26 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
         y += dco.flush()
         self.assertEqual(y, b'foo')
 
+    def test_decompress_eof(self):
+        x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'  # 'foo'
+        dco = zlib.decompressobj()
+        self.assertFalse(dco.eof)
+        dco.decompress(x[:-5])
+        self.assertFalse(dco.eof)
+        dco.decompress(x[-5:])
+        self.assertTrue(dco.eof)
+        dco.flush()
+        self.assertTrue(dco.eof)
+
+    def test_decompress_eof_incomplete_stream(self):
+        x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'  # 'foo'
+        dco = zlib.decompressobj()
+        self.assertFalse(dco.eof)
+        dco.decompress(x[:-5])
+        self.assertFalse(dco.eof)
+        dco.flush()
+        self.assertFalse(dco.eof)
+
     def test_decompress_unused_data(self):
         # Repeated calls to decompress() after EOF should accumulate data in
         # dco.unused_data, instead of just storing the arg to the last call.
@@ -455,6 +508,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
                         data += dco.decompress(
                                 dco.unconsumed_tail + x[i : i + step], maxlen)
                 data += dco.flush()
+                self.assertTrue(dco.eof)
                 self.assertEqual(data, source)
                 self.assertEqual(dco.unconsumed_tail, b'')
                 self.assertEqual(dco.unused_data, remainder)
@@ -547,10 +601,8 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
         decompress = lambda s: d.decompress(s) + d.flush()
         self.check_big_decompress_buffer(size, decompress)
 
-    @bigmemtest(size=_4G + 100, memuse=1)
+    @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
     def test_length_overflow(self, size):
-        if size < _4G + 100:
-            self.skipTest("not enough free memory, need at least 4 GB")
         data = b'x' * size
         c = zlib.compressobj(1)
         d = zlib.decompressobj()
@@ -651,6 +703,7 @@ LAERTES
 
 def test_main():
     support.run_unittest(
+        VersionTestCase,
         ChecksumTestCase,
         ChecksumBigBufferTestCase,
         ExceptionTestCase,
diff --git a/Lib/test/testbz2_bigmem.bz2 b/Lib/test/testbz2_bigmem.bz2
deleted file mode 100644
index c9a4616c80..0000000000
--- a/Lib/test/testbz2_bigmem.bz2
+++ /dev/null
diff --git a/Lib/test/threaded_import_hangers.py b/Lib/test/threaded_import_hangers.py
index adf03e31ff..5484e60a0d 100644
--- a/Lib/test/threaded_import_hangers.py
+++ b/Lib/test/threaded_import_hangers.py
@@ -35,8 +35,11 @@ for name, func, args in [
         ("os.path.abspath", os.path.abspath, ('.',)),
         ]:
 
-    t = Worker(func, args)
-    t.start()
-    t.join(TIMEOUT)
-    if t.is_alive():
-        errors.append("%s appeared to hang" % name)
+    try:
+        t = Worker(func, args)
+        t.start()
+        t.join(TIMEOUT)
+        if t.is_alive():
+            errors.append("%s appeared to hang" % name)
+    finally:
+        del t
diff --git a/Lib/test/tokenize_tests.txt b/Lib/test/tokenize_tests.txt
index 06c83b0a84..2c5fb10576 100644
--- a/Lib/test/tokenize_tests.txt
+++ b/Lib/test/tokenize_tests.txt
@@ -114,8 +114,12 @@ x = b'abc' + B'ABC'
 y = b"abc" + B"ABC"
 x = br'abc' + Br'ABC' + bR'ABC' + BR'ABC'
 y = br"abc" + Br"ABC" + bR"ABC" + BR"ABC"
+x = rb'abc' + rB'ABC' + Rb'ABC' + RB'ABC'
+y = rb"abc" + rB"ABC" + Rb"ABC" + RB"ABC"
 x = br'\\' + BR'\\'
+x = rb'\\' + RB'\\'
 x = br'\'' + ''
+x = rb'\'' + ''
 y = br'''
 foo bar \\
 baz''' + BR'''
@@ -124,6 +128,10 @@ y = Br"""foo
 bar \\ baz
 """ + bR'''spam
 '''
+y = rB"""foo
+bar \\ baz
+""" + Rb'''spam
+'''
 
 # Indentation
 if 1:
diff --git a/Lib/textwrap.py b/Lib/textwrap.py
index dfb400548b..7024d4d245 100644
--- a/Lib/textwrap.py
+++ b/Lib/textwrap.py
@@ -5,9 +5,9 @@
 # Copyright (C) 2002, 2003 Python Software Foundation.
 # Written by Greg Ward <gward@python.net>
 
-import string, re
+import re
 
-__all__ = ['TextWrapper', 'wrap', 'fill', 'dedent']
+__all__ = ['TextWrapper', 'wrap', 'fill', 'dedent', 'indent']
 
 # Hardcode the recognized whitespace characters to the US-ASCII
 # whitespace characters.  The main reason for doing this is that in
@@ -39,8 +39,11 @@ class TextWrapper:
         of wrapped output; also counts towards each line's width.
       expand_tabs (default: true)
         Expand tabs in input text to spaces before further processing.
-        Each tab will become 1 .. 8 spaces, depending on its position in
-        its line.  If false, each tab is treated as a single character.
+        Each tab will become 0 .. 'tabsize' spaces, depending on its position
+        in its line.  If false, each tab is treated as a single character.
+      tabsize (default: 8)
+        Expand tabs in input text to 0 .. 'tabsize' spaces, unless
+        'expand_tabs' is false.
       replace_whitespace (default: true)
         Replace all whitespace characters in the input text by spaces
         after tab expansion.  Note that if expand_tabs is false and
@@ -100,7 +103,8 @@ class TextWrapper:
                  fix_sentence_endings=False,
                  break_long_words=True,
                  drop_whitespace=True,
-                 break_on_hyphens=True):
+                 break_on_hyphens=True,
+                 tabsize=8):
         self.width = width
         self.initial_indent = initial_indent
         self.subsequent_indent = subsequent_indent
@@ -110,6 +114,7 @@ class TextWrapper:
         self.break_long_words = break_long_words
         self.drop_whitespace = drop_whitespace
         self.break_on_hyphens = break_on_hyphens
+        self.tabsize = tabsize
 
 
     # -- Private methods -----------------------------------------------
@@ -123,7 +128,7 @@ class TextWrapper:
         becomes " foo    bar  baz".
         """
         if self.expand_tabs:
-            text = text.expandtabs()
+            text = text.expandtabs(self.tabsize)
         if self.replace_whitespace:
             text = text.translate(self.unicode_whitespace_trans)
         return text
@@ -381,6 +386,25 @@ def dedent(text):
         text = re.sub(r'(?m)^' + margin, '', text)
     return text
 
+
+def indent(text, prefix, predicate=None):
+    """Adds 'prefix' to the beginning of selected lines in 'text'.
+
+    If 'predicate' is provided, 'prefix' will only be added to the lines
+    where 'predicate(line)' is True. If 'predicate' is not provided,
+    it will default to adding 'prefix' to all non-empty lines that do not
+    consist solely of whitespace characters.
+    """
+    if predicate is None:
+        def predicate(line):
+            return line.strip()
+
+    def prefixed_lines():
+        for line in text.splitlines(True):
+            yield (prefix + line if predicate(line) else line)
+    return ''.join(prefixed_lines())
+
+
 if __name__ == "__main__":
     #print dedent("\tfoo\n\tbar")
     #print dedent("  \thello there\n  \t  how are you?")
diff --git a/Lib/threading.py b/Lib/threading.py
index 58ffa7ebc2..6c34d49782 100644
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -3,7 +3,11 @@
 import sys as _sys
 import _thread
 
-from time import time as _time, sleep as _sleep
+from time import sleep as _sleep
+try:
+    from time import monotonic as _time
+except ImportError:
+    from time import time as _time
 from traceback import format_exc as _format_exc
 from _weakrefset import WeakSet
 
@@ -19,12 +23,12 @@ from _weakrefset import WeakSet
 
 __all__ = ['active_count', 'Condition', 'current_thread', 'enumerate', 'Event',
            'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier',
-           'Timer', 'setprofile', 'settrace', 'local', 'stack_size']
+           'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size']
 
 # Rename some stuff so "from threading import *" is safe
 _start_new_thread = _thread.start_new_thread
 _allocate_lock = _thread.allocate_lock
-_get_ident = _thread.get_ident
+get_ident = _thread.get_ident
 ThreadError = _thread.error
 try:
     _CRLock = _thread.RLock
@@ -34,40 +38,6 @@ TIMEOUT_MAX = _thread.TIMEOUT_MAX
 del _thread
 
 
-# Debug support (adapted from ihooks.py).
-
-_VERBOSE = False
-
-if __debug__:
-
-    class _Verbose(object):
-
-        def __init__(self, verbose=None):
-            if verbose is None:
-                verbose = _VERBOSE
-            self._verbose = verbose
-
-        def _note(self, format, *args):
-            if self._verbose:
-                format = format % args
-                # Issue #4188: calling current_thread() can incur an infinite
-                # recursion if it has to create a DummyThread on the fly.
-                ident = _get_ident()
-                try:
-                    name = _active[ident].name
-                except KeyError:
-                    name = "<OS thread %d>" % ident
-                format = "%s: %s\n" % (name, format)
-                _sys.stderr.write(format)
-
-else:
-    # Disable this when using "python -O"
-    class _Verbose(object):
-        def __init__(self, verbose=None):
-            pass
-        def _note(self, *args):
-            pass
-
 # Support for profile and trace hooks
 
 _profile_hook = None
@@ -85,17 +55,14 @@ def settrace(func):
 
 Lock = _allocate_lock
 
-def RLock(verbose=None, *args, **kwargs):
-    if verbose is None:
-        verbose = _VERBOSE
-    if (__debug__ and verbose) or _CRLock is None:
-        return _PyRLock(verbose, *args, **kwargs)
+def RLock(*args, **kwargs):
+    if _CRLock is None:
+        return _PyRLock(*args, **kwargs)
     return _CRLock(*args, **kwargs)
 
-class _RLock(_Verbose):
+class _RLock:
 
-    def __init__(self, verbose=None):
-        _Verbose.__init__(self, verbose)
+    def __init__(self):
         self._block = _allocate_lock()
         self._owner = None
         self._count = 0
@@ -110,37 +77,25 @@ class _RLock(_Verbose):
                 self.__class__.__name__, owner, self._count)
 
     def acquire(self, blocking=True, timeout=-1):
-        me = _get_ident()
+        me = get_ident()
         if self._owner == me:
             self._count = self._count + 1
-            if __debug__:
-                self._note("%s.acquire(%s): recursive success", self, blocking)
             return 1
         rc = self._block.acquire(blocking, timeout)
         if rc:
             self._owner = me
             self._count = 1
-            if __debug__:
-                self._note("%s.acquire(%s): initial success", self, blocking)
-        else:
-            if __debug__:
-                self._note("%s.acquire(%s): failure", self, blocking)
         return rc
 
     __enter__ = acquire
 
     def release(self):
-        if self._owner != _get_ident():
+        if self._owner != get_ident():
             raise RuntimeError("cannot release un-acquired lock")
         self._count = count = self._count - 1
         if not count:
             self._owner = None
             self._block.release()
-            if __debug__:
-                self._note("%s.release(): final release", self)
-        else:
-            if __debug__:
-                self._note("%s.release(): non-final release", self)
 
     def __exit__(self, t, v, tb):
         self.release()
@@ -150,12 +105,10 @@ class _RLock(_Verbose):
     def _acquire_restore(self, state):
         self._block.acquire()
         self._count, self._owner = state
-        if __debug__:
-            self._note("%s._acquire_restore()", self)
 
     def _release_save(self):
-        if __debug__:
-            self._note("%s._release_save()", self)
+        if self._count == 0:
+            raise RuntimeError("cannot release un-acquired lock")
         count = self._count
         self._count = 0
         owner = self._owner
@@ -164,18 +117,14 @@ class _RLock(_Verbose):
         return (count, owner)
 
     def _is_owned(self):
-        return self._owner == _get_ident()
+        return self._owner == get_ident()
 
 _PyRLock = _RLock
 
 
-def Condition(*args, **kwargs):
-    return _Condition(*args, **kwargs)
+class Condition:
 
-class _Condition(_Verbose):
-
-    def __init__(self, lock=None, verbose=None):
-        _Verbose.__init__(self, verbose)
+    def __init__(self, lock=None):
         if lock is None:
             lock = RLock()
         self._lock = lock
@@ -234,23 +183,16 @@ class _Condition(_Verbose):
             if timeout is None:
                 waiter.acquire()
                 gotit = True
-                if __debug__:
-                    self._note("%s.wait(): got it", self)
             else:
                 if timeout > 0:
                     gotit = waiter.acquire(True, timeout)
                 else:
                     gotit = waiter.acquire(False)
                 if not gotit:
-                    if __debug__:
-                        self._note("%s.wait(%s): timed out", self, timeout)
                     try:
                         self._waiters.remove(waiter)
                     except ValueError:
                         pass
-                else:
-                    if __debug__:
-                        self._note("%s.wait(%s): got it", self, timeout)
             return gotit
         finally:
             self._acquire_restore(saved_state)
@@ -266,19 +208,9 @@ class _Condition(_Verbose):
                 else:
                     waittime = endtime - _time()
                     if waittime <= 0:
-                        if __debug__:
-                            self._note("%s.wait_for(%r, %r): Timed out.",
-                                       self, predicate, timeout)
                         break
-            if __debug__:
-                self._note("%s.wait_for(%r, %r): Waiting with timeout=%s.",
-                           self, predicate, timeout, waittime)
             self.wait(waittime)
             result = predicate()
-        else:
-            if __debug__:
-                self._note("%s.wait_for(%r, %r): Success.",
-                           self, predicate, timeout)
         return result
 
     def notify(self, n=1):
@@ -287,11 +219,7 @@ class _Condition(_Verbose):
         __waiters = self._waiters
         waiters = __waiters[:n]
         if not waiters:
-            if __debug__:
-                self._note("%s.notify(): no waiters", self)
             return
-        self._note("%s.notify(): notifying %d waiter%s", self, n,
-                   n!=1 and "s" or "")
         for waiter in waiters:
             waiter.release()
             try:
@@ -305,17 +233,13 @@ class _Condition(_Verbose):
     notifyAll = notify_all
 
 
-def Semaphore(*args, **kwargs):
-    return _Semaphore(*args, **kwargs)
-
-class _Semaphore(_Verbose):
+class Semaphore:
 
     # After Tim Peters' semaphore class, but not quite the same (no maximum)
 
-    def __init__(self, value=1, verbose=None):
+    def __init__(self, value=1):
         if value < 0:
             raise ValueError("semaphore initial value must be >= 0")
-        _Verbose.__init__(self, verbose)
         self._cond = Condition(Lock())
         self._value = value
 
@@ -328,9 +252,6 @@ class _Semaphore(_Verbose):
         while self._value == 0:
             if not blocking:
                 break
-            if __debug__:
-                self._note("%s.acquire(%s): blocked waiting, value=%s",
-                           self, blocking, self._value)
             if timeout is not None:
                 if endtime is None:
                     endtime = _time() + timeout
@@ -341,9 +262,6 @@ class _Semaphore(_Verbose):
             self._cond.wait(timeout)
         else:
             self._value = self._value - 1
-            if __debug__:
-                self._note("%s.acquire: success, value=%s",
-                           self, self._value)
             rc = True
         self._cond.release()
         return rc
@@ -353,9 +271,6 @@ class _Semaphore(_Verbose):
     def release(self):
         self._cond.acquire()
         self._value = self._value + 1
-        if __debug__:
-            self._note("%s.release: success, value=%s",
-                       self, self._value)
         self._cond.notify()
         self._cond.release()
 
@@ -363,30 +278,23 @@ class _Semaphore(_Verbose):
         self.release()
 
 
-def BoundedSemaphore(*args, **kwargs):
-    return _BoundedSemaphore(*args, **kwargs)
-
-class _BoundedSemaphore(_Semaphore):
+class BoundedSemaphore(Semaphore):
     """Semaphore that checks that # releases is <= # acquires"""
-    def __init__(self, value=1, verbose=None):
-        _Semaphore.__init__(self, value, verbose)
+    def __init__(self, value=1):
+        Semaphore.__init__(self, value)
         self._initial_value = value
 
     def release(self):
         if self._value >= self._initial_value:
             raise ValueError("Semaphore released too many times")
-        return _Semaphore.release(self)
-
+        return Semaphore.release(self)
 
-def Event(*args, **kwargs):
-    return _Event(*args, **kwargs)
 
-class _Event(_Verbose):
+class Event:
 
     # After Tim Peters' event class (without is_posted())
 
-    def __init__(self, verbose=None):
-        _Verbose.__init__(self, verbose)
+    def __init__(self):
         self._cond = Condition(Lock())
         self._flag = False
 
@@ -436,13 +344,13 @@ class _Event(_Verbose):
 # since the previous cycle.  In addition, a 'resetting' state exists which is
 # similar to 'draining' except that threads leave with a BrokenBarrierError,
 # and a 'broken' state in which all threads get the exception.
-class Barrier(_Verbose):
+class Barrier:
     """
     Barrier.  Useful for synchronizing a fixed number of threads
     at known synchronization points.  Threads block on 'wait()' and are
     simultaneously once they have all made that call.
     """
-    def __init__(self, parties, action=None, timeout=None, verbose=None):
+    def __init__(self, parties, action=None, timeout=None):
         """
         Create a barrier, initialised to 'parties' threads.
         'action' is a callable which, when supplied, will be called
@@ -451,7 +359,6 @@ class Barrier(_Verbose):
         If a 'timeout' is provided, it is uses as the default for
         all subsequent 'wait()' calls.
         """
-        _Verbose.__init__(self, verbose)
         self._cond = Condition(Lock())
         self._action = action
         self._timeout = timeout
@@ -612,7 +519,7 @@ _dangling = WeakSet()
 
 # Main class for threads
 
-class Thread(_Verbose):
+class Thread:
 
     __initialized = False
     # Need to store a reference to sys.exc_info for printing
@@ -625,16 +532,18 @@ class Thread(_Verbose):
     #XXX __exc_clear = _sys.exc_clear
 
     def __init__(self, group=None, target=None, name=None,
-                 args=(), kwargs=None, verbose=None):
+                 args=(), kwargs=None, *, daemon=None):
         assert group is None, "group argument must be None for now"
-        _Verbose.__init__(self, verbose)
         if kwargs is None:
             kwargs = {}
         self._target = target
         self._name = str(name or _newname())
         self._args = args
         self._kwargs = kwargs
-        self._daemonic = self._set_daemon()
+        if daemon is not None:
+            self._daemonic = daemon
+        else:
+            self._daemonic = current_thread().daemon
         self._ident = None
         self._started = Event()
         self._stopped = False
@@ -652,10 +561,6 @@ class Thread(_Verbose):
             self._block.__init__()
         self._started._reset_internal_locks()
 
-    def _set_daemon(self):
-        # Overridden in _MainThread and _DummyThread
-        return current_thread().daemon
-
     def __repr__(self):
         assert self._initialized, "Thread.__init__() was not called"
         status = "initial"
@@ -675,8 +580,6 @@ class Thread(_Verbose):
 
         if self._started.is_set():
             raise RuntimeError("threads can only be started once")
-        if __debug__:
-            self._note("%s.start(): starting thread", self)
         with _active_limbo_lock:
             _limbo[self] = self
         try:
@@ -717,7 +620,7 @@ class Thread(_Verbose):
             raise
 
     def _set_ident(self):
-        self._ident = _get_ident()
+        self._ident = get_ident()
 
     def _bootstrap_inner(self):
         try:
@@ -726,24 +629,17 @@ class Thread(_Verbose):
             with _active_limbo_lock:
                 _active[self._ident] = self
                 del _limbo[self]
-            if __debug__:
-                self._note("%s._bootstrap(): thread started", self)
 
             if _trace_hook:
-                self._note("%s._bootstrap(): registering trace hook", self)
                 _sys.settrace(_trace_hook)
             if _profile_hook:
-                self._note("%s._bootstrap(): registering profile hook", self)
                 _sys.setprofile(_profile_hook)
 
             try:
                 self.run()
             except SystemExit:
-                if __debug__:
-                    self._note("%s._bootstrap(): raised SystemExit", self)
+                pass
             except:
-                if __debug__:
-                    self._note("%s._bootstrap(): unhandled exception", self)
                 # If sys.stderr is no more (most likely from interpreter
                 # shutdown) use self._stderr.  Otherwise still use sys (as in
                 # _sys) in case sys.stderr was redefined since the creation of
@@ -774,9 +670,6 @@ class Thread(_Verbose):
                     # hog; deleting everything else is just for thoroughness
                     finally:
                         del exc_type, exc_value, exc_tb
-            else:
-                if __debug__:
-                    self._note("%s._bootstrap(): normal return", self)
             finally:
                 # Prevent a race in
                 # test_threading.test_no_refcycle_through_target when
@@ -790,7 +683,7 @@ class Thread(_Verbose):
                 try:
                     # We don't call self._delete() because it also
                     # grabs _active_limbo_lock.
-                    del _active[_get_ident()]
+                    del _active[get_ident()]
                 except:
                     pass
 
@@ -826,7 +719,7 @@ class Thread(_Verbose):
 
         try:
             with _active_limbo_lock:
-                del _active[_get_ident()]
+                del _active[get_ident()]
                 # There must not be any python code between the previous line
                 # and after the lock is released.  Otherwise a tracing function
                 # could try to acquire the lock again in the same thread, (in
@@ -843,29 +736,18 @@ class Thread(_Verbose):
         if self is current_thread():
             raise RuntimeError("cannot join current thread")
 
-        if __debug__:
-            if not self._stopped:
-                self._note("%s.join(): waiting until thread stops", self)
-
         self._block.acquire()
         try:
             if timeout is None:
                 while not self._stopped:
                     self._block.wait()
-                if __debug__:
-                    self._note("%s.join(): thread stopped", self)
             else:
                 deadline = _time() + timeout
                 while not self._stopped:
                     delay = deadline - _time()
                     if delay <= 0:
-                        if __debug__:
-                            self._note("%s.join(): timed out", self)
                         break
                     self._block.wait(delay)
-                else:
-                    if __debug__:
-                        self._note("%s.join(): thread stopped", self)
         finally:
             self._block.release()
 
@@ -917,10 +799,7 @@ class Thread(_Verbose):
 
 # The timer class was contributed by Itamar Shtull-Trauring
 
-def Timer(*args, **kwargs):
-    return _Timer(*args, **kwargs)
-
-class _Timer(Thread):
+class Timer(Thread):
     """Call a function after a specified number of seconds:
 
     t = Timer(30.0, f, args=[], kwargs={})
@@ -952,26 +831,18 @@ class _Timer(Thread):
 class _MainThread(Thread):
 
     def __init__(self):
-        Thread.__init__(self, name="MainThread")
+        Thread.__init__(self, name="MainThread", daemon=False)
         self._started.set()
         self._set_ident()
         with _active_limbo_lock:
             _active[self._ident] = self
 
-    def _set_daemon(self):
-        return False
-
     def _exitfunc(self):
         self._stop()
         t = _pickSomeNonDaemonThread()
-        if t:
-            if __debug__:
-                self._note("%s: waiting for other threads", self)
         while t:
             t.join()
             t = _pickSomeNonDaemonThread()
-        if __debug__:
-            self._note("%s: exiting", self)
         self._delete()
 
 def _pickSomeNonDaemonThread():
@@ -992,7 +863,7 @@ def _pickSomeNonDaemonThread():
 class _DummyThread(Thread):
 
     def __init__(self):
-        Thread.__init__(self, name=_newname("Dummy-%d"))
+        Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True)
 
         # Thread._block consumes an OS-level locking primitive, which
         # can never be used by a _DummyThread.  Since a _DummyThread
@@ -1004,9 +875,6 @@ class _DummyThread(Thread):
         with _active_limbo_lock:
             _active[self._ident] = self
 
-    def _set_daemon(self):
-        return True
-
     def _stop(self):
         pass
 
@@ -1018,9 +886,8 @@ class _DummyThread(Thread):
 
 def current_thread():
     try:
-        return _active[_get_ident()]
+        return _active[get_ident()]
     except KeyError:
-        ##print "current_thread(): no current thread for", _get_ident()
         return _DummyThread()
 
 currentThread = current_thread
@@ -1077,7 +944,7 @@ def _after_fork():
             if thread is current:
                 # There is only one active thread. We reset the ident to
                 # its new value since it can have changed.
-                ident = _get_ident()
+                ident = get_ident()
                 thread._ident = ident
                 new_active[ident] = thread
             else:
diff --git a/Lib/timeit.py b/Lib/timeit.py
index 1ae59e0ccc..4f7d28fbef 100644
--- a/Lib/timeit.py
+++ b/Lib/timeit.py
@@ -9,14 +9,15 @@ the Python Cookbook, published by O'Reilly.
 Library usage: see the Timer class.
 
 Command line usage:
-    python timeit.py [-n N] [-r N] [-s S] [-t] [-c] [-h] [--] [statement]
+    python timeit.py [-n N] [-r N] [-s S] [-t] [-c] [-p] [-h] [--] [statement]
 
 Options:
   -n/--number N: how many times to execute 'statement' (default: see below)
   -r/--repeat N: how many times to repeat the timer (default 3)
   -s/--setup S: statement to be executed once initially (default 'pass')
-  -t/--time: use time.time() (default on Unix)
-  -c/--clock: use time.clock() (default on Windows)
+  -p/--process: use time.process_time() (default is time.perf_counter())
+  -t/--time: use time.time() (deprecated)
+  -c/--clock: use time.clock() (deprecated)
   -v/--verbose: print raw timing results; repeat for more digits precision
   -h/--help: print this usage message and exit
   --: separate options from statement, use when statement starts with -
@@ -66,23 +67,17 @@ __all__ = ["Timer"]
 dummy_src_name = "<timeit-src>"
 default_number = 1000000
 default_repeat = 3
-
-if sys.platform == "win32":
-    # On Windows, the best timer is time.clock()
-    default_timer = time.clock
-else:
-    # On most other platforms the best timer is time.time()
-    default_timer = time.time
+default_timer = time.perf_counter
 
 # Don't change the indentation of the template; the reindent() calls
 # in Timer.__init__() depend on setup being indented 4 spaces and stmt
 # being indented 8 spaces.
 template = """
 def inner(_it, _timer):
-    %(setup)s
+    {setup}
     _t0 = _timer()
     for _i in _it:
-        %(stmt)s
+        {stmt}
     _t1 = _timer()
     return _t1 - _t0
 """
@@ -126,9 +121,9 @@ class Timer:
             stmt = reindent(stmt, 8)
             if isinstance(setup, str):
                 setup = reindent(setup, 4)
-                src = template % {'stmt': stmt, 'setup': setup}
+                src = template.format(stmt=stmt, setup=setup)
             elif callable(setup):
-                src = template % {'stmt': stmt, 'setup': '_setup()'}
+                src = template.format(stmt=stmt, setup='_setup()')
                 ns['_setup'] = setup
             else:
                 raise ValueError("setup is neither a string nor callable")
@@ -255,9 +250,10 @@ def main(args=None, *, _wrap_timer=None):
         args = sys.argv[1:]
     import getopt
     try:
-        opts, args = getopt.getopt(args, "n:s:r:tcvh",
+        opts, args = getopt.getopt(args, "n:s:r:tcpvh",
                                    ["number=", "setup=", "repeat=",
-                                    "time", "clock", "verbose", "help"])
+                                    "time", "clock", "process",
+                                    "verbose", "help"])
     except getopt.error as err:
         print(err)
         print("use -h/--help for command line help")
@@ -282,6 +278,8 @@ def main(args=None, *, _wrap_timer=None):
             timer = time.time
         if o in ("-c", "--clock"):
             timer = time.clock
+        if o in ("-p", "--process"):
+            timer = time.process_time
         if o in ("-v", "--verbose"):
             if verbose:
                 precision += 1
diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py
index a6ad88823a..ea23705699 100644
--- a/Lib/tkinter/__init__.py
+++ b/Lib/tkinter/__init__.py
@@ -30,17 +30,19 @@ button.pack(side=BOTTOM)
 tk.mainloop()
 """
 
-__version__ = "$Revision$"
-
 import sys
 if sys.platform == "win32":
     # Attempt to configure Tcl/Tk without requiring PATH
     from tkinter import _fix
+
+import warnings
+
 import _tkinter # If this fails your Python may not be configured for Tk
 TclError = _tkinter.TclError
 from tkinter.constants import *
 import re
 
+
 wantobjects = 1
 
 TkVersion = float(_tkinter.TK_VERSION)
@@ -190,6 +192,7 @@ class Variable:
     Subclasses StringVar, IntVar, DoubleVar, BooleanVar are specializations
     that constrain the type of the value returned from get()."""
     _default = ""
+    _tk = None
     def __init__(self, master=None, value=None, name=None):
         """Construct a variable
 
@@ -200,6 +203,11 @@ class Variable:
         If NAME matches an existing variable and VALUE is omitted
         then the existing value is retained.
         """
+        # check for type of NAME parameter to override weird error message
+        # raised from Modules/_tkinter.c:SetVar like:
+        # TypeError: setvar() takes exactly 3 arguments (2 given)
+        if name is not None and not isinstance(name, str):
+            raise TypeError("name must be a string")
         global _varnum
         if not master:
             master = _default_root
@@ -211,18 +219,21 @@ class Variable:
             self._name = 'PY_VAR' + repr(_varnum)
             _varnum += 1
         if value is not None:
-            self.set(value)
+            self.initialize(value)
         elif not self._tk.call("info", "exists", self._name):
-            self.set(self._default)
+            self.initialize(self._default)
     def __del__(self):
         """Unset the variable in Tcl."""
-        self._tk.globalunsetvar(self._name)
+        if (self._tk is not None and self._tk.call("info", "exists",
+                                                   self._name)):
+            self._tk.globalunsetvar(self._name)
     def __str__(self):
         """Return the name of the variable in Tcl."""
         return self._name
     def set(self, value):
         """Set the variable to VALUE."""
         return self._tk.globalsetvar(self._name, value)
+    initialize = set
     def get(self):
         """Return value of variable."""
         return self._tk.globalgetvar(self._name)
@@ -297,12 +308,6 @@ class IntVar(Variable):
         """
         Variable.__init__(self, master, value, name)
 
-    def set(self, value):
-        """Set the variable to value, converting booleans to integers."""
-        if isinstance(value, bool):
-            value = int(value)
-        return Variable.set(self, value)
-
     def get(self):
         """Return the value of the variable as an integer."""
         return getint(self._tk.globalgetvar(self._name))
@@ -343,7 +348,10 @@ class BooleanVar(Variable):
 
     def get(self):
         """Return the value of the variable as a bool."""
-        return self._tk.getboolean(self._tk.globalgetvar(self._name))
+        try:
+            return self._tk.getboolean(self._tk.globalgetvar(self._name))
+        except TclError:
+            raise ValueError("invalid literal for getboolean()")
 
 def mainloop(n=0):
     """Run the main loop of Tcl."""
@@ -355,7 +363,10 @@ getdouble = float
 
 def getboolean(s):
     """Convert true and false to integer values 1 and 0."""
-    return _default_root.tk.getboolean(s)
+    try:
+        return _default_root.tk.getboolean(s)
+    except TclError:
+        raise ValueError("invalid literal for getboolean()")
 
 # Methods defined on both toplevel and interior widgets
 class Misc:
@@ -445,7 +456,10 @@ class Misc:
     getdouble = float
     def getboolean(self, s):
         """Return a boolean value for Tcl boolean values true and false given as parameter."""
-        return self.tk.getboolean(s)
+        try:
+            return self.tk.getboolean(s)
+        except TclError:
+            raise ValueError("invalid literal for getboolean()")
     def focus_set(self):
         """Direct input focus to this widget.
 
@@ -1218,7 +1232,6 @@ class Misc:
         return (e,)
     def _report_exception(self):
         """Internal function."""
-        import sys
         exc, val, tb = sys.exc_info()
         root = self._root()
         root.report_callback_exception(exc, val, tb)
@@ -1293,6 +1306,13 @@ class Misc:
                    self.tk.call(
                        'place', 'slaves', self._w))]
     # Grid methods that apply to the master
+    def grid_anchor(self, anchor=None): # new in Tk 8.5
+        """The anchor value controls how to place the grid within the
+        master when no row/column has any weight.
+
+        The default anchor is nw."""
+        self.tk.call('grid', 'anchor', self._w, anchor)
+    anchor = grid_anchor
     def grid_bbox(self, column=None, row=None, col2=None, row2=None):
         """Return a tuple of integer coordinates for the bounding
         box of this widget controlled by the geometry manager grid.
@@ -1311,7 +1331,6 @@ class Misc:
         if col2 is not None and row2 is not None:
             args = args + (col2, row2)
         return self._getints(self.tk.call(*args)) or None
-
     bbox = grid_bbox
     def _grid_configure(self, command, index, cnf, kw):
         """Internal function."""
@@ -1570,6 +1589,14 @@ class Wm:
         the focus. Return current focus model if MODEL is None."""
         return self.tk.call('wm', 'focusmodel', self._w, model)
     focusmodel = wm_focusmodel
+    def wm_forget(self, window): # new in Tk 8.5
+        """The window will be unmappend from the screen and will no longer
+        be managed by wm. toplevel windows will be treated like frame
+        windows once they are no longer managed by wm, however, the menu
+        option configuration will be remembered and the menus will return
+        once the widget is managed again."""
+        self.tk.call('wm', 'forget', window)
+    forget = wm_forget
     def wm_frame(self):
         """Return identifier for decorative frame of this widget if present."""
         return self.tk.call('wm', 'frame', self._w)
@@ -1623,6 +1650,31 @@ class Wm:
         None is given."""
         return self.tk.call('wm', 'iconname', self._w, newName)
     iconname = wm_iconname
+    def wm_iconphoto(self, default=False, *args): # new in Tk 8.5
+        """Sets the titlebar icon for this window based on the named photo
+        images passed through args. If default is True, this is applied to
+        all future created toplevels as well.
+
+        The data in the images is taken as a snapshot at the time of
+        invocation. If the images are later changed, this is not reflected
+        to the titlebar icons. Multiple images are accepted to allow
+        different images sizes to be provided. The window manager may scale
+        provided icons to an appropriate size.
+
+        On Windows, the images are packed into a Windows icon structure.
+        This will override an icon specified to wm_iconbitmap, and vice
+        versa.
+
+        On X, the images are arranged into the _NET_WM_ICON X property,
+        which most modern window managers support. An icon specified by
+        wm_iconbitmap may exist simuultaneously.
+
+        On Macintosh, this currently does nothing."""
+        if default:
+            self.tk.call('wm', 'iconphoto', self._w, "-default", *args)
+        else:
+            self.tk.call('wm', 'iconphoto', self._w, *args)
+    iconphoto = wm_iconphoto
     def wm_iconposition(self, x=None, y=None):
         """Set the position of the icon of this widget to X and Y. Return
         a tuple of the current values of X and X if None is given."""
@@ -1634,6 +1686,12 @@ class Wm:
         value if None is given."""
         return self.tk.call('wm', 'iconwindow', self._w, pathName)
     iconwindow = wm_iconwindow
+    def wm_manage(self, widget): # new in Tk 8.5
+        """The widget specified will become a stand alone top-level window.
+        The window will be decorated with the window managers title bar,
+        etc."""
+        self.tk.call('wm', 'manage', widget)
+    manage = wm_manage
     def wm_maxsize(self, width=None, height=None):
         """Set max WIDTH and HEIGHT for this widget. If the window is gridded
         the values are given in grid units. Return the current values if None
@@ -1722,7 +1780,7 @@ class Tk(Misc, Wm):
         # ensure that self.tk is always _something_.
         self.tk = None
         if baseName is None:
-            import sys, os
+            import os
             baseName = os.path.basename(sys.argv[0])
             baseName, ext = os.path.splitext(baseName)
             if ext not in ('.py', '.pyc', '.pyo'):
@@ -1798,7 +1856,7 @@ class Tk(Misc, Wm):
             exec(open(base_py).read(), dir)
     def report_callback_exception(self, exc, val, tb):
         """Internal function. It reports exception on sys.stderr."""
-        import traceback, sys
+        import traceback
         sys.stderr.write("Exception in Tkinter callback\n")
         sys.last_type = exc
         sys.last_value = val
@@ -2123,25 +2181,45 @@ class Button(Widget):
         """
         return self.tk.call(self._w, 'invoke')
 
+
 # Indices:
 # XXX I don't like these -- take them away
 def AtEnd():
+    warnings.warn("tkinter.AtEnd will be removed in 3.4",
+                  DeprecationWarning, stacklevel=2)
     return 'end'
+
+
 def AtInsert(*args):
+    warnings.warn("tkinter.AtInsert will be removed in 3.4",
+                  DeprecationWarning, stacklevel=2)
     s = 'insert'
     for a in args:
         if a: s = s + (' ' + a)
     return s
+
+
 def AtSelFirst():
+    warnings.warn("tkinter.AtSelFirst will be removed in 3.4",
+                  DeprecationWarning, stacklevel=2)
     return 'sel.first'
+
+
 def AtSelLast():
+    warnings.warn("tkinter.AtSelLast will be removed in 3.4",
+                  DeprecationWarning, stacklevel=2)
     return 'sel.last'
+
+
 def At(x, y=None):
+    warnings.warn("tkinter.At will be removed in 3.4",
+                  DeprecationWarning, stacklevel=2)
     if y is None:
         return '@%r' % (x,)
     else:
         return '@%r,%r' % (x, y)
 
+
 class Canvas(Widget, XView, YView):
     """Canvas widget to display graphical elements like lines or text."""
     def __init__(self, master=None, cnf={}, **kw):
@@ -2727,6 +2805,10 @@ class Menu(Widget):
     def unpost(self):
         """Unmap a menu."""
         self.tk.call(self._w, 'unpost')
+    def xposition(self, index): # new in Tk 8.5
+        """Return the x-position of the leftmost pixel of the menu item
+        at INDEX."""
+        return getint(self.tk.call(self._w, 'xposition', index))
     def yposition(self, index):
         """Return the y-position of the topmost pixel of the menu item at INDEX."""
         return getint(self.tk.call(
@@ -2886,6 +2968,25 @@ class Text(Widget, XView, YView):
         relation OP is satisfied. OP is one of <, <=, ==, >=, >, or !=."""
         return self.tk.getboolean(self.tk.call(
             self._w, 'compare', index1, op, index2))
+    def count(self, index1, index2, *args): # new in Tk 8.5
+        """Counts the number of relevant things between the two indices.
+        If index1 is after index2, the result will be a negative number
+        (and this holds for each of the possible options).
+
+        The actual items which are counted depends on the options given by
+        args. The result is a list of integers, one for the result of each
+        counting option given. Valid counting options are "chars",
+        "displaychars", "displayindices", "displaylines", "indices",
+        "lines", "xpixels" and "ypixels". There is an additional possible
+        option "update", which if given then all subsequent options ensure
+        that any possible out of date information is recalculated."""
+        args = ['-%s' % arg for arg in args if not arg.startswith('-')]
+        args += [index1, index2]
+        res = self.tk.call(self._w, 'count', *args) or None
+        if res is not None and len(args) <= 3:
+            return (res, )
+        else:
+            return res
     def debug(self, boolean=None):
         """Turn on the internal consistency checks of the B-Tree inside the text
         widget according to BOOLEAN."""
@@ -3048,6 +3149,24 @@ class Text(Widget, XView, YView):
     def mark_previous(self, index):
         """Return the name of the previous mark before INDEX."""
         return self.tk.call(self._w, 'mark', 'previous', index) or None
+    def peer_create(self, newPathName, cnf={}, **kw): # new in Tk 8.5
+        """Creates a peer text widget with the given newPathName, and any
+        optional standard configuration options. By default the peer will
+        have the same start and and end line as the parent widget, but
+        these can be overriden with the standard configuration options."""
+        self.tk.call(self._w, 'peer', 'create', newPathName,
+            *self._options(cnf, kw))
+    def peer_names(self): # new in Tk 8.5
+        """Returns a list of peers of this widget (this does not include
+        the widget itself)."""
+        return self.tk.splitlist(self.tk.call(self._w, 'peer', 'names'))
+    def replace(self, index1, index2, chars, *args): # new in Tk 8.5
+        """Replaces the range of characters between index1 and index2 with
+        the given characters and tags specified by args.
+
+        See the method insert for some more information about args, and the
+        method delete for information about the indices."""
+        self.tk.call(self._w, 'replace', index1, index2, chars, *args)
     def scan_mark(self, x, y):
         """Remember the current X, Y coordinates."""
         self.tk.call(self._w, 'scan', 'mark', x, y)
diff --git a/Lib/tkinter/_fix.py b/Lib/tkinter/_fix.py
index 5a69d89787..5f32d25abc 100644
--- a/Lib/tkinter/_fix.py
+++ b/Lib/tkinter/_fix.py
@@ -46,10 +46,10 @@ else:
             s = "\\" + s[3:]
         return s
 
-prefix = os.path.join(sys.prefix,"tcl")
+prefix = os.path.join(sys.base_prefix,"tcl")
 if not os.path.exists(prefix):
     # devdir/../tcltk/lib
-    prefix = os.path.join(sys.prefix, os.path.pardir, "tcltk", "lib")
+    prefix = os.path.join(sys.base_prefix, os.path.pardir, "tcltk", "lib")
     prefix = os.path.abspath(prefix)
 # if this does not exist, no further search is needed
 if os.path.exists(prefix):
diff --git a/Lib/tkinter/filedialog.py b/Lib/tkinter/filedialog.py
index 98d2d5c320..3ffb2528f0 100644
--- a/Lib/tkinter/filedialog.py
+++ b/Lib/tkinter/filedialog.py
@@ -306,7 +306,6 @@ class _Dialog(commondialog.Dialog):
     def _fixresult(self, widget, result):
         if result:
             # keep directory and filename until next time
-            import os
             # convert Tcl path objects to strings
             try:
                 result = result.string
@@ -333,7 +332,6 @@ class Open(_Dialog):
             # multiple results:
             result = tuple([getattr(r, "string", r) for r in result])
             if result:
-                import os
                 path, file = os.path.split(result[0])
                 self.options["initialdir"] = path
                 # don't set initialfile or filename, as we have multiple of these
diff --git a/Lib/tkinter/font.py b/Lib/tkinter/font.py
index 5425b06013..49292412f0 100644
--- a/Lib/tkinter/font.py
+++ b/Lib/tkinter/font.py
@@ -2,27 +2,27 @@
 #
 # written by Fredrik Lundh, February 1998
 #
-# FIXME: should add 'displayof' option where relevant (actual, families,
-#        measure, and metrics)
-#
 
 __version__ = "0.9"
 
+import itertools
 import tkinter
 
+
 # weight/slant
 NORMAL = "normal"
 ROMAN = "roman"
 BOLD   = "bold"
 ITALIC = "italic"
 
+
 def nametofont(name):
     """Given the name of a tk named font, returns a Font representation.
     """
     return Font(name=name, exists=True)
 
-class Font:
 
+class Font:
     """Represents a named font.
 
     Constructor options are:
@@ -44,6 +44,8 @@ class Font:
 
     """
 
+    counter = itertools.count(1)
+
     def _set(self, kw):
         options = []
         for k, v in kw.items():
@@ -63,7 +65,8 @@ class Font:
             options[args[i][1:]] = args[i+1]
         return options
 
-    def __init__(self, root=None, font=None, name=None, exists=False, **options):
+    def __init__(self, root=None, font=None, name=None, exists=False,
+                 **options):
         if not root:
             root = tkinter._default_root
         if font:
@@ -72,7 +75,7 @@ class Font:
         else:
             font = self._set(options)
         if not name:
-            name = "font" + str(id(self))
+            name = "font" + str(next(self.counter))
         self.name = name
 
         if exists:
@@ -118,14 +121,17 @@ class Font:
         "Return a distinct copy of the current font"
         return Font(self._root, **self.actual())
 
-    def actual(self, option=None):
+    def actual(self, option=None, displayof=None):
         "Return actual font attributes"
+        args = ()
+        if displayof:
+            args = ('-displayof', displayof)
         if option:
-            return self._call("font", "actual", self.name, "-"+option)
+            args = args + ('-' + option, )
+            return self._call("font", "actual", self.name, *args)
         else:
             return self._mkdict(
-                self._split(self._call("font", "actual", self.name))
-                )
+                self._split(self._call("font", "actual", self.name, *args)))
 
     def cget(self, option):
         "Get font attribute"
@@ -138,37 +144,47 @@ class Font:
                   *self._set(options))
         else:
             return self._mkdict(
-                self._split(self._call("font", "config", self.name))
-                )
+                self._split(self._call("font", "config", self.name)))
 
     configure = config
 
-    def measure(self, text):
+    def measure(self, text, displayof=None):
         "Return text width"
-        return int(self._call("font", "measure", self.name, text))
+        args = (text,)
+        if displayof:
+            args = ('-displayof', displayof, text)
+        return int(self._call("font", "measure", self.name, *args))
 
-    def metrics(self, *options):
+    def metrics(self, *options, **kw):
         """Return font metrics.
 
         For best performance, create a dummy widget
         using this font before calling this method."""
-
+        args = ()
+        displayof = kw.pop('displayof', None)
+        if displayof:
+            args = ('-displayof', displayof)
         if options:
+            args = args + self._get(options)
             return int(
-                self._call("font", "metrics", self.name, self._get(options))
-                )
+                self._call("font", "metrics", self.name, *args))
         else:
-            res = self._split(self._call("font", "metrics", self.name))
+            res = self._split(self._call("font", "metrics", self.name, *args))
             options = {}
             for i in range(0, len(res), 2):
                 options[res[i][1:]] = int(res[i+1])
             return options
 
-def families(root=None):
+
+def families(root=None, displayof=None):
     "Get font families (as a tuple)"
     if not root:
         root = tkinter._default_root
-    return root.tk.splitlist(root.tk.call("font", "families"))
+    args = ()
+    if displayof:
+        args = ('-displayof', displayof)
+    return root.tk.splitlist(root.tk.call("font", "families", *args))
+
 
 def names(root=None):
     "Get names of defined fonts (as a tuple)"
@@ -176,6 +192,7 @@ def names(root=None):
         root = tkinter._default_root
     return root.tk.splitlist(root.tk.call("font", "names"))
 
+
 # --------------------------------------------------------------------
 # test stuff
 
@@ -198,10 +215,10 @@ if __name__ == "__main__":
 
     print(f.measure("hello"), f.metrics("linespace"))
 
-    print(f.metrics())
+    print(f.metrics(displayof=root))
 
     f = Font(font=("Courier", 20, "bold"))
-    print(f.measure("hello"), f.metrics("linespace"))
+    print(f.measure("hello"), f.metrics("linespace", displayof=root))
 
     w = tkinter.Label(root, text="Hello, world", font=f)
     w.pack()
diff --git a/Lib/tkinter/test/test_tkinter/test_variables.py b/Lib/tkinter/test/test_tkinter/test_variables.py
new file mode 100644
index 0000000000..378cc92ebf
--- /dev/null
+++ b/Lib/tkinter/test/test_tkinter/test_variables.py
@@ -0,0 +1,165 @@
+import unittest
+
+from tkinter import Variable, StringVar, IntVar, DoubleVar, BooleanVar, Tk
+
+
+class Var(Variable):
+
+    _default = "default"
+    side_effect = False
+
+    def set(self, value):
+        self.side_effect = True
+        super().set(value)
+
+
+class TestBase(unittest.TestCase):
+
+    def setUp(self):
+        self.root = Tk()
+
+    def tearDown(self):
+        self.root.destroy()
+
+
+class TestVariable(TestBase):
+
+    def test_default(self):
+        v = Variable(self.root)
+        self.assertEqual("", v.get())
+        self.assertRegex(str(v), r"^PY_VAR(\d+)$")
+
+    def test_name_and_value(self):
+        v = Variable(self.root, "sample string", "varname")
+        self.assertEqual("sample string", v.get())
+        self.assertEqual("varname", str(v))
+
+    def test___del__(self):
+        self.assertFalse(self.root.call("info", "exists", "varname"))
+        v = Variable(self.root, "sample string", "varname")
+        self.assertTrue(self.root.call("info", "exists", "varname"))
+        del v
+        self.assertFalse(self.root.call("info", "exists", "varname"))
+
+    def test_dont_unset_not_existing(self):
+        self.assertFalse(self.root.call("info", "exists", "varname"))
+        v1 = Variable(self.root, name="name")
+        v2 = Variable(self.root, name="name")
+        del v1
+        self.assertFalse(self.root.call("info", "exists", "name"))
+        # shouldn't raise exception
+        del v2
+        self.assertFalse(self.root.call("info", "exists", "name"))
+
+    def test___eq__(self):
+        # values doesn't matter, only class and name are checked
+        v1 = Variable(self.root, name="abc")
+        v2 = Variable(self.root, name="abc")
+        self.assertEqual(v1, v2)
+
+        v3 = Variable(self.root, name="abc")
+        v4 = StringVar(self.root, name="abc")
+        self.assertNotEqual(v3, v4)
+
+    def test_invalid_name(self):
+        with self.assertRaises(TypeError):
+            Variable(self.root, name=123)
+
+    def test_initialize(self):
+        v = Var()
+        self.assertFalse(v.side_effect)
+        v.set("value")
+        self.assertTrue(v.side_effect)
+
+
+class TestStringVar(TestBase):
+
+    def test_default(self):
+        v = StringVar(self.root)
+        self.assertEqual("", v.get())
+
+    def test_get(self):
+        v = StringVar(self.root, "abc", "name")
+        self.assertEqual("abc", v.get())
+        self.root.globalsetvar("name", "value")
+        self.assertEqual("value", v.get())
+
+
+class TestIntVar(TestBase):
+
+    def test_default(self):
+        v = IntVar(self.root)
+        self.assertEqual(0, v.get())
+
+    def test_get(self):
+        v = IntVar(self.root, 123, "name")
+        self.assertEqual(123, v.get())
+        self.root.globalsetvar("name", "345")
+        self.assertEqual(345, v.get())
+
+    def test_invalid_value(self):
+        v = IntVar(self.root, name="name")
+        self.root.globalsetvar("name", "value")
+        with self.assertRaises(ValueError):
+            v.get()
+        self.root.globalsetvar("name", "345.0")
+        with self.assertRaises(ValueError):
+            v.get()
+
+
+class TestDoubleVar(TestBase):
+
+    def test_default(self):
+        v = DoubleVar(self.root)
+        self.assertEqual(0.0, v.get())
+
+    def test_get(self):
+        v = DoubleVar(self.root, 1.23, "name")
+        self.assertAlmostEqual(1.23, v.get())
+        self.root.globalsetvar("name", "3.45")
+        self.assertAlmostEqual(3.45, v.get())
+
+    def test_get_from_int(self):
+        v = DoubleVar(self.root, 1.23, "name")
+        self.assertAlmostEqual(1.23, v.get())
+        self.root.globalsetvar("name", "3.45")
+        self.assertAlmostEqual(3.45, v.get())
+        self.root.globalsetvar("name", "456")
+        self.assertAlmostEqual(456, v.get())
+
+    def test_invalid_value(self):
+        v = DoubleVar(self.root, name="name")
+        self.root.globalsetvar("name", "value")
+        with self.assertRaises(ValueError):
+            v.get()
+
+
+class TestBooleanVar(TestBase):
+
+    def test_default(self):
+        v = BooleanVar(self.root)
+        self.assertEqual(False, v.get())
+
+    def test_get(self):
+        v = BooleanVar(self.root, True, "name")
+        self.assertAlmostEqual(True, v.get())
+        self.root.globalsetvar("name", "0")
+        self.assertAlmostEqual(False, v.get())
+
+    def test_invalid_value_domain(self):
+        v = BooleanVar(self.root, name="name")
+        self.root.globalsetvar("name", "value")
+        with self.assertRaises(ValueError):
+            v.get()
+        self.root.globalsetvar("name", "1.0")
+        with self.assertRaises(ValueError):
+            v.get()
+
+
+tests_gui = (TestVariable, TestStringVar, TestIntVar,
+             TestDoubleVar, TestBooleanVar)
+
+
+if __name__ == "__main__":
+    from test.support import run_unittest
+    run_unittest(*tests_gui)
diff --git a/Lib/token.py b/Lib/token.py
index 6b5320db8c..31fae0a078 100755
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -70,7 +70,7 @@ NT_OFFSET = 256
 
 tok_name = {value: name
             for name, value in globals().items()
-            if isinstance(value, int)}
+            if isinstance(value, int) and not name.startswith('_')}
 __all__.extend(tok_name.values())
 
 def ISTERMINAL(x):
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 29c9e29b30..cbf91ef222 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -45,6 +45,51 @@ tok_name[NL] = 'NL'
 ENCODING = N_TOKENS + 2
 tok_name[ENCODING] = 'ENCODING'
 N_TOKENS += 3
+EXACT_TOKEN_TYPES = {
+    '(':   LPAR,
+    ')':   RPAR,
+    '[':   LSQB,
+    ']':   RSQB,
+    ':':   COLON,
+    ',':   COMMA,
+    ';':   SEMI,
+    '+':   PLUS,
+    '-':   MINUS,
+    '*':   STAR,
+    '/':   SLASH,
+    '|':   VBAR,
+    '&':   AMPER,
+    '<':   LESS,
+    '>':   GREATER,
+    '=':   EQUAL,
+    '.':   DOT,
+    '%':   PERCENT,
+    '{':   LBRACE,
+    '}':   RBRACE,
+    '==':  EQEQUAL,
+    '!=':  NOTEQUAL,
+    '<=':  LESSEQUAL,
+    '>=':  GREATEREQUAL,
+    '~':   TILDE,
+    '^':   CIRCUMFLEX,
+    '<<':  LEFTSHIFT,
+    '>>':  RIGHTSHIFT,
+    '**':  DOUBLESTAR,
+    '+=':  PLUSEQUAL,
+    '-=':  MINEQUAL,
+    '*=':  STAREQUAL,
+    '/=':  SLASHEQUAL,
+    '%=':  PERCENTEQUAL,
+    '&=':  AMPEREQUAL,
+    '|=':  VBAREQUAL,
+    '^=': CIRCUMFLEXEQUAL,
+    '<<=': LEFTSHIFTEQUAL,
+    '>>=': RIGHTSHIFTEQUAL,
+    '**=': DOUBLESTAREQUAL,
+    '//':  DOUBLESLASH,
+    '//=': DOUBLESLASHEQUAL,
+    '@':   AT
+}
 
 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
     def __repr__(self):
@@ -52,6 +97,13 @@ class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line'
         return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
                 self._replace(type=annotated_type))
 
+    @property
+    def exact_type(self):
+        if self.type == OP and self.string in EXACT_TOKEN_TYPES:
+            return EXACT_TOKEN_TYPES[self.string]
+        else:
+            return self.type
+
 def group(*choices): return '(' + '|'.join(choices) + ')'
 def any(*choices): return group(*choices) + '*'
 def maybe(*choices): return group(*choices) + '?'
@@ -75,6 +127,8 @@ Floatnumber = group(Pointfloat, Expfloat)
 Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 
+StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
+
 # Tail end of ' string.
 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
 # Tail end of " string.
@@ -83,10 +137,10 @@ Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
 # Tail end of """ string.
 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-Triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""')
+Triple = group(StringPrefix + "'''", StringPrefix + '"""')
 # Single-line ' or " string.
-String = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
-               r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+               StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
 
 # Because of leftmost-then-longest match semantics, be sure to put the
 # longest operators first (e.g., if = came before ==, == would get
@@ -104,9 +158,9 @@ PlainToken = group(Number, Funny, String, Name)
 Token = Ignore + PlainToken
 
 # First (or only) line of ' or " string.
-ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
                 group("'", r'\\\r?\n'),
-                r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+                StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
 PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
@@ -114,37 +168,49 @@ PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 def _compile(expr):
     return re.compile(expr, re.UNICODE)
 
-tokenprog, pseudoprog, single3prog, double3prog = map(
-    _compile, (Token, PseudoToken, Single3, Double3))
-endprogs = {"'": _compile(Single), '"': _compile(Double),
-            "'''": single3prog, '"""': double3prog,
-            "r'''": single3prog, 'r"""': double3prog,
-            "b'''": single3prog, 'b"""': double3prog,
-            "br'''": single3prog, 'br"""': double3prog,
-            "R'''": single3prog, 'R"""': double3prog,
-            "B'''": single3prog, 'B"""': double3prog,
-            "bR'''": single3prog, 'bR"""': double3prog,
-            "Br'''": single3prog, 'Br"""': double3prog,
-            "BR'''": single3prog, 'BR"""': double3prog,
-            'r': None, 'R': None, 'b': None, 'B': None}
+endpats = {"'": Single, '"': Double,
+           "'''": Single3, '"""': Double3,
+           "r'''": Single3, 'r"""': Double3,
+           "b'''": Single3, 'b"""': Double3,
+           "R'''": Single3, 'R"""': Double3,
+           "B'''": Single3, 'B"""': Double3,
+           "br'''": Single3, 'br"""': Double3,
+           "bR'''": Single3, 'bR"""': Double3,
+           "Br'''": Single3, 'Br"""': Double3,
+           "BR'''": Single3, 'BR"""': Double3,
+           "rb'''": Single3, 'rb"""': Double3,
+           "Rb'''": Single3, 'Rb"""': Double3,
+           "rB'''": Single3, 'rB"""': Double3,
+           "RB'''": Single3, 'RB"""': Double3,
+           "u'''": Single3, 'u"""': Double3,
+           "R'''": Single3, 'R"""': Double3,
+           "U'''": Single3, 'U"""': Double3,
+           'r': None, 'R': None, 'b': None, 'B': None,
+           'u': None, 'U': None}
 
 triple_quoted = {}
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "b'''", 'b"""', "B'''", 'B"""',
           "br'''", 'br"""', "Br'''", 'Br"""',
-          "bR'''", 'bR"""', "BR'''", 'BR"""'):
+          "bR'''", 'bR"""', "BR'''", 'BR"""',
+          "rb'''", 'rb"""', "rB'''", 'rB"""',
+          "Rb'''", 'Rb"""', "RB'''", 'RB"""',
+          "u'''", 'u"""', "U'''", 'U"""',
+          ):
     triple_quoted[t] = t
 single_quoted = {}
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "b'", 'b"', "B'", 'B"',
           "br'", 'br"', "Br'", 'Br"',
-          "bR'", 'bR"', "BR'", 'BR"' ):
+          "bR'", 'bR"', "BR'", 'BR"' ,
+          "rb'", 'rb"', "rB'", 'rB"',
+          "Rb'", 'Rb"', "RB'", 'RB"' ,
+          "u'", 'u"', "U'", 'U"',
+          ):
     single_quoted[t] = t
 
-del _compile
-
 tabsize = 8
 
 class TokenError(Exception): pass
@@ -281,6 +347,10 @@ def detect_encoding(readline):
 
     If no encoding is specified, then the default of 'utf-8' will be returned.
     """
+    try:
+        filename = readline.__self__.name
+    except AttributeError:
+        filename = None
     bom_found = False
     encoding = None
     default = 'utf-8'
@@ -297,7 +367,10 @@ def detect_encoding(readline):
             # per default encoding.
             line_string = line.decode('utf-8')
         except UnicodeDecodeError:
-            raise SyntaxError("invalid or missing encoding declaration")
+            msg = "invalid or missing encoding declaration"
+            if filename is not None:
+                msg = '{} for {!r}'.format(msg, filename)
+            raise SyntaxError(msg)
 
         matches = cookie_re.findall(line_string)
         if not matches:
@@ -307,12 +380,21 @@ def detect_encoding(readline):
             codec = lookup(encoding)
         except LookupError:
             # This behaviour mimics the Python interpreter
-            raise SyntaxError("unknown encoding: " + encoding)
+            if filename is None:
+                msg = "unknown encoding: " + encoding
+            else:
+                msg = "unknown encoding for {!r}: {}".format(filename,
+                        encoding)
+            raise SyntaxError(msg)
 
         if bom_found:
             if encoding != 'utf-8':
                 # This behaviour mimics the Python interpreter
-                raise SyntaxError('encoding problem: utf-8')
+                if filename is None:
+                    msg = 'encoding problem: utf-8'
+                else:
+                    msg = 'encoding problem for {!r}: utf-8'.format(filename)
+                raise SyntaxError(msg)
             encoding += '-sig'
         return encoding
 
@@ -469,7 +551,7 @@ def _tokenize(readline, encoding):
             continued = 0
 
         while pos < max:
-            pseudomatch = pseudoprog.match(line, pos)
+            pseudomatch = _compile(PseudoToken).match(line, pos)
             if pseudomatch:                                # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
@@ -487,7 +569,7 @@ def _tokenize(readline, encoding):
                     assert not token.endswith("\n")
                     yield TokenInfo(COMMENT, token, spos, epos, line)
                 elif token in triple_quoted:
-                    endprog = endprogs[token]
+                    endprog = _compile(endpats[token])
                     endmatch = endprog.match(line, pos)
                     if endmatch:                           # all on one line
                         pos = endmatch.end(0)
@@ -503,8 +585,9 @@ def _tokenize(readline, encoding):
                     token[:3] in single_quoted:
                     if token[-1] == '\n':                  # continued string
                         strstart = (lnum, start)
-                        endprog = (endprogs[initial] or endprogs[token[1]] or
-                                   endprogs[token[2]])
+                        endprog = _compile(endpats[initial] or
+                                           endpats[token[1]] or
+                                           endpats[token[2]])
                         contstr, needcont = line[start:], 1
                         contline = line
                         break
@@ -535,27 +618,65 @@ def _tokenize(readline, encoding):
 def generate_tokens(readline):
     return _tokenize(readline, None)
 
+def main():
+    import argparse
+
+    # Helper error handling routines
+    def perror(message):
+        print(message, file=sys.stderr)
+
+    def error(message, filename=None, location=None):
+        if location:
+            args = (filename,) + location + (message,)
+            perror("%s:%d:%d: error: %s" % args)
+        elif filename:
+            perror("%s: error: %s" % (filename, message))
+        else:
+            perror("error: %s" % message)
+        sys.exit(1)
+
+    # Parse the arguments and options
+    parser = argparse.ArgumentParser(prog='python -m tokenize')
+    parser.add_argument(dest='filename', nargs='?',
+                        metavar='filename.py',
+                        help='the file to tokenize; defaults to stdin')
+    parser.add_argument('-e', '--exact', dest='exact', action='store_true',
+                        help='display token names using the exact type')
+    args = parser.parse_args()
+
+    try:
+        # Tokenize the input
+        if args.filename:
+            filename = args.filename
+            with builtins.open(filename, 'rb') as f:
+                tokens = list(tokenize(f.readline))
+        else:
+            filename = "<stdin>"
+            tokens = _tokenize(sys.stdin.readline, None)
+
+        # Output the tokenization
+        for token in tokens:
+            token_type = token.type
+            if args.exact:
+                token_type = token.exact_type
+            token_range = "%d,%d-%d,%d:" % (token.start + token.end)
+            print("%-20s%-15s%-15r" %
+                  (token_range, tok_name[token_type], token.string))
+    except IndentationError as err:
+        line, column = err.args[1][1:3]
+        error(err.args[0], filename, (line, column))
+    except TokenError as err:
+        line, column = err.args[1]
+        error(err.args[0], filename, (line, column))
+    except SyntaxError as err:
+        error(err, filename)
+    except IOError as err:
+        error(err)
+    except KeyboardInterrupt:
+        print("interrupted\n")
+    except Exception as err:
+        perror("unexpected error: %s" % err)
+        raise
+
 if __name__ == "__main__":
-    # Quick sanity check
-    s = b'''def parseline(self, line):
-            """Parse the line into a command name and a string containing
-            the arguments.  Returns a tuple containing (command, args, line).
-            'command' and 'args' may be None if the line couldn't be parsed.
-            """
-            line = line.strip()
-            if not line:
-                return None, None, line
-            elif line[0] == '?':
-                line = 'help ' + line[1:]
-            elif line[0] == '!':
-                if hasattr(self, 'do_shell'):
-                    line = 'shell ' + line[1:]
-                else:
-                    return None, None, line
-            i, n = 0, len(line)
-            while i < n and line[i] in self.identchars: i = i+1
-            cmd, arg = line[:i], line[i:].strip()
-            return cmd, arg, line
-    '''
-    for tok in tokenize(iter(s.splitlines()).__next__):
-        print(tok)
+    main()
diff --git a/Lib/trace.py b/Lib/trace.py
index 850369b9ff..c09b365a01 100644
--- a/Lib/trace.py
+++ b/Lib/trace.py
@@ -39,8 +39,8 @@ Sample use, programmatically
 
   # create a Trace object, telling it what to ignore, and whether to
   # do tracing or line-counting or both.
-  tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix,], trace=0,
-                    count=1)
+  tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,],
+                       trace=0, count=1)
   # run the new command using the given tracer
   tracer.run('main()')
   # make a report, placing output in /tmp
@@ -48,12 +48,10 @@ Sample use, programmatically
   r.write_results(show_missing=True, coverdir="/tmp")
 """
 __all__ = ['Trace', 'CoverageResults']
-import io
 import linecache
 import os
 import re
 import sys
-import time
 import token
 import tokenize
 import inspect
@@ -61,6 +59,10 @@ import gc
 import dis
 import pickle
 from warnings import warn as _warn
+try:
+    from time import monotonic as _time
+except ImportError:
+    from time import time as _time
 
 try:
     import threading
@@ -244,8 +246,7 @@ class CoverageResults:
         """Return True if the filename does not refer to a file
         we want to have reported.
         """
-        return (filename == "<string>" or
-                filename.startswith("<doctest "))
+        return filename.startswith('<') and filename.endswith('>')
 
     def update(self, other):
         """Merge in the data from another CoverageResults"""
@@ -477,7 +478,7 @@ class Trace:
         self._caller_cache = {}
         self.start_time = None
         if timing:
-            self.start_time = time.time()
+            self.start_time = _time()
         if countcallers:
             self.globaltrace = self.globaltrace_trackcallers
         elif countfuncs:
@@ -615,7 +616,7 @@ class Trace:
             self.counts[key] = self.counts.get(key, 0) + 1
 
             if self.start_time:
-                print('%.2f' % (time.time() - self.start_time), end=' ')
+                print('%.2f' % (_time() - self.start_time), end=' ')
             bname = os.path.basename(filename)
             print("%s(%d): %s" % (bname, lineno,
                                   linecache.getline(filename, lineno)), end='')
@@ -628,7 +629,7 @@ class Trace:
             lineno = frame.f_lineno
 
             if self.start_time:
-                print('%.2f' % (time.time() - self.start_time), end=' ')
+                print('%.2f' % (_time() - self.start_time), end=' ')
             bname = os.path.basename(filename)
             print("%s(%d): %s" % (bname, lineno,
                                   linecache.getline(filename, lineno)), end='')
@@ -750,10 +751,10 @@ def main(argv=None):
                 # should I also call expanduser? (after all, could use $HOME)
 
                 s = s.replace("$prefix",
-                              os.path.join(sys.prefix, "lib",
+                              os.path.join(sys.base_prefix, "lib",
                                            "python" + sys.version[:3]))
                 s = s.replace("$exec_prefix",
-                              os.path.join(sys.exec_prefix, "lib",
+                              os.path.join(sys.base_exec_prefix, "lib",
                                            "python" + sys.version[:3]))
                 s = os.path.normpath(s)
                 ignore_dirs.append(s)
diff --git a/Lib/traceback.py b/Lib/traceback.py
index 8d4e96edcb..eeb9e7387c 100644
--- a/Lib/traceback.py
+++ b/Lib/traceback.py
@@ -119,15 +119,16 @@ def _iter_chain(exc, custom_tb=None, seen=None):
         seen = set()
     seen.add(exc)
     its = []
+    context = exc.__context__
     cause = exc.__cause__
     if cause is not None and cause not in seen:
-        its.append(_iter_chain(cause, None, seen))
+        its.append(_iter_chain(cause, False, seen))
         its.append([(_cause_message, None)])
-    else:
-        context = exc.__context__
-        if context is not None and context not in seen:
-            its.append(_iter_chain(context, None, seen))
-            its.append([(_context_message, None)])
+    elif (context is not None and
+          not exc.__suppress_context__ and
+          context not in seen):
+        its.append(_iter_chain(context, None, seen))
+        its.append([(_context_message, None)])
     its.append([(exc, custom_tb or exc.__traceback__)])
     # itertools.chain is in an extension module and may be unavailable
     for it in its:
diff --git a/Lib/turtle.py b/Lib/turtle.py
index ac0c32cf21..a44743386f 100644
--- a/Lib/turtle.py
+++ b/Lib/turtle.py
@@ -108,7 +108,6 @@ import tkinter as TK
 import types
 import math
 import time
-import os
 import inspect
 
 from os.path import isfile, split, join
diff --git a/Lib/types.py b/Lib/types.py
index ab354d1a71..cfd09eaaff 100644
--- a/Lib/types.py
+++ b/Lib/types.py
@@ -12,6 +12,8 @@ def _f(): pass
 FunctionType = type(_f)
 LambdaType = type(lambda: None)         # Same as FunctionType
 CodeType = type(_f.__code__)
+MappingProxyType = type(type.__dict__)
+SimpleNamespace = type(sys.implementation)
 
 def _g():
     yield 1
@@ -39,3 +41,61 @@ GetSetDescriptorType = type(FunctionType.__code__)
 MemberDescriptorType = type(FunctionType.__globals__)
 
 del sys, _f, _g, _C,                              # Not for export
+
+
+# Provide a PEP 3115 compliant mechanism for class creation
+def new_class(name, bases=(), kwds=None, exec_body=None):
+    """Create a class object dynamically using the appropriate metaclass."""
+    meta, ns, kwds = prepare_class(name, bases, kwds)
+    if exec_body is not None:
+        exec_body(ns)
+    return meta(name, bases, ns, **kwds)
+
+def prepare_class(name, bases=(), kwds=None):
+    """Call the __prepare__ method of the appropriate metaclass.
+
+    Returns (metaclass, namespace, kwds) as a 3-tuple
+
+    *metaclass* is the appropriate metaclass
+    *namespace* is the prepared class namespace
+    *kwds* is an updated copy of the passed in kwds argument with any
+    'metaclass' entry removed. If no kwds argument is passed in, this will
+    be an empty dict.
+    """
+    if kwds is None:
+        kwds = {}
+    else:
+        kwds = dict(kwds) # Don't alter the provided mapping
+    if 'metaclass' in kwds:
+        meta = kwds.pop('metaclass')
+    else:
+        if bases:
+            meta = type(bases[0])
+        else:
+            meta = type
+    if isinstance(meta, type):
+        # when meta is a type, we first determine the most-derived metaclass
+        # instead of invoking the initial candidate directly
+        meta = _calculate_meta(meta, bases)
+    if hasattr(meta, '__prepare__'):
+        ns = meta.__prepare__(name, bases, **kwds)
+    else:
+        ns = {}
+    return meta, ns, kwds
+
+def _calculate_meta(meta, bases):
+    """Calculate the most derived metaclass."""
+    winner = meta
+    for base in bases:
+        base_meta = type(base)
+        if issubclass(winner, base_meta):
+            continue
+        if issubclass(base_meta, winner):
+            winner = base_meta
+            continue
+        # else:
+        raise TypeError("metaclass conflict: "
+                        "the metaclass of a derived class "
+                        "must be a (non-strict) subclass "
+                        "of the metaclasses of all its bases")
+    return winner
diff --git a/Lib/unittest/__main__.py b/Lib/unittest/__main__.py
index 7320050ae9..798ebc0f53 100644
--- a/Lib/unittest/__main__.py
+++ b/Lib/unittest/__main__.py
@@ -2,7 +2,14 @@
 
 import sys
 if sys.argv[0].endswith("__main__.py"):
-    sys.argv[0] = "python -m unittest"
+    import os.path
+    # We change sys.argv[0] to make help message more useful
+    # use executable without path, unquoted
+    # (it's just a hint anyway)
+    # (if you have spaces in your executable you get what you deserve!)
+    executable = os.path.basename(sys.executable)
+    sys.argv[0] = executable + " -m unittest"
+    del os
 
 __unittest = True
 
diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py
index bea810711d..ad1fa8470a 100644
--- a/Lib/unittest/case.py
+++ b/Lib/unittest/case.py
@@ -9,8 +9,7 @@ import warnings
 import collections
 
 from . import result
-from .util import (strclass, safe_repr, sorted_list_difference,
-                   unorderable_list_difference, _count_diff_all_purpose,
+from .util import (strclass, safe_repr, _count_diff_all_purpose,
                    _count_diff_hashable)
 
 __unittest = True
@@ -104,9 +103,9 @@ def expectedFailure(func):
 class _AssertRaisesBaseContext(object):
 
     def __init__(self, expected, test_case, callable_obj=None,
-                  expected_regex=None):
+                 expected_regex=None):
         self.expected = expected
-        self.failureException = test_case.failureException
+        self.test_case = test_case
         if callable_obj is not None:
             try:
                 self.obj_name = callable_obj.__name__
@@ -117,6 +116,24 @@ class _AssertRaisesBaseContext(object):
         if isinstance(expected_regex, (bytes, str)):
             expected_regex = re.compile(expected_regex)
         self.expected_regex = expected_regex
+        self.msg = None
+
+    def _raiseFailure(self, standardMsg):
+        msg = self.test_case._formatMessage(self.msg, standardMsg)
+        raise self.test_case.failureException(msg)
+
+    def handle(self, name, callable_obj, args, kwargs):
+        """
+        If callable_obj is None, assertRaises/Warns is being used as a
+        context manager, so check for a 'msg' kwarg and return self.
+        If callable_obj is not None, call it passing args and kwargs.
+        """
+        if callable_obj is None:
+            self.msg = kwargs.pop('msg', None)
+            return self
+        with self:
+            callable_obj(*args, **kwargs)
+
 
 
 class _AssertRaisesContext(_AssertRaisesBaseContext):
@@ -132,11 +149,10 @@ class _AssertRaisesContext(_AssertRaisesBaseContext):
             except AttributeError:
                 exc_name = str(self.expected)
             if self.obj_name:
-                raise self.failureException("{0} not raised by {1}"
-                    .format(exc_name, self.obj_name))
+                self._raiseFailure("{} not raised by {}".format(exc_name,
+                                                                self.obj_name))
             else:
-                raise self.failureException("{0} not raised"
-                    .format(exc_name))
+                self._raiseFailure("{} not raised".format(exc_name))
         if not issubclass(exc_type, self.expected):
             # let unexpected exceptions pass through
             return False
@@ -147,8 +163,8 @@ class _AssertRaisesContext(_AssertRaisesBaseContext):
 
         expected_regex = self.expected_regex
         if not expected_regex.search(str(exc_value)):
-            raise self.failureException('"%s" does not match "%s"' %
-                     (expected_regex.pattern, str(exc_value)))
+            self._raiseFailure('"{}" does not match "{}"'.format(
+                     expected_regex.pattern, str(exc_value)))
         return True
 
 
@@ -192,14 +208,13 @@ class _AssertWarnsContext(_AssertRaisesBaseContext):
             return
         # Now we simply try to choose a helpful failure message
         if first_matching is not None:
-            raise self.failureException('"%s" does not match "%s"' %
-                     (self.expected_regex.pattern, str(first_matching)))
+            self._raiseFailure('"{}" does not match "{}"'.format(
+                     self.expected_regex.pattern, str(first_matching)))
         if self.obj_name:
-            raise self.failureException("{0} not triggered by {1}"
-                .format(exc_name, self.obj_name))
+            self._raiseFailure("{} not triggered by {}".format(exc_name,
+                                                               self.obj_name))
         else:
-            raise self.failureException("{0} not triggered"
-                .format(exc_name))
+            self._raiseFailure("{} not triggered".format(exc_name))
 
 
 class TestCase(object):
@@ -452,7 +467,7 @@ class TestCase(object):
                         warnings.warn("TestResult has no addExpectedFailure method, reporting as passes",
                                       RuntimeWarning)
                         result.addSuccess(self)
-
+            return result
         finally:
             result.stopTest(self)
             if orig_result is None:
@@ -526,7 +541,6 @@ class TestCase(object):
         except UnicodeDecodeError:
             return  '%s : %s' % (safe_repr(standardMsg), safe_repr(msg))
 
-
     def assertRaises(self, excClass, callableObj=None, *args, **kwargs):
         """Fail unless an exception of class excClass is raised
            by callableObj when invoked with arguments args and keyword
@@ -541,6 +555,9 @@ class TestCase(object):
                 with self.assertRaises(SomeException):
                     do_something()
 
+           An optional keyword argument 'msg' can be provided when assertRaises
+           is used as a context object.
+
            The context manager keeps a reference to the exception as
            the 'exception' attribute. This allows you to inspect the
            exception after the assertion::
@@ -551,25 +568,25 @@ class TestCase(object):
                self.assertEqual(the_exception.error_code, 3)
         """
         context = _AssertRaisesContext(excClass, self, callableObj)
-        if callableObj is None:
-            return context
-        with context:
-            callableObj(*args, **kwargs)
+        return context.handle('assertRaises', callableObj, args, kwargs)
 
     def assertWarns(self, expected_warning, callable_obj=None, *args, **kwargs):
         """Fail unless a warning of class warnClass is triggered
-           by callableObj when invoked with arguments args and keyword
+           by callable_obj when invoked with arguments args and keyword
            arguments kwargs.  If a different type of warning is
            triggered, it will not be handled: depending on the other
            warning filtering rules in effect, it might be silenced, printed
            out, or raised as an exception.
 
-           If called with callableObj omitted or None, will return a
+           If called with callable_obj omitted or None, will return a
            context object used like this::
 
                 with self.assertWarns(SomeWarning):
                     do_something()
 
+           An optional keyword argument 'msg' can be provided when assertWarns
+           is used as a context object.
+
            The context manager keeps a reference to the first matching
            warning as the 'warning' attribute; similarly, the 'filename'
            and 'lineno' attributes give you information about the line
@@ -582,10 +599,7 @@ class TestCase(object):
                self.assertEqual(the_warning.some_attribute, 147)
         """
         context = _AssertWarnsContext(expected_warning, self, callable_obj)
-        if callable_obj is None:
-            return context
-        with context:
-            callable_obj(*args, **kwargs)
+        return context.handle('assertWarns', callable_obj, args, kwargs)
 
     def _getAssertEqualityFunc(self, first, second):
         """Get a detailed comparison function for the types of the two args.
@@ -722,7 +736,7 @@ class TestCase(object):
             msg: Optional message to use on failure instead of a list of
                     differences.
         """
-        if seq_type != None:
+        if seq_type is not None:
             seq_type_name = seq_type.__name__
             if not isinstance(seq1, seq_type):
                 raise self.failureException('First sequence is not a %s: %s'
@@ -951,48 +965,6 @@ class TestCase(object):
 
         self.fail(self._formatMessage(msg, standardMsg))
 
-    def assertSameElements(self, expected_seq, actual_seq, msg=None):
-        """An unordered sequence specific comparison.
-
-        Raises with an error message listing which elements of expected_seq
-        are missing from actual_seq and vice versa if any.
-
-        Duplicate elements are ignored when comparing *expected_seq* and
-        *actual_seq*. It is the equivalent of ``assertEqual(set(expected),
-        set(actual))`` but it works with sequences of unhashable objects as
-        well.
-        """
-        warnings.warn('assertSameElements is deprecated',
-                      DeprecationWarning)
-        try:
-            expected = set(expected_seq)
-            actual = set(actual_seq)
-            missing = sorted(expected.difference(actual))
-            unexpected = sorted(actual.difference(expected))
-        except TypeError:
-            # Fall back to slower list-compare if any of the objects are
-            # not hashable.
-            expected = list(expected_seq)
-            actual = list(actual_seq)
-            try:
-                expected.sort()
-                actual.sort()
-            except TypeError:
-                missing, unexpected = unorderable_list_difference(expected,
-                                                                  actual)
-            else:
-                missing, unexpected = sorted_list_difference(expected, actual)
-        errors = []
-        if missing:
-            errors.append('Expected, but missing:\n    %s' %
-                          safe_repr(missing))
-        if unexpected:
-            errors.append('Unexpected, but present:\n    %s' %
-                          safe_repr(unexpected))
-        if errors:
-            standardMsg = '\n'.join(errors)
-            self.fail(self._formatMessage(msg, standardMsg))
-
 
     def assertCountEqual(self, first, second, msg=None):
         """An unordered sequence comparison asserting that the same elements,
@@ -1037,8 +1009,8 @@ class TestCase(object):
             if (len(first) > self._diffThreshold or
                 len(second) > self._diffThreshold):
                 self._baseAssertEqual(first, second, msg)
-            firstlines = first.splitlines(True)
-            secondlines = second.splitlines(True)
+            firstlines = first.splitlines(keepends=True)
+            secondlines = second.splitlines(keepends=True)
             if len(firstlines) == 1 and first.strip('\r\n') == first:
                 firstlines = [first + '\n']
                 secondlines = [second + '\n']
@@ -1106,15 +1078,15 @@ class TestCase(object):
             expected_regex: Regex (re pattern object or string) expected
                     to be found in error message.
             callable_obj: Function to be called.
+            msg: Optional message used in case of failure. Can only be used
+                    when assertRaisesRegex is used as a context manager.
             args: Extra args.
             kwargs: Extra kwargs.
         """
         context = _AssertRaisesContext(expected_exception, self, callable_obj,
                                        expected_regex)
-        if callable_obj is None:
-            return context
-        with context:
-            callable_obj(*args, **kwargs)
+
+        return context.handle('assertRaisesRegex', callable_obj, args, kwargs)
 
     def assertWarnsRegex(self, expected_warning, expected_regex,
                          callable_obj=None, *args, **kwargs):
@@ -1128,15 +1100,14 @@ class TestCase(object):
             expected_regex: Regex (re pattern object or string) expected
                     to be found in error message.
             callable_obj: Function to be called.
+            msg: Optional message used in case of failure. Can only be used
+                    when assertWarnsRegex is used as a context manager.
             args: Extra args.
             kwargs: Extra kwargs.
         """
         context = _AssertWarnsContext(expected_warning, self, callable_obj,
                                       expected_regex)
-        if callable_obj is None:
-            return context
-        with context:
-            callable_obj(*args, **kwargs)
+        return context.handle('assertWarnsRegex', callable_obj, args, kwargs)
 
     def assertRegex(self, text, expected_regex, msg=None):
         """Fail the test unless the text matches the regular expression."""
diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py
index 63c8139000..ead64936a0 100644
--- a/Lib/unittest/main.py
+++ b/Lib/unittest/main.py
@@ -1,8 +1,8 @@
 """Unittest main program"""
 
 import sys
+import optparse
 import os
-import types
 
 from . import loader, runner
 from .signals import installHandler
@@ -77,6 +77,7 @@ def _convert_name(name):
 def _convert_names(names):
     return [_convert_name(name) for name in names]
 
+
 class TestProgram(object):
     """A command-line program that runs a set of tests; this is primarily
        for making test modules conveniently executable.
@@ -143,33 +144,9 @@ class TestProgram(object):
             self._do_discovery(argv[2:])
             return
 
-        import getopt
-        long_opts = ['help', 'verbose', 'quiet', 'failfast', 'catch', 'buffer']
-        try:
-            options, args = getopt.getopt(argv[1:], 'hHvqfcb', long_opts)
-        except getopt.error as msg:
-            self.usageExit(msg)
-            return
-
-        for opt, value in options:
-            if opt in ('-h','-H','--help'):
-                self.usageExit()
-            if opt in ('-q','--quiet'):
-                self.verbosity = 0
-            if opt in ('-v','--verbose'):
-                self.verbosity = 2
-            if opt in ('-f','--failfast'):
-                if self.failfast is None:
-                    self.failfast = True
-                # Should this raise an exception if -f is not valid?
-            if opt in ('-c','--catch'):
-                if self.catchbreak is None:
-                    self.catchbreak = True
-                # Should this raise an exception if -c is not valid?
-            if opt in ('-b','--buffer'):
-                if self.buffer is None:
-                    self.buffer = True
-                # Should this raise an exception if -b is not valid?
+        parser = self._getOptParser()
+        options, args = parser.parse_args(argv[1:])
+        self._setAttributesFromOptions(options)
 
         if len(args) == 0 and self.module is None:
             # this allows "python -m unittest -v" to still work for
@@ -197,17 +174,15 @@ class TestProgram(object):
             self.test = self.testLoader.loadTestsFromNames(self.testNames,
                                                            self.module)
 
-    def _do_discovery(self, argv, Loader=None):
-        if Loader is None:
-            Loader = lambda: self.testLoader
-
-        # handle command line args for test discovery
-        self.progName = '%s discover' % self.progName
+    def _getOptParser(self):
         import optparse
         parser = optparse.OptionParser()
         parser.prog = self.progName
         parser.add_option('-v', '--verbose', dest='verbose', default=False,
                           help='Verbose output', action='store_true')
+        parser.add_option('-q', '--quiet', dest='quiet', default=False,
+                          help='Quiet output', action='store_true')
+
         if self.failfast != False:
             parser.add_option('-f', '--failfast', dest='failfast', default=False,
                               help='Stop on first fail or error',
@@ -220,6 +195,24 @@ class TestProgram(object):
             parser.add_option('-b', '--buffer', dest='buffer', default=False,
                               help='Buffer stdout and stderr during tests',
                               action='store_true')
+        return parser
+
+    def _setAttributesFromOptions(self, options):
+        # only set options from the parsing here
+        # if they weren't set explicitly in the constructor
+        if self.failfast is None:
+            self.failfast = options.failfast
+        if self.catchbreak is None:
+            self.catchbreak = options.catchbreak
+        if self.buffer is None:
+            self.buffer = options.buffer
+
+        if options.verbose:
+            self.verbosity = 2
+        elif options.quiet:
+            self.verbosity = 0
+
+    def _addDiscoveryOptions(self, parser):
         parser.add_option('-s', '--start-directory', dest='start', default='.',
                           help="Directory to start discovery ('.' default)")
         parser.add_option('-p', '--pattern', dest='pattern', default='test*.py',
@@ -227,6 +220,15 @@ class TestProgram(object):
         parser.add_option('-t', '--top-level-directory', dest='top', default=None,
                           help='Top level directory of project (defaults to start directory)')
 
+    def _do_discovery(self, argv, Loader=None):
+        if Loader is None:
+            Loader = lambda: self.testLoader
+
+        # handle command line args for test discovery
+        self.progName = '%s discover' % self.progName
+        parser = self._getOptParser()
+        self._addDiscoveryOptions(parser)
+
         options, args = parser.parse_args(argv)
         if len(args) > 3:
             self.usageExit()
@@ -234,17 +236,7 @@ class TestProgram(object):
         for name, value in zip(('start', 'pattern', 'top'), args):
             setattr(options, name, value)
 
-        # only set options from the parsing here
-        # if they weren't set explicitly in the constructor
-        if self.failfast is None:
-            self.failfast = options.failfast
-        if self.catchbreak is None:
-            self.catchbreak = options.catchbreak
-        if self.buffer is None:
-            self.buffer = options.buffer
-
-        if options.verbose:
-            self.verbosity = 2
+        self._setAttributesFromOptions(options)
 
         start_dir = options.start
         pattern = options.pattern
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
new file mode 100644
index 0000000000..324cf399b3
--- /dev/null
+++ b/Lib/unittest/mock.py
@@ -0,0 +1,2211 @@
+# mock.py
+# Test tools for mocking and patching.
+# Maintained by Michael Foord
+# Backport for other versions of Python available from
+# http://pypi.python.org/pypi/mock
+
+__all__ = (
+    'Mock',
+    'MagicMock',
+    'patch',
+    'sentinel',
+    'DEFAULT',
+    'ANY',
+    'call',
+    'create_autospec',
+    'FILTER_DIR',
+    'NonCallableMock',
+    'NonCallableMagicMock',
+    'mock_open',
+    'PropertyMock',
+)
+
+
+__version__ = '1.0'
+
+
+import inspect
+import pprint
+import sys
+from functools import wraps
+
+
+BaseExceptions = (BaseException,)
+if 'java' in sys.platform:
+    # jython
+    import java
+    BaseExceptions = (BaseException, java.lang.Throwable)
+
+
+FILTER_DIR = True
+
+# Workaround for issue #12370
+# Without this, the __class__ properties wouldn't be set correctly
+_safe_super = super
+
+def _is_instance_mock(obj):
+    # can't use isinstance on Mock objects because they override __class__
+    # The base class for all mocks is NonCallableMock
+    return issubclass(type(obj), NonCallableMock)
+
+
+def _is_exception(obj):
+    return (
+        isinstance(obj, BaseExceptions) or
+        isinstance(obj, type) and issubclass(obj, BaseExceptions)
+    )
+
+
+class _slotted(object):
+    __slots__ = ['a']
+
+
+DescriptorTypes = (
+    type(_slotted.a),
+    property,
+)
+
+
+def _getsignature(func, skipfirst, instance=False):
+    if isinstance(func, type) and not instance:
+        try:
+            func = func.__init__
+        except AttributeError:
+            return
+        skipfirst = True
+    elif not isinstance(func, FunctionTypes):
+        # for classes where instance is True we end up here too
+        try:
+            func = func.__call__
+        except AttributeError:
+            return
+
+    try:
+        argspec = inspect.getfullargspec(func)
+    except TypeError:
+        # C function / method, possibly inherited object().__init__
+        return
+
+    regargs, varargs, varkw, defaults, kwonly, kwonlydef, ann = argspec
+
+
+    # instance methods and classmethods need to lose the self argument
+    if getattr(func, '__self__', None) is not None:
+        regargs = regargs[1:]
+    if skipfirst:
+        # this condition and the above one are never both True - why?
+        regargs = regargs[1:]
+
+    signature = inspect.formatargspec(
+        regargs, varargs, varkw, defaults,
+        kwonly, kwonlydef, ann, formatvalue=lambda value: "")
+    return signature[1:-1], func
+
+
+def _check_signature(func, mock, skipfirst, instance=False):
+    if not _callable(func):
+        return
+
+    result = _getsignature(func, skipfirst, instance)
+    if result is None:
+        return
+    signature, func = result
+
+    # can't use self because "self" is common as an argument name
+    # unfortunately even not in the first place
+    src = "lambda _mock_self, %s: None" % signature
+    checksig = eval(src, {})
+    _copy_func_details(func, checksig)
+    type(mock)._mock_check_sig = checksig
+
+
+def _copy_func_details(func, funcopy):
+    funcopy.__name__ = func.__name__
+    funcopy.__doc__ = func.__doc__
+    # we explicitly don't copy func.__dict__ into this copy as it would
+    # expose original attributes that should be mocked
+    funcopy.__module__ = func.__module__
+    funcopy.__defaults__ = func.__defaults__
+    funcopy.__kwdefaults__ = func.__kwdefaults__
+
+
+def _callable(obj):
+    if isinstance(obj, type):
+        return True
+    if getattr(obj, '__call__', None) is not None:
+        return True
+    return False
+
+
+def _is_list(obj):
+    # checks for list or tuples
+    # XXXX badly named!
+    return type(obj) in (list, tuple)
+
+
+def _instance_callable(obj):
+    """Given an object, return True if the object is callable.
+    For classes, return True if instances would be callable."""
+    if not isinstance(obj, type):
+        # already an instance
+        return getattr(obj, '__call__', None) is not None
+
+    # *could* be broken by a class overriding __mro__ or __dict__ via
+    # a metaclass
+    for base in (obj,) + obj.__mro__:
+        if base.__dict__.get('__call__') is not None:
+            return True
+    return False
+
+
+def _set_signature(mock, original, instance=False):
+    # creates a function with signature (*args, **kwargs) that delegates to a
+    # mock. It still does signature checking by calling a lambda with the same
+    # signature as the original.
+    if not _callable(original):
+        return
+
+    skipfirst = isinstance(original, type)
+    result = _getsignature(original, skipfirst, instance)
+    if result is None:
+        # was a C function (e.g. object().__init__ ) that can't be mocked
+        return
+
+    signature, func = result
+
+    src = "lambda %s: None" % signature
+    checksig = eval(src, {})
+    _copy_func_details(func, checksig)
+
+    name = original.__name__
+    if not name.isidentifier():
+        name = 'funcopy'
+    context = {'_checksig_': checksig, 'mock': mock}
+    src = """def %s(*args, **kwargs):
+    _checksig_(*args, **kwargs)
+    return mock(*args, **kwargs)""" % name
+    exec (src, context)
+    funcopy = context[name]
+    _setup_func(funcopy, mock)
+    return funcopy
+
+
+def _setup_func(funcopy, mock):
+    funcopy.mock = mock
+
+    # can't use isinstance with mocks
+    if not _is_instance_mock(mock):
+        return
+
+    def assert_called_with(*args, **kwargs):
+        return mock.assert_called_with(*args, **kwargs)
+    def assert_called_once_with(*args, **kwargs):
+        return mock.assert_called_once_with(*args, **kwargs)
+    def assert_has_calls(*args, **kwargs):
+        return mock.assert_has_calls(*args, **kwargs)
+    def assert_any_call(*args, **kwargs):
+        return mock.assert_any_call(*args, **kwargs)
+    def reset_mock():
+        funcopy.method_calls = _CallList()
+        funcopy.mock_calls = _CallList()
+        mock.reset_mock()
+        ret = funcopy.return_value
+        if _is_instance_mock(ret) and not ret is mock:
+            ret.reset_mock()
+
+    funcopy.called = False
+    funcopy.call_count = 0
+    funcopy.call_args = None
+    funcopy.call_args_list = _CallList()
+    funcopy.method_calls = _CallList()
+    funcopy.mock_calls = _CallList()
+
+    funcopy.return_value = mock.return_value
+    funcopy.side_effect = mock.side_effect
+    funcopy._mock_children = mock._mock_children
+
+    funcopy.assert_called_with = assert_called_with
+    funcopy.assert_called_once_with = assert_called_once_with
+    funcopy.assert_has_calls = assert_has_calls
+    funcopy.assert_any_call = assert_any_call
+    funcopy.reset_mock = reset_mock
+
+    mock._mock_delegate = funcopy
+
+
+def _is_magic(name):
+    return '__%s__' % name[2:-2] == name
+
+
+class _SentinelObject(object):
+    "A unique, named, sentinel object."
+    def __init__(self, name):
+        self.name = name
+
+    def __repr__(self):
+        return 'sentinel.%s' % self.name
+
+
+class _Sentinel(object):
+    """Access attributes to return a named object, usable as a sentinel."""
+    def __init__(self):
+        self._sentinels = {}
+
+    def __getattr__(self, name):
+        if name == '__bases__':
+            # Without this help(unittest.mock) raises an exception
+            raise AttributeError
+        return self._sentinels.setdefault(name, _SentinelObject(name))
+
+
+sentinel = _Sentinel()
+
+DEFAULT = sentinel.DEFAULT
+_missing = sentinel.MISSING
+_deleted = sentinel.DELETED
+
+
+def _copy(value):
+    if type(value) in (dict, list, tuple, set):
+        return type(value)(value)
+    return value
+
+
+_allowed_names = set(
+    [
+        'return_value', '_mock_return_value', 'side_effect',
+        '_mock_side_effect', '_mock_parent', '_mock_new_parent',
+        '_mock_name', '_mock_new_name'
+    ]
+)
+
+
+def _delegating_property(name):
+    _allowed_names.add(name)
+    _the_name = '_mock_' + name
+    def _get(self, name=name, _the_name=_the_name):
+        sig = self._mock_delegate
+        if sig is None:
+            return getattr(self, _the_name)
+        return getattr(sig, name)
+    def _set(self, value, name=name, _the_name=_the_name):
+        sig = self._mock_delegate
+        if sig is None:
+            self.__dict__[_the_name] = value
+        else:
+            setattr(sig, name, value)
+
+    return property(_get, _set)
+
+
+
+class _CallList(list):
+
+    def __contains__(self, value):
+        if not isinstance(value, list):
+            return list.__contains__(self, value)
+        len_value = len(value)
+        len_self = len(self)
+        if len_value > len_self:
+            return False
+
+        for i in range(0, len_self - len_value + 1):
+            sub_list = self[i:i+len_value]
+            if sub_list == value:
+                return True
+        return False
+
+    def __repr__(self):
+        return pprint.pformat(list(self))
+
+
+def _check_and_set_parent(parent, value, name, new_name):
+    if not _is_instance_mock(value):
+        return False
+    if ((value._mock_name or value._mock_new_name) or
+        (value._mock_parent is not None) or
+        (value._mock_new_parent is not None)):
+        return False
+
+    _parent = parent
+    while _parent is not None:
+        # setting a mock (value) as a child or return value of itself
+        # should not modify the mock
+        if _parent is value:
+            return False
+        _parent = _parent._mock_new_parent
+
+    if new_name:
+        value._mock_new_parent = parent
+        value._mock_new_name = new_name
+    if name:
+        value._mock_parent = parent
+        value._mock_name = name
+    return True
+
+
+
+class Base(object):
+    _mock_return_value = DEFAULT
+    _mock_side_effect = None
+    def __init__(self, *args, **kwargs):
+        pass
+
+
+
+class NonCallableMock(Base):
+    """A non-callable version of `Mock`"""
+
+    def __new__(cls, *args, **kw):
+        # every instance has its own class
+        # so we can create magic methods on the
+        # class without stomping on other mocks
+        new = type(cls.__name__, (cls,), {'__doc__': cls.__doc__})
+        instance = object.__new__(new)
+        return instance
+
+
+    def __init__(
+            self, spec=None, wraps=None, name=None, spec_set=None,
+            parent=None, _spec_state=None, _new_name='', _new_parent=None,
+            **kwargs
+        ):
+        if _new_parent is None:
+            _new_parent = parent
+
+        __dict__ = self.__dict__
+        __dict__['_mock_parent'] = parent
+        __dict__['_mock_name'] = name
+        __dict__['_mock_new_name'] = _new_name
+        __dict__['_mock_new_parent'] = _new_parent
+
+        if spec_set is not None:
+            spec = spec_set
+            spec_set = True
+
+        self._mock_add_spec(spec, spec_set)
+
+        __dict__['_mock_children'] = {}
+        __dict__['_mock_wraps'] = wraps
+        __dict__['_mock_delegate'] = None
+
+        __dict__['_mock_called'] = False
+        __dict__['_mock_call_args'] = None
+        __dict__['_mock_call_count'] = 0
+        __dict__['_mock_call_args_list'] = _CallList()
+        __dict__['_mock_mock_calls'] = _CallList()
+
+        __dict__['method_calls'] = _CallList()
+
+        if kwargs:
+            self.configure_mock(**kwargs)
+
+        _safe_super(NonCallableMock, self).__init__(
+            spec, wraps, name, spec_set, parent,
+            _spec_state
+        )
+
+
+    def attach_mock(self, mock, attribute):
+        """
+        Attach a mock as an attribute of this one, replacing its name and
+        parent. Calls to the attached mock will be recorded in the
+        `method_calls` and `mock_calls` attributes of this one."""
+        mock._mock_parent = None
+        mock._mock_new_parent = None
+        mock._mock_name = ''
+        mock._mock_new_name = None
+
+        setattr(self, attribute, mock)
+
+
+    def mock_add_spec(self, spec, spec_set=False):
+        """Add a spec to a mock. `spec` can either be an object or a
+        list of strings. Only attributes on the `spec` can be fetched as
+        attributes from the mock.
+
+        If `spec_set` is True then only attributes on the spec can be set."""
+        self._mock_add_spec(spec, spec_set)
+
+
+    def _mock_add_spec(self, spec, spec_set):
+        _spec_class = None
+
+        if spec is not None and not _is_list(spec):
+            if isinstance(spec, type):
+                _spec_class = spec
+            else:
+                _spec_class = _get_class(spec)
+
+            spec = dir(spec)
+
+        __dict__ = self.__dict__
+        __dict__['_spec_class'] = _spec_class
+        __dict__['_spec_set'] = spec_set
+        __dict__['_mock_methods'] = spec
+
+
+    def __get_return_value(self):
+        ret = self._mock_return_value
+        if self._mock_delegate is not None:
+            ret = self._mock_delegate.return_value
+
+        if ret is DEFAULT:
+            ret = self._get_child_mock(
+                _new_parent=self, _new_name='()'
+            )
+            self.return_value = ret
+        return ret
+
+
+    def __set_return_value(self, value):
+        if self._mock_delegate is not None:
+            self._mock_delegate.return_value = value
+        else:
+            self._mock_return_value = value
+            _check_and_set_parent(self, value, None, '()')
+
+    __return_value_doc = "The value to be returned when the mock is called."
+    return_value = property(__get_return_value, __set_return_value,
+                            __return_value_doc)
+
+
+    @property
+    def __class__(self):
+        if self._spec_class is None:
+            return type(self)
+        return self._spec_class
+
+    called = _delegating_property('called')
+    call_count = _delegating_property('call_count')
+    call_args = _delegating_property('call_args')
+    call_args_list = _delegating_property('call_args_list')
+    mock_calls = _delegating_property('mock_calls')
+
+
+    def __get_side_effect(self):
+        delegated = self._mock_delegate
+        if delegated is None:
+            return self._mock_side_effect
+        return delegated.side_effect
+
+    def __set_side_effect(self, value):
+        value = _try_iter(value)
+        delegated = self._mock_delegate
+        if delegated is None:
+            self._mock_side_effect = value
+        else:
+            delegated.side_effect = value
+
+    side_effect = property(__get_side_effect, __set_side_effect)
+
+
+    def reset_mock(self):
+        "Restore the mock object to its initial state."
+        self.called = False
+        self.call_args = None
+        self.call_count = 0
+        self.mock_calls = _CallList()
+        self.call_args_list = _CallList()
+        self.method_calls = _CallList()
+
+        for child in self._mock_children.values():
+            if isinstance(child, _SpecState):
+                continue
+            child.reset_mock()
+
+        ret = self._mock_return_value
+        if _is_instance_mock(ret) and ret is not self:
+            ret.reset_mock()
+
+
+    def configure_mock(self, **kwargs):
+        """Set attributes on the mock through keyword arguments.
+
+        Attributes plus return values and side effects can be set on child
+        mocks using standard dot notation and unpacking a dictionary in the
+        method call:
+
+        >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError}
+        >>> mock.configure_mock(**attrs)"""
+        for arg, val in sorted(kwargs.items(),
+                               # we sort on the number of dots so that
+                               # attributes are set before we set attributes on
+                               # attributes
+                               key=lambda entry: entry[0].count('.')):
+            args = arg.split('.')
+            final = args.pop()
+            obj = self
+            for entry in args:
+                obj = getattr(obj, entry)
+            setattr(obj, final, val)
+
+
+    def __getattr__(self, name):
+        if name == '_mock_methods':
+            raise AttributeError(name)
+        elif self._mock_methods is not None:
+            if name not in self._mock_methods or name in _all_magics:
+                raise AttributeError("Mock object has no attribute %r" % name)
+        elif _is_magic(name):
+            raise AttributeError(name)
+
+        result = self._mock_children.get(name)
+        if result is _deleted:
+            raise AttributeError(name)
+        elif result is None:
+            wraps = None
+            if self._mock_wraps is not None:
+                # XXXX should we get the attribute without triggering code
+                # execution?
+                wraps = getattr(self._mock_wraps, name)
+
+            result = self._get_child_mock(
+                parent=self, name=name, wraps=wraps, _new_name=name,
+                _new_parent=self
+            )
+            self._mock_children[name]  = result
+
+        elif isinstance(result, _SpecState):
+            result = create_autospec(
+                result.spec, result.spec_set, result.instance,
+                result.parent, result.name
+            )
+            self._mock_children[name]  = result
+
+        return result
+
+
+    def __repr__(self):
+        _name_list = [self._mock_new_name]
+        _parent = self._mock_new_parent
+        last = self
+
+        dot = '.'
+        if _name_list == ['()']:
+            dot = ''
+        seen = set()
+        while _parent is not None:
+            last = _parent
+
+            _name_list.append(_parent._mock_new_name + dot)
+            dot = '.'
+            if _parent._mock_new_name == '()':
+                dot = ''
+
+            _parent = _parent._mock_new_parent
+
+            # use ids here so as not to call __hash__ on the mocks
+            if id(_parent) in seen:
+                break
+            seen.add(id(_parent))
+
+        _name_list = list(reversed(_name_list))
+        _first = last._mock_name or 'mock'
+        if len(_name_list) > 1:
+            if _name_list[1] not in ('()', '().'):
+                _first += '.'
+        _name_list[0] = _first
+        name = ''.join(_name_list)
+
+        name_string = ''
+        if name not in ('mock', 'mock.'):
+            name_string = ' name=%r' % name
+
+        spec_string = ''
+        if self._spec_class is not None:
+            spec_string = ' spec=%r'
+            if self._spec_set:
+                spec_string = ' spec_set=%r'
+            spec_string = spec_string % self._spec_class.__name__
+        return "<%s%s%s id='%s'>" % (
+            type(self).__name__,
+            name_string,
+            spec_string,
+            id(self)
+        )
+
+
+    def __dir__(self):
+        """Filter the output of `dir(mock)` to only useful members."""
+        if not FILTER_DIR:
+            return object.__dir__(self)
+
+        extras = self._mock_methods or []
+        from_type = dir(type(self))
+        from_dict = list(self.__dict__)
+
+        from_type = [e for e in from_type if not e.startswith('_')]
+        from_dict = [e for e in from_dict if not e.startswith('_') or
+                     _is_magic(e)]
+        return sorted(set(extras + from_type + from_dict +
+                          list(self._mock_children)))
+
+
+    def __setattr__(self, name, value):
+        if name in _allowed_names:
+            # property setters go through here
+            return object.__setattr__(self, name, value)
+        elif (self._spec_set and self._mock_methods is not None and
+            name not in self._mock_methods and
+            name not in self.__dict__):
+            raise AttributeError("Mock object has no attribute '%s'" % name)
+        elif name in _unsupported_magics:
+            msg = 'Attempting to set unsupported magic method %r.' % name
+            raise AttributeError(msg)
+        elif name in _all_magics:
+            if self._mock_methods is not None and name not in self._mock_methods:
+                raise AttributeError("Mock object has no attribute '%s'" % name)
+
+            if not _is_instance_mock(value):
+                setattr(type(self), name, _get_method(name, value))
+                original = value
+                value = lambda *args, **kw: original(self, *args, **kw)
+            else:
+                # only set _new_name and not name so that mock_calls is tracked
+                # but not method calls
+                _check_and_set_parent(self, value, None, name)
+                setattr(type(self), name, value)
+                self._mock_children[name] = value
+        elif name == '__class__':
+            self._spec_class = value
+            return
+        else:
+            if _check_and_set_parent(self, value, name, name):
+                self._mock_children[name] = value
+        return object.__setattr__(self, name, value)
+
+
+    def __delattr__(self, name):
+        if name in _all_magics and name in type(self).__dict__:
+            delattr(type(self), name)
+            if name not in self.__dict__:
+                # for magic methods that are still MagicProxy objects and
+                # not set on the instance itself
+                return
+
+        if name in self.__dict__:
+            object.__delattr__(self, name)
+
+        obj = self._mock_children.get(name, _missing)
+        if obj is _deleted:
+            raise AttributeError(name)
+        if obj is not _missing:
+            del self._mock_children[name]
+        self._mock_children[name] = _deleted
+
+
+
+    def _format_mock_call_signature(self, args, kwargs):
+        name = self._mock_name or 'mock'
+        return _format_call_signature(name, args, kwargs)
+
+
+    def _format_mock_failure_message(self, args, kwargs):
+        message = 'Expected call: %s\nActual call: %s'
+        expected_string = self._format_mock_call_signature(args, kwargs)
+        call_args = self.call_args
+        if len(call_args) == 3:
+            call_args = call_args[1:]
+        actual_string = self._format_mock_call_signature(*call_args)
+        return message % (expected_string, actual_string)
+
+
+    def assert_called_with(_mock_self, *args, **kwargs):
+        """assert that the mock was called with the specified arguments.
+
+        Raises an AssertionError if the args and keyword args passed in are
+        different to the last call to the mock."""
+        self = _mock_self
+        if self.call_args is None:
+            expected = self._format_mock_call_signature(args, kwargs)
+            raise AssertionError('Expected call: %s\nNot called' % (expected,))
+
+        if self.call_args != (args, kwargs):
+            msg = self._format_mock_failure_message(args, kwargs)
+            raise AssertionError(msg)
+
+
+    def assert_called_once_with(_mock_self, *args, **kwargs):
+        """assert that the mock was called exactly once and with the specified
+        arguments."""
+        self = _mock_self
+        if not self.call_count == 1:
+            msg = ("Expected '%s' to be called once. Called %s times." %
+                   (self._mock_name or 'mock', self.call_count))
+            raise AssertionError(msg)
+        return self.assert_called_with(*args, **kwargs)
+
+
+    def assert_has_calls(self, calls, any_order=False):
+        """assert the mock has been called with the specified calls.
+        The `mock_calls` list is checked for the calls.
+
+        If `any_order` is False (the default) then the calls must be
+        sequential. There can be extra calls before or after the
+        specified calls.
+
+        If `any_order` is True then the calls can be in any order, but
+        they must all appear in `mock_calls`."""
+        if not any_order:
+            if calls not in self.mock_calls:
+                raise AssertionError(
+                    'Calls not found.\nExpected: %r\n'
+                    'Actual: %r' % (calls, self.mock_calls)
+                )
+            return
+
+        all_calls = list(self.mock_calls)
+
+        not_found = []
+        for kall in calls:
+            try:
+                all_calls.remove(kall)
+            except ValueError:
+                not_found.append(kall)
+        if not_found:
+            raise AssertionError(
+                '%r not all found in call list' % (tuple(not_found),)
+            )
+
+
+    def assert_any_call(self, *args, **kwargs):
+        """assert the mock has been called with the specified arguments.
+
+        The assert passes if the mock has *ever* been called, unlike
+        `assert_called_with` and `assert_called_once_with` that only pass if
+        the call is the most recent one."""
+        kall = call(*args, **kwargs)
+        if kall not in self.call_args_list:
+            expected_string = self._format_mock_call_signature(args, kwargs)
+            raise AssertionError(
+                '%s call not found' % expected_string
+            )
+
+
+    def _get_child_mock(self, **kw):
+        """Create the child mocks for attributes and return value.
+        By default child mocks will be the same type as the parent.
+        Subclasses of Mock may want to override this to customize the way
+        child mocks are made.
+
+        For non-callable mocks the callable variant will be used (rather than
+        any custom subclass)."""
+        _type = type(self)
+        if not issubclass(_type, CallableMixin):
+            if issubclass(_type, NonCallableMagicMock):
+                klass = MagicMock
+            elif issubclass(_type, NonCallableMock) :
+                klass = Mock
+        else:
+            klass = _type.__mro__[1]
+        return klass(**kw)
+
+
+
+def _try_iter(obj):
+    if obj is None:
+        return obj
+    if _is_exception(obj):
+        return obj
+    if _callable(obj):
+        return obj
+    try:
+        return iter(obj)
+    except TypeError:
+        # XXXX backwards compatibility
+        # but this will blow up on first call - so maybe we should fail early?
+        return obj
+
+
+
+class CallableMixin(Base):
+
+    def __init__(self, spec=None, side_effect=None, return_value=DEFAULT,
+                 wraps=None, name=None, spec_set=None, parent=None,
+                 _spec_state=None, _new_name='', _new_parent=None, **kwargs):
+        self.__dict__['_mock_return_value'] = return_value
+
+        _safe_super(CallableMixin, self).__init__(
+            spec, wraps, name, spec_set, parent,
+            _spec_state, _new_name, _new_parent, **kwargs
+        )
+
+        self.side_effect = side_effect
+
+
+    def _mock_check_sig(self, *args, **kwargs):
+        # stub method that can be replaced with one with a specific signature
+        pass
+
+
+    def __call__(_mock_self, *args, **kwargs):
+        # can't use self in-case a function / method we are mocking uses self
+        # in the signature
+        _mock_self._mock_check_sig(*args, **kwargs)
+        return _mock_self._mock_call(*args, **kwargs)
+
+
+    def _mock_call(_mock_self, *args, **kwargs):
+        self = _mock_self
+        self.called = True
+        self.call_count += 1
+        self.call_args = _Call((args, kwargs), two=True)
+        self.call_args_list.append(_Call((args, kwargs), two=True))
+
+        _new_name = self._mock_new_name
+        _new_parent = self._mock_new_parent
+        self.mock_calls.append(_Call(('', args, kwargs)))
+
+        seen = set()
+        skip_next_dot = _new_name == '()'
+        do_method_calls = self._mock_parent is not None
+        name = self._mock_name
+        while _new_parent is not None:
+            this_mock_call = _Call((_new_name, args, kwargs))
+            if _new_parent._mock_new_name:
+                dot = '.'
+                if skip_next_dot:
+                    dot = ''
+
+                skip_next_dot = False
+                if _new_parent._mock_new_name == '()':
+                    skip_next_dot = True
+
+                _new_name = _new_parent._mock_new_name + dot + _new_name
+
+            if do_method_calls:
+                if _new_name == name:
+                    this_method_call = this_mock_call
+                else:
+                    this_method_call = _Call((name, args, kwargs))
+                _new_parent.method_calls.append(this_method_call)
+
+                do_method_calls = _new_parent._mock_parent is not None
+                if do_method_calls:
+                    name = _new_parent._mock_name + '.' + name
+
+            _new_parent.mock_calls.append(this_mock_call)
+            _new_parent = _new_parent._mock_new_parent
+
+            # use ids here so as not to call __hash__ on the mocks
+            _new_parent_id = id(_new_parent)
+            if _new_parent_id in seen:
+                break
+            seen.add(_new_parent_id)
+
+        ret_val = DEFAULT
+        effect = self.side_effect
+        if effect is not None:
+            if _is_exception(effect):
+                raise effect
+
+            if not _callable(effect):
+                result = next(effect)
+                if _is_exception(result):
+                    raise result
+                return result
+
+            ret_val = effect(*args, **kwargs)
+            if ret_val is DEFAULT:
+                ret_val = self.return_value
+
+        if (self._mock_wraps is not None and
+             self._mock_return_value is DEFAULT):
+            return self._mock_wraps(*args, **kwargs)
+        if ret_val is DEFAULT:
+            ret_val = self.return_value
+        return ret_val
+
+
+
+class Mock(CallableMixin, NonCallableMock):
+    """
+    Create a new `Mock` object. `Mock` takes several optional arguments
+    that specify the behaviour of the Mock object:
+
+    * `spec`: This can be either a list of strings or an existing object (a
+      class or instance) that acts as the specification for the mock object. If
+      you pass in an object then a list of strings is formed by calling dir on
+      the object (excluding unsupported magic attributes and methods). Accessing
+      any attribute not in this list will raise an `AttributeError`.
+
+      If `spec` is an object (rather than a list of strings) then
+      `mock.__class__` returns the class of the spec object. This allows mocks
+      to pass `isinstance` tests.
+
+    * `spec_set`: A stricter variant of `spec`. If used, attempting to *set*
+      or get an attribute on the mock that isn't on the object passed as
+      `spec_set` will raise an `AttributeError`.
+
+    * `side_effect`: A function to be called whenever the Mock is called. See
+      the `side_effect` attribute. Useful for raising exceptions or
+      dynamically changing return values. The function is called with the same
+      arguments as the mock, and unless it returns `DEFAULT`, the return
+      value of this function is used as the return value.
+
+      If `side_effect` is an iterable then each call to the mock will return
+      the next value from the iterable. If any of the members of the iterable
+      are exceptions they will be raised instead of returned.
+
+      If `side_effect` is an iterable then each call to the mock will return
+      the next value from the iterable.
+
+    * `return_value`: The value returned when the mock is called. By default
+      this is a new Mock (created on first access). See the
+      `return_value` attribute.
+
+    * `wraps`: Item for the mock object to wrap. If `wraps` is not None then
+      calling the Mock will pass the call through to the wrapped object
+      (returning the real result). Attribute access on the mock will return a
+      Mock object that wraps the corresponding attribute of the wrapped object
+      (so attempting to access an attribute that doesn't exist will raise an
+      `AttributeError`).
+
+      If the mock has an explicit `return_value` set then calls are not passed
+      to the wrapped object and the `return_value` is returned instead.
+
+    * `name`: If the mock has a name then it will be used in the repr of the
+      mock. This can be useful for debugging. The name is propagated to child
+      mocks.
+
+    Mocks can also be called with arbitrary keyword arguments. These will be
+    used to set attributes on the mock after it is created.
+    """
+
+
+
+def _dot_lookup(thing, comp, import_path):
+    try:
+        return getattr(thing, comp)
+    except AttributeError:
+        __import__(import_path)
+        return getattr(thing, comp)
+
+
+def _importer(target):
+    components = target.split('.')
+    import_path = components.pop(0)
+    thing = __import__(import_path)
+
+    for comp in components:
+        import_path += ".%s" % comp
+        thing = _dot_lookup(thing, comp, import_path)
+    return thing
+
+
+def _is_started(patcher):
+    # XXXX horrible
+    return hasattr(patcher, 'is_local')
+
+
+class _patch(object):
+
+    attribute_name = None
+    _active_patches = set()
+
+    def __init__(
+            self, getter, attribute, new, spec, create,
+            spec_set, autospec, new_callable, kwargs
+        ):
+        if new_callable is not None:
+            if new is not DEFAULT:
+                raise ValueError(
+                    "Cannot use 'new' and 'new_callable' together"
+                )
+            if autospec is not None:
+                raise ValueError(
+                    "Cannot use 'autospec' and 'new_callable' together"
+                )
+
+        self.getter = getter
+        self.attribute = attribute
+        self.new = new
+        self.new_callable = new_callable
+        self.spec = spec
+        self.create = create
+        self.has_local = False
+        self.spec_set = spec_set
+        self.autospec = autospec
+        self.kwargs = kwargs
+        self.additional_patchers = []
+
+
+    def copy(self):
+        patcher = _patch(
+            self.getter, self.attribute, self.new, self.spec,
+            self.create, self.spec_set,
+            self.autospec, self.new_callable, self.kwargs
+        )
+        patcher.attribute_name = self.attribute_name
+        patcher.additional_patchers = [
+            p.copy() for p in self.additional_patchers
+        ]
+        return patcher
+
+
+    def __call__(self, func):
+        if isinstance(func, type):
+            return self.decorate_class(func)
+        return self.decorate_callable(func)
+
+
+    def decorate_class(self, klass):
+        for attr in dir(klass):
+            if not attr.startswith(patch.TEST_PREFIX):
+                continue
+
+            attr_value = getattr(klass, attr)
+            if not hasattr(attr_value, "__call__"):
+                continue
+
+            patcher = self.copy()
+            setattr(klass, attr, patcher(attr_value))
+        return klass
+
+
+    def decorate_callable(self, func):
+        if hasattr(func, 'patchings'):
+            func.patchings.append(self)
+            return func
+
+        @wraps(func)
+        def patched(*args, **keywargs):
+            extra_args = []
+            entered_patchers = []
+
+            exc_info = tuple()
+            try:
+                for patching in patched.patchings:
+                    arg = patching.__enter__()
+                    entered_patchers.append(patching)
+                    if patching.attribute_name is not None:
+                        keywargs.update(arg)
+                    elif patching.new is DEFAULT:
+                        extra_args.append(arg)
+
+                args += tuple(extra_args)
+                return func(*args, **keywargs)
+            except:
+                if (patching not in entered_patchers and
+                    _is_started(patching)):
+                    # the patcher may have been started, but an exception
+                    # raised whilst entering one of its additional_patchers
+                    entered_patchers.append(patching)
+                # Pass the exception to __exit__
+                exc_info = sys.exc_info()
+                # re-raise the exception
+                raise
+            finally:
+                for patching in reversed(entered_patchers):
+                    patching.__exit__(*exc_info)
+
+        patched.patchings = [self]
+        return patched
+
+
+    def get_original(self):
+        target = self.getter()
+        name = self.attribute
+
+        original = DEFAULT
+        local = False
+
+        try:
+            original = target.__dict__[name]
+        except (AttributeError, KeyError):
+            original = getattr(target, name, DEFAULT)
+        else:
+            local = True
+
+        if not self.create and original is DEFAULT:
+            raise AttributeError(
+                "%s does not have the attribute %r" % (target, name)
+            )
+        return original, local
+
+
+    def __enter__(self):
+        """Perform the patch."""
+        new, spec, spec_set = self.new, self.spec, self.spec_set
+        autospec, kwargs = self.autospec, self.kwargs
+        new_callable = self.new_callable
+        self.target = self.getter()
+
+        # normalise False to None
+        if spec is False:
+            spec = None
+        if spec_set is False:
+            spec_set = None
+        if autospec is False:
+            autospec = None
+
+        if spec is not None and autospec is not None:
+            raise TypeError("Can't specify spec and autospec")
+        if ((spec is not None or autospec is not None) and
+            spec_set not in (True, None)):
+            raise TypeError("Can't provide explicit spec_set *and* spec or autospec")
+
+        original, local = self.get_original()
+
+        if new is DEFAULT and autospec is None:
+            inherit = False
+            if spec is True:
+                # set spec to the object we are replacing
+                spec = original
+                if spec_set is True:
+                    spec_set = original
+                    spec = None
+            elif spec is not None:
+                if spec_set is True:
+                    spec_set = spec
+                    spec = None
+            elif spec_set is True:
+                spec_set = original
+
+            if spec is not None or spec_set is not None:
+                if original is DEFAULT:
+                    raise TypeError("Can't use 'spec' with create=True")
+                if isinstance(original, type):
+                    # If we're patching out a class and there is a spec
+                    inherit = True
+
+            Klass = MagicMock
+            _kwargs = {}
+            if new_callable is not None:
+                Klass = new_callable
+            elif spec is not None or spec_set is not None:
+                this_spec = spec
+                if spec_set is not None:
+                    this_spec = spec_set
+                if _is_list(this_spec):
+                    not_callable = '__call__' not in this_spec
+                else:
+                    not_callable = not callable(this_spec)
+                if not_callable:
+                    Klass = NonCallableMagicMock
+
+            if spec is not None:
+                _kwargs['spec'] = spec
+            if spec_set is not None:
+                _kwargs['spec_set'] = spec_set
+
+            # add a name to mocks
+            if (isinstance(Klass, type) and
+                issubclass(Klass, NonCallableMock) and self.attribute):
+                _kwargs['name'] = self.attribute
+
+            _kwargs.update(kwargs)
+            new = Klass(**_kwargs)
+
+            if inherit and _is_instance_mock(new):
+                # we can only tell if the instance should be callable if the
+                # spec is not a list
+                this_spec = spec
+                if spec_set is not None:
+                    this_spec = spec_set
+                if (not _is_list(this_spec) and not
+                    _instance_callable(this_spec)):
+                    Klass = NonCallableMagicMock
+
+                _kwargs.pop('name')
+                new.return_value = Klass(_new_parent=new, _new_name='()',
+                                         **_kwargs)
+        elif autospec is not None:
+            # spec is ignored, new *must* be default, spec_set is treated
+            # as a boolean. Should we check spec is not None and that spec_set
+            # is a bool?
+            if new is not DEFAULT:
+                raise TypeError(
+                    "autospec creates the mock for you. Can't specify "
+                    "autospec and new."
+                )
+            if original is DEFAULT:
+                raise TypeError("Can't use 'autospec' with create=True")
+            spec_set = bool(spec_set)
+            if autospec is True:
+                autospec = original
+
+            new = create_autospec(autospec, spec_set=spec_set,
+                                  _name=self.attribute, **kwargs)
+        elif kwargs:
+            # can't set keyword args when we aren't creating the mock
+            # XXXX If new is a Mock we could call new.configure_mock(**kwargs)
+            raise TypeError("Can't pass kwargs to a mock we aren't creating")
+
+        new_attr = new
+
+        self.temp_original = original
+        self.is_local = local
+        setattr(self.target, self.attribute, new_attr)
+        if self.attribute_name is not None:
+            extra_args = {}
+            if self.new is DEFAULT:
+                extra_args[self.attribute_name] =  new
+            for patching in self.additional_patchers:
+                arg = patching.__enter__()
+                if patching.new is DEFAULT:
+                    extra_args.update(arg)
+            return extra_args
+
+        return new
+
+
+    def __exit__(self, *exc_info):
+        """Undo the patch."""
+        if not _is_started(self):
+            raise RuntimeError('stop called on unstarted patcher')
+
+        if self.is_local and self.temp_original is not DEFAULT:
+            setattr(self.target, self.attribute, self.temp_original)
+        else:
+            delattr(self.target, self.attribute)
+            if not self.create and not hasattr(self.target, self.attribute):
+                # needed for proxy objects like django settings
+                setattr(self.target, self.attribute, self.temp_original)
+
+        del self.temp_original
+        del self.is_local
+        del self.target
+        for patcher in reversed(self.additional_patchers):
+            if _is_started(patcher):
+                patcher.__exit__(*exc_info)
+
+
+    def start(self):
+        """Activate a patch, returning any created mock."""
+        result = self.__enter__()
+        self._active_patches.add(self)
+        return result
+
+
+    def stop(self):
+        """Stop an active patch."""
+        self._active_patches.discard(self)
+        return self.__exit__()
+
+
+
+def _get_target(target):
+    try:
+        target, attribute = target.rsplit('.', 1)
+    except (TypeError, ValueError):
+        raise TypeError("Need a valid target to patch. You supplied: %r" %
+                        (target,))
+    getter = lambda: _importer(target)
+    return getter, attribute
+
+
+def _patch_object(
+        target, attribute, new=DEFAULT, spec=None,
+        create=False, spec_set=None, autospec=None,
+        new_callable=None, **kwargs
+    ):
+    """
+    patch the named member (`attribute`) on an object (`target`) with a mock
+    object.
+
+    `patch.object` can be used as a decorator, class decorator or a context
+    manager. Arguments `new`, `spec`, `create`, `spec_set`,
+    `autospec` and `new_callable` have the same meaning as for `patch`. Like
+    `patch`, `patch.object` takes arbitrary keyword arguments for configuring
+    the mock object it creates.
+
+    When used as a class decorator `patch.object` honours `patch.TEST_PREFIX`
+    for choosing which methods to wrap.
+    """
+    getter = lambda: target
+    return _patch(
+        getter, attribute, new, spec, create,
+        spec_set, autospec, new_callable, kwargs
+    )
+
+
+def _patch_multiple(target, spec=None, create=False, spec_set=None,
+                    autospec=None, new_callable=None, **kwargs):
+    """Perform multiple patches in a single call. It takes the object to be
+    patched (either as an object or a string to fetch the object by importing)
+    and keyword arguments for the patches::
+
+        with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'):
+            ...
+
+    Use `DEFAULT` as the value if you want `patch.multiple` to create
+    mocks for you. In this case the created mocks are passed into a decorated
+    function by keyword, and a dictionary is returned when `patch.multiple` is
+    used as a context manager.
+
+    `patch.multiple` can be used as a decorator, class decorator or a context
+    manager. The arguments `spec`, `spec_set`, `create`,
+    `autospec` and `new_callable` have the same meaning as for `patch`. These
+    arguments will be applied to *all* patches done by `patch.multiple`.
+
+    When used as a class decorator `patch.multiple` honours `patch.TEST_PREFIX`
+    for choosing which methods to wrap.
+    """
+    if type(target) is str:
+        getter = lambda: _importer(target)
+    else:
+        getter = lambda: target
+
+    if not kwargs:
+        raise ValueError(
+            'Must supply at least one keyword argument with patch.multiple'
+        )
+    # need to wrap in a list for python 3, where items is a view
+    items = list(kwargs.items())
+    attribute, new = items[0]
+    patcher = _patch(
+        getter, attribute, new, spec, create, spec_set,
+        autospec, new_callable, {}
+    )
+    patcher.attribute_name = attribute
+    for attribute, new in items[1:]:
+        this_patcher = _patch(
+            getter, attribute, new, spec, create, spec_set,
+            autospec, new_callable, {}
+        )
+        this_patcher.attribute_name = attribute
+        patcher.additional_patchers.append(this_patcher)
+    return patcher
+
+
+def patch(
+        target, new=DEFAULT, spec=None, create=False,
+        spec_set=None, autospec=None, new_callable=None, **kwargs
+    ):
+    """
+    `patch` acts as a function decorator, class decorator or a context
+    manager. Inside the body of the function or with statement, the `target`
+    is patched with a `new` object. When the function/with statement exits
+    the patch is undone.
+
+    If `new` is omitted, then the target is replaced with a
+    `MagicMock`. If `patch` is used as a decorator and `new` is
+    omitted, the created mock is passed in as an extra argument to the
+    decorated function. If `patch` is used as a context manager the created
+    mock is returned by the context manager.
+
+    `target` should be a string in the form `'package.module.ClassName'`. The
+    `target` is imported and the specified object replaced with the `new`
+    object, so the `target` must be importable from the environment you are
+    calling `patch` from. The target is imported when the decorated function
+    is executed, not at decoration time.
+
+    The `spec` and `spec_set` keyword arguments are passed to the `MagicMock`
+    if patch is creating one for you.
+
+    In addition you can pass `spec=True` or `spec_set=True`, which causes
+    patch to pass in the object being mocked as the spec/spec_set object.
+
+    `new_callable` allows you to specify a different class, or callable object,
+    that will be called to create the `new` object. By default `MagicMock` is
+    used.
+
+    A more powerful form of `spec` is `autospec`. If you set `autospec=True`
+    then the mock with be created with a spec from the object being replaced.
+    All attributes of the mock will also have the spec of the corresponding
+    attribute of the object being replaced. Methods and functions being
+    mocked will have their arguments checked and will raise a `TypeError` if
+    they are called with the wrong signature. For mocks replacing a class,
+    their return value (the 'instance') will have the same spec as the class.
+
+    Instead of `autospec=True` you can pass `autospec=some_object` to use an
+    arbitrary object as the spec instead of the one being replaced.
+
+    By default `patch` will fail to replace attributes that don't exist. If
+    you pass in `create=True`, and the attribute doesn't exist, patch will
+    create the attribute for you when the patched function is called, and
+    delete it again afterwards. This is useful for writing tests against
+    attributes that your production code creates at runtime. It is off by by
+    default because it can be dangerous. With it switched on you can write
+    passing tests against APIs that don't actually exist!
+
+    Patch can be used as a `TestCase` class decorator. It works by
+    decorating each test method in the class. This reduces the boilerplate
+    code when your test methods share a common patchings set. `patch` finds
+    tests by looking for method names that start with `patch.TEST_PREFIX`.
+    By default this is `test`, which matches the way `unittest` finds tests.
+    You can specify an alternative prefix by setting `patch.TEST_PREFIX`.
+
+    Patch can be used as a context manager, with the with statement. Here the
+    patching applies to the indented block after the with statement. If you
+    use "as" then the patched object will be bound to the name after the
+    "as"; very useful if `patch` is creating a mock object for you.
+
+    `patch` takes arbitrary keyword arguments. These will be passed to
+    the `Mock` (or `new_callable`) on construction.
+
+    `patch.dict(...)`, `patch.multiple(...)` and `patch.object(...)` are
+    available for alternate use-cases.
+    """
+    getter, attribute = _get_target(target)
+    return _patch(
+        getter, attribute, new, spec, create,
+        spec_set, autospec, new_callable, kwargs
+    )
+
+
+class _patch_dict(object):
+    """
+    Patch a dictionary, or dictionary like object, and restore the dictionary
+    to its original state after the test.
+
+    `in_dict` can be a dictionary or a mapping like container. If it is a
+    mapping then it must at least support getting, setting and deleting items
+    plus iterating over keys.
+
+    `in_dict` can also be a string specifying the name of the dictionary, which
+    will then be fetched by importing it.
+
+    `values` can be a dictionary of values to set in the dictionary. `values`
+    can also be an iterable of `(key, value)` pairs.
+
+    If `clear` is True then the dictionary will be cleared before the new
+    values are set.
+
+    `patch.dict` can also be called with arbitrary keyword arguments to set
+    values in the dictionary::
+
+        with patch.dict('sys.modules', mymodule=Mock(), other_module=Mock()):
+            ...
+
+    `patch.dict` can be used as a context manager, decorator or class
+    decorator. When used as a class decorator `patch.dict` honours
+    `patch.TEST_PREFIX` for choosing which methods to wrap.
+    """
+
+    def __init__(self, in_dict, values=(), clear=False, **kwargs):
+        if isinstance(in_dict, str):
+            in_dict = _importer(in_dict)
+        self.in_dict = in_dict
+        # support any argument supported by dict(...) constructor
+        self.values = dict(values)
+        self.values.update(kwargs)
+        self.clear = clear
+        self._original = None
+
+
+    def __call__(self, f):
+        if isinstance(f, type):
+            return self.decorate_class(f)
+        @wraps(f)
+        def _inner(*args, **kw):
+            self._patch_dict()
+            try:
+                return f(*args, **kw)
+            finally:
+                self._unpatch_dict()
+
+        return _inner
+
+
+    def decorate_class(self, klass):
+        for attr in dir(klass):
+            attr_value = getattr(klass, attr)
+            if (attr.startswith(patch.TEST_PREFIX) and
+                 hasattr(attr_value, "__call__")):
+                decorator = _patch_dict(self.in_dict, self.values, self.clear)
+                decorated = decorator(attr_value)
+                setattr(klass, attr, decorated)
+        return klass
+
+
+    def __enter__(self):
+        """Patch the dict."""
+        self._patch_dict()
+
+
+    def _patch_dict(self):
+        values = self.values
+        in_dict = self.in_dict
+        clear = self.clear
+
+        try:
+            original = in_dict.copy()
+        except AttributeError:
+            # dict like object with no copy method
+            # must support iteration over keys
+            original = {}
+            for key in in_dict:
+                original[key] = in_dict[key]
+        self._original = original
+
+        if clear:
+            _clear_dict(in_dict)
+
+        try:
+            in_dict.update(values)
+        except AttributeError:
+            # dict like object with no update method
+            for key in values:
+                in_dict[key] = values[key]
+
+
+    def _unpatch_dict(self):
+        in_dict = self.in_dict
+        original = self._original
+
+        _clear_dict(in_dict)
+
+        try:
+            in_dict.update(original)
+        except AttributeError:
+            for key in original:
+                in_dict[key] = original[key]
+
+
+    def __exit__(self, *args):
+        """Unpatch the dict."""
+        self._unpatch_dict()
+        return False
+
+    start = __enter__
+    stop = __exit__
+
+
+def _clear_dict(in_dict):
+    try:
+        in_dict.clear()
+    except AttributeError:
+        keys = list(in_dict)
+        for key in keys:
+            del in_dict[key]
+
+
+def _patch_stopall():
+    """Stop all active patches."""
+    for patch in list(_patch._active_patches):
+        patch.stop()
+
+
+patch.object = _patch_object
+patch.dict = _patch_dict
+patch.multiple = _patch_multiple
+patch.stopall = _patch_stopall
+patch.TEST_PREFIX = 'test'
+
+magic_methods = (
+    "lt le gt ge eq ne "
+    "getitem setitem delitem "
+    "len contains iter "
+    "hash str sizeof "
+    "enter exit "
+    "divmod neg pos abs invert "
+    "complex int float index "
+    "trunc floor ceil "
+    "bool next "
+)
+
+numerics = "add sub mul div floordiv mod lshift rshift and xor or pow "
+inplace = ' '.join('i%s' % n for n in numerics.split())
+right = ' '.join('r%s' % n for n in numerics.split())
+
+# not including __prepare__, __instancecheck__, __subclasscheck__
+# (as they are metaclass methods)
+# __del__ is not supported at all as it causes problems if it exists
+
+_non_defaults = set('__%s__' % method for method in [
+    'get', 'set', 'delete', 'reversed', 'missing', 'reduce', 'reduce_ex',
+    'getinitargs', 'getnewargs', 'getstate', 'setstate', 'getformat',
+    'setformat', 'repr', 'dir', 'subclasses', 'format',
+])
+
+
+def _get_method(name, func):
+    "Turns a callable object (like a mock) into a real function"
+    def method(self, *args, **kw):
+        return func(self, *args, **kw)
+    method.__name__ = name
+    return method
+
+
+_magics = set(
+    '__%s__' % method for method in
+    ' '.join([magic_methods, numerics, inplace, right]).split()
+)
+
+_all_magics = _magics | _non_defaults
+
+_unsupported_magics = set([
+    '__getattr__', '__setattr__',
+    '__init__', '__new__', '__prepare__'
+    '__instancecheck__', '__subclasscheck__',
+    '__del__'
+])
+
+_calculate_return_value = {
+    '__hash__': lambda self: object.__hash__(self),
+    '__str__': lambda self: object.__str__(self),
+    '__sizeof__': lambda self: object.__sizeof__(self),
+}
+
+_return_values = {
+    '__lt__': NotImplemented,
+    '__gt__': NotImplemented,
+    '__le__': NotImplemented,
+    '__ge__': NotImplemented,
+    '__int__': 1,
+    '__contains__': False,
+    '__len__': 0,
+    '__exit__': False,
+    '__complex__': 1j,
+    '__float__': 1.0,
+    '__bool__': True,
+    '__index__': 1,
+}
+
+
+def _get_eq(self):
+    def __eq__(other):
+        ret_val = self.__eq__._mock_return_value
+        if ret_val is not DEFAULT:
+            return ret_val
+        return self is other
+    return __eq__
+
+def _get_ne(self):
+    def __ne__(other):
+        if self.__ne__._mock_return_value is not DEFAULT:
+            return DEFAULT
+        return self is not other
+    return __ne__
+
+def _get_iter(self):
+    def __iter__():
+        ret_val = self.__iter__._mock_return_value
+        if ret_val is DEFAULT:
+            return iter([])
+        # if ret_val was already an iterator, then calling iter on it should
+        # return the iterator unchanged
+        return iter(ret_val)
+    return __iter__
+
+_side_effect_methods = {
+    '__eq__': _get_eq,
+    '__ne__': _get_ne,
+    '__iter__': _get_iter,
+}
+
+
+
+def _set_return_value(mock, method, name):
+    fixed = _return_values.get(name, DEFAULT)
+    if fixed is not DEFAULT:
+        method.return_value = fixed
+        return
+
+    return_calulator = _calculate_return_value.get(name)
+    if return_calulator is not None:
+        try:
+            return_value = return_calulator(mock)
+        except AttributeError:
+            # XXXX why do we return AttributeError here?
+            #      set it as a side_effect instead?
+            return_value = AttributeError(name)
+        method.return_value = return_value
+        return
+
+    side_effector = _side_effect_methods.get(name)
+    if side_effector is not None:
+        method.side_effect = side_effector(mock)
+
+
+
+class MagicMixin(object):
+    def __init__(self, *args, **kw):
+        _safe_super(MagicMixin, self).__init__(*args, **kw)
+        self._mock_set_magics()
+
+
+    def _mock_set_magics(self):
+        these_magics = _magics
+
+        if self._mock_methods is not None:
+            these_magics = _magics.intersection(self._mock_methods)
+
+            remove_magics = set()
+            remove_magics = _magics - these_magics
+
+            for entry in remove_magics:
+                if entry in type(self).__dict__:
+                    # remove unneeded magic methods
+                    delattr(self, entry)
+
+        # don't overwrite existing attributes if called a second time
+        these_magics = these_magics - set(type(self).__dict__)
+
+        _type = type(self)
+        for entry in these_magics:
+            setattr(_type, entry, MagicProxy(entry, self))
+
+
+
+class NonCallableMagicMock(MagicMixin, NonCallableMock):
+    """A version of `MagicMock` that isn't callable."""
+    def mock_add_spec(self, spec, spec_set=False):
+        """Add a spec to a mock. `spec` can either be an object or a
+        list of strings. Only attributes on the `spec` can be fetched as
+        attributes from the mock.
+
+        If `spec_set` is True then only attributes on the spec can be set."""
+        self._mock_add_spec(spec, spec_set)
+        self._mock_set_magics()
+
+
+
+class MagicMock(MagicMixin, Mock):
+    """
+    MagicMock is a subclass of Mock with default implementations
+    of most of the magic methods. You can use MagicMock without having to
+    configure the magic methods yourself.
+
+    If you use the `spec` or `spec_set` arguments then *only* magic
+    methods that exist in the spec will be created.
+
+    Attributes and the return value of a `MagicMock` will also be `MagicMocks`.
+    """
+    def mock_add_spec(self, spec, spec_set=False):
+        """Add a spec to a mock. `spec` can either be an object or a
+        list of strings. Only attributes on the `spec` can be fetched as
+        attributes from the mock.
+
+        If `spec_set` is True then only attributes on the spec can be set."""
+        self._mock_add_spec(spec, spec_set)
+        self._mock_set_magics()
+
+
+
+class MagicProxy(object):
+    def __init__(self, name, parent):
+        self.name = name
+        self.parent = parent
+
+    def __call__(self, *args, **kwargs):
+        m = self.create_mock()
+        return m(*args, **kwargs)
+
+    def create_mock(self):
+        entry = self.name
+        parent = self.parent
+        m = parent._get_child_mock(name=entry, _new_name=entry,
+                                   _new_parent=parent)
+        setattr(parent, entry, m)
+        _set_return_value(parent, m, entry)
+        return m
+
+    def __get__(self, obj, _type=None):
+        return self.create_mock()
+
+
+
+class _ANY(object):
+    "A helper object that compares equal to everything."
+
+    def __eq__(self, other):
+        return True
+
+    def __ne__(self, other):
+        return False
+
+    def __repr__(self):
+        return '<ANY>'
+
+ANY = _ANY()
+
+
+
+def _format_call_signature(name, args, kwargs):
+    message = '%s(%%s)' % name
+    formatted_args = ''
+    args_string = ', '.join([repr(arg) for arg in args])
+    kwargs_string = ', '.join([
+        '%s=%r' % (key, value) for key, value in kwargs.items()
+    ])
+    if args_string:
+        formatted_args = args_string
+    if kwargs_string:
+        if formatted_args:
+            formatted_args += ', '
+        formatted_args += kwargs_string
+
+    return message % formatted_args
+
+
+
+class _Call(tuple):
+    """
+    A tuple for holding the results of a call to a mock, either in the form
+    `(args, kwargs)` or `(name, args, kwargs)`.
+
+    If args or kwargs are empty then a call tuple will compare equal to
+    a tuple without those values. This makes comparisons less verbose::
+
+        _Call(('name', (), {})) == ('name',)
+        _Call(('name', (1,), {})) == ('name', (1,))
+        _Call(((), {'a': 'b'})) == ({'a': 'b'},)
+
+    The `_Call` object provides a useful shortcut for comparing with call::
+
+        _Call(((1, 2), {'a': 3})) == call(1, 2, a=3)
+        _Call(('foo', (1, 2), {'a': 3})) == call.foo(1, 2, a=3)
+
+    If the _Call has no name then it will match any name.
+    """
+    def __new__(cls, value=(), name=None, parent=None, two=False,
+                from_kall=True):
+        name = ''
+        args = ()
+        kwargs = {}
+        _len = len(value)
+        if _len == 3:
+            name, args, kwargs = value
+        elif _len == 2:
+            first, second = value
+            if isinstance(first, str):
+                name = first
+                if isinstance(second, tuple):
+                    args = second
+                else:
+                    kwargs = second
+            else:
+                args, kwargs = first, second
+        elif _len == 1:
+            value, = value
+            if isinstance(value, str):
+                name = value
+            elif isinstance(value, tuple):
+                args = value
+            else:
+                kwargs = value
+
+        if two:
+            return tuple.__new__(cls, (args, kwargs))
+
+        return tuple.__new__(cls, (name, args, kwargs))
+
+
+    def __init__(self, value=(), name=None, parent=None, two=False,
+                 from_kall=True):
+        self.name = name
+        self.parent = parent
+        self.from_kall = from_kall
+
+
+    def __eq__(self, other):
+        if other is ANY:
+            return True
+        try:
+            len_other = len(other)
+        except TypeError:
+            return False
+
+        self_name = ''
+        if len(self) == 2:
+            self_args, self_kwargs = self
+        else:
+            self_name, self_args, self_kwargs = self
+
+        other_name = ''
+        if len_other == 0:
+            other_args, other_kwargs = (), {}
+        elif len_other == 3:
+            other_name, other_args, other_kwargs = other
+        elif len_other == 1:
+            value, = other
+            if isinstance(value, tuple):
+                other_args = value
+                other_kwargs = {}
+            elif isinstance(value, str):
+                other_name = value
+                other_args, other_kwargs = (), {}
+            else:
+                other_args = ()
+                other_kwargs = value
+        else:
+            # len 2
+            # could be (name, args) or (name, kwargs) or (args, kwargs)
+            first, second = other
+            if isinstance(first, str):
+                other_name = first
+                if isinstance(second, tuple):
+                    other_args, other_kwargs = second, {}
+                else:
+                    other_args, other_kwargs = (), second
+            else:
+                other_args, other_kwargs = first, second
+
+        if self_name and other_name != self_name:
+            return False
+
+        # this order is important for ANY to work!
+        return (other_args, other_kwargs) == (self_args, self_kwargs)
+
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+    def __call__(self, *args, **kwargs):
+        if self.name is None:
+            return _Call(('', args, kwargs), name='()')
+
+        name = self.name + '()'
+        return _Call((self.name, args, kwargs), name=name, parent=self)
+
+
+    def __getattr__(self, attr):
+        if self.name is None:
+            return _Call(name=attr, from_kall=False)
+        name = '%s.%s' % (self.name, attr)
+        return _Call(name=name, parent=self, from_kall=False)
+
+
+    def __repr__(self):
+        if not self.from_kall:
+            name = self.name or 'call'
+            if name.startswith('()'):
+                name = 'call%s' % name
+            return name
+
+        if len(self) == 2:
+            name = 'call'
+            args, kwargs = self
+        else:
+            name, args, kwargs = self
+            if not name:
+                name = 'call'
+            elif not name.startswith('()'):
+                name = 'call.%s' % name
+            else:
+                name = 'call%s' % name
+        return _format_call_signature(name, args, kwargs)
+
+
+    def call_list(self):
+        """For a call object that represents multiple calls, `call_list`
+        returns a list of all the intermediate calls as well as the
+        final call."""
+        vals = []
+        thing = self
+        while thing is not None:
+            if thing.from_kall:
+                vals.append(thing)
+            thing = thing.parent
+        return _CallList(reversed(vals))
+
+
+call = _Call(from_kall=False)
+
+
+
+def create_autospec(spec, spec_set=False, instance=False, _parent=None,
+                    _name=None, **kwargs):
+    """Create a mock object using another object as a spec. Attributes on the
+    mock will use the corresponding attribute on the `spec` object as their
+    spec.
+
+    Functions or methods being mocked will have their arguments checked
+    to check that they are called with the correct signature.
+
+    If `spec_set` is True then attempting to set attributes that don't exist
+    on the spec object will raise an `AttributeError`.
+
+    If a class is used as a spec then the return value of the mock (the
+    instance of the class) will have the same spec. You can use a class as the
+    spec for an instance object by passing `instance=True`. The returned mock
+    will only be callable if instances of the mock are callable.
+
+    `create_autospec` also takes arbitrary keyword arguments that are passed to
+    the constructor of the created mock."""
+    if _is_list(spec):
+        # can't pass a list instance to the mock constructor as it will be
+        # interpreted as a list of strings
+        spec = type(spec)
+
+    is_type = isinstance(spec, type)
+
+    _kwargs = {'spec': spec}
+    if spec_set:
+        _kwargs = {'spec_set': spec}
+    elif spec is None:
+        # None we mock with a normal mock without a spec
+        _kwargs = {}
+
+    _kwargs.update(kwargs)
+
+    Klass = MagicMock
+    if type(spec) in DescriptorTypes:
+        # descriptors don't have a spec
+        # because we don't know what type they return
+        _kwargs = {}
+    elif not _callable(spec):
+        Klass = NonCallableMagicMock
+    elif is_type and instance and not _instance_callable(spec):
+        Klass = NonCallableMagicMock
+
+    _new_name = _name
+    if _parent is None:
+        # for a top level object no _new_name should be set
+        _new_name = ''
+
+    mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name,
+                 name=_name, **_kwargs)
+
+    if isinstance(spec, FunctionTypes):
+        # should only happen at the top level because we don't
+        # recurse for functions
+        mock = _set_signature(mock, spec)
+    else:
+        _check_signature(spec, mock, is_type, instance)
+
+    if _parent is not None and not instance:
+        _parent._mock_children[_name] = mock
+
+    if is_type and not instance and 'return_value' not in kwargs:
+        mock.return_value = create_autospec(spec, spec_set, instance=True,
+                                            _name='()', _parent=mock)
+
+    for entry in dir(spec):
+        if _is_magic(entry):
+            # MagicMock already does the useful magic methods for us
+            continue
+
+        # XXXX do we need a better way of getting attributes without
+        # triggering code execution (?) Probably not - we need the actual
+        # object to mock it so we would rather trigger a property than mock
+        # the property descriptor. Likewise we want to mock out dynamically
+        # provided attributes.
+        # XXXX what about attributes that raise exceptions other than
+        # AttributeError on being fetched?
+        # we could be resilient against it, or catch and propagate the
+        # exception when the attribute is fetched from the mock
+        try:
+            original = getattr(spec, entry)
+        except AttributeError:
+            continue
+
+        kwargs = {'spec': original}
+        if spec_set:
+            kwargs = {'spec_set': original}
+
+        if not isinstance(original, FunctionTypes):
+            new = _SpecState(original, spec_set, mock, entry, instance)
+            mock._mock_children[entry] = new
+        else:
+            parent = mock
+            if isinstance(spec, FunctionTypes):
+                parent = mock.mock
+
+            new = MagicMock(parent=parent, name=entry, _new_name=entry,
+                            _new_parent=parent, **kwargs)
+            mock._mock_children[entry] = new
+            skipfirst = _must_skip(spec, entry, is_type)
+            _check_signature(original, new, skipfirst=skipfirst)
+
+        # so functions created with _set_signature become instance attributes,
+        # *plus* their underlying mock exists in _mock_children of the parent
+        # mock. Adding to _mock_children may be unnecessary where we are also
+        # setting as an instance attribute?
+        if isinstance(new, FunctionTypes):
+            setattr(mock, entry, new)
+
+    return mock
+
+
+def _must_skip(spec, entry, is_type):
+    if not isinstance(spec, type):
+        if entry in getattr(spec, '__dict__', {}):
+            # instance attribute - shouldn't skip
+            return False
+        spec = spec.__class__
+
+    for klass in spec.__mro__:
+        result = klass.__dict__.get(entry, DEFAULT)
+        if result is DEFAULT:
+            continue
+        if isinstance(result, (staticmethod, classmethod)):
+            return False
+        return is_type
+
+    # shouldn't get here unless function is a dynamically provided attribute
+    # XXXX untested behaviour
+    return is_type
+
+
+def _get_class(obj):
+    try:
+        return obj.__class__
+    except AttributeError:
+        # it is possible for objects to have no __class__
+        return type(obj)
+
+
+class _SpecState(object):
+
+    def __init__(self, spec, spec_set=False, parent=None,
+                 name=None, ids=None, instance=False):
+        self.spec = spec
+        self.ids = ids
+        self.spec_set = spec_set
+        self.parent = parent
+        self.instance = instance
+        self.name = name
+
+
+FunctionTypes = (
+    # python function
+    type(create_autospec),
+    # instance method
+    type(ANY.__eq__),
+)
+
+
+file_spec = None
+
+
+def mock_open(mock=None, read_data=''):
+    """
+    A helper function to create a mock to replace the use of `open`. It works
+    for `open` called directly or used as a context manager.
+
+    The `mock` argument is the mock object to configure. If `None` (the
+    default) then a `MagicMock` will be created for you, with the API limited
+    to methods or attributes available on standard file handles.
+
+    `read_data` is a string for the `read` method of the file handle to return.
+    This is an empty string by default.
+    """
+    global file_spec
+    if file_spec is None:
+        import _io
+        file_spec = list(set(dir(_io.TextIOWrapper)).union(set(dir(_io.BytesIO))))
+
+    if mock is None:
+        mock = MagicMock(name='open', spec=open)
+
+    handle = MagicMock(spec=file_spec)
+    handle.write.return_value = None
+    handle.__enter__.return_value = handle
+    handle.read.return_value = read_data
+
+    mock.return_value = handle
+    return mock
+
+
+class PropertyMock(Mock):
+    """
+    A mock intended to be used as a property, or other descriptor, on a class.
+    `PropertyMock` provides `__get__` and `__set__` methods so you can specify
+    a return value when it is fetched.
+
+    Fetching a `PropertyMock` instance from an object calls the mock, with
+    no args. Setting it calls the mock with the value being set.
+    """
+    def _get_child_mock(self, **kwargs):
+        return MagicMock(**kwargs)
+
+    def __get__(self, obj, obj_type):
+        return self()
+    def __set__(self, obj, val):
+        self(val)
diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py
index 44bf1862e4..97e5426927 100644
--- a/Lib/unittest/result.py
+++ b/Lib/unittest/result.py
@@ -1,6 +1,5 @@
 """Test result object"""
 
-import os
 import io
 import sys
 import traceback
diff --git a/Lib/unittest/test/__init__.py b/Lib/unittest/test/__init__.py
index 99b730b154..cdae8a7442 100644
--- a/Lib/unittest/test/__init__.py
+++ b/Lib/unittest/test/__init__.py
@@ -14,6 +14,7 @@ def suite():
             __import__(modname)
             module = sys.modules[modname]
             suite.addTest(loader.loadTestsFromModule(module))
+    suite.addTest(loader.loadTestsFromName('unittest.test.testmock'))
     return suite
 
 
diff --git a/Lib/unittest/test/_test_warnings.py b/Lib/unittest/test/_test_warnings.py
index d0be18d4ad..5cbfb532ad 100644
--- a/Lib/unittest/test/_test_warnings.py
+++ b/Lib/unittest/test/_test_warnings.py
@@ -10,7 +10,6 @@ combinations of warnings args and -W flags and check that the output is correct.
 See #10535.
 """
 
-import io
 import sys
 import unittest
 import warnings
diff --git a/Lib/unittest/test/test_assertions.py b/Lib/unittest/test/test_assertions.py
index a1d20ebf2d..d43fe5a601 100644
--- a/Lib/unittest/test/test_assertions.py
+++ b/Lib/unittest/test/test_assertions.py
@@ -1,6 +1,7 @@
 import datetime
 import warnings
 import unittest
+from itertools import product
 
 
 class Test_Assertions(unittest.TestCase):
@@ -145,6 +146,14 @@ class TestLongMessage(unittest.TestCase):
         self.testableTrue._formatMessage(one, '\uFFFD')
 
     def assertMessages(self, methodName, args, errors):
+        """
+        Check that methodName(*args) raises the correct error messages.
+        errors should be a list of 4 regex that match the error when:
+          1) longMessage = False and no msg passed;
+          2) longMessage = False and msg passed;
+          3) longMessage = True and no msg passed;
+          4) longMessage = True and msg passed;
+        """
         def getMethod(i):
             useTestableFalse  = i < 2
             if useTestableFalse:
@@ -284,3 +293,67 @@ class TestLongMessage(unittest.TestCase):
                             ["^unexpectedly identical: None$", "^oops$",
                              "^unexpectedly identical: None$",
                              "^unexpectedly identical: None : oops$"])
+
+
+    def assertMessagesCM(self, methodName, args, func, errors):
+        """
+        Check that the correct error messages are raised while executing:
+          with method(*args):
+              func()
+        *errors* should be a list of 4 regex that match the error when:
+          1) longMessage = False and no msg passed;
+          2) longMessage = False and msg passed;
+          3) longMessage = True and no msg passed;
+          4) longMessage = True and msg passed;
+        """
+        p = product((self.testableFalse, self.testableTrue),
+                    ({}, {"msg": "oops"}))
+        for (cls, kwargs), err in zip(p, errors):
+            method = getattr(cls, methodName)
+            with self.assertRaisesRegex(cls.failureException, err):
+                with method(*args, **kwargs) as cm:
+                    func()
+
+    def testAssertRaises(self):
+        self.assertMessagesCM('assertRaises', (TypeError,), lambda: None,
+                              ['^TypeError not raised$', '^oops$',
+                               '^TypeError not raised$',
+                               '^TypeError not raised : oops$'])
+
+    def testAssertRaisesRegex(self):
+        # test error not raised
+        self.assertMessagesCM('assertRaisesRegex', (TypeError, 'unused regex'),
+                              lambda: None,
+                              ['^TypeError not raised$', '^oops$',
+                               '^TypeError not raised$',
+                               '^TypeError not raised : oops$'])
+        # test error raised but with wrong message
+        def raise_wrong_message():
+            raise TypeError('foo')
+        self.assertMessagesCM('assertRaisesRegex', (TypeError, 'regex'),
+                              raise_wrong_message,
+                              ['^"regex" does not match "foo"$', '^oops$',
+                               '^"regex" does not match "foo"$',
+                               '^"regex" does not match "foo" : oops$'])
+
+    def testAssertWarns(self):
+        self.assertMessagesCM('assertWarns', (UserWarning,), lambda: None,
+                              ['^UserWarning not triggered$', '^oops$',
+                               '^UserWarning not triggered$',
+                               '^UserWarning not triggered : oops$'])
+
+    def testAssertWarnsRegex(self):
+        # test error not raised
+        self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'),
+                              lambda: None,
+                              ['^UserWarning not triggered$', '^oops$',
+                               '^UserWarning not triggered$',
+                               '^UserWarning not triggered : oops$'])
+        # test warning raised but with wrong message
+        def raise_wrong_message():
+            warnings.warn('foo')
+        self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'regex'),
+                              raise_wrong_message,
+                              ['^"regex" does not match "foo"$', '^oops$',
+                               '^"regex" does not match "foo"$',
+                               '^"regex" does not match "foo" : oops$'])
diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py
index c74a539515..fdb2e78462 100644
--- a/Lib/unittest/test/test_case.py
+++ b/Lib/unittest/test/test_case.py
@@ -4,6 +4,7 @@ import pickle
 import re
 import sys
 import warnings
+import weakref
 import inspect
 
 from copy import deepcopy
@@ -386,27 +387,62 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
         self.assertIsInstance(Foo().id(), str)
 
 
-    # "If result is omitted or None, a temporary result object is created
-    # and used, but is not made available to the caller. As TestCase owns the
+    # "If result is omitted or None, a temporary result object is created,
+    # used, and is made available to the caller. As TestCase owns the
     # temporary result startTestRun and stopTestRun are called.
 
     def test_run__uses_defaultTestResult(self):
         events = []
+        defaultResult = LoggingResult(events)
 
         class Foo(unittest.TestCase):
             def test(self):
                 events.append('test')
 
             def defaultTestResult(self):
-                return LoggingResult(events)
+                return defaultResult
 
         # Make run() find a result object on its own
-        Foo('test').run()
+        result = Foo('test').run()
 
+        self.assertIs(result, defaultResult)
         expected = ['startTestRun', 'startTest', 'test', 'addSuccess',
             'stopTest', 'stopTestRun']
         self.assertEqual(events, expected)
 
+
+    # "The result object is returned to run's caller"
+    def test_run__returns_given_result(self):
+
+        class Foo(unittest.TestCase):
+            def test(self):
+                pass
+
+        result = unittest.TestResult()
+
+        retval = Foo('test').run(result)
+        self.assertIs(retval, result)
+
+
+    # "The same effect [as method run] may be had by simply calling the
+    # TestCase instance."
+    def test_call__invoking_an_instance_delegates_to_run(self):
+        resultIn = unittest.TestResult()
+        resultOut = unittest.TestResult()
+
+        class Foo(unittest.TestCase):
+            def test(self):
+                pass
+
+            def run(self, result):
+                self.assertIs(result, resultIn)
+                return resultOut
+
+        retval = Foo('test')(resultIn)
+
+        self.assertIs(retval, resultOut)
+
+
     def testShortDescriptionWithoutDocstring(self):
         self.assertIsNone(self.shortDescription())
 
@@ -1140,7 +1176,6 @@ test case
             (self.assert_, (True,)),
             (self.failUnlessRaises, (TypeError, lambda _: 3.14 + 'spam')),
             (self.failIf, (False,)),
-            (self.assertSameElements, ([1, 1, 2, 3], [1, 2, 3])),
             (self.assertDictContainsSubset, (dict(a=1, b=2), dict(a=1, b=2, c=3))),
             (self.assertRaisesRegexp, (KeyError, 'foo', lambda: {}['foo'])),
             (self.assertRegexpMatches, ('bar', 'bar')),
@@ -1149,18 +1184,20 @@ test case
             with self.assertWarns(DeprecationWarning):
                 meth(*args)
 
-    def testDeprecatedFailMethods(self):
-        """Test that the deprecated fail* methods get removed in 3.3"""
+    # disable this test for now. When the version where the fail* methods will
+    # be removed is decided, re-enable it and update the version
+    def _testDeprecatedFailMethods(self):
+        """Test that the deprecated fail* methods get removed in 3.x"""
         if sys.version_info[:2] < (3, 3):
             return
         deprecated_names = [
             'failIfEqual', 'failUnlessEqual', 'failUnlessAlmostEqual',
             'failIfAlmostEqual', 'failUnless', 'failUnlessRaises', 'failIf',
-            'assertSameElements', 'assertDictContainsSubset',
+            'assertDictContainsSubset',
         ]
         for deprecated_name in deprecated_names:
             with self.assertRaises(AttributeError):
-                getattr(self, deprecated_name)  # remove these in 3.3
+                getattr(self, deprecated_name)  # remove these in 3.x
 
     def testDeepcopy(self):
         # Issue: 5660
@@ -1268,3 +1305,11 @@ test case
             klass('test_something').run(result)
             self.assertEqual(len(result.errors), 1)
             self.assertEqual(result.testsRun, 1)
+
+    @support.cpython_only
+    def testNoCycles(self):
+        case = unittest.TestCase()
+        wr = weakref.ref(case)
+        with support.disable_gc():
+            del case
+            self.assertFalse(wr())
diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py
index f7e31a57f1..d1b9ef5a0a 100644
--- a/Lib/unittest/test/test_loader.py
+++ b/Lib/unittest/test/test_loader.py
@@ -239,7 +239,7 @@ class Test_TestLoader(unittest.TestCase):
         try:
             loader.loadTestsFromName('sdasfasfasdf')
         except ImportError as e:
-            self.assertEqual(str(e), "No module named sdasfasfasdf")
+            self.assertEqual(str(e), "No module named 'sdasfasfasdf'")
         else:
             self.fail("TestLoader.loadTestsFromName failed to raise ImportError")
 
@@ -619,7 +619,7 @@ class Test_TestLoader(unittest.TestCase):
         try:
             loader.loadTestsFromNames(['sdasfasfasdf'])
         except ImportError as e:
-            self.assertEqual(str(e), "No module named sdasfasfasdf")
+            self.assertEqual(str(e), "No module named 'sdasfasfasdf'")
         else:
             self.fail("TestLoader.loadTestsFromNames failed to raise ImportError")
 
diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py
index d5d0f5a71c..9794868f6d 100644
--- a/Lib/unittest/test/test_program.py
+++ b/Lib/unittest/test/test_program.py
@@ -131,23 +131,6 @@ class TestCommandLineArgs(unittest.TestCase):
         FakeRunner.test = None
         FakeRunner.raiseError = False
 
-    def testHelpAndUnknown(self):
-        program = self.program
-        def usageExit(msg=None):
-            program.msg = msg
-            program.exit = True
-        program.usageExit = usageExit
-
-        for opt in '-h', '-H', '--help':
-            program.exit = False
-            program.parseArgs([None, opt])
-            self.assertTrue(program.exit)
-            self.assertIsNone(program.msg)
-
-        program.parseArgs([None, '-$'])
-        self.assertTrue(program.exit)
-        self.assertIsNotNone(program.msg)
-
     def testVerbosity(self):
         program = self.program
 
diff --git a/Lib/unittest/test/testmock/__init__.py b/Lib/unittest/test/testmock/__init__.py
new file mode 100644
index 0000000000..87d7ae994d
--- /dev/null
+++ b/Lib/unittest/test/testmock/__init__.py
@@ -0,0 +1,17 @@
+import os
+import sys
+import unittest
+
+
+here = os.path.dirname(__file__)
+loader = unittest.defaultTestLoader
+
+def load_tests(*args):
+    suite = unittest.TestSuite()
+    for fn in os.listdir(here):
+        if fn.startswith("test") and fn.endswith(".py"):
+            modname = "unittest.test.testmock." + fn[:-3]
+            __import__(modname)
+            module = sys.modules[modname]
+            suite.addTest(loader.loadTestsFromModule(module))
+    return suite
diff --git a/Lib/unittest/test/testmock/support.py b/Lib/unittest/test/testmock/support.py
new file mode 100644
index 0000000000..f4738793b3
--- /dev/null
+++ b/Lib/unittest/test/testmock/support.py
@@ -0,0 +1,23 @@
+import sys
+
+def is_instance(obj, klass):
+    """Version of is_instance that doesn't access __class__"""
+    return issubclass(type(obj), klass)
+
+
+class SomeClass(object):
+    class_attribute = None
+
+    def wibble(self):
+        pass
+
+
+class X(object):
+    pass
+
+
+def examine_warnings(func):
+    def wrapper():
+        with catch_warnings(record=True) as ws:
+            func(ws)
+    return wrapper
diff --git a/Lib/unittest/test/testmock/testcallable.py b/Lib/unittest/test/testmock/testcallable.py
new file mode 100644
index 0000000000..7b2dd003ea
--- /dev/null
+++ b/Lib/unittest/test/testmock/testcallable.py
@@ -0,0 +1,147 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+import unittest
+from unittest.test.testmock.support import is_instance, X, SomeClass
+
+from unittest.mock import (
+    Mock, MagicMock, NonCallableMagicMock,
+    NonCallableMock, patch, create_autospec,
+    CallableMixin
+)
+
+
+
+class TestCallable(unittest.TestCase):
+
+    def assertNotCallable(self, mock):
+        self.assertTrue(is_instance(mock, NonCallableMagicMock))
+        self.assertFalse(is_instance(mock, CallableMixin))
+
+
+    def test_non_callable(self):
+        for mock in NonCallableMagicMock(), NonCallableMock():
+            self.assertRaises(TypeError, mock)
+            self.assertFalse(hasattr(mock, '__call__'))
+            self.assertIn(mock.__class__.__name__, repr(mock))
+
+
+    def test_heirarchy(self):
+        self.assertTrue(issubclass(MagicMock, Mock))
+        self.assertTrue(issubclass(NonCallableMagicMock, NonCallableMock))
+
+
+    def test_attributes(self):
+        one = NonCallableMock()
+        self.assertTrue(issubclass(type(one.one), Mock))
+
+        two = NonCallableMagicMock()
+        self.assertTrue(issubclass(type(two.two), MagicMock))
+
+
+    def test_subclasses(self):
+        class MockSub(Mock):
+            pass
+
+        one = MockSub()
+        self.assertTrue(issubclass(type(one.one), MockSub))
+
+        class MagicSub(MagicMock):
+            pass
+
+        two = MagicSub()
+        self.assertTrue(issubclass(type(two.two), MagicSub))
+
+
+    def test_patch_spec(self):
+        patcher = patch('%s.X' % __name__, spec=True)
+        mock = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        instance = mock()
+        mock.assert_called_once_with()
+
+        self.assertNotCallable(instance)
+        self.assertRaises(TypeError, instance)
+
+
+    def test_patch_spec_set(self):
+        patcher = patch('%s.X' % __name__, spec_set=True)
+        mock = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        instance = mock()
+        mock.assert_called_once_with()
+
+        self.assertNotCallable(instance)
+        self.assertRaises(TypeError, instance)
+
+
+    def test_patch_spec_instance(self):
+        patcher = patch('%s.X' % __name__, spec=X())
+        mock = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        self.assertNotCallable(mock)
+        self.assertRaises(TypeError, mock)
+
+
+    def test_patch_spec_set_instance(self):
+        patcher = patch('%s.X' % __name__, spec_set=X())
+        mock = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        self.assertNotCallable(mock)
+        self.assertRaises(TypeError, mock)
+
+
+    def test_patch_spec_callable_class(self):
+        class CallableX(X):
+            def __call__(self):
+                pass
+
+        class Sub(CallableX):
+            pass
+
+        class Multi(SomeClass, Sub):
+            pass
+
+        for arg in 'spec', 'spec_set':
+            for Klass in CallableX, Sub, Multi:
+                with patch('%s.X' % __name__, **{arg: Klass}) as mock:
+                    instance = mock()
+                    mock.assert_called_once_with()
+
+                    self.assertTrue(is_instance(instance, MagicMock))
+                    # inherited spec
+                    self.assertRaises(AttributeError, getattr, instance,
+                                      'foobarbaz')
+
+                    result = instance()
+                    # instance is callable, result has no spec
+                    instance.assert_called_once_with()
+
+                    result(3, 2, 1)
+                    result.assert_called_once_with(3, 2, 1)
+                    result.foo(3, 2, 1)
+                    result.foo.assert_called_once_with(3, 2, 1)
+
+
+    def test_create_autopsec(self):
+        mock = create_autospec(X)
+        instance = mock()
+        self.assertRaises(TypeError, instance)
+
+        mock = create_autospec(X())
+        self.assertRaises(TypeError, mock)
+
+
+    def test_create_autospec_instance(self):
+        mock = create_autospec(SomeClass, instance=True)
+
+        self.assertRaises(TypeError, mock)
+        mock.wibble()
+        mock.wibble.assert_called_once_with()
+
+        self.assertRaises(TypeError, mock.wibble, 'some',  'args')
diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py
new file mode 100644
index 0000000000..8bfb29391d
--- /dev/null
+++ b/Lib/unittest/test/testmock/testhelpers.py
@@ -0,0 +1,889 @@
+import unittest
+
+from unittest.mock import (
+    call, _Call, create_autospec, MagicMock,
+    Mock, ANY, _CallList, patch, PropertyMock
+)
+
+from datetime import datetime
+
+class SomeClass(object):
+    def one(self, a, b):
+        pass
+    def two(self):
+        pass
+    def three(self, a=None):
+        pass
+
+
+
+class AnyTest(unittest.TestCase):
+
+    def test_any(self):
+        self.assertEqual(ANY, object())
+
+        mock = Mock()
+        mock(ANY)
+        mock.assert_called_with(ANY)
+
+        mock = Mock()
+        mock(foo=ANY)
+        mock.assert_called_with(foo=ANY)
+
+    def test_repr(self):
+        self.assertEqual(repr(ANY), '<ANY>')
+        self.assertEqual(str(ANY), '<ANY>')
+
+
+    def test_any_and_datetime(self):
+        mock = Mock()
+        mock(datetime.now(), foo=datetime.now())
+
+        mock.assert_called_with(ANY, foo=ANY)
+
+
+    def test_any_mock_calls_comparison_order(self):
+        mock = Mock()
+        d = datetime.now()
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+            def __ne__(self, other):
+                return True
+
+        for d in datetime.now(), Foo():
+            mock.reset_mock()
+
+            mock(d, foo=d, bar=d)
+            mock.method(d, zinga=d, alpha=d)
+            mock().method(a1=d, z99=d)
+
+            expected = [
+                call(ANY, foo=ANY, bar=ANY),
+                call.method(ANY, zinga=ANY, alpha=ANY),
+                call(), call().method(a1=ANY, z99=ANY)
+            ]
+            self.assertEqual(expected, mock.mock_calls)
+            self.assertEqual(mock.mock_calls, expected)
+
+
+
+class CallTest(unittest.TestCase):
+
+    def test_call_with_call(self):
+        kall = _Call()
+        self.assertEqual(kall, _Call())
+        self.assertEqual(kall, _Call(('',)))
+        self.assertEqual(kall, _Call(((),)))
+        self.assertEqual(kall, _Call(({},)))
+        self.assertEqual(kall, _Call(('', ())))
+        self.assertEqual(kall, _Call(('', {})))
+        self.assertEqual(kall, _Call(('', (), {})))
+        self.assertEqual(kall, _Call(('foo',)))
+        self.assertEqual(kall, _Call(('bar', ())))
+        self.assertEqual(kall, _Call(('baz', {})))
+        self.assertEqual(kall, _Call(('spam', (), {})))
+
+        kall = _Call(((1, 2, 3),))
+        self.assertEqual(kall, _Call(((1, 2, 3),)))
+        self.assertEqual(kall, _Call(('', (1, 2, 3))))
+        self.assertEqual(kall, _Call(((1, 2, 3), {})))
+        self.assertEqual(kall, _Call(('', (1, 2, 3), {})))
+
+        kall = _Call(((1, 2, 4),))
+        self.assertNotEqual(kall, _Call(('', (1, 2, 3))))
+        self.assertNotEqual(kall, _Call(('', (1, 2, 3), {})))
+
+        kall = _Call(('foo', (1, 2, 4),))
+        self.assertNotEqual(kall, _Call(('', (1, 2, 4))))
+        self.assertNotEqual(kall, _Call(('', (1, 2, 4), {})))
+        self.assertNotEqual(kall, _Call(('bar', (1, 2, 4))))
+        self.assertNotEqual(kall, _Call(('bar', (1, 2, 4), {})))
+
+        kall = _Call(({'a': 3},))
+        self.assertEqual(kall, _Call(('', (), {'a': 3})))
+        self.assertEqual(kall, _Call(('', {'a': 3})))
+        self.assertEqual(kall, _Call(((), {'a': 3})))
+        self.assertEqual(kall, _Call(({'a': 3},)))
+
+
+    def test_empty__Call(self):
+        args = _Call()
+
+        self.assertEqual(args, ())
+        self.assertEqual(args, ('foo',))
+        self.assertEqual(args, ((),))
+        self.assertEqual(args, ('foo', ()))
+        self.assertEqual(args, ('foo',(), {}))
+        self.assertEqual(args, ('foo', {}))
+        self.assertEqual(args, ({},))
+
+
+    def test_named_empty_call(self):
+        args = _Call(('foo', (), {}))
+
+        self.assertEqual(args, ('foo',))
+        self.assertEqual(args, ('foo', ()))
+        self.assertEqual(args, ('foo',(), {}))
+        self.assertEqual(args, ('foo', {}))
+
+        self.assertNotEqual(args, ((),))
+        self.assertNotEqual(args, ())
+        self.assertNotEqual(args, ({},))
+        self.assertNotEqual(args, ('bar',))
+        self.assertNotEqual(args, ('bar', ()))
+        self.assertNotEqual(args, ('bar', {}))
+
+
+    def test_call_with_args(self):
+        args = _Call(((1, 2, 3), {}))
+
+        self.assertEqual(args, ((1, 2, 3),))
+        self.assertEqual(args, ('foo', (1, 2, 3)))
+        self.assertEqual(args, ('foo', (1, 2, 3), {}))
+        self.assertEqual(args, ((1, 2, 3), {}))
+
+
+    def test_named_call_with_args(self):
+        args = _Call(('foo', (1, 2, 3), {}))
+
+        self.assertEqual(args, ('foo', (1, 2, 3)))
+        self.assertEqual(args, ('foo', (1, 2, 3), {}))
+
+        self.assertNotEqual(args, ((1, 2, 3),))
+        self.assertNotEqual(args, ((1, 2, 3), {}))
+
+
+    def test_call_with_kwargs(self):
+        args = _Call(((), dict(a=3, b=4)))
+
+        self.assertEqual(args, (dict(a=3, b=4),))
+        self.assertEqual(args, ('foo', dict(a=3, b=4)))
+        self.assertEqual(args, ('foo', (), dict(a=3, b=4)))
+        self.assertEqual(args, ((), dict(a=3, b=4)))
+
+
+    def test_named_call_with_kwargs(self):
+        args = _Call(('foo', (), dict(a=3, b=4)))
+
+        self.assertEqual(args, ('foo', dict(a=3, b=4)))
+        self.assertEqual(args, ('foo', (), dict(a=3, b=4)))
+
+        self.assertNotEqual(args, (dict(a=3, b=4),))
+        self.assertNotEqual(args, ((), dict(a=3, b=4)))
+
+
+    def test_call_with_args_call_empty_name(self):
+        args = _Call(((1, 2, 3), {}))
+        self.assertEqual(args, call(1, 2, 3))
+        self.assertEqual(call(1, 2, 3), args)
+        self.assertTrue(call(1, 2, 3) in [args])
+
+
+    def test_call_ne(self):
+        self.assertNotEqual(_Call(((1, 2, 3),)), call(1, 2))
+        self.assertFalse(_Call(((1, 2, 3),)) != call(1, 2, 3))
+        self.assertTrue(_Call(((1, 2), {})) != call(1, 2, 3))
+
+
+    def test_call_non_tuples(self):
+        kall = _Call(((1, 2, 3),))
+        for value in 1, None, self, int:
+            self.assertNotEqual(kall, value)
+            self.assertFalse(kall == value)
+
+
+    def test_repr(self):
+        self.assertEqual(repr(_Call()), 'call()')
+        self.assertEqual(repr(_Call(('foo',))), 'call.foo()')
+
+        self.assertEqual(repr(_Call(((1, 2, 3), {'a': 'b'}))),
+                         "call(1, 2, 3, a='b')")
+        self.assertEqual(repr(_Call(('bar', (1, 2, 3), {'a': 'b'}))),
+                         "call.bar(1, 2, 3, a='b')")
+
+        self.assertEqual(repr(call), 'call')
+        self.assertEqual(str(call), 'call')
+
+        self.assertEqual(repr(call()), 'call()')
+        self.assertEqual(repr(call(1)), 'call(1)')
+        self.assertEqual(repr(call(zz='thing')), "call(zz='thing')")
+
+        self.assertEqual(repr(call().foo), 'call().foo')
+        self.assertEqual(repr(call(1).foo.bar(a=3).bing),
+                         'call().foo.bar().bing')
+        self.assertEqual(
+            repr(call().foo(1, 2, a=3)),
+            "call().foo(1, 2, a=3)"
+        )
+        self.assertEqual(repr(call()()), "call()()")
+        self.assertEqual(repr(call(1)(2)), "call()(2)")
+        self.assertEqual(
+            repr(call()().bar().baz.beep(1)),
+            "call()().bar().baz.beep(1)"
+        )
+
+
+    def test_call(self):
+        self.assertEqual(call(), ('', (), {}))
+        self.assertEqual(call('foo', 'bar', one=3, two=4),
+                         ('', ('foo', 'bar'), {'one': 3, 'two': 4}))
+
+        mock = Mock()
+        mock(1, 2, 3)
+        mock(a=3, b=6)
+        self.assertEqual(mock.call_args_list,
+                         [call(1, 2, 3), call(a=3, b=6)])
+
+    def test_attribute_call(self):
+        self.assertEqual(call.foo(1), ('foo', (1,), {}))
+        self.assertEqual(call.bar.baz(fish='eggs'),
+                         ('bar.baz', (), {'fish': 'eggs'}))
+
+        mock = Mock()
+        mock.foo(1, 2 ,3)
+        mock.bar.baz(a=3, b=6)
+        self.assertEqual(mock.method_calls,
+                         [call.foo(1, 2, 3), call.bar.baz(a=3, b=6)])
+
+
+    def test_extended_call(self):
+        result = call(1).foo(2).bar(3, a=4)
+        self.assertEqual(result, ('().foo().bar', (3,), dict(a=4)))
+
+        mock = MagicMock()
+        mock(1, 2, a=3, b=4)
+        self.assertEqual(mock.call_args, call(1, 2, a=3, b=4))
+        self.assertNotEqual(mock.call_args, call(1, 2, 3))
+
+        self.assertEqual(mock.call_args_list, [call(1, 2, a=3, b=4)])
+        self.assertEqual(mock.mock_calls, [call(1, 2, a=3, b=4)])
+
+        mock = MagicMock()
+        mock.foo(1).bar()().baz.beep(a=6)
+
+        last_call = call.foo(1).bar()().baz.beep(a=6)
+        self.assertEqual(mock.mock_calls[-1], last_call)
+        self.assertEqual(mock.mock_calls, last_call.call_list())
+
+
+    def test_call_list(self):
+        mock = MagicMock()
+        mock(1)
+        self.assertEqual(call(1).call_list(), mock.mock_calls)
+
+        mock = MagicMock()
+        mock(1).method(2)
+        self.assertEqual(call(1).method(2).call_list(),
+                         mock.mock_calls)
+
+        mock = MagicMock()
+        mock(1).method(2)(3)
+        self.assertEqual(call(1).method(2)(3).call_list(),
+                         mock.mock_calls)
+
+        mock = MagicMock()
+        int(mock(1).method(2)(3).foo.bar.baz(4)(5))
+        kall = call(1).method(2)(3).foo.bar.baz(4)(5).__int__()
+        self.assertEqual(kall.call_list(), mock.mock_calls)
+
+
+    def test_call_any(self):
+        self.assertEqual(call, ANY)
+
+        m = MagicMock()
+        int(m)
+        self.assertEqual(m.mock_calls, [ANY])
+        self.assertEqual([ANY], m.mock_calls)
+
+
+    def test_two_args_call(self):
+        args = _Call(((1, 2), {'a': 3}), two=True)
+        self.assertEqual(len(args), 2)
+        self.assertEqual(args[0], (1, 2))
+        self.assertEqual(args[1], {'a': 3})
+
+        other_args = _Call(((1, 2), {'a': 3}))
+        self.assertEqual(args, other_args)
+
+
+class SpecSignatureTest(unittest.TestCase):
+
+    def _check_someclass_mock(self, mock):
+        self.assertRaises(AttributeError, getattr, mock, 'foo')
+        mock.one(1, 2)
+        mock.one.assert_called_with(1, 2)
+        self.assertRaises(AssertionError,
+                          mock.one.assert_called_with, 3, 4)
+        self.assertRaises(TypeError, mock.one, 1)
+
+        mock.two()
+        mock.two.assert_called_with()
+        self.assertRaises(AssertionError,
+                          mock.two.assert_called_with, 3)
+        self.assertRaises(TypeError, mock.two, 1)
+
+        mock.three()
+        mock.three.assert_called_with()
+        self.assertRaises(AssertionError,
+                          mock.three.assert_called_with, 3)
+        self.assertRaises(TypeError, mock.three, 3, 2)
+
+        mock.three(1)
+        mock.three.assert_called_with(1)
+
+        mock.three(a=1)
+        mock.three.assert_called_with(a=1)
+
+
+    def test_basic(self):
+        for spec in (SomeClass, SomeClass()):
+            mock = create_autospec(spec)
+            self._check_someclass_mock(mock)
+
+
+    def test_create_autospec_return_value(self):
+        def f():
+            pass
+        mock = create_autospec(f, return_value='foo')
+        self.assertEqual(mock(), 'foo')
+
+        class Foo(object):
+            pass
+
+        mock = create_autospec(Foo, return_value='foo')
+        self.assertEqual(mock(), 'foo')
+
+
+    def test_autospec_reset_mock(self):
+        m = create_autospec(int)
+        int(m)
+        m.reset_mock()
+        self.assertEqual(m.__int__.call_count, 0)
+
+
+    def test_mocking_unbound_methods(self):
+        class Foo(object):
+            def foo(self, foo):
+                pass
+        p = patch.object(Foo, 'foo')
+        mock_foo = p.start()
+        Foo().foo(1)
+
+        mock_foo.assert_called_with(1)
+
+
+    def test_create_autospec_unbound_methods(self):
+        # see mock issue 128
+        # this is expected to fail until the issue is fixed
+        return
+        class Foo(object):
+            def foo(self):
+                pass
+
+        klass = create_autospec(Foo)
+        instance = klass()
+        self.assertRaises(TypeError, instance.foo, 1)
+
+        # Note: no type checking on the "self" parameter
+        klass.foo(1)
+        klass.foo.assert_called_with(1)
+        self.assertRaises(TypeError, klass.foo)
+
+
+    def test_create_autospec_keyword_arguments(self):
+        class Foo(object):
+            a = 3
+        m = create_autospec(Foo, a='3')
+        self.assertEqual(m.a, '3')
+
+
+    def test_create_autospec_keyword_only_arguments(self):
+        def foo(a, *, b=None):
+            pass
+
+        m = create_autospec(foo)
+        m(1)
+        m.assert_called_with(1)
+        self.assertRaises(TypeError, m, 1, 2)
+
+        m(2, b=3)
+        m.assert_called_with(2, b=3)
+
+
+    def test_function_as_instance_attribute(self):
+        obj = SomeClass()
+        def f(a):
+            pass
+        obj.f = f
+
+        mock = create_autospec(obj)
+        mock.f('bing')
+        mock.f.assert_called_with('bing')
+
+
+    def test_spec_as_list(self):
+        # because spec as a list of strings in the mock constructor means
+        # something very different we treat a list instance as the type.
+        mock = create_autospec([])
+        mock.append('foo')
+        mock.append.assert_called_with('foo')
+
+        self.assertRaises(AttributeError, getattr, mock, 'foo')
+
+        class Foo(object):
+            foo = []
+
+        mock = create_autospec(Foo)
+        mock.foo.append(3)
+        mock.foo.append.assert_called_with(3)
+        self.assertRaises(AttributeError, getattr, mock.foo, 'foo')
+
+
+    def test_attributes(self):
+        class Sub(SomeClass):
+            attr = SomeClass()
+
+        sub_mock = create_autospec(Sub)
+
+        for mock in (sub_mock, sub_mock.attr):
+            self._check_someclass_mock(mock)
+
+
+    def test_builtin_functions_types(self):
+        # we could replace builtin functions / methods with a function
+        # with *args / **kwargs signature. Using the builtin method type
+        # as a spec seems to work fairly well though.
+        class BuiltinSubclass(list):
+            def bar(self, arg):
+                pass
+            sorted = sorted
+            attr = {}
+
+        mock = create_autospec(BuiltinSubclass)
+        mock.append(3)
+        mock.append.assert_called_with(3)
+        self.assertRaises(AttributeError, getattr, mock.append, 'foo')
+
+        mock.bar('foo')
+        mock.bar.assert_called_with('foo')
+        self.assertRaises(TypeError, mock.bar, 'foo', 'bar')
+        self.assertRaises(AttributeError, getattr, mock.bar, 'foo')
+
+        mock.sorted([1, 2])
+        mock.sorted.assert_called_with([1, 2])
+        self.assertRaises(AttributeError, getattr, mock.sorted, 'foo')
+
+        mock.attr.pop(3)
+        mock.attr.pop.assert_called_with(3)
+        self.assertRaises(AttributeError, getattr, mock.attr, 'foo')
+
+
+    def test_method_calls(self):
+        class Sub(SomeClass):
+            attr = SomeClass()
+
+        mock = create_autospec(Sub)
+        mock.one(1, 2)
+        mock.two()
+        mock.three(3)
+
+        expected = [call.one(1, 2), call.two(), call.three(3)]
+        self.assertEqual(mock.method_calls, expected)
+
+        mock.attr.one(1, 2)
+        mock.attr.two()
+        mock.attr.three(3)
+
+        expected.extend(
+            [call.attr.one(1, 2), call.attr.two(), call.attr.three(3)]
+        )
+        self.assertEqual(mock.method_calls, expected)
+
+
+    def test_magic_methods(self):
+        class BuiltinSubclass(list):
+            attr = {}
+
+        mock = create_autospec(BuiltinSubclass)
+        self.assertEqual(list(mock), [])
+        self.assertRaises(TypeError, int, mock)
+        self.assertRaises(TypeError, int, mock.attr)
+        self.assertEqual(list(mock), [])
+
+        self.assertIsInstance(mock['foo'], MagicMock)
+        self.assertIsInstance(mock.attr['foo'], MagicMock)
+
+
+    def test_spec_set(self):
+        class Sub(SomeClass):
+            attr = SomeClass()
+
+        for spec in (Sub, Sub()):
+            mock = create_autospec(spec, spec_set=True)
+            self._check_someclass_mock(mock)
+
+            self.assertRaises(AttributeError, setattr, mock, 'foo', 'bar')
+            self.assertRaises(AttributeError, setattr, mock.attr, 'foo', 'bar')
+
+
+    def test_descriptors(self):
+        class Foo(object):
+            @classmethod
+            def f(cls, a, b):
+                pass
+            @staticmethod
+            def g(a, b):
+                pass
+
+        class Bar(Foo):
+            pass
+
+        class Baz(SomeClass, Bar):
+            pass
+
+        for spec in (Foo, Foo(), Bar, Bar(), Baz, Baz()):
+            mock = create_autospec(spec)
+            mock.f(1, 2)
+            mock.f.assert_called_once_with(1, 2)
+
+            mock.g(3, 4)
+            mock.g.assert_called_once_with(3, 4)
+
+
+    def test_recursive(self):
+        class A(object):
+            def a(self):
+                pass
+            foo = 'foo bar baz'
+            bar = foo
+
+        A.B = A
+        mock = create_autospec(A)
+
+        mock()
+        self.assertFalse(mock.B.called)
+
+        mock.a()
+        mock.B.a()
+        self.assertEqual(mock.method_calls, [call.a(), call.B.a()])
+
+        self.assertIs(A.foo, A.bar)
+        self.assertIsNot(mock.foo, mock.bar)
+        mock.foo.lower()
+        self.assertRaises(AssertionError, mock.bar.lower.assert_called_with)
+
+
+    def test_spec_inheritance_for_classes(self):
+        class Foo(object):
+            def a(self):
+                pass
+            class Bar(object):
+                def f(self):
+                    pass
+
+        class_mock = create_autospec(Foo)
+
+        self.assertIsNot(class_mock, class_mock())
+
+        for this_mock in class_mock, class_mock():
+            this_mock.a()
+            this_mock.a.assert_called_with()
+            self.assertRaises(TypeError, this_mock.a, 'foo')
+            self.assertRaises(AttributeError, getattr, this_mock, 'b')
+
+        instance_mock = create_autospec(Foo())
+        instance_mock.a()
+        instance_mock.a.assert_called_with()
+        self.assertRaises(TypeError, instance_mock.a, 'foo')
+        self.assertRaises(AttributeError, getattr, instance_mock, 'b')
+
+        # The return value isn't isn't callable
+        self.assertRaises(TypeError, instance_mock)
+
+        instance_mock.Bar.f()
+        instance_mock.Bar.f.assert_called_with()
+        self.assertRaises(AttributeError, getattr, instance_mock.Bar, 'g')
+
+        instance_mock.Bar().f()
+        instance_mock.Bar().f.assert_called_with()
+        self.assertRaises(AttributeError, getattr, instance_mock.Bar(), 'g')
+
+
+    def test_inherit(self):
+        class Foo(object):
+            a = 3
+
+        Foo.Foo = Foo
+
+        # class
+        mock = create_autospec(Foo)
+        instance = mock()
+        self.assertRaises(AttributeError, getattr, instance, 'b')
+
+        attr_instance = mock.Foo()
+        self.assertRaises(AttributeError, getattr, attr_instance, 'b')
+
+        # instance
+        mock = create_autospec(Foo())
+        self.assertRaises(AttributeError, getattr, mock, 'b')
+        self.assertRaises(TypeError, mock)
+
+        # attribute instance
+        call_result = mock.Foo()
+        self.assertRaises(AttributeError, getattr, call_result, 'b')
+
+
+    def test_builtins(self):
+        # used to fail with infinite recursion
+        create_autospec(1)
+
+        create_autospec(int)
+        create_autospec('foo')
+        create_autospec(str)
+        create_autospec({})
+        create_autospec(dict)
+        create_autospec([])
+        create_autospec(list)
+        create_autospec(set())
+        create_autospec(set)
+        create_autospec(1.0)
+        create_autospec(float)
+        create_autospec(1j)
+        create_autospec(complex)
+        create_autospec(False)
+        create_autospec(True)
+
+
+    def test_function(self):
+        def f(a, b):
+            pass
+
+        mock = create_autospec(f)
+        self.assertRaises(TypeError, mock)
+        mock(1, 2)
+        mock.assert_called_with(1, 2)
+
+        f.f = f
+        mock = create_autospec(f)
+        self.assertRaises(TypeError, mock.f)
+        mock.f(3, 4)
+        mock.f.assert_called_with(3, 4)
+
+
+    def test_skip_attributeerrors(self):
+        class Raiser(object):
+            def __get__(self, obj, type=None):
+                if obj is None:
+                    raise AttributeError('Can only be accessed via an instance')
+
+        class RaiserClass(object):
+            raiser = Raiser()
+
+            @staticmethod
+            def existing(a, b):
+                return a + b
+
+        s = create_autospec(RaiserClass)
+        self.assertRaises(TypeError, lambda x: s.existing(1, 2, 3))
+        s.existing(1, 2)
+        self.assertRaises(AttributeError, lambda: s.nonexisting)
+
+        # check we can fetch the raiser attribute and it has no spec
+        obj = s.raiser
+        obj.foo, obj.bar
+
+
+    def test_signature_class(self):
+        class Foo(object):
+            def __init__(self, a, b=3):
+                pass
+
+        mock = create_autospec(Foo)
+
+        self.assertRaises(TypeError, mock)
+        mock(1)
+        mock.assert_called_once_with(1)
+
+        mock(4, 5)
+        mock.assert_called_with(4, 5)
+
+
+    def test_class_with_no_init(self):
+        # this used to raise an exception
+        # due to trying to get a signature from object.__init__
+        class Foo(object):
+            pass
+        create_autospec(Foo)
+
+
+    def test_signature_callable(self):
+        class Callable(object):
+            def __init__(self):
+                pass
+            def __call__(self, a):
+                pass
+
+        mock = create_autospec(Callable)
+        mock()
+        mock.assert_called_once_with()
+        self.assertRaises(TypeError, mock, 'a')
+
+        instance = mock()
+        self.assertRaises(TypeError, instance)
+        instance(a='a')
+        instance.assert_called_once_with(a='a')
+        instance('a')
+        instance.assert_called_with('a')
+
+        mock = create_autospec(Callable())
+        mock(a='a')
+        mock.assert_called_once_with(a='a')
+        self.assertRaises(TypeError, mock)
+        mock('a')
+        mock.assert_called_with('a')
+
+
+    def test_signature_noncallable(self):
+        class NonCallable(object):
+            def __init__(self):
+                pass
+
+        mock = create_autospec(NonCallable)
+        instance = mock()
+        mock.assert_called_once_with()
+        self.assertRaises(TypeError, mock, 'a')
+        self.assertRaises(TypeError, instance)
+        self.assertRaises(TypeError, instance, 'a')
+
+        mock = create_autospec(NonCallable())
+        self.assertRaises(TypeError, mock)
+        self.assertRaises(TypeError, mock, 'a')
+
+
+    def test_create_autospec_none(self):
+        class Foo(object):
+            bar = None
+
+        mock = create_autospec(Foo)
+        none = mock.bar
+        self.assertNotIsInstance(none, type(None))
+
+        none.foo()
+        none.foo.assert_called_once_with()
+
+
+    def test_autospec_functions_with_self_in_odd_place(self):
+        class Foo(object):
+            def f(a, self):
+                pass
+
+        a = create_autospec(Foo)
+        a.f(self=10)
+        a.f.assert_called_with(self=10)
+
+
+    def test_autospec_property(self):
+        class Foo(object):
+            @property
+            def foo(self):
+                return 3
+
+        foo = create_autospec(Foo)
+        mock_property = foo.foo
+
+        # no spec on properties
+        self.assertTrue(isinstance(mock_property, MagicMock))
+        mock_property(1, 2, 3)
+        mock_property.abc(4, 5, 6)
+        mock_property.assert_called_once_with(1, 2, 3)
+        mock_property.abc.assert_called_once_with(4, 5, 6)
+
+
+    def test_autospec_slots(self):
+        class Foo(object):
+            __slots__ = ['a']
+
+        foo = create_autospec(Foo)
+        mock_slot = foo.a
+
+        # no spec on slots
+        mock_slot(1, 2, 3)
+        mock_slot.abc(4, 5, 6)
+        mock_slot.assert_called_once_with(1, 2, 3)
+        mock_slot.abc.assert_called_once_with(4, 5, 6)
+
+
+class TestCallList(unittest.TestCase):
+
+    def test_args_list_contains_call_list(self):
+        mock = Mock()
+        self.assertIsInstance(mock.call_args_list, _CallList)
+
+        mock(1, 2)
+        mock(a=3)
+        mock(3, 4)
+        mock(b=6)
+
+        for kall in call(1, 2), call(a=3), call(3, 4), call(b=6):
+            self.assertTrue(kall in mock.call_args_list)
+
+        calls = [call(a=3), call(3, 4)]
+        self.assertTrue(calls in mock.call_args_list)
+        calls = [call(1, 2), call(a=3)]
+        self.assertTrue(calls in mock.call_args_list)
+        calls = [call(3, 4), call(b=6)]
+        self.assertTrue(calls in mock.call_args_list)
+        calls = [call(3, 4)]
+        self.assertTrue(calls in mock.call_args_list)
+
+        self.assertFalse(call('fish') in mock.call_args_list)
+        self.assertFalse([call('fish')] in mock.call_args_list)
+
+
+    def test_call_list_str(self):
+        mock = Mock()
+        mock(1, 2)
+        mock.foo(a=3)
+        mock.foo.bar().baz('fish', cat='dog')
+
+        expected = (
+            "[call(1, 2),\n"
+            " call.foo(a=3),\n"
+            " call.foo.bar(),\n"
+            " call.foo.bar().baz('fish', cat='dog')]"
+        )
+        self.assertEqual(str(mock.mock_calls), expected)
+
+
+    def test_propertymock(self):
+        p = patch('%s.SomeClass.one' % __name__, new_callable=PropertyMock)
+        mock = p.start()
+        try:
+            SomeClass.one
+            mock.assert_called_once_with()
+
+            s = SomeClass()
+            s.one
+            mock.assert_called_with()
+            self.assertEqual(mock.mock_calls, [call(), call()])
+
+            s.one = 3
+            self.assertEqual(mock.mock_calls, [call(), call(), call(3)])
+        finally:
+            p.stop()
+
+
+    def test_propertymock_returnvalue(self):
+        m = MagicMock()
+        p = PropertyMock()
+        type(m).foo = p
+
+        returned = m.foo
+        p.assert_called_once_with()
+        self.assertIsInstance(returned, MagicMock)
+        self.assertNotIsInstance(returned, PropertyMock)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/unittest/test/testmock/testmagicmethods.py
new file mode 100644
index 0000000000..2bcf08801e
--- /dev/null
+++ b/Lib/unittest/test/testmock/testmagicmethods.py
@@ -0,0 +1,403 @@
+import unittest
+import inspect
+import sys
+from unittest.mock import Mock, MagicMock, _magics
+
+
+
+class TestMockingMagicMethods(unittest.TestCase):
+
+    def test_deleting_magic_methods(self):
+        mock = Mock()
+        self.assertFalse(hasattr(mock, '__getitem__'))
+
+        mock.__getitem__ = Mock()
+        self.assertTrue(hasattr(mock, '__getitem__'))
+
+        del mock.__getitem__
+        self.assertFalse(hasattr(mock, '__getitem__'))
+
+
+    def test_magicmock_del(self):
+        mock = MagicMock()
+        # before using getitem
+        del mock.__getitem__
+        self.assertRaises(TypeError, lambda: mock['foo'])
+
+        mock = MagicMock()
+        # this time use it first
+        mock['foo']
+        del mock.__getitem__
+        self.assertRaises(TypeError, lambda: mock['foo'])
+
+
+    def test_magic_method_wrapping(self):
+        mock = Mock()
+        def f(self, name):
+            return self, 'fish'
+
+        mock.__getitem__ = f
+        self.assertFalse(mock.__getitem__ is f)
+        self.assertEqual(mock['foo'], (mock, 'fish'))
+        self.assertEqual(mock.__getitem__('foo'), (mock, 'fish'))
+
+        mock.__getitem__ = mock
+        self.assertTrue(mock.__getitem__ is mock)
+
+
+    def test_magic_methods_isolated_between_mocks(self):
+        mock1 = Mock()
+        mock2 = Mock()
+
+        mock1.__iter__ = Mock(return_value=iter([]))
+        self.assertEqual(list(mock1), [])
+        self.assertRaises(TypeError, lambda: list(mock2))
+
+
+    def test_repr(self):
+        mock = Mock()
+        self.assertEqual(repr(mock), "<Mock id='%s'>" % id(mock))
+        mock.__repr__ = lambda s: 'foo'
+        self.assertEqual(repr(mock), 'foo')
+
+
+    def test_str(self):
+        mock = Mock()
+        self.assertEqual(str(mock), object.__str__(mock))
+        mock.__str__ = lambda s: 'foo'
+        self.assertEqual(str(mock), 'foo')
+
+
+    def test_dict_methods(self):
+        mock = Mock()
+
+        self.assertRaises(TypeError, lambda: mock['foo'])
+        def _del():
+            del mock['foo']
+        def _set():
+            mock['foo'] = 3
+        self.assertRaises(TypeError, _del)
+        self.assertRaises(TypeError, _set)
+
+        _dict = {}
+        def getitem(s, name):
+            return _dict[name]
+        def setitem(s, name, value):
+            _dict[name] = value
+        def delitem(s, name):
+            del _dict[name]
+
+        mock.__setitem__ = setitem
+        mock.__getitem__ = getitem
+        mock.__delitem__ = delitem
+
+        self.assertRaises(KeyError, lambda: mock['foo'])
+        mock['foo'] = 'bar'
+        self.assertEqual(_dict, {'foo': 'bar'})
+        self.assertEqual(mock['foo'], 'bar')
+        del mock['foo']
+        self.assertEqual(_dict, {})
+
+
+    def test_numeric(self):
+        original = mock = Mock()
+        mock.value = 0
+
+        self.assertRaises(TypeError, lambda: mock + 3)
+
+        def add(self, other):
+            mock.value += other
+            return self
+        mock.__add__ = add
+        self.assertEqual(mock + 3, mock)
+        self.assertEqual(mock.value, 3)
+
+        del mock.__add__
+        def iadd(mock):
+            mock += 3
+        self.assertRaises(TypeError, iadd, mock)
+        mock.__iadd__ = add
+        mock += 6
+        self.assertEqual(mock, original)
+        self.assertEqual(mock.value, 9)
+
+        self.assertRaises(TypeError, lambda: 3 + mock)
+        mock.__radd__ = add
+        self.assertEqual(7 + mock, mock)
+        self.assertEqual(mock.value, 16)
+
+
+    def test_hash(self):
+        mock = Mock()
+        # test delegation
+        self.assertEqual(hash(mock), Mock.__hash__(mock))
+
+        def _hash(s):
+            return 3
+        mock.__hash__ = _hash
+        self.assertEqual(hash(mock), 3)
+
+
+    def test_nonzero(self):
+        m = Mock()
+        self.assertTrue(bool(m))
+
+        m.__bool__ = lambda s: False
+        self.assertFalse(bool(m))
+
+
+    def test_comparison(self):
+        mock = Mock()
+        def comp(s, o):
+            return True
+        mock.__lt__ = mock.__gt__ = mock.__le__ = mock.__ge__ = comp
+        self. assertTrue(mock < 3)
+        self. assertTrue(mock > 3)
+        self. assertTrue(mock <= 3)
+        self. assertTrue(mock >= 3)
+
+        self.assertRaises(TypeError, lambda: MagicMock() < object())
+        self.assertRaises(TypeError, lambda: object() < MagicMock())
+        self.assertRaises(TypeError, lambda: MagicMock() < MagicMock())
+        self.assertRaises(TypeError, lambda: MagicMock() > object())
+        self.assertRaises(TypeError, lambda: object() > MagicMock())
+        self.assertRaises(TypeError, lambda: MagicMock() > MagicMock())
+        self.assertRaises(TypeError, lambda: MagicMock() <= object())
+        self.assertRaises(TypeError, lambda: object() <= MagicMock())
+        self.assertRaises(TypeError, lambda: MagicMock() <= MagicMock())
+        self.assertRaises(TypeError, lambda: MagicMock() >= object())
+        self.assertRaises(TypeError, lambda: object() >= MagicMock())
+        self.assertRaises(TypeError, lambda: MagicMock() >= MagicMock())
+
+
+    def test_equality(self):
+        for mock in Mock(), MagicMock():
+            self.assertEqual(mock == mock, True)
+            self.assertIsInstance(mock == mock, bool)
+            self.assertEqual(mock != mock, False)
+            self.assertIsInstance(mock != mock, bool)
+            self.assertEqual(mock == object(), False)
+            self.assertEqual(mock != object(), True)
+
+            def eq(self, other):
+                return other == 3
+            mock.__eq__ = eq
+            self.assertTrue(mock == 3)
+            self.assertFalse(mock == 4)
+
+            def ne(self, other):
+                return other == 3
+            mock.__ne__ = ne
+            self.assertTrue(mock != 3)
+            self.assertFalse(mock != 4)
+
+        mock = MagicMock()
+        mock.__eq__.return_value = True
+        self.assertIsInstance(mock == 3, bool)
+        self.assertEqual(mock == 3, True)
+
+        mock.__ne__.return_value = False
+        self.assertIsInstance(mock != 3, bool)
+        self.assertEqual(mock != 3, False)
+
+
+    def test_len_contains_iter(self):
+        mock = Mock()
+
+        self.assertRaises(TypeError, len, mock)
+        self.assertRaises(TypeError, iter, mock)
+        self.assertRaises(TypeError, lambda: 'foo' in mock)
+
+        mock.__len__ = lambda s: 6
+        self.assertEqual(len(mock), 6)
+
+        mock.__contains__ = lambda s, o: o == 3
+        self.assertTrue(3 in mock)
+        self.assertFalse(6 in mock)
+
+        mock.__iter__ = lambda s: iter('foobarbaz')
+        self.assertEqual(list(mock), list('foobarbaz'))
+
+
+    def test_magicmock(self):
+        mock = MagicMock()
+
+        mock.__iter__.return_value = iter([1, 2, 3])
+        self.assertEqual(list(mock), [1, 2, 3])
+
+        getattr(mock, '__bool__').return_value = False
+        self.assertFalse(hasattr(mock, '__nonzero__'))
+        self.assertFalse(bool(mock))
+
+        for entry in _magics:
+            self.assertTrue(hasattr(mock, entry))
+        self.assertFalse(hasattr(mock, '__imaginery__'))
+
+
+    def test_magic_mock_equality(self):
+        mock = MagicMock()
+        self.assertIsInstance(mock == object(), bool)
+        self.assertIsInstance(mock != object(), bool)
+
+        self.assertEqual(mock == object(), False)
+        self.assertEqual(mock != object(), True)
+        self.assertEqual(mock == mock, True)
+        self.assertEqual(mock != mock, False)
+
+
+    def test_magicmock_defaults(self):
+        mock = MagicMock()
+        self.assertEqual(int(mock), 1)
+        self.assertEqual(complex(mock), 1j)
+        self.assertEqual(float(mock), 1.0)
+        self.assertNotIn(object(), mock)
+        self.assertEqual(len(mock), 0)
+        self.assertEqual(list(mock), [])
+        self.assertEqual(hash(mock), object.__hash__(mock))
+        self.assertEqual(str(mock), object.__str__(mock))
+        self.assertTrue(bool(mock))
+
+        # in Python 3 oct and hex use __index__
+        # so these tests are for __index__ in py3k
+        self.assertEqual(oct(mock), '0o1')
+        self.assertEqual(hex(mock), '0x1')
+        # how to test __sizeof__ ?
+
+
+    def test_magic_methods_and_spec(self):
+        class Iterable(object):
+            def __iter__(self):
+                pass
+
+        mock = Mock(spec=Iterable)
+        self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+        mock.__iter__ = Mock(return_value=iter([]))
+        self.assertEqual(list(mock), [])
+
+        class NonIterable(object):
+            pass
+        mock = Mock(spec=NonIterable)
+        self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+        def set_int():
+            mock.__int__ = Mock(return_value=iter([]))
+        self.assertRaises(AttributeError, set_int)
+
+        mock = MagicMock(spec=Iterable)
+        self.assertEqual(list(mock), [])
+        self.assertRaises(AttributeError, set_int)
+
+
+    def test_magic_methods_and_spec_set(self):
+        class Iterable(object):
+            def __iter__(self):
+                pass
+
+        mock = Mock(spec_set=Iterable)
+        self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+        mock.__iter__ = Mock(return_value=iter([]))
+        self.assertEqual(list(mock), [])
+
+        class NonIterable(object):
+            pass
+        mock = Mock(spec_set=NonIterable)
+        self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+        def set_int():
+            mock.__int__ = Mock(return_value=iter([]))
+        self.assertRaises(AttributeError, set_int)
+
+        mock = MagicMock(spec_set=Iterable)
+        self.assertEqual(list(mock), [])
+        self.assertRaises(AttributeError, set_int)
+
+
+    def test_setting_unsupported_magic_method(self):
+        mock = MagicMock()
+        def set_setattr():
+            mock.__setattr__ = lambda self, name: None
+        self.assertRaisesRegex(AttributeError,
+            "Attempting to set unsupported magic method '__setattr__'.",
+            set_setattr
+        )
+
+
+    def test_attributes_and_return_value(self):
+        mock = MagicMock()
+        attr = mock.foo
+        def _get_type(obj):
+            # the type of every mock (or magicmock) is a custom subclass
+            # so the real type is the second in the mro
+            return type(obj).__mro__[1]
+        self.assertEqual(_get_type(attr), MagicMock)
+
+        returned = mock()
+        self.assertEqual(_get_type(returned), MagicMock)
+
+
+    def test_magic_methods_are_magic_mocks(self):
+        mock = MagicMock()
+        self.assertIsInstance(mock.__getitem__, MagicMock)
+
+        mock[1][2].__getitem__.return_value = 3
+        self.assertEqual(mock[1][2][3], 3)
+
+
+    def test_magic_method_reset_mock(self):
+        mock = MagicMock()
+        str(mock)
+        self.assertTrue(mock.__str__.called)
+        mock.reset_mock()
+        self.assertFalse(mock.__str__.called)
+
+
+    def test_dir(self):
+        # overriding the default implementation
+        for mock in Mock(), MagicMock():
+            def _dir(self):
+                return ['foo']
+            mock.__dir__ = _dir
+            self.assertEqual(dir(mock), ['foo'])
+
+
+    @unittest.skipIf('PyPy' in sys.version, "This fails differently on pypy")
+    def test_bound_methods(self):
+        m = Mock()
+
+        # XXXX should this be an expected failure instead?
+
+        # this seems like it should work, but is hard to do without introducing
+        # other api inconsistencies. Failure message could be better though.
+        m.__iter__ = [3].__iter__
+        self.assertRaises(TypeError, iter, m)
+
+
+    def test_magic_method_type(self):
+        class Foo(MagicMock):
+            pass
+
+        foo = Foo()
+        self.assertIsInstance(foo.__int__, Foo)
+
+
+    def test_descriptor_from_class(self):
+        m = MagicMock()
+        type(m).__str__.return_value = 'foo'
+        self.assertEqual(str(m), 'foo')
+
+
+    def test_iterable_as_iter_return_value(self):
+        m = MagicMock()
+        m.__iter__.return_value = [1, 2, 3]
+        self.assertEqual(list(m), [1, 2, 3])
+        self.assertEqual(list(m), [1, 2, 3])
+
+        m.__iter__.return_value = iter([4, 5, 6])
+        self.assertEqual(list(m), [4, 5, 6])
+        self.assertEqual(list(m), [])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
new file mode 100644
index 0000000000..2c6f128975
--- /dev/null
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -0,0 +1,1275 @@
+import copy
+import sys
+
+import unittest
+from unittest.test.testmock.support import is_instance
+from unittest import mock
+from unittest.mock import (
+    call, DEFAULT, patch, sentinel,
+    MagicMock, Mock, NonCallableMock,
+    NonCallableMagicMock, _CallList,
+    create_autospec
+)
+
+
+class Iter(object):
+    def __init__(self):
+        self.thing = iter(['this', 'is', 'an', 'iter'])
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return next(self.thing)
+
+    __next__ = next
+
+
+
+class MockTest(unittest.TestCase):
+
+    def test_all(self):
+        # if __all__ is badly defined then import * will raise an error
+        # We have to exec it because you can't import * inside a method
+        # in Python 3
+        exec("from unittest.mock import *")
+
+
+    def test_constructor(self):
+        mock = Mock()
+
+        self.assertFalse(mock.called, "called not initialised correctly")
+        self.assertEqual(mock.call_count, 0,
+                         "call_count not initialised correctly")
+        self.assertTrue(is_instance(mock.return_value, Mock),
+                        "return_value not initialised correctly")
+
+        self.assertEqual(mock.call_args, None,
+                         "call_args not initialised correctly")
+        self.assertEqual(mock.call_args_list, [],
+                         "call_args_list not initialised correctly")
+        self.assertEqual(mock.method_calls, [],
+                          "method_calls not initialised correctly")
+
+        # Can't use hasattr for this test as it always returns True on a mock
+        self.assertFalse('_items' in mock.__dict__,
+                         "default mock should not have '_items' attribute")
+
+        self.assertIsNone(mock._mock_parent,
+                          "parent not initialised correctly")
+        self.assertIsNone(mock._mock_methods,
+                          "methods not initialised correctly")
+        self.assertEqual(mock._mock_children, {},
+                         "children not initialised incorrectly")
+
+
+    def test_return_value_in_constructor(self):
+        mock = Mock(return_value=None)
+        self.assertIsNone(mock.return_value,
+                          "return value in constructor not honoured")
+
+
+    def test_repr(self):
+        mock = Mock(name='foo')
+        self.assertIn('foo', repr(mock))
+        self.assertIn("'%s'" % id(mock), repr(mock))
+
+        mocks = [(Mock(), 'mock'), (Mock(name='bar'), 'bar')]
+        for mock, name in mocks:
+            self.assertIn('%s.bar' % name, repr(mock.bar))
+            self.assertIn('%s.foo()' % name, repr(mock.foo()))
+            self.assertIn('%s.foo().bing' % name, repr(mock.foo().bing))
+            self.assertIn('%s()' % name, repr(mock()))
+            self.assertIn('%s()()' % name, repr(mock()()))
+            self.assertIn('%s()().foo.bar.baz().bing' % name,
+                          repr(mock()().foo.bar.baz().bing))
+
+
+    def test_repr_with_spec(self):
+        class X(object):
+            pass
+
+        mock = Mock(spec=X)
+        self.assertIn(" spec='X' ", repr(mock))
+
+        mock = Mock(spec=X())
+        self.assertIn(" spec='X' ", repr(mock))
+
+        mock = Mock(spec_set=X)
+        self.assertIn(" spec_set='X' ", repr(mock))
+
+        mock = Mock(spec_set=X())
+        self.assertIn(" spec_set='X' ", repr(mock))
+
+        mock = Mock(spec=X, name='foo')
+        self.assertIn(" spec='X' ", repr(mock))
+        self.assertIn(" name='foo' ", repr(mock))
+
+        mock = Mock(name='foo')
+        self.assertNotIn("spec", repr(mock))
+
+        mock = Mock()
+        self.assertNotIn("spec", repr(mock))
+
+        mock = Mock(spec=['foo'])
+        self.assertNotIn("spec", repr(mock))
+
+
+    def test_side_effect(self):
+        mock = Mock()
+
+        def effect(*args, **kwargs):
+            raise SystemError('kablooie')
+
+        mock.side_effect = effect
+        self.assertRaises(SystemError, mock, 1, 2, fish=3)
+        mock.assert_called_with(1, 2, fish=3)
+
+        results = [1, 2, 3]
+        def effect():
+            return results.pop()
+        mock.side_effect = effect
+
+        self.assertEqual([mock(), mock(), mock()], [3, 2, 1],
+                          "side effect not used correctly")
+
+        mock = Mock(side_effect=sentinel.SideEffect)
+        self.assertEqual(mock.side_effect, sentinel.SideEffect,
+                          "side effect in constructor not used")
+
+        def side_effect():
+            return DEFAULT
+        mock = Mock(side_effect=side_effect, return_value=sentinel.RETURN)
+        self.assertEqual(mock(), sentinel.RETURN)
+
+
+    @unittest.skipUnless('java' in sys.platform,
+                          'This test only applies to Jython')
+    def test_java_exception_side_effect(self):
+        import java
+        mock = Mock(side_effect=java.lang.RuntimeException("Boom!"))
+
+        # can't use assertRaises with java exceptions
+        try:
+            mock(1, 2, fish=3)
+        except java.lang.RuntimeException:
+            pass
+        else:
+            self.fail('java exception not raised')
+        mock.assert_called_with(1,2, fish=3)
+
+
+    def test_reset_mock(self):
+        parent = Mock()
+        spec = ["something"]
+        mock = Mock(name="child", parent=parent, spec=spec)
+        mock(sentinel.Something, something=sentinel.SomethingElse)
+        something = mock.something
+        mock.something()
+        mock.side_effect = sentinel.SideEffect
+        return_value = mock.return_value
+        return_value()
+
+        mock.reset_mock()
+
+        self.assertEqual(mock._mock_name, "child",
+                         "name incorrectly reset")
+        self.assertEqual(mock._mock_parent, parent,
+                         "parent incorrectly reset")
+        self.assertEqual(mock._mock_methods, spec,
+                         "methods incorrectly reset")
+
+        self.assertFalse(mock.called, "called not reset")
+        self.assertEqual(mock.call_count, 0, "call_count not reset")
+        self.assertEqual(mock.call_args, None, "call_args not reset")
+        self.assertEqual(mock.call_args_list, [], "call_args_list not reset")
+        self.assertEqual(mock.method_calls, [],
+                        "method_calls not initialised correctly: %r != %r" %
+                        (mock.method_calls, []))
+        self.assertEqual(mock.mock_calls, [])
+
+        self.assertEqual(mock.side_effect, sentinel.SideEffect,
+                          "side_effect incorrectly reset")
+        self.assertEqual(mock.return_value, return_value,
+                          "return_value incorrectly reset")
+        self.assertFalse(return_value.called, "return value mock not reset")
+        self.assertEqual(mock._mock_children, {'something': something},
+                          "children reset incorrectly")
+        self.assertEqual(mock.something, something,
+                          "children incorrectly cleared")
+        self.assertFalse(mock.something.called, "child not reset")
+
+
+    def test_reset_mock_recursion(self):
+        mock = Mock()
+        mock.return_value = mock
+
+        # used to cause recursion
+        mock.reset_mock()
+
+
+    def test_call(self):
+        mock = Mock()
+        self.assertTrue(is_instance(mock.return_value, Mock),
+                        "Default return_value should be a Mock")
+
+        result = mock()
+        self.assertEqual(mock(), result,
+                         "different result from consecutive calls")
+        mock.reset_mock()
+
+        ret_val = mock(sentinel.Arg)
+        self.assertTrue(mock.called, "called not set")
+        self.assertEqual(mock.call_count, 1, "call_count incoreect")
+        self.assertEqual(mock.call_args, ((sentinel.Arg,), {}),
+                         "call_args not set")
+        self.assertEqual(mock.call_args_list, [((sentinel.Arg,), {})],
+                         "call_args_list not initialised correctly")
+
+        mock.return_value = sentinel.ReturnValue
+        ret_val = mock(sentinel.Arg, key=sentinel.KeyArg)
+        self.assertEqual(ret_val, sentinel.ReturnValue,
+                         "incorrect return value")
+
+        self.assertEqual(mock.call_count, 2, "call_count incorrect")
+        self.assertEqual(mock.call_args,
+                         ((sentinel.Arg,), {'key': sentinel.KeyArg}),
+                         "call_args not set")
+        self.assertEqual(mock.call_args_list, [
+            ((sentinel.Arg,), {}),
+            ((sentinel.Arg,), {'key': sentinel.KeyArg})
+        ],
+            "call_args_list not set")
+
+
+    def test_call_args_comparison(self):
+        mock = Mock()
+        mock()
+        mock(sentinel.Arg)
+        mock(kw=sentinel.Kwarg)
+        mock(sentinel.Arg, kw=sentinel.Kwarg)
+        self.assertEqual(mock.call_args_list, [
+            (),
+            ((sentinel.Arg,),),
+            ({"kw": sentinel.Kwarg},),
+            ((sentinel.Arg,), {"kw": sentinel.Kwarg})
+        ])
+        self.assertEqual(mock.call_args,
+                         ((sentinel.Arg,), {"kw": sentinel.Kwarg}))
+
+
+    def test_assert_called_with(self):
+        mock = Mock()
+        mock()
+
+        # Will raise an exception if it fails
+        mock.assert_called_with()
+        self.assertRaises(AssertionError, mock.assert_called_with, 1)
+
+        mock.reset_mock()
+        self.assertRaises(AssertionError, mock.assert_called_with)
+
+        mock(1, 2, 3, a='fish', b='nothing')
+        mock.assert_called_with(1, 2, 3, a='fish', b='nothing')
+
+
+    def test_assert_called_once_with(self):
+        mock = Mock()
+        mock()
+
+        # Will raise an exception if it fails
+        mock.assert_called_once_with()
+
+        mock()
+        self.assertRaises(AssertionError, mock.assert_called_once_with)
+
+        mock.reset_mock()
+        self.assertRaises(AssertionError, mock.assert_called_once_with)
+
+        mock('foo', 'bar', baz=2)
+        mock.assert_called_once_with('foo', 'bar', baz=2)
+
+        mock.reset_mock()
+        mock('foo', 'bar', baz=2)
+        self.assertRaises(
+            AssertionError,
+            lambda: mock.assert_called_once_with('bob', 'bar', baz=2)
+        )
+
+
+    def test_attribute_access_returns_mocks(self):
+        mock = Mock()
+        something = mock.something
+        self.assertTrue(is_instance(something, Mock), "attribute isn't a mock")
+        self.assertEqual(mock.something, something,
+                         "different attributes returned for same name")
+
+        # Usage example
+        mock = Mock()
+        mock.something.return_value = 3
+
+        self.assertEqual(mock.something(), 3, "method returned wrong value")
+        self.assertTrue(mock.something.called,
+                        "method didn't record being called")
+
+
+    def test_attributes_have_name_and_parent_set(self):
+        mock = Mock()
+        something = mock.something
+
+        self.assertEqual(something._mock_name, "something",
+                         "attribute name not set correctly")
+        self.assertEqual(something._mock_parent, mock,
+                         "attribute parent not set correctly")
+
+
+    def test_method_calls_recorded(self):
+        mock = Mock()
+        mock.something(3, fish=None)
+        mock.something_else.something(6, cake=sentinel.Cake)
+
+        self.assertEqual(mock.something_else.method_calls,
+                          [("something", (6,), {'cake': sentinel.Cake})],
+                          "method calls not recorded correctly")
+        self.assertEqual(mock.method_calls, [
+            ("something", (3,), {'fish': None}),
+            ("something_else.something", (6,), {'cake': sentinel.Cake})
+        ],
+            "method calls not recorded correctly")
+
+
+    def test_method_calls_compare_easily(self):
+        mock = Mock()
+        mock.something()
+        self.assertEqual(mock.method_calls, [('something',)])
+        self.assertEqual(mock.method_calls, [('something', (), {})])
+
+        mock = Mock()
+        mock.something('different')
+        self.assertEqual(mock.method_calls, [('something', ('different',))])
+        self.assertEqual(mock.method_calls,
+                         [('something', ('different',), {})])
+
+        mock = Mock()
+        mock.something(x=1)
+        self.assertEqual(mock.method_calls, [('something', {'x': 1})])
+        self.assertEqual(mock.method_calls, [('something', (), {'x': 1})])
+
+        mock = Mock()
+        mock.something('different', some='more')
+        self.assertEqual(mock.method_calls, [
+            ('something', ('different',), {'some': 'more'})
+        ])
+
+
+    def test_only_allowed_methods_exist(self):
+        for spec in ['something'], ('something',):
+            for arg in 'spec', 'spec_set':
+                mock = Mock(**{arg: spec})
+
+                # this should be allowed
+                mock.something
+                self.assertRaisesRegex(
+                    AttributeError,
+                    "Mock object has no attribute 'something_else'",
+                    getattr, mock, 'something_else'
+                )
+
+
+    def test_from_spec(self):
+        class Something(object):
+            x = 3
+            __something__ = None
+            def y(self):
+                pass
+
+        def test_attributes(mock):
+            # should work
+            mock.x
+            mock.y
+            mock.__something__
+            self.assertRaisesRegex(
+                AttributeError,
+                "Mock object has no attribute 'z'",
+                getattr, mock, 'z'
+            )
+            self.assertRaisesRegex(
+                AttributeError,
+                "Mock object has no attribute '__foobar__'",
+                getattr, mock, '__foobar__'
+            )
+
+        test_attributes(Mock(spec=Something))
+        test_attributes(Mock(spec=Something()))
+
+
+    def test_wraps_calls(self):
+        real = Mock()
+
+        mock = Mock(wraps=real)
+        self.assertEqual(mock(), real())
+
+        real.reset_mock()
+
+        mock(1, 2, fish=3)
+        real.assert_called_with(1, 2, fish=3)
+
+
+    def test_wraps_call_with_nondefault_return_value(self):
+        real = Mock()
+
+        mock = Mock(wraps=real)
+        mock.return_value = 3
+
+        self.assertEqual(mock(), 3)
+        self.assertFalse(real.called)
+
+
+    def test_wraps_attributes(self):
+        class Real(object):
+            attribute = Mock()
+
+        real = Real()
+
+        mock = Mock(wraps=real)
+        self.assertEqual(mock.attribute(), real.attribute())
+        self.assertRaises(AttributeError, lambda: mock.fish)
+
+        self.assertNotEqual(mock.attribute, real.attribute)
+        result = mock.attribute.frog(1, 2, fish=3)
+        Real.attribute.frog.assert_called_with(1, 2, fish=3)
+        self.assertEqual(result, Real.attribute.frog())
+
+
+    def test_exceptional_side_effect(self):
+        mock = Mock(side_effect=AttributeError)
+        self.assertRaises(AttributeError, mock)
+
+        mock = Mock(side_effect=AttributeError('foo'))
+        self.assertRaises(AttributeError, mock)
+
+
+    def test_baseexceptional_side_effect(self):
+        mock = Mock(side_effect=KeyboardInterrupt)
+        self.assertRaises(KeyboardInterrupt, mock)
+
+        mock = Mock(side_effect=KeyboardInterrupt('foo'))
+        self.assertRaises(KeyboardInterrupt, mock)
+
+
+    def test_assert_called_with_message(self):
+        mock = Mock()
+        self.assertRaisesRegex(AssertionError, 'Not called',
+                                mock.assert_called_with)
+
+
+    def test_assert_called_once_with_message(self):
+        mock = Mock(name='geoffrey')
+        self.assertRaisesRegex(AssertionError,
+                     r"Expected 'geoffrey' to be called once\.",
+                     mock.assert_called_once_with)
+
+
+    def test__name__(self):
+        mock = Mock()
+        self.assertRaises(AttributeError, lambda: mock.__name__)
+
+        mock.__name__ = 'foo'
+        self.assertEqual(mock.__name__, 'foo')
+
+
+    def test_spec_list_subclass(self):
+        class Sub(list):
+            pass
+        mock = Mock(spec=Sub(['foo']))
+
+        mock.append(3)
+        mock.append.assert_called_with(3)
+        self.assertRaises(AttributeError, getattr, mock, 'foo')
+
+
+    def test_spec_class(self):
+        class X(object):
+            pass
+
+        mock = Mock(spec=X)
+        self.assertTrue(isinstance(mock, X))
+
+        mock = Mock(spec=X())
+        self.assertTrue(isinstance(mock, X))
+
+        self.assertIs(mock.__class__, X)
+        self.assertEqual(Mock().__class__.__name__, 'Mock')
+
+        mock = Mock(spec_set=X)
+        self.assertTrue(isinstance(mock, X))
+
+        mock = Mock(spec_set=X())
+        self.assertTrue(isinstance(mock, X))
+
+
+    def test_setting_attribute_with_spec_set(self):
+        class X(object):
+            y = 3
+
+        mock = Mock(spec=X)
+        mock.x = 'foo'
+
+        mock = Mock(spec_set=X)
+        def set_attr():
+            mock.x = 'foo'
+
+        mock.y = 'foo'
+        self.assertRaises(AttributeError, set_attr)
+
+
+    def test_copy(self):
+        current = sys.getrecursionlimit()
+        self.addCleanup(sys.setrecursionlimit, current)
+
+        # can't use sys.maxint as this doesn't exist in Python 3
+        sys.setrecursionlimit(int(10e8))
+        # this segfaults without the fix in place
+        copy.copy(Mock())
+
+
+    def test_subclass_with_properties(self):
+        class SubClass(Mock):
+            def _get(self):
+                return 3
+            def _set(self, value):
+                raise NameError('strange error')
+            some_attribute = property(_get, _set)
+
+        s = SubClass(spec_set=SubClass)
+        self.assertEqual(s.some_attribute, 3)
+
+        def test():
+            s.some_attribute = 3
+        self.assertRaises(NameError, test)
+
+        def test():
+            s.foo = 'bar'
+        self.assertRaises(AttributeError, test)
+
+
+    def test_setting_call(self):
+        mock = Mock()
+        def __call__(self, a):
+            return self._mock_call(a)
+
+        type(mock).__call__ = __call__
+        mock('one')
+        mock.assert_called_with('one')
+
+        self.assertRaises(TypeError, mock, 'one', 'two')
+
+
+    def test_dir(self):
+        mock = Mock()
+        attrs = set(dir(mock))
+        type_attrs = set([m for m in dir(Mock) if not m.startswith('_')])
+
+        # all public attributes from the type are included
+        self.assertEqual(set(), type_attrs - attrs)
+
+        # creates these attributes
+        mock.a, mock.b
+        self.assertIn('a', dir(mock))
+        self.assertIn('b', dir(mock))
+
+        # instance attributes
+        mock.c = mock.d = None
+        self.assertIn('c', dir(mock))
+        self.assertIn('d', dir(mock))
+
+        # magic methods
+        mock.__iter__ = lambda s: iter([])
+        self.assertIn('__iter__', dir(mock))
+
+
+    def test_dir_from_spec(self):
+        mock = Mock(spec=unittest.TestCase)
+        testcase_attrs = set(dir(unittest.TestCase))
+        attrs = set(dir(mock))
+
+        # all attributes from the spec are included
+        self.assertEqual(set(), testcase_attrs - attrs)
+
+        # shadow a sys attribute
+        mock.version = 3
+        self.assertEqual(dir(mock).count('version'), 1)
+
+
+    def test_filter_dir(self):
+        patcher = patch.object(mock, 'FILTER_DIR', False)
+        patcher.start()
+        try:
+            attrs = set(dir(Mock()))
+            type_attrs = set(dir(Mock))
+
+            # ALL attributes from the type are included
+            self.assertEqual(set(), type_attrs - attrs)
+        finally:
+            patcher.stop()
+
+
+    def test_configure_mock(self):
+        mock = Mock(foo='bar')
+        self.assertEqual(mock.foo, 'bar')
+
+        mock = MagicMock(foo='bar')
+        self.assertEqual(mock.foo, 'bar')
+
+        kwargs = {'side_effect': KeyError, 'foo.bar.return_value': 33,
+                  'foo': MagicMock()}
+        mock = Mock(**kwargs)
+        self.assertRaises(KeyError, mock)
+        self.assertEqual(mock.foo.bar(), 33)
+        self.assertIsInstance(mock.foo, MagicMock)
+
+        mock = Mock()
+        mock.configure_mock(**kwargs)
+        self.assertRaises(KeyError, mock)
+        self.assertEqual(mock.foo.bar(), 33)
+        self.assertIsInstance(mock.foo, MagicMock)
+
+
+    def assertRaisesWithMsg(self, exception, message, func, *args, **kwargs):
+        # needed because assertRaisesRegex doesn't work easily with newlines
+        try:
+            func(*args, **kwargs)
+        except:
+            instance = sys.exc_info()[1]
+            self.assertIsInstance(instance, exception)
+        else:
+            self.fail('Exception %r not raised' % (exception,))
+
+        msg = str(instance)
+        self.assertEqual(msg, message)
+
+
+    def test_assert_called_with_failure_message(self):
+        mock = NonCallableMock()
+
+        expected = "mock(1, '2', 3, bar='foo')"
+        message = 'Expected call: %s\nNot called'
+        self.assertRaisesWithMsg(
+            AssertionError, message % (expected,),
+            mock.assert_called_with, 1, '2', 3, bar='foo'
+        )
+
+        mock.foo(1, '2', 3, foo='foo')
+
+
+        asserters = [
+            mock.foo.assert_called_with, mock.foo.assert_called_once_with
+        ]
+        for meth in asserters:
+            actual = "foo(1, '2', 3, foo='foo')"
+            expected = "foo(1, '2', 3, bar='foo')"
+            message = 'Expected call: %s\nActual call: %s'
+            self.assertRaisesWithMsg(
+                AssertionError, message % (expected, actual),
+                meth, 1, '2', 3, bar='foo'
+            )
+
+        # just kwargs
+        for meth in asserters:
+            actual = "foo(1, '2', 3, foo='foo')"
+            expected = "foo(bar='foo')"
+            message = 'Expected call: %s\nActual call: %s'
+            self.assertRaisesWithMsg(
+                AssertionError, message % (expected, actual),
+                meth, bar='foo'
+            )
+
+        # just args
+        for meth in asserters:
+            actual = "foo(1, '2', 3, foo='foo')"
+            expected = "foo(1, 2, 3)"
+            message = 'Expected call: %s\nActual call: %s'
+            self.assertRaisesWithMsg(
+                AssertionError, message % (expected, actual),
+                meth, 1, 2, 3
+            )
+
+        # empty
+        for meth in asserters:
+            actual = "foo(1, '2', 3, foo='foo')"
+            expected = "foo()"
+            message = 'Expected call: %s\nActual call: %s'
+            self.assertRaisesWithMsg(
+                AssertionError, message % (expected, actual), meth
+            )
+
+
+    def test_mock_calls(self):
+        mock = MagicMock()
+
+        # need to do this because MagicMock.mock_calls used to just return
+        # a MagicMock which also returned a MagicMock when __eq__ was called
+        self.assertIs(mock.mock_calls == [], True)
+
+        mock = MagicMock()
+        mock()
+        expected = [('', (), {})]
+        self.assertEqual(mock.mock_calls, expected)
+
+        mock.foo()
+        expected.append(call.foo())
+        self.assertEqual(mock.mock_calls, expected)
+        # intermediate mock_calls work too
+        self.assertEqual(mock.foo.mock_calls, [('', (), {})])
+
+        mock = MagicMock()
+        mock().foo(1, 2, 3, a=4, b=5)
+        expected = [
+            ('', (), {}), ('().foo', (1, 2, 3), dict(a=4, b=5))
+        ]
+        self.assertEqual(mock.mock_calls, expected)
+        self.assertEqual(mock.return_value.foo.mock_calls,
+                         [('', (1, 2, 3), dict(a=4, b=5))])
+        self.assertEqual(mock.return_value.mock_calls,
+                         [('foo', (1, 2, 3), dict(a=4, b=5))])
+
+        mock = MagicMock()
+        mock().foo.bar().baz()
+        expected = [
+            ('', (), {}), ('().foo.bar', (), {}),
+            ('().foo.bar().baz', (), {})
+        ]
+        self.assertEqual(mock.mock_calls, expected)
+        self.assertEqual(mock().mock_calls,
+                         call.foo.bar().baz().call_list())
+
+        for kwargs in dict(), dict(name='bar'):
+            mock = MagicMock(**kwargs)
+            int(mock.foo)
+            expected = [('foo.__int__', (), {})]
+            self.assertEqual(mock.mock_calls, expected)
+
+            mock = MagicMock(**kwargs)
+            mock.a()()
+            expected = [('a', (), {}), ('a()', (), {})]
+            self.assertEqual(mock.mock_calls, expected)
+            self.assertEqual(mock.a().mock_calls, [call()])
+
+            mock = MagicMock(**kwargs)
+            mock(1)(2)(3)
+            self.assertEqual(mock.mock_calls, call(1)(2)(3).call_list())
+            self.assertEqual(mock().mock_calls, call(2)(3).call_list())
+            self.assertEqual(mock()().mock_calls, call(3).call_list())
+
+            mock = MagicMock(**kwargs)
+            mock(1)(2)(3).a.b.c(4)
+            self.assertEqual(mock.mock_calls,
+                             call(1)(2)(3).a.b.c(4).call_list())
+            self.assertEqual(mock().mock_calls,
+                             call(2)(3).a.b.c(4).call_list())
+            self.assertEqual(mock()().mock_calls,
+                             call(3).a.b.c(4).call_list())
+
+            mock = MagicMock(**kwargs)
+            int(mock().foo.bar().baz())
+            last_call = ('().foo.bar().baz().__int__', (), {})
+            self.assertEqual(mock.mock_calls[-1], last_call)
+            self.assertEqual(mock().mock_calls,
+                             call.foo.bar().baz().__int__().call_list())
+            self.assertEqual(mock().foo.bar().mock_calls,
+                             call.baz().__int__().call_list())
+            self.assertEqual(mock().foo.bar().baz.mock_calls,
+                             call().__int__().call_list())
+
+
+    def test_subclassing(self):
+        class Subclass(Mock):
+            pass
+
+        mock = Subclass()
+        self.assertIsInstance(mock.foo, Subclass)
+        self.assertIsInstance(mock(), Subclass)
+
+        class Subclass(Mock):
+            def _get_child_mock(self, **kwargs):
+                return Mock(**kwargs)
+
+        mock = Subclass()
+        self.assertNotIsInstance(mock.foo, Subclass)
+        self.assertNotIsInstance(mock(), Subclass)
+
+
+    def test_arg_lists(self):
+        mocks = [
+            Mock(),
+            MagicMock(),
+            NonCallableMock(),
+            NonCallableMagicMock()
+        ]
+
+        def assert_attrs(mock):
+            names = 'call_args_list', 'method_calls', 'mock_calls'
+            for name in names:
+                attr = getattr(mock, name)
+                self.assertIsInstance(attr, _CallList)
+                self.assertIsInstance(attr, list)
+                self.assertEqual(attr, [])
+
+        for mock in mocks:
+            assert_attrs(mock)
+
+            if callable(mock):
+                mock()
+                mock(1, 2)
+                mock(a=3)
+
+                mock.reset_mock()
+                assert_attrs(mock)
+
+            mock.foo()
+            mock.foo.bar(1, a=3)
+            mock.foo(1).bar().baz(3)
+
+            mock.reset_mock()
+            assert_attrs(mock)
+
+
+    def test_call_args_two_tuple(self):
+        mock = Mock()
+        mock(1, a=3)
+        mock(2, b=4)
+
+        self.assertEqual(len(mock.call_args), 2)
+        args, kwargs = mock.call_args
+        self.assertEqual(args, (2,))
+        self.assertEqual(kwargs, dict(b=4))
+
+        expected_list = [((1,), dict(a=3)), ((2,), dict(b=4))]
+        for expected, call_args in zip(expected_list, mock.call_args_list):
+            self.assertEqual(len(call_args), 2)
+            self.assertEqual(expected[0], call_args[0])
+            self.assertEqual(expected[1], call_args[1])
+
+
+    def test_side_effect_iterator(self):
+        mock = Mock(side_effect=iter([1, 2, 3]))
+        self.assertEqual([mock(), mock(), mock()], [1, 2, 3])
+        self.assertRaises(StopIteration, mock)
+
+        mock = MagicMock(side_effect=['a', 'b', 'c'])
+        self.assertEqual([mock(), mock(), mock()], ['a', 'b', 'c'])
+        self.assertRaises(StopIteration, mock)
+
+        mock = Mock(side_effect='ghi')
+        self.assertEqual([mock(), mock(), mock()], ['g', 'h', 'i'])
+        self.assertRaises(StopIteration, mock)
+
+        class Foo(object):
+            pass
+        mock = MagicMock(side_effect=Foo)
+        self.assertIsInstance(mock(), Foo)
+
+        mock = Mock(side_effect=Iter())
+        self.assertEqual([mock(), mock(), mock(), mock()],
+                         ['this', 'is', 'an', 'iter'])
+        self.assertRaises(StopIteration, mock)
+
+
+    def test_side_effect_iterator_exceptions(self):
+        for Klass in Mock, MagicMock:
+            iterable = (ValueError, 3, KeyError, 6)
+            m = Klass(side_effect=iterable)
+            self.assertRaises(ValueError, m)
+            self.assertEqual(m(), 3)
+            self.assertRaises(KeyError, m)
+            self.assertEqual(m(), 6)
+
+
+    def test_side_effect_setting_iterator(self):
+        mock = Mock()
+        mock.side_effect = iter([1, 2, 3])
+        self.assertEqual([mock(), mock(), mock()], [1, 2, 3])
+        self.assertRaises(StopIteration, mock)
+        side_effect = mock.side_effect
+        self.assertIsInstance(side_effect, type(iter([])))
+
+        mock.side_effect = ['a', 'b', 'c']
+        self.assertEqual([mock(), mock(), mock()], ['a', 'b', 'c'])
+        self.assertRaises(StopIteration, mock)
+        side_effect = mock.side_effect
+        self.assertIsInstance(side_effect, type(iter([])))
+
+        this_iter = Iter()
+        mock.side_effect = this_iter
+        self.assertEqual([mock(), mock(), mock(), mock()],
+                         ['this', 'is', 'an', 'iter'])
+        self.assertRaises(StopIteration, mock)
+        self.assertIs(mock.side_effect, this_iter)
+
+
+    def test_assert_has_calls_any_order(self):
+        mock = Mock()
+        mock(1, 2)
+        mock(a=3)
+        mock(3, 4)
+        mock(b=6)
+        mock(b=6)
+
+        kalls = [
+            call(1, 2), ({'a': 3},),
+            ((3, 4),), ((), {'a': 3}),
+            ('', (1, 2)), ('', {'a': 3}),
+            ('', (1, 2), {}), ('', (), {'a': 3})
+        ]
+        for kall in kalls:
+            mock.assert_has_calls([kall], any_order=True)
+
+        for kall in call(1, '2'), call(b=3), call(), 3, None, 'foo':
+            self.assertRaises(
+                AssertionError, mock.assert_has_calls,
+                [kall], any_order=True
+            )
+
+        kall_lists = [
+            [call(1, 2), call(b=6)],
+            [call(3, 4), call(1, 2)],
+            [call(b=6), call(b=6)],
+        ]
+
+        for kall_list in kall_lists:
+            mock.assert_has_calls(kall_list, any_order=True)
+
+        kall_lists = [
+            [call(b=6), call(b=6), call(b=6)],
+            [call(1, 2), call(1, 2)],
+            [call(3, 4), call(1, 2), call(5, 7)],
+            [call(b=6), call(3, 4), call(b=6), call(1, 2), call(b=6)],
+        ]
+        for kall_list in kall_lists:
+            self.assertRaises(
+                AssertionError, mock.assert_has_calls,
+                kall_list, any_order=True
+            )
+
+    def test_assert_has_calls(self):
+        kalls1 = [
+                call(1, 2), ({'a': 3},),
+                ((3, 4),), call(b=6),
+                ('', (1,), {'b': 6}),
+        ]
+        kalls2 = [call.foo(), call.bar(1)]
+        kalls2.extend(call.spam().baz(a=3).call_list())
+        kalls2.extend(call.bam(set(), foo={}).fish([1]).call_list())
+
+        mocks = []
+        for mock in Mock(), MagicMock():
+            mock(1, 2)
+            mock(a=3)
+            mock(3, 4)
+            mock(b=6)
+            mock(1, b=6)
+            mocks.append((mock, kalls1))
+
+        mock = Mock()
+        mock.foo()
+        mock.bar(1)
+        mock.spam().baz(a=3)
+        mock.bam(set(), foo={}).fish([1])
+        mocks.append((mock, kalls2))
+
+        for mock, kalls in mocks:
+            for i in range(len(kalls)):
+                for step in 1, 2, 3:
+                    these = kalls[i:i+step]
+                    mock.assert_has_calls(these)
+
+                    if len(these) > 1:
+                        self.assertRaises(
+                            AssertionError,
+                            mock.assert_has_calls,
+                            list(reversed(these))
+                        )
+
+
+    def test_assert_any_call(self):
+        mock = Mock()
+        mock(1, 2)
+        mock(a=3)
+        mock(1, b=6)
+
+        mock.assert_any_call(1, 2)
+        mock.assert_any_call(a=3)
+        mock.assert_any_call(1, b=6)
+
+        self.assertRaises(
+            AssertionError,
+            mock.assert_any_call
+        )
+        self.assertRaises(
+            AssertionError,
+            mock.assert_any_call,
+            1, 3
+        )
+        self.assertRaises(
+            AssertionError,
+            mock.assert_any_call,
+            a=4
+        )
+
+
+    def test_mock_calls_create_autospec(self):
+        def f(a, b):
+            pass
+        obj = Iter()
+        obj.f = f
+
+        funcs = [
+            create_autospec(f),
+            create_autospec(obj).f
+        ]
+        for func in funcs:
+            func(1, 2)
+            func(3, 4)
+
+            self.assertEqual(
+                func.mock_calls, [call(1, 2), call(3, 4)]
+            )
+
+
+    def test_mock_add_spec(self):
+        class _One(object):
+            one = 1
+        class _Two(object):
+            two = 2
+        class Anything(object):
+            one = two = three = 'four'
+
+        klasses = [
+            Mock, MagicMock, NonCallableMock, NonCallableMagicMock
+        ]
+        for Klass in list(klasses):
+            klasses.append(lambda K=Klass: K(spec=Anything))
+            klasses.append(lambda K=Klass: K(spec_set=Anything))
+
+        for Klass in klasses:
+            for kwargs in dict(), dict(spec_set=True):
+                mock = Klass()
+                #no error
+                mock.one, mock.two, mock.three
+
+                for One, Two in [(_One, _Two), (['one'], ['two'])]:
+                    for kwargs in dict(), dict(spec_set=True):
+                        mock.mock_add_spec(One, **kwargs)
+
+                        mock.one
+                        self.assertRaises(
+                            AttributeError, getattr, mock, 'two'
+                        )
+                        self.assertRaises(
+                            AttributeError, getattr, mock, 'three'
+                        )
+                        if 'spec_set' in kwargs:
+                            self.assertRaises(
+                                AttributeError, setattr, mock, 'three', None
+                            )
+
+                        mock.mock_add_spec(Two, **kwargs)
+                        self.assertRaises(
+                            AttributeError, getattr, mock, 'one'
+                        )
+                        mock.two
+                        self.assertRaises(
+                            AttributeError, getattr, mock, 'three'
+                        )
+                        if 'spec_set' in kwargs:
+                            self.assertRaises(
+                                AttributeError, setattr, mock, 'three', None
+                            )
+            # note that creating a mock, setting an instance attribute, and
+            # *then* setting a spec doesn't work. Not the intended use case
+
+
+    def test_mock_add_spec_magic_methods(self):
+        for Klass in MagicMock, NonCallableMagicMock:
+            mock = Klass()
+            int(mock)
+
+            mock.mock_add_spec(object)
+            self.assertRaises(TypeError, int, mock)
+
+            mock = Klass()
+            mock['foo']
+            mock.__int__.return_value =4
+
+            mock.mock_add_spec(int)
+            self.assertEqual(int(mock), 4)
+            self.assertRaises(TypeError, lambda: mock['foo'])
+
+
+    def test_adding_child_mock(self):
+        for Klass in NonCallableMock, Mock, MagicMock, NonCallableMagicMock:
+            mock = Klass()
+
+            mock.foo = Mock()
+            mock.foo()
+
+            self.assertEqual(mock.method_calls, [call.foo()])
+            self.assertEqual(mock.mock_calls, [call.foo()])
+
+            mock = Klass()
+            mock.bar = Mock(name='name')
+            mock.bar()
+            self.assertEqual(mock.method_calls, [])
+            self.assertEqual(mock.mock_calls, [])
+
+            # mock with an existing _new_parent but no name
+            mock = Klass()
+            mock.baz = MagicMock()()
+            mock.baz()
+            self.assertEqual(mock.method_calls, [])
+            self.assertEqual(mock.mock_calls, [])
+
+
+    def test_adding_return_value_mock(self):
+        for Klass in Mock, MagicMock:
+            mock = Klass()
+            mock.return_value = MagicMock()
+
+            mock()()
+            self.assertEqual(mock.mock_calls, [call(), call()()])
+
+
+    def test_manager_mock(self):
+        class Foo(object):
+            one = 'one'
+            two = 'two'
+        manager = Mock()
+        p1 = patch.object(Foo, 'one')
+        p2 = patch.object(Foo, 'two')
+
+        mock_one = p1.start()
+        self.addCleanup(p1.stop)
+        mock_two = p2.start()
+        self.addCleanup(p2.stop)
+
+        manager.attach_mock(mock_one, 'one')
+        manager.attach_mock(mock_two, 'two')
+
+        Foo.two()
+        Foo.one()
+
+        self.assertEqual(manager.mock_calls, [call.two(), call.one()])
+
+
+    def test_magic_methods_mock_calls(self):
+        for Klass in Mock, MagicMock:
+            m = Klass()
+            m.__int__ = Mock(return_value=3)
+            m.__float__ = MagicMock(return_value=3.0)
+            int(m)
+            float(m)
+
+            self.assertEqual(m.mock_calls, [call.__int__(), call.__float__()])
+            self.assertEqual(m.method_calls, [])
+
+
+    def test_attribute_deletion(self):
+        # this behaviour isn't *useful*, but at least it's now tested...
+        for Klass in Mock, MagicMock, NonCallableMagicMock, NonCallableMock:
+            m = Klass()
+            original = m.foo
+            m.foo = 3
+            del m.foo
+            self.assertEqual(m.foo, original)
+
+            new = m.foo = Mock()
+            del m.foo
+            self.assertEqual(m.foo, new)
+
+
+    def test_mock_parents(self):
+        for Klass in Mock, MagicMock:
+            m = Klass()
+            original_repr = repr(m)
+            m.return_value = m
+            self.assertIs(m(), m)
+            self.assertEqual(repr(m), original_repr)
+
+            m.reset_mock()
+            self.assertIs(m(), m)
+            self.assertEqual(repr(m), original_repr)
+
+            m = Klass()
+            m.b = m.a
+            self.assertIn("name='mock.a'", repr(m.b))
+            self.assertIn("name='mock.a'", repr(m.a))
+            m.reset_mock()
+            self.assertIn("name='mock.a'", repr(m.b))
+            self.assertIn("name='mock.a'", repr(m.a))
+
+            m = Klass()
+            original_repr = repr(m)
+            m.a = m()
+            m.a.return_value = m
+
+            self.assertEqual(repr(m), original_repr)
+            self.assertEqual(repr(m.a()), original_repr)
+
+
+    def test_attach_mock(self):
+        classes = Mock, MagicMock, NonCallableMagicMock, NonCallableMock
+        for Klass in classes:
+            for Klass2 in classes:
+                m = Klass()
+
+                m2 = Klass2(name='foo')
+                m.attach_mock(m2, 'bar')
+
+                self.assertIs(m.bar, m2)
+                self.assertIn("name='mock.bar'", repr(m2))
+
+                m.bar.baz(1)
+                self.assertEqual(m.mock_calls, [call.bar.baz(1)])
+                self.assertEqual(m.method_calls, [call.bar.baz(1)])
+
+
+    def test_attach_mock_return_value(self):
+        classes = Mock, MagicMock, NonCallableMagicMock, NonCallableMock
+        for Klass in Mock, MagicMock:
+            for Klass2 in classes:
+                m = Klass()
+
+                m2 = Klass2(name='foo')
+                m.attach_mock(m2, 'return_value')
+
+                self.assertIs(m(), m2)
+                self.assertIn("name='mock()'", repr(m2))
+
+                m2.foo()
+                self.assertEqual(m.mock_calls, call().foo().call_list())
+
+
+    def test_attribute_deletion(self):
+        for mock in Mock(), MagicMock():
+            self.assertTrue(hasattr(mock, 'm'))
+
+            del mock.m
+            self.assertFalse(hasattr(mock, 'm'))
+
+            del mock.f
+            self.assertFalse(hasattr(mock, 'f'))
+            self.assertRaises(AttributeError, getattr, mock, 'f')
+
+
+    def test_class_assignable(self):
+        for mock in Mock(), MagicMock():
+            self.assertNotIsInstance(mock, int)
+
+            mock.__class__ = int
+            self.assertIsInstance(mock, int)
+            mock.foo
+
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/unittest/test/testmock/testpatch.py b/Lib/unittest/test/testmock/testpatch.py
new file mode 100644
index 0000000000..c1091b4e9b
--- /dev/null
+++ b/Lib/unittest/test/testmock/testpatch.py
@@ -0,0 +1,1785 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+import os
+import sys
+
+import unittest
+from unittest.test.testmock import support
+from unittest.test.testmock.support import SomeClass, is_instance
+
+from unittest.mock import (
+    NonCallableMock, CallableMixin, patch, sentinel,
+    MagicMock, Mock, NonCallableMagicMock, patch, _patch,
+    DEFAULT, call, _get_target
+)
+
+
+builtin_string = 'builtins'
+
+PTModule = sys.modules[__name__]
+MODNAME = '%s.PTModule' % __name__
+
+
+def _get_proxy(obj, get_only=True):
+    class Proxy(object):
+        def __getattr__(self, name):
+            return getattr(obj, name)
+    if not get_only:
+        def __setattr__(self, name, value):
+            setattr(obj, name, value)
+        def __delattr__(self, name):
+            delattr(obj, name)
+        Proxy.__setattr__ = __setattr__
+        Proxy.__delattr__ = __delattr__
+    return Proxy()
+
+
+# for use in the test
+something  = sentinel.Something
+something_else  = sentinel.SomethingElse
+
+
+class Foo(object):
+    def __init__(self, a):
+        pass
+    def f(self, a):
+        pass
+    def g(self):
+        pass
+    foo = 'bar'
+
+    class Bar(object):
+        def a(self):
+            pass
+
+foo_name = '%s.Foo' % __name__
+
+
+def function(a, b=Foo):
+    pass
+
+
+class Container(object):
+    def __init__(self):
+        self.values = {}
+
+    def __getitem__(self, name):
+        return self.values[name]
+
+    def __setitem__(self, name, value):
+        self.values[name] = value
+
+    def __delitem__(self, name):
+        del self.values[name]
+
+    def __iter__(self):
+        return iter(self.values)
+
+
+
+class PatchTest(unittest.TestCase):
+
+    def assertNotCallable(self, obj, magic=True):
+        MockClass = NonCallableMagicMock
+        if not magic:
+            MockClass = NonCallableMock
+
+        self.assertRaises(TypeError, obj)
+        self.assertTrue(is_instance(obj, MockClass))
+        self.assertFalse(is_instance(obj, CallableMixin))
+
+
+    def test_single_patchobject(self):
+        class Something(object):
+            attribute = sentinel.Original
+
+        @patch.object(Something, 'attribute', sentinel.Patched)
+        def test():
+            self.assertEqual(Something.attribute, sentinel.Patched, "unpatched")
+
+        test()
+        self.assertEqual(Something.attribute, sentinel.Original,
+                         "patch not restored")
+
+
+    def test_patchobject_with_none(self):
+        class Something(object):
+            attribute = sentinel.Original
+
+        @patch.object(Something, 'attribute', None)
+        def test():
+            self.assertIsNone(Something.attribute, "unpatched")
+
+        test()
+        self.assertEqual(Something.attribute, sentinel.Original,
+                         "patch not restored")
+
+
+    def test_multiple_patchobject(self):
+        class Something(object):
+            attribute = sentinel.Original
+            next_attribute = sentinel.Original2
+
+        @patch.object(Something, 'attribute', sentinel.Patched)
+        @patch.object(Something, 'next_attribute', sentinel.Patched2)
+        def test():
+            self.assertEqual(Something.attribute, sentinel.Patched,
+                             "unpatched")
+            self.assertEqual(Something.next_attribute, sentinel.Patched2,
+                             "unpatched")
+
+        test()
+        self.assertEqual(Something.attribute, sentinel.Original,
+                         "patch not restored")
+        self.assertEqual(Something.next_attribute, sentinel.Original2,
+                         "patch not restored")
+
+
+    def test_object_lookup_is_quite_lazy(self):
+        global something
+        original = something
+        @patch('%s.something' % __name__, sentinel.Something2)
+        def test():
+            pass
+
+        try:
+            something = sentinel.replacement_value
+            test()
+            self.assertEqual(something, sentinel.replacement_value)
+        finally:
+            something = original
+
+
+    def test_patch(self):
+        @patch('%s.something' % __name__, sentinel.Something2)
+        def test():
+            self.assertEqual(PTModule.something, sentinel.Something2,
+                             "unpatched")
+
+        test()
+        self.assertEqual(PTModule.something, sentinel.Something,
+                         "patch not restored")
+
+        @patch('%s.something' % __name__, sentinel.Something2)
+        @patch('%s.something_else' % __name__, sentinel.SomethingElse)
+        def test():
+            self.assertEqual(PTModule.something, sentinel.Something2,
+                             "unpatched")
+            self.assertEqual(PTModule.something_else, sentinel.SomethingElse,
+                             "unpatched")
+
+        self.assertEqual(PTModule.something, sentinel.Something,
+                         "patch not restored")
+        self.assertEqual(PTModule.something_else, sentinel.SomethingElse,
+                         "patch not restored")
+
+        # Test the patching and restoring works a second time
+        test()
+
+        self.assertEqual(PTModule.something, sentinel.Something,
+                         "patch not restored")
+        self.assertEqual(PTModule.something_else, sentinel.SomethingElse,
+                         "patch not restored")
+
+        mock = Mock()
+        mock.return_value = sentinel.Handle
+        @patch('%s.open' % builtin_string, mock)
+        def test():
+            self.assertEqual(open('filename', 'r'), sentinel.Handle,
+                             "open not patched")
+        test()
+        test()
+
+        self.assertNotEqual(open, mock, "patch not restored")
+
+
+    def test_patch_class_attribute(self):
+        @patch('%s.SomeClass.class_attribute' % __name__,
+               sentinel.ClassAttribute)
+        def test():
+            self.assertEqual(PTModule.SomeClass.class_attribute,
+                             sentinel.ClassAttribute, "unpatched")
+        test()
+
+        self.assertIsNone(PTModule.SomeClass.class_attribute,
+                          "patch not restored")
+
+
+    def test_patchobject_with_default_mock(self):
+        class Test(object):
+            something = sentinel.Original
+            something2 = sentinel.Original2
+
+        @patch.object(Test, 'something')
+        def test(mock):
+            self.assertEqual(mock, Test.something,
+                             "Mock not passed into test function")
+            self.assertIsInstance(mock, MagicMock,
+                            "patch with two arguments did not create a mock")
+
+        test()
+
+        @patch.object(Test, 'something')
+        @patch.object(Test, 'something2')
+        def test(this1, this2, mock1, mock2):
+            self.assertEqual(this1, sentinel.this1,
+                             "Patched function didn't receive initial argument")
+            self.assertEqual(this2, sentinel.this2,
+                             "Patched function didn't receive second argument")
+            self.assertEqual(mock1, Test.something2,
+                             "Mock not passed into test function")
+            self.assertEqual(mock2, Test.something,
+                             "Second Mock not passed into test function")
+            self.assertIsInstance(mock2, MagicMock,
+                            "patch with two arguments did not create a mock")
+            self.assertIsInstance(mock2, MagicMock,
+                            "patch with two arguments did not create a mock")
+
+            # A hack to test that new mocks are passed the second time
+            self.assertNotEqual(outerMock1, mock1, "unexpected value for mock1")
+            self.assertNotEqual(outerMock2, mock2, "unexpected value for mock1")
+            return mock1, mock2
+
+        outerMock1 = outerMock2 = None
+        outerMock1, outerMock2 = test(sentinel.this1, sentinel.this2)
+
+        # Test that executing a second time creates new mocks
+        test(sentinel.this1, sentinel.this2)
+
+
+    def test_patch_with_spec(self):
+        @patch('%s.SomeClass' % __name__, spec=SomeClass)
+        def test(MockSomeClass):
+            self.assertEqual(SomeClass, MockSomeClass)
+            self.assertTrue(is_instance(SomeClass.wibble, MagicMock))
+            self.assertRaises(AttributeError, lambda: SomeClass.not_wibble)
+
+        test()
+
+
+    def test_patchobject_with_spec(self):
+        @patch.object(SomeClass, 'class_attribute', spec=SomeClass)
+        def test(MockAttribute):
+            self.assertEqual(SomeClass.class_attribute, MockAttribute)
+            self.assertTrue(is_instance(SomeClass.class_attribute.wibble,
+                                       MagicMock))
+            self.assertRaises(AttributeError,
+                              lambda: SomeClass.class_attribute.not_wibble)
+
+        test()
+
+
+    def test_patch_with_spec_as_list(self):
+        @patch('%s.SomeClass' % __name__, spec=['wibble'])
+        def test(MockSomeClass):
+            self.assertEqual(SomeClass, MockSomeClass)
+            self.assertTrue(is_instance(SomeClass.wibble, MagicMock))
+            self.assertRaises(AttributeError, lambda: SomeClass.not_wibble)
+
+        test()
+
+
+    def test_patchobject_with_spec_as_list(self):
+        @patch.object(SomeClass, 'class_attribute', spec=['wibble'])
+        def test(MockAttribute):
+            self.assertEqual(SomeClass.class_attribute, MockAttribute)
+            self.assertTrue(is_instance(SomeClass.class_attribute.wibble,
+                                       MagicMock))
+            self.assertRaises(AttributeError,
+                              lambda: SomeClass.class_attribute.not_wibble)
+
+        test()
+
+
+    def test_nested_patch_with_spec_as_list(self):
+        # regression test for nested decorators
+        @patch('%s.open' % builtin_string)
+        @patch('%s.SomeClass' % __name__, spec=['wibble'])
+        def test(MockSomeClass, MockOpen):
+            self.assertEqual(SomeClass, MockSomeClass)
+            self.assertTrue(is_instance(SomeClass.wibble, MagicMock))
+            self.assertRaises(AttributeError, lambda: SomeClass.not_wibble)
+        test()
+
+
+    def test_patch_with_spec_as_boolean(self):
+        @patch('%s.SomeClass' % __name__, spec=True)
+        def test(MockSomeClass):
+            self.assertEqual(SomeClass, MockSomeClass)
+            # Should not raise attribute error
+            MockSomeClass.wibble
+
+            self.assertRaises(AttributeError, lambda: MockSomeClass.not_wibble)
+
+        test()
+
+
+    def test_patch_object_with_spec_as_boolean(self):
+        @patch.object(PTModule, 'SomeClass', spec=True)
+        def test(MockSomeClass):
+            self.assertEqual(SomeClass, MockSomeClass)
+            # Should not raise attribute error
+            MockSomeClass.wibble
+
+            self.assertRaises(AttributeError, lambda: MockSomeClass.not_wibble)
+
+        test()
+
+
+    def test_patch_class_acts_with_spec_is_inherited(self):
+        @patch('%s.SomeClass' % __name__, spec=True)
+        def test(MockSomeClass):
+            self.assertTrue(is_instance(MockSomeClass, MagicMock))
+            instance = MockSomeClass()
+            self.assertNotCallable(instance)
+            # Should not raise attribute error
+            instance.wibble
+
+            self.assertRaises(AttributeError, lambda: instance.not_wibble)
+
+        test()
+
+
+    def test_patch_with_create_mocks_non_existent_attributes(self):
+        @patch('%s.frooble' % builtin_string, sentinel.Frooble, create=True)
+        def test():
+            self.assertEqual(frooble, sentinel.Frooble)
+
+        test()
+        self.assertRaises(NameError, lambda: frooble)
+
+
+    def test_patchobject_with_create_mocks_non_existent_attributes(self):
+        @patch.object(SomeClass, 'frooble', sentinel.Frooble, create=True)
+        def test():
+            self.assertEqual(SomeClass.frooble, sentinel.Frooble)
+
+        test()
+        self.assertFalse(hasattr(SomeClass, 'frooble'))
+
+
+    def test_patch_wont_create_by_default(self):
+        try:
+            @patch('%s.frooble' % builtin_string, sentinel.Frooble)
+            def test():
+                self.assertEqual(frooble, sentinel.Frooble)
+
+            test()
+        except AttributeError:
+            pass
+        else:
+            self.fail('Patching non existent attributes should fail')
+
+        self.assertRaises(NameError, lambda: frooble)
+
+
+    def test_patchobject_wont_create_by_default(self):
+        try:
+            @patch.object(SomeClass, 'frooble', sentinel.Frooble)
+            def test():
+                self.fail('Patching non existent attributes should fail')
+
+            test()
+        except AttributeError:
+            pass
+        else:
+            self.fail('Patching non existent attributes should fail')
+        self.assertFalse(hasattr(SomeClass, 'frooble'))
+
+
+    def test_patch_with_static_methods(self):
+        class Foo(object):
+            @staticmethod
+            def woot():
+                return sentinel.Static
+
+        @patch.object(Foo, 'woot', staticmethod(lambda: sentinel.Patched))
+        def anonymous():
+            self.assertEqual(Foo.woot(), sentinel.Patched)
+        anonymous()
+
+        self.assertEqual(Foo.woot(), sentinel.Static)
+
+
+    def test_patch_local(self):
+        foo = sentinel.Foo
+        @patch.object(sentinel, 'Foo', 'Foo')
+        def anonymous():
+            self.assertEqual(sentinel.Foo, 'Foo')
+        anonymous()
+
+        self.assertEqual(sentinel.Foo, foo)
+
+
+    def test_patch_slots(self):
+        class Foo(object):
+            __slots__ = ('Foo',)
+
+        foo = Foo()
+        foo.Foo = sentinel.Foo
+
+        @patch.object(foo, 'Foo', 'Foo')
+        def anonymous():
+            self.assertEqual(foo.Foo, 'Foo')
+        anonymous()
+
+        self.assertEqual(foo.Foo, sentinel.Foo)
+
+
+    def test_patchobject_class_decorator(self):
+        class Something(object):
+            attribute = sentinel.Original
+
+        class Foo(object):
+            def test_method(other_self):
+                self.assertEqual(Something.attribute, sentinel.Patched,
+                                 "unpatched")
+            def not_test_method(other_self):
+                self.assertEqual(Something.attribute, sentinel.Original,
+                                 "non-test method patched")
+
+        Foo = patch.object(Something, 'attribute', sentinel.Patched)(Foo)
+
+        f = Foo()
+        f.test_method()
+        f.not_test_method()
+
+        self.assertEqual(Something.attribute, sentinel.Original,
+                         "patch not restored")
+
+
+    def test_patch_class_decorator(self):
+        class Something(object):
+            attribute = sentinel.Original
+
+        class Foo(object):
+            def test_method(other_self, mock_something):
+                self.assertEqual(PTModule.something, mock_something,
+                                 "unpatched")
+            def not_test_method(other_self):
+                self.assertEqual(PTModule.something, sentinel.Something,
+                                 "non-test method patched")
+        Foo = patch('%s.something' % __name__)(Foo)
+
+        f = Foo()
+        f.test_method()
+        f.not_test_method()
+
+        self.assertEqual(Something.attribute, sentinel.Original,
+                         "patch not restored")
+        self.assertEqual(PTModule.something, sentinel.Something,
+                         "patch not restored")
+
+
+    def test_patchobject_twice(self):
+        class Something(object):
+            attribute = sentinel.Original
+            next_attribute = sentinel.Original2
+
+        @patch.object(Something, 'attribute', sentinel.Patched)
+        @patch.object(Something, 'attribute', sentinel.Patched)
+        def test():
+            self.assertEqual(Something.attribute, sentinel.Patched, "unpatched")
+
+        test()
+
+        self.assertEqual(Something.attribute, sentinel.Original,
+                         "patch not restored")
+
+
+    def test_patch_dict(self):
+        foo = {'initial': object(), 'other': 'something'}
+        original = foo.copy()
+
+        @patch.dict(foo)
+        def test():
+            foo['a'] = 3
+            del foo['initial']
+            foo['other'] = 'something else'
+
+        test()
+
+        self.assertEqual(foo, original)
+
+        @patch.dict(foo, {'a': 'b'})
+        def test():
+            self.assertEqual(len(foo), 3)
+            self.assertEqual(foo['a'], 'b')
+
+        test()
+
+        self.assertEqual(foo, original)
+
+        @patch.dict(foo, [('a', 'b')])
+        def test():
+            self.assertEqual(len(foo), 3)
+            self.assertEqual(foo['a'], 'b')
+
+        test()
+
+        self.assertEqual(foo, original)
+
+
+    def test_patch_dict_with_container_object(self):
+        foo = Container()
+        foo['initial'] = object()
+        foo['other'] =  'something'
+
+        original = foo.values.copy()
+
+        @patch.dict(foo)
+        def test():
+            foo['a'] = 3
+            del foo['initial']
+            foo['other'] = 'something else'
+
+        test()
+
+        self.assertEqual(foo.values, original)
+
+        @patch.dict(foo, {'a': 'b'})
+        def test():
+            self.assertEqual(len(foo.values), 3)
+            self.assertEqual(foo['a'], 'b')
+
+        test()
+
+        self.assertEqual(foo.values, original)
+
+
+    def test_patch_dict_with_clear(self):
+        foo = {'initial': object(), 'other': 'something'}
+        original = foo.copy()
+
+        @patch.dict(foo, clear=True)
+        def test():
+            self.assertEqual(foo, {})
+            foo['a'] = 3
+            foo['other'] = 'something else'
+
+        test()
+
+        self.assertEqual(foo, original)
+
+        @patch.dict(foo, {'a': 'b'}, clear=True)
+        def test():
+            self.assertEqual(foo, {'a': 'b'})
+
+        test()
+
+        self.assertEqual(foo, original)
+
+        @patch.dict(foo, [('a', 'b')], clear=True)
+        def test():
+            self.assertEqual(foo, {'a': 'b'})
+
+        test()
+
+        self.assertEqual(foo, original)
+
+
+    def test_patch_dict_with_container_object_and_clear(self):
+        foo = Container()
+        foo['initial'] = object()
+        foo['other'] =  'something'
+
+        original = foo.values.copy()
+
+        @patch.dict(foo, clear=True)
+        def test():
+            self.assertEqual(foo.values, {})
+            foo['a'] = 3
+            foo['other'] = 'something else'
+
+        test()
+
+        self.assertEqual(foo.values, original)
+
+        @patch.dict(foo, {'a': 'b'}, clear=True)
+        def test():
+            self.assertEqual(foo.values, {'a': 'b'})
+
+        test()
+
+        self.assertEqual(foo.values, original)
+
+
+    def test_name_preserved(self):
+        foo = {}
+
+        @patch('%s.SomeClass' % __name__, object())
+        @patch('%s.SomeClass' % __name__, object(), autospec=True)
+        @patch.object(SomeClass, object())
+        @patch.dict(foo)
+        def some_name():
+            pass
+
+        self.assertEqual(some_name.__name__, 'some_name')
+
+
+    def test_patch_with_exception(self):
+        foo = {}
+
+        @patch.dict(foo, {'a': 'b'})
+        def test():
+            raise NameError('Konrad')
+        try:
+            test()
+        except NameError:
+            pass
+        else:
+            self.fail('NameError not raised by test')
+
+        self.assertEqual(foo, {})
+
+
+    def test_patch_dict_with_string(self):
+        @patch.dict('os.environ', {'konrad_delong': 'some value'})
+        def test():
+            self.assertIn('konrad_delong', os.environ)
+
+        test()
+
+
+    def test_patch_descriptor(self):
+        # would be some effort to fix this - we could special case the
+        # builtin descriptors: classmethod, property, staticmethod
+        return
+        class Nothing(object):
+            foo = None
+
+        class Something(object):
+            foo = {}
+
+            @patch.object(Nothing, 'foo', 2)
+            @classmethod
+            def klass(cls):
+                self.assertIs(cls, Something)
+
+            @patch.object(Nothing, 'foo', 2)
+            @staticmethod
+            def static(arg):
+                return arg
+
+            @patch.dict(foo)
+            @classmethod
+            def klass_dict(cls):
+                self.assertIs(cls, Something)
+
+            @patch.dict(foo)
+            @staticmethod
+            def static_dict(arg):
+                return arg
+
+        # these will raise exceptions if patching descriptors is broken
+        self.assertEqual(Something.static('f00'), 'f00')
+        Something.klass()
+        self.assertEqual(Something.static_dict('f00'), 'f00')
+        Something.klass_dict()
+
+        something = Something()
+        self.assertEqual(something.static('f00'), 'f00')
+        something.klass()
+        self.assertEqual(something.static_dict('f00'), 'f00')
+        something.klass_dict()
+
+
+    def test_patch_spec_set(self):
+        @patch('%s.SomeClass' % __name__, spec=SomeClass, spec_set=True)
+        def test(MockClass):
+            MockClass.z = 'foo'
+
+        self.assertRaises(AttributeError, test)
+
+        @patch.object(support, 'SomeClass', spec=SomeClass, spec_set=True)
+        def test(MockClass):
+            MockClass.z = 'foo'
+
+        self.assertRaises(AttributeError, test)
+        @patch('%s.SomeClass' % __name__, spec_set=True)
+        def test(MockClass):
+            MockClass.z = 'foo'
+
+        self.assertRaises(AttributeError, test)
+
+        @patch.object(support, 'SomeClass', spec_set=True)
+        def test(MockClass):
+            MockClass.z = 'foo'
+
+        self.assertRaises(AttributeError, test)
+
+
+    def test_spec_set_inherit(self):
+        @patch('%s.SomeClass' % __name__, spec_set=True)
+        def test(MockClass):
+            instance = MockClass()
+            instance.z = 'foo'
+
+        self.assertRaises(AttributeError, test)
+
+
+    def test_patch_start_stop(self):
+        original = something
+        patcher = patch('%s.something' % __name__)
+        self.assertIs(something, original)
+        mock = patcher.start()
+        try:
+            self.assertIsNot(mock, original)
+            self.assertIs(something, mock)
+        finally:
+            patcher.stop()
+        self.assertIs(something, original)
+
+
+    def test_stop_without_start(self):
+        patcher = patch(foo_name, 'bar', 3)
+
+        # calling stop without start used to produce a very obscure error
+        self.assertRaises(RuntimeError, patcher.stop)
+
+
+    def test_patchobject_start_stop(self):
+        original = something
+        patcher = patch.object(PTModule, 'something', 'foo')
+        self.assertIs(something, original)
+        replaced = patcher.start()
+        try:
+            self.assertEqual(replaced, 'foo')
+            self.assertIs(something, replaced)
+        finally:
+            patcher.stop()
+        self.assertIs(something, original)
+
+
+    def test_patch_dict_start_stop(self):
+        d = {'foo': 'bar'}
+        original = d.copy()
+        patcher = patch.dict(d, [('spam', 'eggs')], clear=True)
+        self.assertEqual(d, original)
+
+        patcher.start()
+        try:
+            self.assertEqual(d, {'spam': 'eggs'})
+        finally:
+            patcher.stop()
+        self.assertEqual(d, original)
+
+
+    def test_patch_dict_class_decorator(self):
+        this = self
+        d = {'spam': 'eggs'}
+        original = d.copy()
+
+        class Test(object):
+            def test_first(self):
+                this.assertEqual(d, {'foo': 'bar'})
+            def test_second(self):
+                this.assertEqual(d, {'foo': 'bar'})
+
+        Test = patch.dict(d, {'foo': 'bar'}, clear=True)(Test)
+        self.assertEqual(d, original)
+
+        test = Test()
+
+        test.test_first()
+        self.assertEqual(d, original)
+
+        test.test_second()
+        self.assertEqual(d, original)
+
+        test = Test()
+
+        test.test_first()
+        self.assertEqual(d, original)
+
+        test.test_second()
+        self.assertEqual(d, original)
+
+
+    def test_get_only_proxy(self):
+        class Something(object):
+            foo = 'foo'
+        class SomethingElse:
+            foo = 'foo'
+
+        for thing in Something, SomethingElse, Something(), SomethingElse:
+            proxy = _get_proxy(thing)
+
+            @patch.object(proxy, 'foo', 'bar')
+            def test():
+                self.assertEqual(proxy.foo, 'bar')
+            test()
+            self.assertEqual(proxy.foo, 'foo')
+            self.assertEqual(thing.foo, 'foo')
+            self.assertNotIn('foo', proxy.__dict__)
+
+
+    def test_get_set_delete_proxy(self):
+        class Something(object):
+            foo = 'foo'
+        class SomethingElse:
+            foo = 'foo'
+
+        for thing in Something, SomethingElse, Something(), SomethingElse:
+            proxy = _get_proxy(Something, get_only=False)
+
+            @patch.object(proxy, 'foo', 'bar')
+            def test():
+                self.assertEqual(proxy.foo, 'bar')
+            test()
+            self.assertEqual(proxy.foo, 'foo')
+            self.assertEqual(thing.foo, 'foo')
+            self.assertNotIn('foo', proxy.__dict__)
+
+
+    def test_patch_keyword_args(self):
+        kwargs = {'side_effect': KeyError, 'foo.bar.return_value': 33,
+                  'foo': MagicMock()}
+
+        patcher = patch(foo_name, **kwargs)
+        mock = patcher.start()
+        patcher.stop()
+
+        self.assertRaises(KeyError, mock)
+        self.assertEqual(mock.foo.bar(), 33)
+        self.assertIsInstance(mock.foo, MagicMock)
+
+
+    def test_patch_object_keyword_args(self):
+        kwargs = {'side_effect': KeyError, 'foo.bar.return_value': 33,
+                  'foo': MagicMock()}
+
+        patcher = patch.object(Foo, 'f', **kwargs)
+        mock = patcher.start()
+        patcher.stop()
+
+        self.assertRaises(KeyError, mock)
+        self.assertEqual(mock.foo.bar(), 33)
+        self.assertIsInstance(mock.foo, MagicMock)
+
+
+    def test_patch_dict_keyword_args(self):
+        original = {'foo': 'bar'}
+        copy = original.copy()
+
+        patcher = patch.dict(original, foo=3, bar=4, baz=5)
+        patcher.start()
+
+        try:
+            self.assertEqual(original, dict(foo=3, bar=4, baz=5))
+        finally:
+            patcher.stop()
+
+        self.assertEqual(original, copy)
+
+
+    def test_autospec(self):
+        class Boo(object):
+            def __init__(self, a):
+                pass
+            def f(self, a):
+                pass
+            def g(self):
+                pass
+            foo = 'bar'
+
+            class Bar(object):
+                def a(self):
+                    pass
+
+        def _test(mock):
+            mock(1)
+            mock.assert_called_with(1)
+            self.assertRaises(TypeError, mock)
+
+        def _test2(mock):
+            mock.f(1)
+            mock.f.assert_called_with(1)
+            self.assertRaises(TypeError, mock.f)
+
+            mock.g()
+            mock.g.assert_called_with()
+            self.assertRaises(TypeError, mock.g, 1)
+
+            self.assertRaises(AttributeError, getattr, mock, 'h')
+
+            mock.foo.lower()
+            mock.foo.lower.assert_called_with()
+            self.assertRaises(AttributeError, getattr, mock.foo, 'bar')
+
+            mock.Bar()
+            mock.Bar.assert_called_with()
+
+            mock.Bar.a()
+            mock.Bar.a.assert_called_with()
+            self.assertRaises(TypeError, mock.Bar.a, 1)
+
+            mock.Bar().a()
+            mock.Bar().a.assert_called_with()
+            self.assertRaises(TypeError, mock.Bar().a, 1)
+
+            self.assertRaises(AttributeError, getattr, mock.Bar, 'b')
+            self.assertRaises(AttributeError, getattr, mock.Bar(), 'b')
+
+        def function(mock):
+            _test(mock)
+            _test2(mock)
+            _test2(mock(1))
+            self.assertIs(mock, Foo)
+            return mock
+
+        test = patch(foo_name, autospec=True)(function)
+
+        mock = test()
+        self.assertIsNot(Foo, mock)
+        # test patching a second time works
+        test()
+
+        module = sys.modules[__name__]
+        test = patch.object(module, 'Foo', autospec=True)(function)
+
+        mock = test()
+        self.assertIsNot(Foo, mock)
+        # test patching a second time works
+        test()
+
+
+    def test_autospec_function(self):
+        @patch('%s.function' % __name__, autospec=True)
+        def test(mock):
+            function(1)
+            function.assert_called_with(1)
+            function(2, 3)
+            function.assert_called_with(2, 3)
+
+            self.assertRaises(TypeError, function)
+            self.assertRaises(AttributeError, getattr, function, 'foo')
+
+        test()
+
+
+    def test_autospec_keywords(self):
+        @patch('%s.function' % __name__, autospec=True,
+               return_value=3)
+        def test(mock_function):
+            #self.assertEqual(function.abc, 'foo')
+            return function(1, 2)
+
+        result = test()
+        self.assertEqual(result, 3)
+
+
+    def test_autospec_with_new(self):
+        patcher = patch('%s.function' % __name__, new=3, autospec=True)
+        self.assertRaises(TypeError, patcher.start)
+
+        module = sys.modules[__name__]
+        patcher = patch.object(module, 'function', new=3, autospec=True)
+        self.assertRaises(TypeError, patcher.start)
+
+
+    def test_autospec_with_object(self):
+        class Bar(Foo):
+            extra = []
+
+        patcher = patch(foo_name, autospec=Bar)
+        mock = patcher.start()
+        try:
+            self.assertIsInstance(mock, Bar)
+            self.assertIsInstance(mock.extra, list)
+        finally:
+            patcher.stop()
+
+
+    def test_autospec_inherits(self):
+        FooClass = Foo
+        patcher = patch(foo_name, autospec=True)
+        mock = patcher.start()
+        try:
+            self.assertIsInstance(mock, FooClass)
+            self.assertIsInstance(mock(3), FooClass)
+        finally:
+            patcher.stop()
+
+
+    def test_autospec_name(self):
+        patcher = patch(foo_name, autospec=True)
+        mock = patcher.start()
+
+        try:
+            self.assertIn(" name='Foo'", repr(mock))
+            self.assertIn(" name='Foo.f'", repr(mock.f))
+            self.assertIn(" name='Foo()'", repr(mock(None)))
+            self.assertIn(" name='Foo().f'", repr(mock(None).f))
+        finally:
+            patcher.stop()
+
+
+    def test_tracebacks(self):
+        @patch.object(Foo, 'f', object())
+        def test():
+            raise AssertionError
+        try:
+            test()
+        except:
+            err = sys.exc_info()
+
+        result = unittest.TextTestResult(None, None, 0)
+        traceback = result._exc_info_to_string(err, self)
+        self.assertIn('raise AssertionError', traceback)
+
+
+    def test_new_callable_patch(self):
+        patcher = patch(foo_name, new_callable=NonCallableMagicMock)
+
+        m1 = patcher.start()
+        patcher.stop()
+        m2 = patcher.start()
+        patcher.stop()
+
+        self.assertIsNot(m1, m2)
+        for mock in m1, m2:
+            self.assertNotCallable(m1)
+
+
+    def test_new_callable_patch_object(self):
+        patcher = patch.object(Foo, 'f', new_callable=NonCallableMagicMock)
+
+        m1 = patcher.start()
+        patcher.stop()
+        m2 = patcher.start()
+        patcher.stop()
+
+        self.assertIsNot(m1, m2)
+        for mock in m1, m2:
+            self.assertNotCallable(m1)
+
+
+    def test_new_callable_keyword_arguments(self):
+        class Bar(object):
+            kwargs = None
+            def __init__(self, **kwargs):
+                Bar.kwargs = kwargs
+
+        patcher = patch(foo_name, new_callable=Bar, arg1=1, arg2=2)
+        m = patcher.start()
+        try:
+            self.assertIs(type(m), Bar)
+            self.assertEqual(Bar.kwargs, dict(arg1=1, arg2=2))
+        finally:
+            patcher.stop()
+
+
+    def test_new_callable_spec(self):
+        class Bar(object):
+            kwargs = None
+            def __init__(self, **kwargs):
+                Bar.kwargs = kwargs
+
+        patcher = patch(foo_name, new_callable=Bar, spec=Bar)
+        patcher.start()
+        try:
+            self.assertEqual(Bar.kwargs, dict(spec=Bar))
+        finally:
+            patcher.stop()
+
+        patcher = patch(foo_name, new_callable=Bar, spec_set=Bar)
+        patcher.start()
+        try:
+            self.assertEqual(Bar.kwargs, dict(spec_set=Bar))
+        finally:
+            patcher.stop()
+
+
+    def test_new_callable_create(self):
+        non_existent_attr = '%s.weeeee' % foo_name
+        p = patch(non_existent_attr, new_callable=NonCallableMock)
+        self.assertRaises(AttributeError, p.start)
+
+        p = patch(non_existent_attr, new_callable=NonCallableMock,
+                  create=True)
+        m = p.start()
+        try:
+            self.assertNotCallable(m, magic=False)
+        finally:
+            p.stop()
+
+
+    def test_new_callable_incompatible_with_new(self):
+        self.assertRaises(
+            ValueError, patch, foo_name, new=object(), new_callable=MagicMock
+        )
+        self.assertRaises(
+            ValueError, patch.object, Foo, 'f', new=object(),
+            new_callable=MagicMock
+        )
+
+
+    def test_new_callable_incompatible_with_autospec(self):
+        self.assertRaises(
+            ValueError, patch, foo_name, new_callable=MagicMock,
+            autospec=True
+        )
+        self.assertRaises(
+            ValueError, patch.object, Foo, 'f', new_callable=MagicMock,
+            autospec=True
+        )
+
+
+    def test_new_callable_inherit_for_mocks(self):
+        class MockSub(Mock):
+            pass
+
+        MockClasses = (
+            NonCallableMock, NonCallableMagicMock, MagicMock, Mock, MockSub
+        )
+        for Klass in MockClasses:
+            for arg in 'spec', 'spec_set':
+                kwargs = {arg: True}
+                p = patch(foo_name, new_callable=Klass, **kwargs)
+                m = p.start()
+                try:
+                    instance = m.return_value
+                    self.assertRaises(AttributeError, getattr, instance, 'x')
+                finally:
+                    p.stop()
+
+
+    def test_new_callable_inherit_non_mock(self):
+        class NotAMock(object):
+            def __init__(self, spec):
+                self.spec = spec
+
+        p = patch(foo_name, new_callable=NotAMock, spec=True)
+        m = p.start()
+        try:
+            self.assertTrue(is_instance(m, NotAMock))
+            self.assertRaises(AttributeError, getattr, m, 'return_value')
+        finally:
+            p.stop()
+
+        self.assertEqual(m.spec, Foo)
+
+
+    def test_new_callable_class_decorating(self):
+        test = self
+        original = Foo
+        class SomeTest(object):
+
+            def _test(self, mock_foo):
+                test.assertIsNot(Foo, original)
+                test.assertIs(Foo, mock_foo)
+                test.assertIsInstance(Foo, SomeClass)
+
+            def test_two(self, mock_foo):
+                self._test(mock_foo)
+            def test_one(self, mock_foo):
+                self._test(mock_foo)
+
+        SomeTest = patch(foo_name, new_callable=SomeClass)(SomeTest)
+        SomeTest().test_one()
+        SomeTest().test_two()
+        self.assertIs(Foo, original)
+
+
+    def test_patch_multiple(self):
+        original_foo = Foo
+        original_f = Foo.f
+        original_g = Foo.g
+
+        patcher1 = patch.multiple(foo_name, f=1, g=2)
+        patcher2 = patch.multiple(Foo, f=1, g=2)
+
+        for patcher in patcher1, patcher2:
+            patcher.start()
+            try:
+                self.assertIs(Foo, original_foo)
+                self.assertEqual(Foo.f, 1)
+                self.assertEqual(Foo.g, 2)
+            finally:
+                patcher.stop()
+
+            self.assertIs(Foo, original_foo)
+            self.assertEqual(Foo.f, original_f)
+            self.assertEqual(Foo.g, original_g)
+
+
+        @patch.multiple(foo_name, f=3, g=4)
+        def test():
+            self.assertIs(Foo, original_foo)
+            self.assertEqual(Foo.f, 3)
+            self.assertEqual(Foo.g, 4)
+
+        test()
+
+
+    def test_patch_multiple_no_kwargs(self):
+        self.assertRaises(ValueError, patch.multiple, foo_name)
+        self.assertRaises(ValueError, patch.multiple, Foo)
+
+
+    def test_patch_multiple_create_mocks(self):
+        original_foo = Foo
+        original_f = Foo.f
+        original_g = Foo.g
+
+        @patch.multiple(foo_name, f=DEFAULT, g=3, foo=DEFAULT)
+        def test(f, foo):
+            self.assertIs(Foo, original_foo)
+            self.assertIs(Foo.f, f)
+            self.assertEqual(Foo.g, 3)
+            self.assertIs(Foo.foo, foo)
+            self.assertTrue(is_instance(f, MagicMock))
+            self.assertTrue(is_instance(foo, MagicMock))
+
+        test()
+        self.assertEqual(Foo.f, original_f)
+        self.assertEqual(Foo.g, original_g)
+
+
+    def test_patch_multiple_create_mocks_different_order(self):
+        # bug revealed by Jython!
+        original_f = Foo.f
+        original_g = Foo.g
+
+        patcher = patch.object(Foo, 'f', 3)
+        patcher.attribute_name = 'f'
+
+        other = patch.object(Foo, 'g', DEFAULT)
+        other.attribute_name = 'g'
+        patcher.additional_patchers = [other]
+
+        @patcher
+        def test(g):
+            self.assertIs(Foo.g, g)
+            self.assertEqual(Foo.f, 3)
+
+        test()
+        self.assertEqual(Foo.f, original_f)
+        self.assertEqual(Foo.g, original_g)
+
+
+    def test_patch_multiple_stacked_decorators(self):
+        original_foo = Foo
+        original_f = Foo.f
+        original_g = Foo.g
+
+        @patch.multiple(foo_name, f=DEFAULT)
+        @patch.multiple(foo_name, foo=DEFAULT)
+        @patch(foo_name + '.g')
+        def test1(g, **kwargs):
+            _test(g, **kwargs)
+
+        @patch.multiple(foo_name, f=DEFAULT)
+        @patch(foo_name + '.g')
+        @patch.multiple(foo_name, foo=DEFAULT)
+        def test2(g, **kwargs):
+            _test(g, **kwargs)
+
+        @patch(foo_name + '.g')
+        @patch.multiple(foo_name, f=DEFAULT)
+        @patch.multiple(foo_name, foo=DEFAULT)
+        def test3(g, **kwargs):
+            _test(g, **kwargs)
+
+        def _test(g, **kwargs):
+            f = kwargs.pop('f')
+            foo = kwargs.pop('foo')
+            self.assertFalse(kwargs)
+
+            self.assertIs(Foo, original_foo)
+            self.assertIs(Foo.f, f)
+            self.assertIs(Foo.g, g)
+            self.assertIs(Foo.foo, foo)
+            self.assertTrue(is_instance(f, MagicMock))
+            self.assertTrue(is_instance(g, MagicMock))
+            self.assertTrue(is_instance(foo, MagicMock))
+
+        test1()
+        test2()
+        test3()
+        self.assertEqual(Foo.f, original_f)
+        self.assertEqual(Foo.g, original_g)
+
+
+    def test_patch_multiple_create_mocks_patcher(self):
+        original_foo = Foo
+        original_f = Foo.f
+        original_g = Foo.g
+
+        patcher = patch.multiple(foo_name, f=DEFAULT, g=3, foo=DEFAULT)
+
+        result = patcher.start()
+        try:
+            f = result['f']
+            foo = result['foo']
+            self.assertEqual(set(result), set(['f', 'foo']))
+
+            self.assertIs(Foo, original_foo)
+            self.assertIs(Foo.f, f)
+            self.assertIs(Foo.foo, foo)
+            self.assertTrue(is_instance(f, MagicMock))
+            self.assertTrue(is_instance(foo, MagicMock))
+        finally:
+            patcher.stop()
+
+        self.assertEqual(Foo.f, original_f)
+        self.assertEqual(Foo.g, original_g)
+
+
+    def test_patch_multiple_decorating_class(self):
+        test = self
+        original_foo = Foo
+        original_f = Foo.f
+        original_g = Foo.g
+
+        class SomeTest(object):
+
+            def _test(self, f, foo):
+                test.assertIs(Foo, original_foo)
+                test.assertIs(Foo.f, f)
+                test.assertEqual(Foo.g, 3)
+                test.assertIs(Foo.foo, foo)
+                test.assertTrue(is_instance(f, MagicMock))
+                test.assertTrue(is_instance(foo, MagicMock))
+
+            def test_two(self, f, foo):
+                self._test(f, foo)
+            def test_one(self, f, foo):
+                self._test(f, foo)
+
+        SomeTest = patch.multiple(
+            foo_name, f=DEFAULT, g=3, foo=DEFAULT
+        )(SomeTest)
+
+        thing = SomeTest()
+        thing.test_one()
+        thing.test_two()
+
+        self.assertEqual(Foo.f, original_f)
+        self.assertEqual(Foo.g, original_g)
+
+
+    def test_patch_multiple_create(self):
+        patcher = patch.multiple(Foo, blam='blam')
+        self.assertRaises(AttributeError, patcher.start)
+
+        patcher = patch.multiple(Foo, blam='blam', create=True)
+        patcher.start()
+        try:
+            self.assertEqual(Foo.blam, 'blam')
+        finally:
+            patcher.stop()
+
+        self.assertFalse(hasattr(Foo, 'blam'))
+
+
+    def test_patch_multiple_spec_set(self):
+        # if spec_set works then we can assume that spec and autospec also
+        # work as the underlying machinery is the same
+        patcher = patch.multiple(Foo, foo=DEFAULT, spec_set=['a', 'b'])
+        result = patcher.start()
+        try:
+            self.assertEqual(Foo.foo, result['foo'])
+            Foo.foo.a(1)
+            Foo.foo.b(2)
+            Foo.foo.a.assert_called_with(1)
+            Foo.foo.b.assert_called_with(2)
+            self.assertRaises(AttributeError, setattr, Foo.foo, 'c', None)
+        finally:
+            patcher.stop()
+
+
+    def test_patch_multiple_new_callable(self):
+        class Thing(object):
+            pass
+
+        patcher = patch.multiple(
+            Foo, f=DEFAULT, g=DEFAULT, new_callable=Thing
+        )
+        result = patcher.start()
+        try:
+            self.assertIs(Foo.f, result['f'])
+            self.assertIs(Foo.g, result['g'])
+            self.assertIsInstance(Foo.f, Thing)
+            self.assertIsInstance(Foo.g, Thing)
+            self.assertIsNot(Foo.f, Foo.g)
+        finally:
+            patcher.stop()
+
+
+    def test_nested_patch_failure(self):
+        original_f = Foo.f
+        original_g = Foo.g
+
+        @patch.object(Foo, 'g', 1)
+        @patch.object(Foo, 'missing', 1)
+        @patch.object(Foo, 'f', 1)
+        def thing1():
+            pass
+
+        @patch.object(Foo, 'missing', 1)
+        @patch.object(Foo, 'g', 1)
+        @patch.object(Foo, 'f', 1)
+        def thing2():
+            pass
+
+        @patch.object(Foo, 'g', 1)
+        @patch.object(Foo, 'f', 1)
+        @patch.object(Foo, 'missing', 1)
+        def thing3():
+            pass
+
+        for func in thing1, thing2, thing3:
+            self.assertRaises(AttributeError, func)
+            self.assertEqual(Foo.f, original_f)
+            self.assertEqual(Foo.g, original_g)
+
+
+    def test_new_callable_failure(self):
+        original_f = Foo.f
+        original_g = Foo.g
+        original_foo = Foo.foo
+
+        def crasher():
+            raise NameError('crasher')
+
+        @patch.object(Foo, 'g', 1)
+        @patch.object(Foo, 'foo', new_callable=crasher)
+        @patch.object(Foo, 'f', 1)
+        def thing1():
+            pass
+
+        @patch.object(Foo, 'foo', new_callable=crasher)
+        @patch.object(Foo, 'g', 1)
+        @patch.object(Foo, 'f', 1)
+        def thing2():
+            pass
+
+        @patch.object(Foo, 'g', 1)
+        @patch.object(Foo, 'f', 1)
+        @patch.object(Foo, 'foo', new_callable=crasher)
+        def thing3():
+            pass
+
+        for func in thing1, thing2, thing3:
+            self.assertRaises(NameError, func)
+            self.assertEqual(Foo.f, original_f)
+            self.assertEqual(Foo.g, original_g)
+            self.assertEqual(Foo.foo, original_foo)
+
+
+    def test_patch_multiple_failure(self):
+        original_f = Foo.f
+        original_g = Foo.g
+
+        patcher = patch.object(Foo, 'f', 1)
+        patcher.attribute_name = 'f'
+
+        good = patch.object(Foo, 'g', 1)
+        good.attribute_name = 'g'
+
+        bad = patch.object(Foo, 'missing', 1)
+        bad.attribute_name = 'missing'
+
+        for additionals in [good, bad], [bad, good]:
+            patcher.additional_patchers = additionals
+
+            @patcher
+            def func():
+                pass
+
+            self.assertRaises(AttributeError, func)
+            self.assertEqual(Foo.f, original_f)
+            self.assertEqual(Foo.g, original_g)
+
+
+    def test_patch_multiple_new_callable_failure(self):
+        original_f = Foo.f
+        original_g = Foo.g
+        original_foo = Foo.foo
+
+        def crasher():
+            raise NameError('crasher')
+
+        patcher = patch.object(Foo, 'f', 1)
+        patcher.attribute_name = 'f'
+
+        good = patch.object(Foo, 'g', 1)
+        good.attribute_name = 'g'
+
+        bad = patch.object(Foo, 'foo', new_callable=crasher)
+        bad.attribute_name = 'foo'
+
+        for additionals in [good, bad], [bad, good]:
+            patcher.additional_patchers = additionals
+
+            @patcher
+            def func():
+                pass
+
+            self.assertRaises(NameError, func)
+            self.assertEqual(Foo.f, original_f)
+            self.assertEqual(Foo.g, original_g)
+            self.assertEqual(Foo.foo, original_foo)
+
+
+    def test_patch_multiple_string_subclasses(self):
+        Foo = type('Foo', (str,), {'fish': 'tasty'})
+        foo = Foo()
+        @patch.multiple(foo, fish='nearly gone')
+        def test():
+            self.assertEqual(foo.fish, 'nearly gone')
+
+        test()
+        self.assertEqual(foo.fish, 'tasty')
+
+
+    @patch('unittest.mock.patch.TEST_PREFIX', 'foo')
+    def test_patch_test_prefix(self):
+        class Foo(object):
+            thing = 'original'
+
+            def foo_one(self):
+                return self.thing
+            def foo_two(self):
+                return self.thing
+            def test_one(self):
+                return self.thing
+            def test_two(self):
+                return self.thing
+
+        Foo = patch.object(Foo, 'thing', 'changed')(Foo)
+
+        foo = Foo()
+        self.assertEqual(foo.foo_one(), 'changed')
+        self.assertEqual(foo.foo_two(), 'changed')
+        self.assertEqual(foo.test_one(), 'original')
+        self.assertEqual(foo.test_two(), 'original')
+
+
+    @patch('unittest.mock.patch.TEST_PREFIX', 'bar')
+    def test_patch_dict_test_prefix(self):
+        class Foo(object):
+            def bar_one(self):
+                return dict(the_dict)
+            def bar_two(self):
+                return dict(the_dict)
+            def test_one(self):
+                return dict(the_dict)
+            def test_two(self):
+                return dict(the_dict)
+
+        the_dict = {'key': 'original'}
+        Foo = patch.dict(the_dict, key='changed')(Foo)
+
+        foo =Foo()
+        self.assertEqual(foo.bar_one(), {'key': 'changed'})
+        self.assertEqual(foo.bar_two(), {'key': 'changed'})
+        self.assertEqual(foo.test_one(), {'key': 'original'})
+        self.assertEqual(foo.test_two(), {'key': 'original'})
+
+
+    def test_patch_with_spec_mock_repr(self):
+        for arg in ('spec', 'autospec', 'spec_set'):
+            p = patch('%s.SomeClass' % __name__, **{arg: True})
+            m = p.start()
+            try:
+                self.assertIn(" name='SomeClass'", repr(m))
+                self.assertIn(" name='SomeClass.class_attribute'",
+                              repr(m.class_attribute))
+                self.assertIn(" name='SomeClass()'", repr(m()))
+                self.assertIn(" name='SomeClass().class_attribute'",
+                              repr(m().class_attribute))
+            finally:
+                p.stop()
+
+
+    def test_patch_nested_autospec_repr(self):
+        with patch('unittest.test.testmock.support', autospec=True) as m:
+            self.assertIn(" name='support.SomeClass.wibble()'",
+                          repr(m.SomeClass.wibble()))
+            self.assertIn(" name='support.SomeClass().wibble()'",
+                          repr(m.SomeClass().wibble()))
+
+
+
+    def test_mock_calls_with_patch(self):
+        for arg in ('spec', 'autospec', 'spec_set'):
+            p = patch('%s.SomeClass' % __name__, **{arg: True})
+            m = p.start()
+            try:
+                m.wibble()
+
+                kalls = [call.wibble()]
+                self.assertEqual(m.mock_calls, kalls)
+                self.assertEqual(m.method_calls, kalls)
+                self.assertEqual(m.wibble.mock_calls, [call()])
+
+                result = m()
+                kalls.append(call())
+                self.assertEqual(m.mock_calls, kalls)
+
+                result.wibble()
+                kalls.append(call().wibble())
+                self.assertEqual(m.mock_calls, kalls)
+
+                self.assertEqual(result.mock_calls, [call.wibble()])
+                self.assertEqual(result.wibble.mock_calls, [call()])
+                self.assertEqual(result.method_calls, [call.wibble()])
+            finally:
+                p.stop()
+
+
+    def test_patch_imports_lazily(self):
+        sys.modules.pop('squizz', None)
+
+        p1 = patch('squizz.squozz')
+        self.assertRaises(ImportError, p1.start)
+
+        squizz = Mock()
+        squizz.squozz = 6
+        sys.modules['squizz'] = squizz
+        p1 = patch('squizz.squozz')
+        squizz.squozz = 3
+        p1.start()
+        p1.stop()
+        self.assertEqual(squizz.squozz, 3)
+
+
+    def test_patch_propogrates_exc_on_exit(self):
+        class holder:
+            exc_info = None, None, None
+
+        class custom_patch(_patch):
+            def __exit__(self, etype=None, val=None, tb=None):
+                _patch.__exit__(self, etype, val, tb)
+                holder.exc_info = etype, val, tb
+            stop = __exit__
+
+        def with_custom_patch(target):
+            getter, attribute = _get_target(target)
+            return custom_patch(
+                getter, attribute, DEFAULT, None, False, None,
+                None, None, {}
+            )
+
+        @with_custom_patch('squizz.squozz')
+        def test(mock):
+            raise RuntimeError
+
+        self.assertRaises(RuntimeError, test)
+        self.assertIs(holder.exc_info[0], RuntimeError)
+        self.assertIsNotNone(holder.exc_info[1],
+                            'exception value not propgated')
+        self.assertIsNotNone(holder.exc_info[2],
+                            'exception traceback not propgated')
+
+
+    def test_create_and_specs(self):
+        for kwarg in ('spec', 'spec_set', 'autospec'):
+            p = patch('%s.doesnotexist' % __name__, create=True,
+                      **{kwarg: True})
+            self.assertRaises(TypeError, p.start)
+            self.assertRaises(NameError, lambda: doesnotexist)
+
+            # check that spec with create is innocuous if the original exists
+            p = patch(MODNAME, create=True, **{kwarg: True})
+            p.start()
+            p.stop()
+
+
+    def test_multiple_specs(self):
+        original = PTModule
+        for kwarg in ('spec', 'spec_set'):
+            p = patch(MODNAME, autospec=0, **{kwarg: 0})
+            self.assertRaises(TypeError, p.start)
+            self.assertIs(PTModule, original)
+
+        for kwarg in ('spec', 'autospec'):
+            p = patch(MODNAME, spec_set=0, **{kwarg: 0})
+            self.assertRaises(TypeError, p.start)
+            self.assertIs(PTModule, original)
+
+        for kwarg in ('spec_set', 'autospec'):
+            p = patch(MODNAME, spec=0, **{kwarg: 0})
+            self.assertRaises(TypeError, p.start)
+            self.assertIs(PTModule, original)
+
+
+    def test_specs_false_instead_of_none(self):
+        p = patch(MODNAME, spec=False, spec_set=False, autospec=False)
+        mock = p.start()
+        try:
+            # no spec should have been set, so attribute access should not fail
+            mock.does_not_exist
+            mock.does_not_exist = 3
+        finally:
+            p.stop()
+
+
+    def test_falsey_spec(self):
+        for kwarg in ('spec', 'autospec', 'spec_set'):
+            p = patch(MODNAME, **{kwarg: 0})
+            m = p.start()
+            try:
+                self.assertRaises(AttributeError, getattr, m, 'doesnotexit')
+            finally:
+                p.stop()
+
+
+    def test_spec_set_true(self):
+        for kwarg in ('spec', 'autospec'):
+            p = patch(MODNAME, spec_set=True, **{kwarg: True})
+            m = p.start()
+            try:
+                self.assertRaises(AttributeError, setattr, m,
+                                  'doesnotexist', 'something')
+                self.assertRaises(AttributeError, getattr, m, 'doesnotexist')
+            finally:
+                p.stop()
+
+
+    def test_callable_spec_as_list(self):
+        spec = ('__call__',)
+        p = patch(MODNAME, spec=spec)
+        m = p.start()
+        try:
+            self.assertTrue(callable(m))
+        finally:
+            p.stop()
+
+
+    def test_not_callable_spec_as_list(self):
+        spec = ('foo', 'bar')
+        p = patch(MODNAME, spec=spec)
+        m = p.start()
+        try:
+            self.assertFalse(callable(m))
+        finally:
+            p.stop()
+
+
+    def test_patch_stopall(self):
+        unlink = os.unlink
+        chdir = os.chdir
+        path = os.path
+        patch('os.unlink', something).start()
+        patch('os.chdir', something_else).start()
+
+        @patch('os.path')
+        def patched(mock_path):
+            patch.stopall()
+            self.assertIs(os.path, mock_path)
+            self.assertIs(os.unlink, unlink)
+            self.assertIs(os.chdir, chdir)
+
+        patched()
+        self.assertIs(os.path, path)
+
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/unittest/test/testmock/testsentinel.py b/Lib/unittest/test/testmock/testsentinel.py
new file mode 100644
index 0000000000..bfda68ece3
--- /dev/null
+++ b/Lib/unittest/test/testmock/testsentinel.py
@@ -0,0 +1,28 @@
+import unittest
+from unittest.mock import sentinel, DEFAULT
+
+
+class SentinelTest(unittest.TestCase):
+
+    def testSentinels(self):
+        self.assertEqual(sentinel.whatever, sentinel.whatever,
+                         'sentinel not stored')
+        self.assertNotEqual(sentinel.whatever, sentinel.whateverelse,
+                            'sentinel should be unique')
+
+
+    def testSentinelName(self):
+        self.assertEqual(str(sentinel.whatever), 'sentinel.whatever',
+                         'sentinel name incorrect')
+
+
+    def testDEFAULT(self):
+        self.assertTrue(DEFAULT is sentinel.DEFAULT)
+
+    def testBases(self):
+        # If this doesn't raise an AttributeError then help(mock) is broken
+        self.assertRaises(AttributeError, lambda: sentinel.__bases__)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/unittest/test/testmock/testwith.py b/Lib/unittest/test/testmock/testwith.py
new file mode 100644
index 0000000000..0a0cfad120
--- /dev/null
+++ b/Lib/unittest/test/testmock/testwith.py
@@ -0,0 +1,176 @@
+import unittest
+from warnings import catch_warnings
+
+from unittest.test.testmock.support import is_instance
+from unittest.mock import MagicMock, Mock, patch, sentinel, mock_open, call
+
+
+
+something  = sentinel.Something
+something_else  = sentinel.SomethingElse
+
+
+
+class WithTest(unittest.TestCase):
+
+    def test_with_statement(self):
+        with patch('%s.something' % __name__, sentinel.Something2):
+            self.assertEqual(something, sentinel.Something2, "unpatched")
+        self.assertEqual(something, sentinel.Something)
+
+
+    def test_with_statement_exception(self):
+        try:
+            with patch('%s.something' % __name__, sentinel.Something2):
+                self.assertEqual(something, sentinel.Something2, "unpatched")
+                raise Exception('pow')
+        except Exception:
+            pass
+        else:
+            self.fail("patch swallowed exception")
+        self.assertEqual(something, sentinel.Something)
+
+
+    def test_with_statement_as(self):
+        with patch('%s.something' % __name__) as mock_something:
+            self.assertEqual(something, mock_something, "unpatched")
+            self.assertTrue(is_instance(mock_something, MagicMock),
+                            "patching wrong type")
+        self.assertEqual(something, sentinel.Something)
+
+
+    def test_patch_object_with_statement(self):
+        class Foo(object):
+            something = 'foo'
+        original = Foo.something
+        with patch.object(Foo, 'something'):
+            self.assertNotEqual(Foo.something, original, "unpatched")
+        self.assertEqual(Foo.something, original)
+
+
+    def test_with_statement_nested(self):
+        with catch_warnings(record=True):
+            with patch('%s.something' % __name__) as mock_something, patch('%s.something_else' % __name__) as mock_something_else:
+                self.assertEqual(something, mock_something, "unpatched")
+                self.assertEqual(something_else, mock_something_else,
+                                 "unpatched")
+
+        self.assertEqual(something, sentinel.Something)
+        self.assertEqual(something_else, sentinel.SomethingElse)
+
+
+    def test_with_statement_specified(self):
+        with patch('%s.something' % __name__, sentinel.Patched) as mock_something:
+            self.assertEqual(something, mock_something, "unpatched")
+            self.assertEqual(mock_something, sentinel.Patched, "wrong patch")
+        self.assertEqual(something, sentinel.Something)
+
+
+    def testContextManagerMocking(self):
+        mock = Mock()
+        mock.__enter__ = Mock()
+        mock.__exit__ = Mock()
+        mock.__exit__.return_value = False
+
+        with mock as m:
+            self.assertEqual(m, mock.__enter__.return_value)
+        mock.__enter__.assert_called_with()
+        mock.__exit__.assert_called_with(None, None, None)
+
+
+    def test_context_manager_with_magic_mock(self):
+        mock = MagicMock()
+
+        with self.assertRaises(TypeError):
+            with mock:
+                'foo' + 3
+        mock.__enter__.assert_called_with()
+        self.assertTrue(mock.__exit__.called)
+
+
+    def test_with_statement_same_attribute(self):
+        with patch('%s.something' % __name__, sentinel.Patched) as mock_something:
+            self.assertEqual(something, mock_something, "unpatched")
+
+            with patch('%s.something' % __name__) as mock_again:
+                self.assertEqual(something, mock_again, "unpatched")
+
+            self.assertEqual(something, mock_something,
+                             "restored with wrong instance")
+
+        self.assertEqual(something, sentinel.Something, "not restored")
+
+
+    def test_with_statement_imbricated(self):
+        with patch('%s.something' % __name__) as mock_something:
+            self.assertEqual(something, mock_something, "unpatched")
+
+            with patch('%s.something_else' % __name__) as mock_something_else:
+                self.assertEqual(something_else, mock_something_else,
+                                 "unpatched")
+
+        self.assertEqual(something, sentinel.Something)
+        self.assertEqual(something_else, sentinel.SomethingElse)
+
+
+    def test_dict_context_manager(self):
+        foo = {}
+        with patch.dict(foo, {'a': 'b'}):
+            self.assertEqual(foo, {'a': 'b'})
+        self.assertEqual(foo, {})
+
+        with self.assertRaises(NameError):
+            with patch.dict(foo, {'a': 'b'}):
+                self.assertEqual(foo, {'a': 'b'})
+                raise NameError('Konrad')
+
+        self.assertEqual(foo, {})
+
+
+
+class TestMockOpen(unittest.TestCase):
+
+    def test_mock_open(self):
+        mock = mock_open()
+        with patch('%s.open' % __name__, mock, create=True) as patched:
+            self.assertIs(patched, mock)
+            open('foo')
+
+        mock.assert_called_once_with('foo')
+
+
+    def test_mock_open_context_manager(self):
+        mock = mock_open()
+        handle = mock.return_value
+        with patch('%s.open' % __name__, mock, create=True):
+            with open('foo') as f:
+                f.read()
+
+        expected_calls = [call('foo'), call().__enter__(), call().read(),
+                          call().__exit__(None, None, None)]
+        self.assertEqual(mock.mock_calls, expected_calls)
+        self.assertIs(f, handle)
+
+
+    def test_explicit_mock(self):
+        mock = MagicMock()
+        mock_open(mock)
+
+        with patch('%s.open' % __name__, mock, create=True) as patched:
+            self.assertIs(patched, mock)
+            open('foo')
+
+        mock.assert_called_once_with('foo')
+
+
+    def test_read_data(self):
+        mock = mock_open(read_data='foo')
+        with patch('%s.open' % __name__, mock, create=True):
+            h = open('bar')
+            result = h.read()
+
+        self.assertEqual(result, 'foo')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py
index eb14c1d8b4..b712ebb5cb 100644
--- a/Lib/urllib/error.py
+++ b/Lib/urllib/error.py
@@ -13,6 +13,9 @@ response.
 
 import urllib.response
 
+__all__ = ['URLError', 'HTTPError', 'ContentTooShortError']
+
+
 # do these error classes make sense?
 # make sure all of the IOError stuff is overridden.  we just want to be
 # subtypes.
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index eb45c7eac6..5ddec5f14e 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -89,14 +89,16 @@ import http.client
 import io
 import os
 import posixpath
-import random
 import re
 import socket
 import sys
 import time
 import collections
+import tempfile
+import contextlib
 import warnings
 
+
 from urllib.error import URLError, HTTPError, ContentTooShortError
 from urllib.parse import (
     urlparse, urlsplit, urljoin, unwrap, quote, unquote,
@@ -112,21 +114,40 @@ except ImportError:
 else:
     _have_ssl = True
 
+__all__ = [
+    # Classes
+    'Request', 'OpenerDirector', 'BaseHandler', 'HTTPDefaultErrorHandler',
+    'HTTPRedirectHandler', 'HTTPCookieProcessor', 'ProxyHandler',
+    'HTTPPasswordMgr', 'HTTPPasswordMgrWithDefaultRealm',
+    'AbstractBasicAuthHandler', 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler',
+    'AbstractDigestAuthHandler', 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler',
+    'HTTPHandler', 'FileHandler', 'FTPHandler', 'CacheFTPHandler',
+    'UnknownHandler', 'HTTPErrorProcessor',
+    # Functions
+    'urlopen', 'install_opener', 'build_opener',
+    'pathname2url', 'url2pathname', 'getproxies',
+    # Legacy interface
+    'urlretrieve', 'urlcleanup', 'URLopener', 'FancyURLopener',
+]
+
 # used in User-Agent header sent
 __version__ = sys.version[:3]
 
 _opener = None
 def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-            *, cafile=None, capath=None):
+            *, cafile=None, capath=None, cadefault=False):
     global _opener
-    if cafile or capath:
+    if cafile or capath or cadefault:
         if not _have_ssl:
             raise ValueError('SSL support not available')
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.options |= ssl.OP_NO_SSLv2
-        if cafile or capath:
+        if cafile or capath or cadefault:
             context.verify_mode = ssl.CERT_REQUIRED
-            context.load_verify_locations(cafile, capath)
+            if cafile or capath:
+                context.load_verify_locations(cafile, capath)
+            else:
+                context.set_default_verify_paths()
             check_hostname = True
         else:
             check_hostname = False
@@ -142,17 +163,78 @@ def install_opener(opener):
     global _opener
     _opener = opener
 
-# TODO(jhylton): Make this work with the same global opener.
-_urlopener = None
+_url_tempfiles = []
 def urlretrieve(url, filename=None, reporthook=None, data=None):
-    global _urlopener
-    if not _urlopener:
-        _urlopener = FancyURLopener()
-    return _urlopener.retrieve(url, filename, reporthook, data)
+    """
+    Retrieve a URL into a temporary location on disk.
+
+    Requires a URL argument. If a filename is passed, it is used as
+    the temporary file location. The reporthook argument should be
+    a callable that accepts a block number, a read size, and the
+    total file size of the URL target. The data argument should be
+    valid URL encoded data.
+
+    If a filename is passed and the URL points to a local resource,
+    the result is a copy from local file to new file.
+
+    Returns a tuple containing the path to the newly created
+    data file as well as the resulting HTTPMessage object.
+    """
+    url_type, path = splittype(url)
+
+    with contextlib.closing(urlopen(url, data)) as fp:
+        headers = fp.info()
+
+        # Just return the local path and the "headers" for file://
+        # URLs. No sense in performing a copy unless requested.
+        if url_type == "file" and not filename:
+            return os.path.normpath(path), headers
+
+        # Handle temporary file setup.
+        if filename:
+            tfp = open(filename, 'wb')
+        else:
+            tfp = tempfile.NamedTemporaryFile(delete=False)
+            filename = tfp.name
+            _url_tempfiles.append(filename)
+
+        with tfp:
+            result = filename, headers
+            bs = 1024*8
+            size = -1
+            read = 0
+            blocknum = 0
+            if "content-length" in headers:
+                size = int(headers["Content-Length"])
+
+            if reporthook:
+                reporthook(blocknum, bs, size)
+
+            while True:
+                block = fp.read(bs)
+                if not block:
+                    break
+                read += len(block)
+                tfp.write(block)
+                blocknum += 1
+                if reporthook:
+                    reporthook(blocknum, bs, size)
+
+    if size >= 0 and read < size:
+        raise ContentTooShortError(
+            "retrieval incomplete: got only %i out of %i bytes"
+            % (read, size), result)
+
+    return result
 
 def urlcleanup():
-    if _urlopener:
-        _urlopener.cleanup()
+    for temp_file in _url_tempfiles:
+        try:
+            os.unlink(temp_file)
+        except EnvironmentError:
+            pass
+
+    del _url_tempfiles[:]
     global _opener
     if _opener:
         _opener = None
@@ -178,7 +260,8 @@ def request_host(request):
 class Request:
 
     def __init__(self, url, data=None, headers={},
-                 origin_req_host=None, unverifiable=False):
+                 origin_req_host=None, unverifiable=False,
+                 method=None):
         # unwrap('<URL:type://host/path>') --> 'type://host/path'
         self.full_url = unwrap(url)
         self.full_url, self.fragment = splittag(self.full_url)
@@ -192,6 +275,7 @@ class Request:
             origin_req_host = request_host(self)
         self.origin_req_host = origin_req_host
         self.unverifiable = unverifiable
+        self.method = method
         self._parse()
 
     def _parse(self):
@@ -203,41 +287,60 @@ class Request:
             self.host = unquote(self.host)
 
     def get_method(self):
-        if self.data is not None:
+        """Return a string indicating the HTTP request method."""
+        if self.method is not None:
+            return self.method
+        elif self.data is not None:
             return "POST"
         else:
             return "GET"
 
+    def get_full_url(self):
+        if self.fragment:
+            return '%s#%s' % (self.full_url, self.fragment)
+        else:
+            return self.full_url
+
     # Begin deprecated methods
 
     def add_data(self, data):
+        msg = "Request.add_data method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         self.data = data
 
     def has_data(self):
+        msg = "Request.has_data method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         return self.data is not None
 
     def get_data(self):
+        msg = "Request.get_data method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         return self.data
 
-    def get_full_url(self):
-        if self.fragment:
-            return '%s#%s' % (self.full_url, self.fragment)
-        else:
-            return self.full_url
-
     def get_type(self):
+        msg = "Request.get_type method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         return self.type
 
     def get_host(self):
+        msg = "Request.get_host method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         return self.host
 
     def get_selector(self):
+        msg = "Request.get_selector method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         return self.selector
 
     def is_unverifiable(self):
+        msg = "Request.is_unverifiable method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         return self.unverifiable
 
     def get_origin_req_host(self):
+        msg = "Request.get_origin_req_host method is deprecated."
+        warnings.warn(msg, DeprecationWarning, stacklevel=1)
         return self.origin_req_host
 
     # End deprecated methods
@@ -682,8 +785,8 @@ class ProxyHandler(BaseHandler):
         self.proxies = proxies
         for type, url in proxies.items():
             setattr(self, '%s_open' % type,
-                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
-                    meth(r, proxy, type))
+                    lambda r, proxy=url, type=type, meth=self.proxy_open:
+                        meth(r, proxy, type))
 
     def proxy_open(self, req, proxy, type):
         orig_type = req.type
@@ -825,17 +928,23 @@ class AbstractBasicAuthHandler:
             self.retried += 1
 
         if authreq:
-            mo = AbstractBasicAuthHandler.rx.search(authreq)
-            if mo:
-                scheme, quote, realm = mo.groups()
-                if quote not in ["'", '"']:
-                    warnings.warn("Basic Auth Realm was unquoted",
-                                  UserWarning, 2)
-                if scheme.lower() == 'basic':
-                    response = self.retry_http_basic_auth(host, req, realm)
-                    if response and response.code != 401:
-                        self.retried = 0
-                    return response
+            scheme = authreq.split()[0]
+            if scheme.lower() != 'basic':
+                raise ValueError("AbstractBasicAuthHandler does not"
+                                 " support the following scheme: '%s'" %
+                                 scheme)
+            else:
+                mo = AbstractBasicAuthHandler.rx.search(authreq)
+                if mo:
+                    scheme, quote, realm = mo.groups()
+                    if quote not in ['"',"'"]:
+                        warnings.warn("Basic Auth Realm was unquoted",
+                                      UserWarning, 2)
+                    if scheme.lower() == 'basic':
+                        response = self.retry_http_basic_auth(host, req, realm)
+                        if response and response.code != 401:
+                            self.retried = 0
+                        return response
 
     def retry_http_basic_auth(self, host, req, realm):
         user, pw = self.passwd.find_user_password(realm, host)
@@ -878,9 +987,9 @@ class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
         return response
 
 
-def randombytes(n):
-    """Return n random bytes."""
-    return os.urandom(n)
+# Return n random bytes.
+_randombytes = os.urandom
+
 
 class AbstractDigestAuthHandler:
     # Digest authentication is specified in RFC 2617.
@@ -921,6 +1030,9 @@ class AbstractDigestAuthHandler:
             scheme = authreq.split()[0]
             if scheme.lower() == 'digest':
                 return self.retry_http_digest_auth(req, authreq)
+            elif scheme.lower() != 'basic':
+                raise ValueError("AbstractDigestAuthHandler does not support"
+                                 " the following scheme: '%s'" % scheme)
 
     def retry_http_digest_auth(self, req, auth):
         token, challenge = auth.split(' ', 1)
@@ -941,7 +1053,7 @@ class AbstractDigestAuthHandler:
         # authentication, and to provide some message integrity protection.
         # This isn't a fabulous effort, but it's probably Good Enough.
         s = "%s:%s:%s:" % (self.nonce_count, nonce, time.ctime())
-        b = s.encode("ascii") + randombytes(8)
+        b = s.encode("ascii") + _randombytes(8)
         dig = hashlib.sha1(b).hexdigest()
         return dig[:16]
 
@@ -1066,7 +1178,7 @@ class AbstractHTTPHandler(BaseHandler):
         if request.data is not None:  # POST
             data = request.data
             if isinstance(data, str):
-                msg = "POST data should be bytes or an iterable of bytes. "\
+                msg = "POST data should be bytes or an iterable of bytes. " \
                       "It cannot be of type str."
                 raise TypeError(msg)
             if not request.has_header('Content-type'):
@@ -1162,7 +1274,6 @@ class HTTPHandler(AbstractHTTPHandler):
     http_request = AbstractHTTPHandler.do_request_
 
 if hasattr(http.client, 'HTTPSConnection'):
-    import ssl
 
     class HTTPSHandler(AbstractHTTPHandler):
 
@@ -1177,6 +1288,8 @@ if hasattr(http.client, 'HTTPSConnection'):
 
         https_request = AbstractHTTPHandler.do_request_
 
+    __all__.append('HTTPSHandler')
+
 class HTTPCookieProcessor(BaseHandler):
     def __init__(self, cookiejar=None):
         import http.cookiejar
@@ -1463,6 +1576,9 @@ class URLopener:
 
     # Constructor
     def __init__(self, proxies=None, **x509):
+        msg = "%(class)s style of invoking requests is deprecated. " \
+              "Use newer urlopen functions/methods" % {'class': self.__class__.__name__}
+        warnings.warn(msg, DeprecationWarning, stacklevel=3)
         if proxies is None:
             proxies = getproxies()
         assert hasattr(proxies, 'keys'), "proxies must be a mapping"
@@ -1542,6 +1658,8 @@ class URLopener:
                 return getattr(self, name)(url)
             else:
                 return getattr(self, name)(url, data)
+        except HTTPError:
+            raise
         except socket.error as msg:
             raise IOError('socket error', msg).with_traceback(sys.exc_info()[2])
 
@@ -1760,8 +1878,8 @@ class URLopener:
 
     def open_local_file(self, url):
         """Use local file."""
-        import mimetypes, email.utils
-        from io import StringIO
+        import email.utils
+        import mimetypes
         host, file = splithost(url)
         localname = url2pathname(file)
         try:
@@ -1877,7 +1995,7 @@ class URLopener:
         msg.append('Content-type: %s' % type)
         if encoding == 'base64':
             # XXX is this encoding/decoding ok?
-            data = base64.decodebytes(data.encode('ascii')).decode('latin1')
+            data = base64.decodebytes(data.encode('ascii')).decode('latin-1')
         else:
             data = unquote(data)
         msg.append('Content-Length: %d' % len(data))
@@ -1970,7 +2088,6 @@ class FancyURLopener(URLopener):
             URLopener.http_error_default(self, url, fp,
                                          errcode, errmsg, headers)
         stuff = headers['www-authenticate']
-        import re
         match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
         if not match:
             URLopener.http_error_default(self, url, fp,
@@ -1996,7 +2113,6 @@ class FancyURLopener(URLopener):
             URLopener.http_error_default(self, url, fp,
                                          errcode, errmsg, headers)
         stuff = headers['proxy-authenticate']
-        import re
         match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
         if not match:
             URLopener.http_error_default(self, url, fp,
@@ -2282,8 +2398,6 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings):
       'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.1', '10.0/16']
     }
     """
-    import re
-    import socket
     from fnmatch import fnmatch
 
     hostonly, port = splitport(host)
@@ -2386,7 +2500,6 @@ elif os.name == 'nt':
                     for p in proxyServer.split(';'):
                         protocol, address = p.split('=', 1)
                         # See if address has a type:// prefix
-                        import re
                         if not re.match('^([^/:]+)://', address):
                             address = '%s://%s' % (protocol, address)
                         proxies[protocol] = address
@@ -2418,7 +2531,6 @@ elif os.name == 'nt':
     def proxy_bypass_registry(host):
         try:
             import winreg
-            import re
         except ImportError:
             # Std modules, so should be around - but you never know!
             return 0
diff --git a/Lib/urllib/response.py b/Lib/urllib/response.py
index ffaa5fa6fc..1cf1d1aaed 100644
--- a/Lib/urllib/response.py
+++ b/Lib/urllib/response.py
@@ -37,12 +37,15 @@ class addbase(object):
                                              id(self), self.fp)
 
     def close(self):
+        if self.fp:
+            self.fp.close()
+        self.fp = None
         self.read = None
         self.readline = None
         self.readlines = None
         self.fileno = None
-        if self.fp: self.fp.close()
-        self.fp = None
+        self.__iter__ = None
+        self.__next__ = None
 
     def __enter__(self):
         if self.fp is None:
diff --git a/Lib/uuid.py b/Lib/uuid.py
index 5684ad7ace..0df0743c4c 100644
--- a/Lib/uuid.py
+++ b/Lib/uuid.py
@@ -440,7 +440,7 @@ try:
     import sys
     if sys.platform == 'darwin':
         import os
-        if int(os.uname()[2].split('.')[0]) >= 9:
+        if int(os.uname().release.split('.')[0]) >= 9:
             _uuid_generate_random = _uuid_generate_time = None
 
     # On Windows prior to 2000, UuidCreate gives a UUID containing the
diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py
new file mode 100644
index 0000000000..3920747931
--- /dev/null
+++ b/Lib/venv/__init__.py
@@ -0,0 +1,407 @@
+"""
+Virtual environment (venv) package for Python. Based on PEP 405.
+
+Copyright (C) 2011-2012 Vinay Sajip.
+Licensed to the PSF under a contributor agreement.
+
+usage: python -m venv [-h] [--system-site-packages] [--symlinks] [--clear]
+            [--upgrade]
+            ENV_DIR [ENV_DIR ...]
+
+Creates virtual Python environments in one or more target directories.
+
+positional arguments:
+  ENV_DIR               A directory to create the environment in.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --system-site-packages
+                        Give the virtual environment access to the system
+                        site-packages dir.
+  --symlinks            Attempt to symlink rather than copy.
+  --clear               Delete the environment directory if it already exists.
+                        If not specified and the directory exists, an error is
+                        raised.
+  --upgrade             Upgrade the environment directory to use this version
+                        of Python, assuming Python has been upgraded in-place.
+"""
+import base64
+import io
+import logging
+import os
+import os.path
+import shutil
+import sys
+import sysconfig
+try:
+    import threading
+except ImportError:
+    threading = None
+
+logger = logging.getLogger(__name__)
+
+class Context:
+    """
+    Holds information about a current venv creation/upgrade request.
+    """
+    pass
+
+
+class EnvBuilder:
+    """
+    This class exists to allow virtual environment creation to be
+    customised. The constructor parameters determine the builder's
+    behaviour when called upon to create a virtual environment.
+
+    By default, the builder makes the system (global) site-packages dir
+    *un*available to the created environment.
+
+    If invoked using the Python -m option, the default is to use copying
+    on Windows platforms but symlinks elsewhere. If instantiated some
+    other way, the default is to *not* use symlinks.
+
+    :param system_site_packages: If True, the system (global) site-packages
+                                 dir is available to created environments.
+    :param clear: If True and the target directory exists, it is deleted.
+                  Otherwise, if the target directory exists, an error is
+                  raised.
+    :param symlinks: If True, attempt to symlink rather than copy files into
+                     virtual environment.
+    :param upgrade: If True, upgrade an existing virtual environment.
+    """
+
+    def __init__(self, system_site_packages=False, clear=False,
+                 symlinks=False, upgrade=False):
+        self.system_site_packages = system_site_packages
+        self.clear = clear
+        self.symlinks = symlinks
+        self.upgrade = upgrade
+
+    def create(self, env_dir):
+        """
+        Create a virtual environment in a directory.
+
+        :param env_dir: The target directory to create an environment in.
+
+        """
+        env_dir = os.path.abspath(env_dir)
+        context = self.ensure_directories(env_dir)
+        self.create_configuration(context)
+        self.setup_python(context)
+        if not self.upgrade:
+            self.setup_scripts(context)
+            self.post_setup(context)
+
+    def ensure_directories(self, env_dir):
+        """
+        Create the directories for the environment.
+
+        Returns a context object which holds paths in the environment,
+        for use by subsequent logic.
+        """
+
+        def create_if_needed(d):
+            if not os.path.exists(d):
+                os.makedirs(d)
+
+        if os.path.exists(env_dir) and not (self.clear or self.upgrade):
+            raise ValueError('Directory exists: %s' % env_dir)
+        if os.path.exists(env_dir) and self.clear:
+            shutil.rmtree(env_dir)
+        context = Context()
+        context.env_dir = env_dir
+        context.env_name = os.path.split(env_dir)[1]
+        context.prompt = '(%s) ' % context.env_name
+        create_if_needed(env_dir)
+        env = os.environ
+        if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
+            executable = os.environ['__PYVENV_LAUNCHER__']
+        else:
+            executable = sys.executable
+        dirname, exename = os.path.split(os.path.abspath(executable))
+        context.executable = executable
+        context.python_dir = dirname
+        context.python_exe = exename
+        if sys.platform == 'win32':
+            binname = 'Scripts'
+            incpath = 'Include'
+            libpath = os.path.join(env_dir, 'Lib', 'site-packages')
+        else:
+            binname = 'bin'
+            incpath = 'include'
+            libpath = os.path.join(env_dir, 'lib', 'python%d.%d' % sys.version_info[:2], 'site-packages')
+        context.inc_path = path = os.path.join(env_dir, incpath)
+        create_if_needed(path)
+        create_if_needed(libpath)
+        context.bin_path = binpath = os.path.join(env_dir, binname)
+        context.bin_name = binname
+        context.env_exe = os.path.join(binpath, exename)
+        create_if_needed(binpath)
+        return context
+
+    def create_configuration(self, context):
+        """
+        Create a configuration file indicating where the environment's Python
+        was copied from, and whether the system site-packages should be made
+        available in the environment.
+
+        :param context: The information for the environment creation request
+                        being processed.
+        """
+        context.cfg_path = path = os.path.join(context.env_dir, 'pyvenv.cfg')
+        with open(path, 'w', encoding='utf-8') as f:
+            f.write('home = %s\n' % context.python_dir)
+            if self.system_site_packages:
+                incl = 'true'
+            else:
+                incl = 'false'
+            f.write('include-system-site-packages = %s\n' % incl)
+            f.write('version = %d.%d.%d\n' % sys.version_info[:3])
+
+    if os.name == 'nt':
+        def include_binary(self, f):
+            if f.endswith(('.pyd', '.dll')):
+                result = True
+            else:
+                result = f.startswith('python') and f.endswith('.exe')
+            return result
+
+    def symlink_or_copy(self, src, dst):
+        """
+        Try symlinking a file, and if that fails, fall back to copying.
+        """
+        force_copy = not self.symlinks
+        if not force_copy:
+            try:
+                if not os.path.islink(dst): # can't link to itself!
+                    os.symlink(src, dst)
+            except Exception:   # may need to use a more specific exception
+                logger.warning('Unable to symlink %r to %r', src, dst)
+                force_copy = True
+        if force_copy:
+            shutil.copyfile(src, dst)
+
+    def setup_python(self, context):
+        """
+        Set up a Python executable in the environment.
+
+        :param context: The information for the environment creation request
+                        being processed.
+        """
+        binpath = context.bin_path
+        exename = context.python_exe
+        path = context.env_exe
+        copier = self.symlink_or_copy
+        copier(context.executable, path)
+        dirname = context.python_dir
+        if os.name != 'nt':
+            if not os.path.islink(path):
+                os.chmod(path, 0o755)
+            for suffix in ('python', 'python3'):
+                path = os.path.join(binpath, suffix)
+                if not os.path.exists(path):
+                    os.symlink(exename, path)
+        else:
+            subdir = 'DLLs'
+            include = self.include_binary
+            files = [f for f in os.listdir(dirname) if include(f)]
+            for f in files:
+                src = os.path.join(dirname, f)
+                dst = os.path.join(binpath, f)
+                if dst != context.env_exe:  # already done, above
+                    copier(src, dst)
+            dirname = os.path.join(dirname, subdir)
+            if os.path.isdir(dirname):
+                files = [f for f in os.listdir(dirname) if include(f)]
+                for f in files:
+                    src = os.path.join(dirname, f)
+                    dst = os.path.join(binpath, f)
+                    copier(src, dst)
+            # copy init.tcl over
+            for root, dirs, files in os.walk(context.python_dir):
+                if 'init.tcl' in files:
+                    tcldir = os.path.basename(root)
+                    tcldir = os.path.join(context.env_dir, 'Lib', tcldir)
+                    os.makedirs(tcldir)
+                    src = os.path.join(root, 'init.tcl')
+                    dst = os.path.join(tcldir, 'init.tcl')
+                    shutil.copyfile(src, dst)
+                    break
+
+    def setup_scripts(self, context):
+        """
+        Set up scripts into the created environment from a directory.
+
+        This method installs the default scripts into the environment
+        being created. You can prevent the default installation by overriding
+        this method if you really need to, or if you need to specify
+        a different location for the scripts to install. By default, the
+        'scripts' directory in the venv package is used as the source of
+        scripts to install.
+        """
+        path = os.path.abspath(os.path.dirname(__file__))
+        path = os.path.join(path, 'scripts')
+        self.install_scripts(context, path)
+
+    def post_setup(self, context):
+        """
+        Hook for post-setup modification of the venv. Subclasses may install
+        additional packages or scripts here, add activation shell scripts, etc.
+
+        :param context: The information for the environment creation request
+                        being processed.
+        """
+        pass
+
+    def replace_variables(self, text, context):
+        """
+        Replace variable placeholders in script text with context-specific
+        variables.
+
+        Return the text passed in , but with variables replaced.
+
+        :param text: The text in which to replace placeholder variables.
+        :param context: The information for the environment creation request
+                        being processed.
+        """
+        text = text.replace('__VENV_DIR__', context.env_dir)
+        text = text.replace('__VENV_NAME__', context.prompt)
+        text = text.replace('__VENV_BIN_NAME__', context.bin_name)
+        text = text.replace('__VENV_PYTHON__', context.env_exe)
+        return text
+
+    def install_scripts(self, context, path):
+        """
+        Install scripts into the created environment from a directory.
+
+        :param context: The information for the environment creation request
+                        being processed.
+        :param path:    Absolute pathname of a directory containing script.
+                        Scripts in the 'common' subdirectory of this directory,
+                        and those in the directory named for the platform
+                        being run on, are installed in the created environment.
+                        Placeholder variables are replaced with environment-
+                        specific values.
+        """
+        binpath = context.bin_path
+        plen = len(path)
+        for root, dirs, files in os.walk(path):
+            if root == path: # at top-level, remove irrelevant dirs
+                for d in dirs[:]:
+                    if d not in ('common', os.name):
+                        dirs.remove(d)
+                continue # ignore files in top level
+            for f in files:
+                srcfile = os.path.join(root, f)
+                suffix = root[plen:].split(os.sep)[2:]
+                if not suffix:
+                    dstdir = binpath
+                else:
+                    dstdir = os.path.join(binpath, *suffix)
+                if not os.path.exists(dstdir):
+                    os.makedirs(dstdir)
+                dstfile = os.path.join(dstdir, f)
+                with open(srcfile, 'rb') as f:
+                    data = f.read()
+                if srcfile.endswith('.exe'):
+                    mode = 'wb'
+                else:
+                    mode = 'w'
+                    try:
+                        data = data.decode('utf-8')
+                        data = self.replace_variables(data, context)
+                    except UnicodeDecodeError as e:
+                        data = None
+                        logger.warning('unable to copy script %r, '
+                                       'may be binary: %s', srcfile, e)
+                if data is not None:
+                    with open(dstfile, mode) as f:
+                        f.write(data)
+                    shutil.copymode(srcfile, dstfile)
+
+
+def create(env_dir, system_site_packages=False, clear=False, symlinks=False):
+    """
+    Create a virtual environment in a directory.
+
+    By default, makes the system (global) site-packages dir *un*available to
+    the created environment, and uses copying rather than symlinking for files
+    obtained from the source Python installation.
+
+    :param env_dir: The target directory to create an environment in.
+    :param system_site_packages: If True, the system (global) site-packages
+                                 dir is available to the environment.
+    :param clear: If True and the target directory exists, it is deleted.
+                  Otherwise, if the target directory exists, an error is
+                  raised.
+    :param symlinks: If True, attempt to symlink rather than copy files into
+                     virtual environment.
+    """
+    builder = EnvBuilder(system_site_packages=system_site_packages,
+                                   clear=clear, symlinks=symlinks)
+    builder.create(env_dir)
+
+def main(args=None):
+    compatible = True
+    if sys.version_info < (3, 3):
+        compatible = False
+    elif not hasattr(sys, 'base_prefix'):
+        compatible = False
+    if not compatible:
+        raise ValueError('This script is only for use with Python 3.3')
+    else:
+        import argparse
+
+        parser = argparse.ArgumentParser(prog=__name__,
+                                         description='Creates virtual Python '
+                                                     'environments in one or '
+                                                     'more target '
+                                                     'directories.',
+                                         epilog='Once an environment has been '
+                                                'created, you may wish to '
+                                                'activate it, e.g. by '
+                                                'sourcing an activate script '
+                                                'in its bin directory.')
+        parser.add_argument('dirs', metavar='ENV_DIR', nargs='+',
+                            help='A directory to create the environment in.')
+        parser.add_argument('--system-site-packages', default=False,
+                            action='store_true', dest='system_site',
+                            help='Give the virtual environment access to the '
+                                 'system site-packages dir.')
+        if os.name == 'nt':
+            use_symlinks = False
+        else:
+            use_symlinks = True
+        parser.add_argument('--symlinks', default=use_symlinks,
+                            action='store_true', dest='symlinks',
+                            help='Try to use symlinks rather than copies, '
+                                 'when symlinks are not the default for '
+                                 'the platform.')
+        parser.add_argument('--clear', default=False, action='store_true',
+                            dest='clear', help='Delete the environment '
+                                               'directory if it already '
+                                               'exists. If not specified and '
+                                               'the directory exists, an error'
+                                               ' is raised.')
+        parser.add_argument('--upgrade', default=False, action='store_true',
+                            dest='upgrade', help='Upgrade the environment '
+                                               'directory to use this version '
+                                               'of Python, assuming Python '
+                                               'has been upgraded in-place.')
+        options = parser.parse_args(args)
+        if options.upgrade and options.clear:
+            raise ValueError('you cannot supply --upgrade and --clear together.')
+        builder = EnvBuilder(system_site_packages=options.system_site,
+                             clear=options.clear, symlinks=options.symlinks,
+                             upgrade=options.upgrade)
+        for d in options.dirs:
+            builder.create(d)
+
+if __name__ == '__main__':
+    rc = 1
+    try:
+        main()
+        rc = 0
+    except Exception as e:
+        print('Error: %s' % e, file=sys.stderr)
+    sys.exit(rc)
diff --git a/Lib/venv/__main__.py b/Lib/venv/__main__.py
new file mode 100644
index 0000000000..912423e4a7
--- /dev/null
+++ b/Lib/venv/__main__.py
@@ -0,0 +1,10 @@
+import sys
+from . import main
+
+rc = 1
+try:
+    main()
+    rc = 0
+except Exception as e:
+    print('Error: %s' % e, file=sys.stderr)
+sys.exit(rc)
diff --git a/Lib/venv/scripts/nt/Activate.ps1 b/Lib/venv/scripts/nt/Activate.ps1
new file mode 100644
index 0000000000..1c5ef98726
--- /dev/null
+++ b/Lib/venv/scripts/nt/Activate.ps1
@@ -0,0 +1,34 @@
+$env:VIRTUAL_ENV="__VENV_DIR__"

+

+# Revert to original values

+if (Test-Path function:_OLD_VIRTUAL_PROMPT) {

+    copy-item function:_OLD_VIRTUAL_PROMPT function:prompt

+    remove-item function:_OLD_VIRTUAL_PROMPT

+}

+

+if (Test-Path env:_OLD_VIRTUAL_PYTHONHOME) {

+    copy-item env:_OLD_VIRTUAL_PYTHONHOME env:PYTHONHOME

+    remove-item env:_OLD_VIRTUAL_PYTHONHOME

+}

+

+if (Test-Path env:_OLD_VIRTUAL_PATH) {

+    copy-item env:_OLD_VIRTUAL_PATH env:PATH

+    remove-item env:_OLD_VIRTUAL_PATH

+}

+

+# Set the prompt to include the env name

+copy-item function:prompt function:_OLD_VIRTUAL_PROMPT

+function prompt {

+    Write-Host -NoNewline -ForegroundColor Green '[__VENV_NAME__]'

+    _OLD_VIRTUAL_PROMPT

+}

+

+# Clear PYTHONHOME

+if (Test-Path env:PYTHONHOME) {

+    copy-item env:PYTHONHOME env:_OLD_VIRTUAL_PYTHONHOME

+    remove-item env:PYTHONHOME

+}

+

+# Add the venv to the PATH

+copy-item env:PATH env:_OLD_VIRTUAL_PATH

+$env:PATH = "$env:VIRTUAL_ENV\__VENV_BIN_NAME__;$env:PATH"

diff --git a/Lib/venv/scripts/nt/Deactivate.ps1 b/Lib/venv/scripts/nt/Deactivate.ps1
new file mode 100644
index 0000000000..3d1e96bc8c
--- /dev/null
+++ b/Lib/venv/scripts/nt/Deactivate.ps1
@@ -0,0 +1,19 @@
+# Revert to original values

+if (Test-Path function:_OLD_VIRTUAL_PROMPT) {

+    copy-item function:_OLD_VIRTUAL_PROMPT function:prompt

+    remove-item function:_OLD_VIRTUAL_PROMPT

+}

+

+if (Test-Path env:_OLD_VIRTUAL_PYTHONHOME) {

+    copy-item env:_OLD_VIRTUAL_PYTHONHOME env:PYTHONHOME

+    remove-item env:_OLD_VIRTUAL_PYTHONHOME

+}

+

+if (Test-Path env:_OLD_VIRTUAL_PATH) {

+    copy-item env:_OLD_VIRTUAL_PATH env:PATH

+    remove-item env:_OLD_VIRTUAL_PATH

+}

+

+if (Test-Path env:VIRTUAL_ENV) {

+    remove-item env:VIRTUAL_ENV

+}

diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat
new file mode 100644
index 0000000000..c45e65a26f
--- /dev/null
+++ b/Lib/venv/scripts/nt/activate.bat
@@ -0,0 +1,31 @@
+@echo off

+set VIRTUAL_ENV=__VENV_DIR__

+

+if not defined PROMPT (

+    set PROMPT=$P$G

+)

+

+if defined _OLD_VIRTUAL_PROMPT (

+    set PROMPT=%_OLD_VIRTUAL_PROMPT%

+)

+

+if defined _OLD_VIRTUAL_PYTHONHOME (

+     set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%

+)

+

+set _OLD_VIRTUAL_PROMPT=%PROMPT%

+set PROMPT=__VENV_NAME__%PROMPT%

+

+if defined PYTHONHOME (

+     set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%

+     set PYTHONHOME=

+)

+

+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%; goto SKIPPATH

+

+set _OLD_VIRTUAL_PATH=%PATH%

+

+:SKIPPATH

+set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%

+

+:END

diff --git a/Lib/venv/scripts/nt/deactivate.bat b/Lib/venv/scripts/nt/deactivate.bat
new file mode 100644
index 0000000000..62da5b1308
--- /dev/null
+++ b/Lib/venv/scripts/nt/deactivate.bat
@@ -0,0 +1,17 @@
+@echo off

+

+if defined _OLD_VIRTUAL_PROMPT (

+    set PROMPT=%_OLD_VIRTUAL_PROMPT%

+)

+set _OLD_VIRTUAL_PROMPT=

+

+if defined _OLD_VIRTUAL_PYTHONHOME (

+     set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%

+     set _OLD_VIRTUAL_PYTHONHOME=

+)

+

+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%

+

+set _OLD_VIRTUAL_PATH=

+

+:END

diff --git a/Lib/venv/scripts/nt/pydoc.py b/Lib/venv/scripts/nt/pydoc.py
new file mode 100644
index 0000000000..dbf2db71b4
--- /dev/null
+++ b/Lib/venv/scripts/nt/pydoc.py
@@ -0,0 +1,4 @@
+#!__VENV_PYTHON__
+if __name__ == '__main__':
+    import sys, pydoc
+    sys.exit(pydoc.cli())
diff --git a/Lib/venv/scripts/posix/activate b/Lib/venv/scripts/posix/activate
new file mode 100644
index 0000000000..c241450c0b
--- /dev/null
+++ b/Lib/venv/scripts/posix/activate
@@ -0,0 +1,76 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+deactivate () {
+    # reset old environment variables
+    if [ -n "$_OLD_VIRTUAL_PATH" ] ; then
+        PATH="$_OLD_VIRTUAL_PATH"
+        export PATH
+        unset _OLD_VIRTUAL_PATH
+    fi
+    if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then
+        PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
+        export PYTHONHOME
+        unset _OLD_VIRTUAL_PYTHONHOME
+    fi
+
+    # This should detect bash and zsh, which have a hash command that must
+    # be called to get it to forget past commands.  Without forgetting
+    # past commands the $PATH changes we made may not be respected
+    if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then
+        hash -r
+    fi
+
+    if [ -n "$_OLD_VIRTUAL_PS1" ] ; then
+        PS1="$_OLD_VIRTUAL_PS1"
+        export PS1
+        unset _OLD_VIRTUAL_PS1
+    fi
+
+    unset VIRTUAL_ENV
+    if [ ! "$1" = "nondestructive" ] ; then
+    # Self destruct!
+        unset -f deactivate
+    fi
+}
+
+# unset irrelavent variables
+deactivate nondestructive
+
+VIRTUAL_ENV="__VENV_DIR__"
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH"
+export PATH
+
+# unset PYTHONHOME if set
+# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
+# could use `if (set -u; : $PYTHONHOME) ;` in bash
+if [ -n "$PYTHONHOME" ] ; then
+    _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
+    unset PYTHONHOME
+fi
+
+if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then
+    _OLD_VIRTUAL_PS1="$PS1"
+    if [ "x__VENV_NAME__" != x ] ; then
+	PS1="__VENV_NAME__$PS1"
+    else
+    if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
+        # special case for Aspen magic directories
+        # see http://www.zetadev.com/software/aspen/
+        PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
+    else
+        PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
+    fi
+    fi
+    export PS1
+fi
+
+# This should detect bash and zsh, which have a hash command that must
+# be called to get it to forget past commands.  Without forgetting
+# past commands the $PATH changes we made may not be respected
+if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then
+    hash -r
+fi
diff --git a/Lib/venv/scripts/posix/pydoc b/Lib/venv/scripts/posix/pydoc
new file mode 100755
index 0000000000..9233771128
--- /dev/null
+++ b/Lib/venv/scripts/posix/pydoc
@@ -0,0 +1,5 @@
+#!__VENV_PYTHON__
+if __name__ == '__main__':
+    import sys, pydoc
+    sys.exit(pydoc.cli())
+
diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py
index 202f34a1a2..94d4ad42e6 100644
--- a/Lib/webbrowser.py
+++ b/Lib/webbrowser.py
@@ -230,7 +230,7 @@ class UnixBrowser(BaseBrowser):
         cmdline = [self.name] + raise_opt + args
 
         if remote or self.background:
-            inout = io.open(os.devnull, "r+")
+            inout = subprocess.DEVNULL
         else:
             # for TTY browsers, we need stdin/out
             inout = None
@@ -238,17 +238,14 @@ class UnixBrowser(BaseBrowser):
                              stdout=(self.redirect_stdout and inout or None),
                              stderr=inout, start_new_session=True)
         if remote:
-            # wait five seconds. If the subprocess is not finished, the
+            # wait at most five seconds. If the subprocess is not finished, the
             # remote invocation has (hopefully) started a new instance.
-            time.sleep(1)
-            rc = p.poll()
-            if rc is None:
-                time.sleep(4)
-                rc = p.poll()
-                if rc is None:
-                    return True
-            # if remote call failed, open() will try direct invocation
-            return not rc
+            try:
+                rc = p.wait(5)
+                # if remote call failed, open() will try direct invocation
+                return not rc
+            except subprocess.TimeoutExpired:
+                return True
         elif self.background:
             if p.poll() is None:
                 return True
@@ -306,6 +303,18 @@ class Galeon(UnixBrowser):
     background = True
 
 
+class Chrome(UnixBrowser):
+    "Launcher class for Google Chrome browser."
+
+    remote_args = ['%action', '%s']
+    remote_action = ""
+    remote_action_newwin = "--new-window"
+    remote_action_newtab = ""
+    background = True
+
+Chromium = Chrome
+
+
 class Opera(UnixBrowser):
     "Launcher class for Opera browser."
 
@@ -345,7 +354,7 @@ class Konqueror(BaseBrowser):
         else:
             action = "openURL"
 
-        devnull = io.open(os.devnull, "r+")
+        devnull = subprocess.DEVNULL
         # if possible, put browser in separate process group, so
         # keyboard interrupts don't affect browser as well as Python
         setsid = getattr(os, 'setsid', None)
@@ -443,6 +452,14 @@ class Grail(BaseBrowser):
 
 def register_X_browsers():
 
+    # use xdg-open if around
+    if _iscommand("xdg-open"):
+        register("xdg-open", None, BackgroundBrowser("xdg-open"))
+
+    # The default GNOME3 browser
+    if "GNOME_DESKTOP_SESSION_ID" in os.environ and _iscommand("gvfs-open"):
+        register("gvfs-open", None, BackgroundBrowser("gvfs-open"))
+
     # The default GNOME browser
     if "GNOME_DESKTOP_SESSION_ID" in os.environ and _iscommand("gnome-open"):
         register("gnome-open", None, BackgroundBrowser("gnome-open"))
@@ -473,6 +490,11 @@ def register_X_browsers():
     if _iscommand("skipstone"):
         register("skipstone", None, BackgroundBrowser("skipstone"))
 
+    # Google Chrome/Chromium browsers
+    for browser in ("google-chrome", "chrome", "chromium", "chromium-browser"):
+        if _iscommand(browser):
+            register(browser, None, Chrome(browser))
+
     # Opera, quite popular
     if _iscommand("opera"):
         register("opera", None, Opera("opera"))
diff --git a/Lib/wsgiref.egg-info b/Lib/wsgiref.egg-info
deleted file mode 100644
index c0b7893c34..0000000000
--- a/Lib/wsgiref.egg-info
+++ /dev/null
@@ -1,8 +0,0 @@
-Metadata-Version: 1.0
-Name: wsgiref
-Version: 0.1.2
-Summary: WSGI (PEP 333) Reference Library
-Author: Phillip J. Eby
-Author-email: web-sig@python.org
-License: PSF or ZPL
-Platform: UNKNOWN
diff --git a/Lib/wsgiref/simple_server.py b/Lib/wsgiref/simple_server.py
index af82f953c5..a6015fb5ef 100644
--- a/Lib/wsgiref/simple_server.py
+++ b/Lib/wsgiref/simple_server.py
@@ -14,13 +14,14 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 import sys
 import urllib.parse
 from wsgiref.handlers import SimpleHandler
+from platform import python_implementation
 
 __version__ = "0.2"
 __all__ = ['WSGIServer', 'WSGIRequestHandler', 'demo_app', 'make_server']
 
 
 server_version = "WSGIServer/" + __version__
-sys_version = "Python/" + sys.version.split()[0]
+sys_version = python_implementation() + "/" + sys.version.split()[0]
 software_version = server_version + ' ' + sys_version
 
 
diff --git a/Lib/xdrlib.py b/Lib/xdrlib.py
index 29639159cb..c05cf87e4a 100644
--- a/Lib/xdrlib.py
+++ b/Lib/xdrlib.py
@@ -141,11 +141,7 @@ class Unpacker:
         data = self.__buf[i:j]
         if len(data) < 4:
             raise EOFError
-        x = struct.unpack('>L', data)[0]
-        try:
-            return int(x)
-        except OverflowError:
-            return x
+        return struct.unpack('>L', data)[0]
 
     def unpack_int(self):
         i = self.__pos
diff --git a/Lib/xml/dom/__init__.py b/Lib/xml/dom/__init__.py
index 4401bdfcf3..97cf9a6429 100644
--- a/Lib/xml/dom/__init__.py
+++ b/Lib/xml/dom/__init__.py
@@ -17,6 +17,7 @@ pulldom -- DOM builder supporting on-demand tree-building for selected
 
 class Node:
     """Class giving the NodeType constants."""
+    __slots__ = ()
 
     # DOM implementations may use this as a base class for their own
     # Node implementations.  If they don't, the constants defined here
diff --git a/Lib/xml/dom/domreg.py b/Lib/xml/dom/domreg.py
index cb35bb0596..8c3d901acb 100644
--- a/Lib/xml/dom/domreg.py
+++ b/Lib/xml/dom/domreg.py
@@ -2,8 +2,6 @@
 directly. Instead, the functions getDOMImplementation and
 registerDOMImplementation should be imported from xml.dom."""
 
-from xml.dom.minicompat import *  # isinstance, StringTypes
-
 # This is a list of well-known implementations.  Well-known names
 # should be published by posting to xml-sig@python.org, and are
 # subsequently recorded in this file.
diff --git a/Lib/xml/dom/expatbuilder.py b/Lib/xml/dom/expatbuilder.py
index a98fe03235..f074ab932f 100644
--- a/Lib/xml/dom/expatbuilder.py
+++ b/Lib/xml/dom/expatbuilder.py
@@ -33,8 +33,6 @@ from xml.parsers import expat
 from xml.dom.minidom import _append_child, _set_attribute_node
 from xml.dom.NodeFilter import NodeFilter
 
-from xml.dom.minicompat import *
-
 TEXT_NODE = Node.TEXT_NODE
 CDATA_SECTION_NODE = Node.CDATA_SECTION_NODE
 DOCUMENT_NODE = Node.DOCUMENT_NODE
@@ -283,27 +281,23 @@ class ExpatBuilder:
         elif childNodes and childNodes[-1].nodeType == TEXT_NODE:
             node = childNodes[-1]
             value = node.data + data
-            d = node.__dict__
-            d['data'] = d['nodeValue'] = value
+            node.data = value
             return
         else:
             node = minidom.Text()
-            d = node.__dict__
-            d['data'] = d['nodeValue'] = data
-            d['ownerDocument'] = self.document
+            node.data = data
+            node.ownerDocument = self.document
         _append_child(self.curNode, node)
 
     def character_data_handler(self, data):
         childNodes = self.curNode.childNodes
         if childNodes and childNodes[-1].nodeType == TEXT_NODE:
             node = childNodes[-1]
-            d = node.__dict__
-            d['data'] = d['nodeValue'] = node.data + data
+            node.data = node.data + data
             return
         node = minidom.Text()
-        d = node.__dict__
-        d['data'] = d['nodeValue'] = node.data + data
-        d['ownerDocument'] = self.document
+        node.data = node.data + data
+        node.ownerDocument = self.document
         _append_child(self.curNode, node)
 
     def entity_decl_handler(self, entityName, is_parameter_entity, value,
@@ -363,11 +357,8 @@ class ExpatBuilder:
                 a = minidom.Attr(attributes[i], EMPTY_NAMESPACE,
                                  None, EMPTY_PREFIX)
                 value = attributes[i+1]
-                d = a.childNodes[0].__dict__
-                d['data'] = d['nodeValue'] = value
-                d = a.__dict__
-                d['value'] = d['nodeValue'] = value
-                d['ownerDocument'] = self.document
+                a.value = value
+                a.ownerDocument = self.document
                 _set_attribute_node(node, a)
 
         if node is not self.document.documentElement:
@@ -761,15 +752,13 @@ class Namespaces:
                 else:
                     a = minidom.Attr("xmlns", XMLNS_NAMESPACE,
                                      "xmlns", EMPTY_PREFIX)
-                d = a.childNodes[0].__dict__
-                d['data'] = d['nodeValue'] = uri
-                d = a.__dict__
-                d['value'] = d['nodeValue'] = uri
-                d['ownerDocument'] = self.document
+                a.value = uri
+                a.ownerDocument = self.document
                 _set_attribute_node(node, a)
             del self._ns_ordered_prefixes[:]
 
         if attributes:
+            node._ensure_attributes()
             _attrs = node._attrs
             _attrsNS = node._attrsNS
             for i in range(0, len(attributes), 2):
@@ -785,12 +774,9 @@ class Namespaces:
                                      aname, EMPTY_PREFIX)
                     _attrs[aname] = a
                     _attrsNS[(EMPTY_NAMESPACE, aname)] = a
-                d = a.childNodes[0].__dict__
-                d['data'] = d['nodeValue'] = value
-                d = a.__dict__
-                d['ownerDocument'] = self.document
-                d['value'] = d['nodeValue'] = value
-                d['ownerElement'] = node
+                a.ownerDocument = self.document
+                a.value = value
+                a.ownerElement = node
 
     if __debug__:
         # This only adds some asserts to the original
diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py
index 1870ebd8a2..55c4ce11c2 100644
--- a/Lib/xml/dom/minidom.py
+++ b/Lib/xml/dom/minidom.py
@@ -15,7 +15,6 @@ Todo:
  * SAX 2 namespaces
 """
 
-import codecs
 import io
 import xml.dom
 
@@ -48,25 +47,25 @@ class Node(xml.dom.Node):
         return self.toprettyxml("", "", encoding)
 
     def toprettyxml(self, indent="\t", newl="\n", encoding=None):
-        # indent = the indentation string to prepend, per level
-        # newl = the newline string to append
-        use_encoding = "utf-8" if encoding is None else encoding
-        writer = codecs.getwriter(use_encoding)(io.BytesIO())
+        if encoding is None:
+            writer = io.StringIO()
+        else:
+            writer = io.TextIOWrapper(io.BytesIO(),
+                                      encoding=encoding,
+                                      errors="xmlcharrefreplace",
+                                      newline='\n')
         if self.nodeType == Node.DOCUMENT_NODE:
             # Can pass encoding only to document, to put it into XML header
             self.writexml(writer, "", indent, newl, encoding)
         else:
             self.writexml(writer, "", indent, newl)
         if encoding is None:
-            return writer.stream.getvalue().decode(use_encoding)
+            return writer.getvalue()
         else:
-            return writer.stream.getvalue()
+            return writer.detach().getvalue()
 
     def hasChildNodes(self):
-        if self.childNodes:
-            return True
-        else:
-            return False
+        return bool(self.childNodes)
 
     def _get_childNodes(self):
         return self.childNodes
@@ -287,10 +286,10 @@ def _append_child(self, node):
     childNodes = self.childNodes
     if childNodes:
         last = childNodes[-1]
-        node.__dict__["previousSibling"] = last
-        last.__dict__["nextSibling"] = node
+        node.previousSibling = last
+        last.nextSibling = node
     childNodes.append(node)
-    node.__dict__["parentNode"] = self
+    node.parentNode = self
 
 def _in_document(node):
     # return True iff node is part of a document tree
@@ -343,9 +342,10 @@ class DocumentFragment(Node):
 
 
 class Attr(Node):
+    __slots__=('_name', '_value', 'namespaceURI',
+               '_prefix', 'childNodes', '_localName', 'ownerDocument', 'ownerElement')
     nodeType = Node.ATTRIBUTE_NODE
     attributes = None
-    ownerElement = None
     specified = False
     _is_id = False
 
@@ -353,12 +353,11 @@ class Attr(Node):
 
     def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None,
                  prefix=None):
-        # skip setattr for performance
-        d = self.__dict__
-        d["nodeName"] = d["name"] = qName
-        d["namespaceURI"] = namespaceURI
-        d["prefix"] = prefix
-        d['childNodes'] = NodeList()
+        self.ownerElement = None
+        self._name = qName
+        self.namespaceURI = namespaceURI
+        self._prefix = prefix
+        self.childNodes = NodeList()
 
         # Add the single child node that represents the value of the attr
         self.childNodes.append(Text())
@@ -366,9 +365,10 @@ class Attr(Node):
         # nodeValue and value are set elsewhere
 
     def _get_localName(self):
-        if 'localName' in self.__dict__:
-            return self.__dict__['localName']
-        return self.nodeName.split(":", 1)[-1]
+        try:
+            return self._localName
+        except AttributeError:
+            return self.nodeName.split(":", 1)[-1]
 
     def _get_name(self):
         return self.name
@@ -376,20 +376,30 @@ class Attr(Node):
     def _get_specified(self):
         return self.specified
 
-    def __setattr__(self, name, value):
-        d = self.__dict__
-        if name in ("value", "nodeValue"):
-            d["value"] = d["nodeValue"] = value
-            d2 = self.childNodes[0].__dict__
-            d2["data"] = d2["nodeValue"] = value
-            if self.ownerElement is not None:
-                _clear_id_cache(self.ownerElement)
-        elif name in ("name", "nodeName"):
-            d["name"] = d["nodeName"] = value
-            if self.ownerElement is not None:
-                _clear_id_cache(self.ownerElement)
-        else:
-            d[name] = value
+    def _get_name(self):
+        return self._name
+
+    def _set_name(self, value):
+        self._name = value
+        if self.ownerElement is not None:
+            _clear_id_cache(self.ownerElement)
+
+    nodeName = name = property(_get_name, _set_name)
+
+    def _get_value(self):
+        return self._value
+
+    def _set_value(self, value):
+        self._value = value
+        self.childNodes[0].data = value
+        if self.ownerElement is not None:
+            _clear_id_cache(self.ownerElement)
+        self.childNodes[0].data = value
+
+    nodeValue = value = property(_get_value, _set_value)
+
+    def _get_prefix(self):
+        return self._prefix
 
     def _set_prefix(self, prefix):
         nsuri = self.namespaceURI
@@ -397,22 +407,16 @@ class Attr(Node):
             if nsuri and nsuri != XMLNS_NAMESPACE:
                 raise xml.dom.NamespaceErr(
                     "illegal use of 'xmlns' prefix for the wrong namespace")
-        d = self.__dict__
-        d['prefix'] = prefix
+        self._prefix = prefix
         if prefix is None:
             newName = self.localName
         else:
             newName = "%s:%s" % (prefix, self.localName)
         if self.ownerElement:
             _clear_id_cache(self.ownerElement)
-        d['nodeName'] = d['name'] = newName
+        self.name = newName
 
-    def _set_value(self, value):
-        d = self.__dict__
-        d['value'] = d['nodeValue'] = value
-        if self.ownerElement:
-            _clear_id_cache(self.ownerElement)
-        self.childNodes[0].data = value
+    prefix = property(_get_prefix, _set_prefix)
 
     def unlink(self):
         # This implementation does not call the base implementation
@@ -587,8 +591,8 @@ class NamedNodeMap(object):
             _clear_id_cache(self._ownerElement)
             del self._attrs[n.nodeName]
             del self._attrsNS[(n.namespaceURI, n.localName)]
-            if 'ownerElement' in n.__dict__:
-                n.__dict__['ownerElement'] = None
+            if hasattr(n, 'ownerElement'):
+                n.ownerElement = None
             return n
         else:
             raise xml.dom.NotFoundErr()
@@ -599,8 +603,8 @@ class NamedNodeMap(object):
             _clear_id_cache(self._ownerElement)
             del self._attrsNS[(n.namespaceURI, n.localName)]
             del self._attrs[n.nodeName]
-            if 'ownerElement' in n.__dict__:
-                n.__dict__['ownerElement'] = None
+            if hasattr(n, 'ownerElement'):
+                n.ownerElement = None
             return n
         else:
             raise xml.dom.NotFoundErr()
@@ -660,6 +664,9 @@ class TypeInfo(object):
 _no_type = TypeInfo(None, None)
 
 class Element(Node):
+    __slots__=('ownerDocument', 'parentNode', 'tagName', 'nodeName', 'prefix',
+               'namespaceURI', '_localName', 'childNodes', '_attrs', '_attrsNS',
+               'nextSibling', 'previousSibling')
     nodeType = Node.ELEMENT_NODE
     nodeValue = None
     schemaType = _no_type
@@ -675,41 +682,57 @@ class Element(Node):
 
     def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None,
                  localName=None):
+        self.parentNode = None
         self.tagName = self.nodeName = tagName
         self.prefix = prefix
         self.namespaceURI = namespaceURI
         self.childNodes = NodeList()
+        self.nextSibling = self.previousSibling = None
+
+        # Attribute dictionaries are lazily created
+        # attributes are double-indexed:
+        #    tagName -> Attribute
+        #    URI,localName -> Attribute
+        # in the future: consider lazy generation
+        # of attribute objects this is too tricky
+        # for now because of headaches with
+        # namespaces.
+        self._attrs = None
+        self._attrsNS = None
 
-        self._attrs = {}   # attributes are double-indexed:
-        self._attrsNS = {} #    tagName -> Attribute
-                           #    URI,localName -> Attribute
-                           # in the future: consider lazy generation
-                           # of attribute objects this is too tricky
-                           # for now because of headaches with
-                           # namespaces.
+    def _ensure_attributes(self):
+        if self._attrs is None:
+            self._attrs = {}
+            self._attrsNS = {}
 
     def _get_localName(self):
-        if 'localName' in self.__dict__:
-            return self.__dict__['localName']
-        return self.tagName.split(":", 1)[-1]
+        try:
+            return self._localName
+        except AttributeError:
+            return self.tagName.split(":", 1)[-1]
 
     def _get_tagName(self):
         return self.tagName
 
     def unlink(self):
-        for attr in list(self._attrs.values()):
-            attr.unlink()
+        if self._attrs is not None:
+            for attr in list(self._attrs.values()):
+                attr.unlink()
         self._attrs = None
         self._attrsNS = None
         Node.unlink(self)
 
     def getAttribute(self, attname):
+        if self._attrs is None:
+            return ""
         try:
             return self._attrs[attname].value
         except KeyError:
             return ""
 
     def getAttributeNS(self, namespaceURI, localName):
+        if self._attrsNS is None:
+            return ""
         try:
             return self._attrsNS[(namespaceURI, localName)].value
         except KeyError:
@@ -719,14 +742,11 @@ class Element(Node):
         attr = self.getAttributeNode(attname)
         if attr is None:
             attr = Attr(attname)
-            # for performance
-            d = attr.__dict__
-            d["value"] = d["nodeValue"] = value
-            d["ownerDocument"] = self.ownerDocument
+            attr.value = value # also sets nodeValue
+            attr.ownerDocument = self.ownerDocument
             self.setAttributeNode(attr)
         elif value != attr.value:
-            d = attr.__dict__
-            d["value"] = d["nodeValue"] = value
+            attr.value = value
             if attr.isId:
                 _clear_id_cache(self)
 
@@ -734,33 +754,33 @@ class Element(Node):
         prefix, localname = _nssplit(qualifiedName)
         attr = self.getAttributeNodeNS(namespaceURI, localname)
         if attr is None:
-            # for performance
             attr = Attr(qualifiedName, namespaceURI, localname, prefix)
-            d = attr.__dict__
-            d["prefix"] = prefix
-            d["nodeName"] = qualifiedName
-            d["value"] = d["nodeValue"] = value
-            d["ownerDocument"] = self.ownerDocument
+            attr.value = value
+            attr.ownerDocument = self.ownerDocument
             self.setAttributeNode(attr)
         else:
-            d = attr.__dict__
             if value != attr.value:
-                d["value"] = d["nodeValue"] = value
+                attr.value = value
                 if attr.isId:
                     _clear_id_cache(self)
             if attr.prefix != prefix:
-                d["prefix"] = prefix
-                d["nodeName"] = qualifiedName
+                attr.prefix = prefix
+                attr.nodeName = qualifiedName
 
     def getAttributeNode(self, attrname):
+        if self._attrs is None:
+            return None
         return self._attrs.get(attrname)
 
     def getAttributeNodeNS(self, namespaceURI, localName):
+        if self._attrsNS is None:
+            return None
         return self._attrsNS.get((namespaceURI, localName))
 
     def setAttributeNode(self, attr):
         if attr.ownerElement not in (None, self):
             raise xml.dom.InuseAttributeErr("attribute node already owned")
+        self._ensure_attributes()
         old1 = self._attrs.get(attr.name, None)
         if old1 is not None:
             self.removeAttributeNode(old1)
@@ -779,6 +799,8 @@ class Element(Node):
     setAttributeNodeNS = setAttributeNode
 
     def removeAttribute(self, name):
+        if self._attrsNS is None:
+            raise xml.dom.NotFoundErr()
         try:
             attr = self._attrs[name]
         except KeyError:
@@ -786,6 +808,8 @@ class Element(Node):
         self.removeAttributeNode(attr)
 
     def removeAttributeNS(self, namespaceURI, localName):
+        if self._attrsNS is None:
+            raise xml.dom.NotFoundErr()
         try:
             attr = self._attrsNS[(namespaceURI, localName)]
         except KeyError:
@@ -808,9 +832,13 @@ class Element(Node):
     removeAttributeNodeNS = removeAttributeNode
 
     def hasAttribute(self, name):
+        if self._attrs is None:
+            return False
         return name in self._attrs
 
     def hasAttributeNS(self, namespaceURI, localName):
+        if self._attrsNS is None:
+            return False
         return (namespaceURI, localName) in self._attrsNS
 
     def getElementsByTagName(self, name):
@@ -851,6 +879,7 @@ class Element(Node):
             writer.write("/>%s"%(newl))
 
     def _get_attributes(self):
+        self._ensure_attributes()
         return NamedNodeMap(self._attrs, self._attrsNS, self)
 
     def hasAttributes(self):
@@ -875,7 +904,7 @@ class Element(Node):
         if _get_containing_entref(self) is not None:
             raise xml.dom.NoModificationAllowedErr()
         if not idAttr._is_id:
-            idAttr.__dict__['_is_id'] = True
+            idAttr._is_id = True
             self._magic_id_nodes += 1
             self.ownerDocument._magic_id_count += 1
             _clear_id_cache(self)
@@ -888,19 +917,20 @@ defproperty(Element, "localName",
 
 def _set_attribute_node(element, attr):
     _clear_id_cache(element)
+    element._ensure_attributes()
     element._attrs[attr.name] = attr
     element._attrsNS[(attr.namespaceURI, attr.localName)] = attr
 
     # This creates a circular reference, but Element.unlink()
     # breaks the cycle since the references to the attribute
     # dictionaries are tossed.
-    attr.__dict__['ownerElement'] = element
-
+    attr.ownerElement = element
 
 class Childless:
     """Mixin that makes childless-ness easy to implement and avoids
     the complexity of the Node methods that deal with children.
     """
+    __slots__ = ()
 
     attributes = None
     childNodes = EmptyNodeList()
@@ -939,54 +969,49 @@ class Childless:
 
 class ProcessingInstruction(Childless, Node):
     nodeType = Node.PROCESSING_INSTRUCTION_NODE
+    __slots__ = ('target', 'data')
 
     def __init__(self, target, data):
-        self.target = self.nodeName = target
-        self.data = self.nodeValue = data
+        self.target = target
+        self.data = data
 
-    def _get_data(self):
+    # nodeValue is an alias for data
+    def _get_nodeValue(self):
         return self.data
-    def _set_data(self, value):
-        d = self.__dict__
-        d['data'] = d['nodeValue'] = value
+    def _set_nodeValue(self, value):
+        self.data = data
+    nodeValue = property(_get_nodeValue, _set_nodeValue)
 
-    def _get_target(self):
+    # nodeName is an alias for target
+    def _get_nodeName(self):
         return self.target
-    def _set_target(self, value):
-        d = self.__dict__
-        d['target'] = d['nodeName'] = value
-
-    def __setattr__(self, name, value):
-        if name == "data" or name == "nodeValue":
-            self.__dict__['data'] = self.__dict__['nodeValue'] = value
-        elif name == "target" or name == "nodeName":
-            self.__dict__['target'] = self.__dict__['nodeName'] = value
-        else:
-            self.__dict__[name] = value
+    def _set_nodeName(self, value):
+        self.target = value
+    nodeName = property(_get_nodeName, _set_nodeName)
 
     def writexml(self, writer, indent="", addindent="", newl=""):
         writer.write("%s<?%s %s?>%s" % (indent,self.target, self.data, newl))
 
 
 class CharacterData(Childless, Node):
+    __slots__=('_data', 'ownerDocument','parentNode', 'previousSibling', 'nextSibling')
+
+    def __init__(self):
+        self.ownerDocument = self.parentNode = None
+        self.previousSibling = self.nextSibling = None
+        self._data = ''
+        Node.__init__(self)
+
     def _get_length(self):
         return len(self.data)
     __len__ = _get_length
 
     def _get_data(self):
-        return self.__dict__['data']
+        return self._data
     def _set_data(self, data):
-        d = self.__dict__
-        d['data'] = d['nodeValue'] = data
-
-    _get_nodeValue = _get_data
-    _set_nodeValue = _set_data
+        self._data = data
 
-    def __setattr__(self, name, value):
-        if name == "data" or name == "nodeValue":
-            self.__dict__['data'] = self.__dict__['nodeValue'] = value
-        else:
-            self.__dict__[name] = value
+    data = nodeValue = property(_get_data, _set_data)
 
     def __repr__(self):
         data = self.data
@@ -1043,10 +1068,7 @@ defproperty(CharacterData, "length", doc="Length of the string data.")
 
 
 class Text(CharacterData):
-    # Make sure we don't add an instance __dict__ if we don't already
-    # have one, at least when that's possible:
-    # XXX this does not work, CharacterData is an old-style class
-    # __slots__ = ()
+    __slots__ = ()
 
     nodeType = Node.TEXT_NODE
     nodeName = "#text"
@@ -1113,9 +1135,7 @@ class Text(CharacterData):
             else:
                 break
         if content:
-            d = self.__dict__
-            d['data'] = content
-            d['nodeValue'] = content
+            self.data = content
             return self
         else:
             return None
@@ -1161,7 +1181,8 @@ class Comment(CharacterData):
     nodeName = "#comment"
 
     def __init__(self, data):
-        self.data = self.nodeValue = data
+        CharacterData.__init__(self)
+        self._data = data
 
     def writexml(self, writer, indent="", addindent="", newl=""):
         if "--" in self.data:
@@ -1170,10 +1191,7 @@ class Comment(CharacterData):
 
 
 class CDATASection(Text):
-    # Make sure we don't add an instance __dict__ if we don't already
-    # have one, at least when that's possible:
-    # XXX this does not work, Text is an old-style class
-    # __slots__ = ()
+    __slots__ = ()
 
     nodeType = Node.CDATA_SECTION_NODE
     nodeName = "#cdata-section"
@@ -1253,8 +1271,7 @@ defproperty(ReadOnlySequentialNamedNodeMap, "length",
 class Identified:
     """Mix-in class that supports the publicId and systemId attributes."""
 
-    # XXX this does not work, this is an old-style class
-    # __slots__ = 'publicId', 'systemId'
+    __slots__ = 'publicId', 'systemId'
 
     def _identified_mixin_init(self, publicId, systemId):
         self.publicId = publicId
@@ -1505,18 +1522,19 @@ def _clear_id_cache(node):
         node.ownerDocument._id_search_stack= None
 
 class Document(Node, DocumentLS):
+    __slots__ = ('_elem_info', 'doctype',
+                 '_id_search_stack', 'childNodes', '_id_cache')
     _child_node_types = (Node.ELEMENT_NODE, Node.PROCESSING_INSTRUCTION_NODE,
                          Node.COMMENT_NODE, Node.DOCUMENT_TYPE_NODE)
 
+    implementation = DOMImplementation()
     nodeType = Node.DOCUMENT_NODE
     nodeName = "#document"
     nodeValue = None
     attributes = None
-    doctype = None
     parentNode = None
     previousSibling = nextSibling = None
 
-    implementation = DOMImplementation()
 
     # Document attributes from Level 3 (WD 9 April 2002)
 
@@ -1531,6 +1549,7 @@ class Document(Node, DocumentLS):
     _magic_id_count = 0
 
     def __init__(self):
+        self.doctype = None
         self.childNodes = NodeList()
         # mapping of (namespaceURI, localName) -> ElementInfo
         #        and tagName -> ElementInfo
@@ -1772,12 +1791,12 @@ class Document(Node, DocumentLS):
             raise xml.dom.NotSupportedErr("cannot import document type nodes")
         return _clone_node(node, deep, self)
 
-    def writexml(self, writer, indent="", addindent="", newl="",
-                 encoding = None):
+    def writexml(self, writer, indent="", addindent="", newl="", encoding=None):
         if encoding is None:
             writer.write('<?xml version="1.0" ?>'+newl)
         else:
-            writer.write('<?xml version="1.0" encoding="%s"?>%s' % (encoding, newl))
+            writer.write('<?xml version="1.0" encoding="%s"?>%s' % (
+                encoding, newl))
         for node in self.childNodes:
             node.writexml(writer, indent, addindent, newl)
 
@@ -1816,17 +1835,15 @@ class Document(Node, DocumentLS):
                 element.removeAttributeNode(n)
         else:
             element = None
-        # avoid __setattr__
-        d = n.__dict__
-        d['prefix'] = prefix
-        d['localName'] = localName
-        d['namespaceURI'] = namespaceURI
-        d['nodeName'] = name
+        n.prefix = prefix
+        n._localName = localName
+        n.namespaceURI = namespaceURI
+        n.nodeName = name
         if n.nodeType == Node.ELEMENT_NODE:
-            d['tagName'] = name
+            n.tagName = name
         else:
             # attribute node
-            d['name'] = name
+            n.name = name
             if element is not None:
                 element.setAttributeNode(n)
                 if is_id:
diff --git a/Lib/xml/dom/pulldom.py b/Lib/xml/dom/pulldom.py
index d5ac8b2b98..43504f7656 100644
--- a/Lib/xml/dom/pulldom.py
+++ b/Lib/xml/dom/pulldom.py
@@ -1,6 +1,5 @@
 import xml.sax
 import xml.sax.handler
-import types
 
 START_ELEMENT = "START_ELEMENT"
 END_ELEMENT = "END_ELEMENT"
@@ -334,10 +333,7 @@ def parse(stream_or_string, parser=None, bufsize=None):
     return DOMEventStream(stream, parser, bufsize)
 
 def parseString(string, parser=None):
-    try:
-        from io import StringIO
-    except ImportError:
-        from io import StringIO
+    from io import StringIO
 
     bufsize = len(string)
     buf = StringIO(string)
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
index ff8ff7d78f..641d787dab 100644
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -68,8 +68,9 @@ __all__ = [
     "tostring", "tostringlist",
     "TreeBuilder",
     "VERSION",
-    "XML",
+    "XML", "XMLID",
     "XMLParser", "XMLTreeBuilder",
+    "register_namespace",
     ]
 
 VERSION = "1.3.0"
@@ -99,34 +100,11 @@ VERSION = "1.3.0"
 import sys
 import re
 import warnings
+import io
+import contextlib
 
+from . import ElementPath
 
-class _SimpleElementPath:
-    # emulate pre-1.2 find/findtext/findall behaviour
-    def find(self, element, tag, namespaces=None):
-        for elem in element:
-            if elem.tag == tag:
-                return elem
-        return None
-    def findtext(self, element, tag, default=None, namespaces=None):
-        elem = self.find(element, tag)
-        if elem is None:
-            return default
-        return elem.text or ""
-    def iterfind(self, element, tag, namespaces=None):
-        if tag[:3] == ".//":
-            for elem in element.iter(tag[3:]):
-                yield elem
-        for elem in element:
-            if elem.tag == tag:
-                yield elem
-    def findall(self, element, tag, namespaces=None):
-        return list(self.iterfind(element, tag, namespaces))
-
-try:
-    from . import ElementPath
-except ImportError:
-    ElementPath = _SimpleElementPath()
 
 ##
 # Parser error.  This is a subclass of <b>SyntaxError</b>.
@@ -148,9 +126,9 @@ class ParseError(SyntaxError):
 # @defreturn flag
 
 def iselement(element):
-    # FIXME: not sure about this; might be a better idea to look
-    # for tag/attrib/text attributes
-    return isinstance(element, Element) or hasattr(element, "tag")
+    # FIXME: not sure about this;
+    # isinstance(element, Element) or look for tag/attrib/text attributes
+    return hasattr(element, 'tag')
 
 ##
 # Element class.  This class defines the Element interface, and
@@ -205,6 +183,9 @@ class Element:
     # constructor
 
     def __init__(self, tag, attrib={}, **extra):
+        if not isinstance(attrib, dict):
+            raise TypeError("attrib must be dict, not %s" % (
+                attrib.__class__.__name__,))
         attrib = attrib.copy()
         attrib.update(extra)
         self.tag = tag
@@ -298,7 +279,7 @@ class Element:
     # @param element The element to add.
 
     def append(self, element):
-        # assert iselement(element)
+        self._assert_is_element(element)
         self._children.append(element)
 
     ##
@@ -308,8 +289,8 @@ class Element:
     # @since 1.3
 
     def extend(self, elements):
-        # for element in elements:
-        #     assert iselement(element)
+        for element in elements:
+            self._assert_is_element(element)
         self._children.extend(elements)
 
     ##
@@ -318,9 +299,15 @@ class Element:
     # @param index Where to insert the new subelement.
 
     def insert(self, index, element):
-        # assert iselement(element)
+        self._assert_is_element(element)
         self._children.insert(index, element)
 
+    def _assert_is_element(self, e):
+        # Need to refer to the actual Python implementation, not the
+        # shadowing C implementation.
+        if not isinstance(e, _Element):
+            raise TypeError('expected an Element, not %s' % type(e).__name__)
+
     ##
     # Removes a matching subelement.  Unlike the <b>find</b> methods,
     # this method compares elements based on identity, not on tag
@@ -810,59 +797,38 @@ class ElementTree:
     #     "c14n"; default is "xml").
 
     def write(self, file_or_filename,
-              # keyword arguments
               encoding=None,
               xml_declaration=None,
               default_namespace=None,
               method=None):
-        # assert self._root is not None
         if not method:
             method = "xml"
         elif method not in _serialize:
-            # FIXME: raise an ImportError for c14n if ElementC14N is missing?
             raise ValueError("unknown method %r" % method)
         if not encoding:
             if method == "c14n":
                 encoding = "utf-8"
             else:
                 encoding = "us-ascii"
-        elif encoding == str:  # lxml.etree compatibility.
-            encoding = "unicode"
         else:
             encoding = encoding.lower()
-        if hasattr(file_or_filename, "write"):
-            file = file_or_filename
-        else:
-            if encoding != "unicode":
-                file = open(file_or_filename, "wb")
+        with _get_writer(file_or_filename, encoding) as write:
+            if method == "xml" and (xml_declaration or
+                    (xml_declaration is None and
+                     encoding not in ("utf-8", "us-ascii", "unicode"))):
+                declared_encoding = encoding
+                if encoding == "unicode":
+                    # Retrieve the default encoding for the xml declaration
+                    import locale
+                    declared_encoding = locale.getpreferredencoding()
+                write("<?xml version='1.0' encoding='%s'?>\n" % (
+                    declared_encoding,))
+            if method == "text":
+                _serialize_text(write, self._root)
             else:
-                file = open(file_or_filename, "w")
-        if encoding != "unicode":
-            def write(text):
-                try:
-                    return file.write(text.encode(encoding,
-                                                  "xmlcharrefreplace"))
-                except (TypeError, AttributeError):
-                    _raise_serialization_error(text)
-        else:
-            write = file.write
-        if method == "xml" and (xml_declaration or
-                (xml_declaration is None and
-                 encoding not in ("utf-8", "us-ascii", "unicode"))):
-            declared_encoding = encoding
-            if encoding == "unicode":
-                # Retrieve the default encoding for the xml declaration
-                import locale
-                declared_encoding = locale.getpreferredencoding()
-            write("<?xml version='1.0' encoding='%s'?>\n" % declared_encoding)
-        if method == "text":
-            _serialize_text(write, self._root)
-        else:
-            qnames, namespaces = _namespaces(self._root, default_namespace)
-            serialize = _serialize[method]
-            serialize(write, self._root, qnames, namespaces)
-        if file_or_filename is not file:
-            file.close()
+                qnames, namespaces = _namespaces(self._root, default_namespace)
+                serialize = _serialize[method]
+                serialize(write, self._root, qnames, namespaces)
 
     def write_c14n(self, file):
         # lxml.etree compatibility.  use output method instead
@@ -871,6 +837,58 @@ class ElementTree:
 # --------------------------------------------------------------------
 # serialization support
 
+@contextlib.contextmanager
+def _get_writer(file_or_filename, encoding):
+    # returns text write method and release all resourses after using
+    try:
+        write = file_or_filename.write
+    except AttributeError:
+        # file_or_filename is a file name
+        if encoding == "unicode":
+            file = open(file_or_filename, "w")
+        else:
+            file = open(file_or_filename, "w", encoding=encoding,
+                        errors="xmlcharrefreplace")
+        with file:
+            yield file.write
+    else:
+        # file_or_filename is a file-like object
+        # encoding determines if it is a text or binary writer
+        if encoding == "unicode":
+            # use a text writer as is
+            yield write
+        else:
+            # wrap a binary writer with TextIOWrapper
+            with contextlib.ExitStack() as stack:
+                if isinstance(file_or_filename, io.BufferedIOBase):
+                    file = file_or_filename
+                elif isinstance(file_or_filename, io.RawIOBase):
+                    file = io.BufferedWriter(file_or_filename)
+                    # Keep the original file open when the BufferedWriter is
+                    # destroyed
+                    stack.callback(file.detach)
+                else:
+                    # This is to handle passed objects that aren't in the
+                    # IOBase hierarchy, but just have a write method
+                    file = io.BufferedIOBase()
+                    file.writable = lambda: True
+                    file.write = write
+                    try:
+                        # TextIOWrapper uses this methods to determine
+                        # if BOM (for UTF-16, etc) should be added
+                        file.seekable = file_or_filename.seekable
+                        file.tell = file_or_filename.tell
+                    except AttributeError:
+                        pass
+                file = io.TextIOWrapper(file,
+                                        encoding=encoding,
+                                        errors="xmlcharrefreplace",
+                                        newline="\n")
+                # Keep the original file open when the TextIOWrapper is
+                # destroyed
+                stack.callback(file.detach)
+                yield file.write
+
 def _namespaces(elem, default_namespace=None):
     # identify namespaces used in this tree
 
@@ -910,11 +928,7 @@ def _namespaces(elem, default_namespace=None):
             _raise_serialization_error(qname)
 
     # populate qname and namespaces table
-    try:
-        iterate = elem.iter
-    except AttributeError:
-        iterate = elem.getiterator # cET compatibility
-    for elem in iterate():
+    for elem in elem.iter():
         tag = elem.tag
         if isinstance(tag, QName):
             if tag.text not in qnames:
@@ -1086,6 +1100,8 @@ _namespace_map = {
     # dublin core
     "http://purl.org/dc/elements/1.1/": "dc",
 }
+# For tests and troubleshooting
+register_namespace._namespace_map = _namespace_map
 
 def _raise_serialization_error(text):
     raise TypeError(
@@ -1154,22 +1170,13 @@ def _escape_attrib_html(text):
 # @defreturn string
 
 def tostring(element, encoding=None, method=None):
-    class dummy:
-        pass
-    data = []
-    file = dummy()
-    file.write = data.append
-    ElementTree(element).write(file, encoding, method=method)
-    if encoding in (str, "unicode"):
-        return "".join(data)
-    else:
-        return b"".join(data)
+    stream = io.StringIO() if encoding == 'unicode' else io.BytesIO()
+    ElementTree(element).write(stream, encoding, method=method)
+    return stream.getvalue()
 
 ##
 # Generates a string representation of an XML element, including all
-# subelements.  If encoding is False, the string is returned as a
-# sequence of string fragments; otherwise it is a sequence of
-# bytestrings.
+# subelements.
 #
 # @param element An Element instance.
 # @keyparam encoding Optional output encoding (default is US-ASCII).
@@ -1180,15 +1187,29 @@ def tostring(element, encoding=None, method=None):
 # @defreturn sequence
 # @since 1.3
 
+class _ListDataStream(io.BufferedIOBase):
+    """ An auxiliary stream accumulating into a list reference
+    """
+    def __init__(self, lst):
+        self.lst = lst
+
+    def writable(self):
+        return True
+
+    def seekable(self):
+        return True
+
+    def write(self, b):
+        self.lst.append(b)
+
+    def tell(self):
+        return len(self.lst)
+
 def tostringlist(element, encoding=None, method=None):
-    class dummy:
-        pass
-    data = []
-    file = dummy()
-    file.write = data.append
-    ElementTree(element).write(file, encoding, method=method)
-    # FIXME: merge small fragments into larger parts
-    return data
+    lst = []
+    stream = _ListDataStream(lst)
+    ElementTree(element).write(stream, encoding, method=method)
+    return lst
 
 ##
 # Writes an element tree or element structure to sys.stdout.  This
@@ -1510,24 +1531,30 @@ class XMLParser:
         self.target = self._target = target
         self._error = expat.error
         self._names = {} # name memo cache
-        # callbacks
+        # main callbacks
         parser.DefaultHandlerExpand = self._default
-        parser.StartElementHandler = self._start
-        parser.EndElementHandler = self._end
-        parser.CharacterDataHandler = self._data
-        # optional callbacks
-        parser.CommentHandler = self._comment
-        parser.ProcessingInstructionHandler = self._pi
+        if hasattr(target, 'start'):
+            parser.StartElementHandler = self._start
+        if hasattr(target, 'end'):
+            parser.EndElementHandler = self._end
+        if hasattr(target, 'data'):
+            parser.CharacterDataHandler = target.data
+        # miscellaneous callbacks
+        if hasattr(target, 'comment'):
+            parser.CommentHandler = target.comment
+        if hasattr(target, 'pi'):
+            parser.ProcessingInstructionHandler = target.pi
         # let expat do the buffering, if supported
         try:
-            self._parser.buffer_text = 1
+            parser.buffer_text = 1
         except AttributeError:
             pass
         # use new-style attribute handling, if supported
         try:
-            self._parser.ordered_attributes = 1
-            self._parser.specified_attributes = 1
-            parser.StartElementHandler = self._start_list
+            parser.ordered_attributes = 1
+            parser.specified_attributes = 1
+            if hasattr(target, 'start'):
+                parser.StartElementHandler = self._start_list
         except AttributeError:
             pass
         self._doctype = None
@@ -1571,44 +1598,29 @@ class XMLParser:
                 attrib[fixname(attrib_in[i])] = attrib_in[i+1]
         return self.target.start(tag, attrib)
 
-    def _data(self, text):
-        return self.target.data(text)
-
     def _end(self, tag):
         return self.target.end(self._fixname(tag))
 
-    def _comment(self, data):
-        try:
-            comment = self.target.comment
-        except AttributeError:
-            pass
-        else:
-            return comment(data)
-
-    def _pi(self, target, data):
-        try:
-            pi = self.target.pi
-        except AttributeError:
-            pass
-        else:
-            return pi(target, data)
-
     def _default(self, text):
         prefix = text[:1]
         if prefix == "&":
             # deal with undefined entities
             try:
-                self.target.data(self.entity[text[1:-1]])
+                data_handler = self.target.data
+            except AttributeError:
+                return
+            try:
+                data_handler(self.entity[text[1:-1]])
             except KeyError:
                 from xml.parsers import expat
                 err = expat.error(
                     "undefined entity %s: line %d, column %d" %
-                    (text, self._parser.ErrorLineNumber,
-                    self._parser.ErrorColumnNumber)
+                    (text, self.parser.ErrorLineNumber,
+                    self.parser.ErrorColumnNumber)
                     )
                 err.code = 11 # XML_ERROR_UNDEFINED_ENTITY
-                err.lineno = self._parser.ErrorLineNumber
-                err.offset = self._parser.ErrorColumnNumber
+                err.lineno = self.parser.ErrorLineNumber
+                err.offset = self.parser.ErrorColumnNumber
                 raise err
         elif prefix == "<" and text[:9] == "<!DOCTYPE":
             self._doctype = [] # inside a doctype declaration
@@ -1626,16 +1638,16 @@ class XMLParser:
                 type = self._doctype[1]
                 if type == "PUBLIC" and n == 4:
                     name, type, pubid, system = self._doctype
+                    if pubid:
+                        pubid = pubid[1:-1]
                 elif type == "SYSTEM" and n == 3:
                     name, type, system = self._doctype
                     pubid = None
                 else:
                     return
-                if pubid:
-                    pubid = pubid[1:-1]
                 if hasattr(self.target, "doctype"):
                     self.target.doctype(name, pubid, system[1:-1])
-                elif self.doctype is not self._XMLParser__doctype:
+                elif self.doctype != self._XMLParser__doctype:
                     # warn about deprecated call
                     self._XMLParser__doctype(name, pubid, system[1:-1])
                     self.doctype(name, pubid, system[1:-1])
@@ -1666,7 +1678,7 @@ class XMLParser:
 
     def feed(self, data):
         try:
-            self._parser.Parse(data, 0)
+            self.parser.Parse(data, 0)
         except self._error as v:
             self._raiseerror(v)
 
@@ -1678,12 +1690,113 @@ class XMLParser:
 
     def close(self):
         try:
-            self._parser.Parse("", 1) # end of data
+            self.parser.Parse("", 1) # end of data
         except self._error as v:
             self._raiseerror(v)
-        tree = self.target.close()
-        del self.target, self._parser # get rid of circular references
-        return tree
+        try:
+            close_handler = self.target.close
+        except AttributeError:
+            pass
+        else:
+            return close_handler()
+        finally:
+            # get rid of circular references
+            del self.parser, self._parser
+            del self.target, self._target
+
+
+# Import the C accelerators
+try:
+    # Element, SubElement, ParseError, TreeBuilder, XMLParser
+    from _elementtree import *
+except ImportError:
+    pass
+else:
+    # Overwrite 'ElementTree.parse' and 'iterparse' to use the C XMLParser
+
+    class ElementTree(ElementTree):
+        def parse(self, source, parser=None):
+            close_source = False
+            if not hasattr(source, 'read'):
+                source = open(source, 'rb')
+                close_source = True
+            try:
+                if parser is not None:
+                    while True:
+                        data = source.read(65536)
+                        if not data:
+                            break
+                        parser.feed(data)
+                    self._root = parser.close()
+                else:
+                    parser = XMLParser()
+                    self._root = parser._parse(source)
+                return self._root
+            finally:
+                if close_source:
+                    source.close()
+
+    class iterparse:
+        """Parses an XML section into an element tree incrementally.
+
+        Reports what’s going on to the user. 'source' is a filename or file
+        object containing XML data. 'events' is a list of events to report back.
+        The supported events are the strings "start", "end", "start-ns" and
+        "end-ns" (the "ns" events are used to get detailed namespace
+        information). If 'events' is omitted, only "end" events are reported.
+        'parser' is an optional parser instance. If not given, the standard
+        XMLParser parser is used. Returns an iterator providing
+        (event, elem) pairs.
+        """
+
+        root = None
+        def __init__(self, file, events=None, parser=None):
+            self._close_file = False
+            if not hasattr(file, 'read'):
+                file = open(file, 'rb')
+                self._close_file = True
+            self._file = file
+            self._events = []
+            self._index = 0
+            self._error = None
+            self.root = self._root = None
+            if parser is None:
+                parser = XMLParser(target=TreeBuilder())
+            self._parser = parser
+            self._parser._setevents(self._events, events)
+
+        def __next__(self):
+            while True:
+                try:
+                    item = self._events[self._index]
+                    self._index += 1
+                    return item
+                except IndexError:
+                    pass
+                if self._error:
+                    e = self._error
+                    self._error = None
+                    raise e
+                if self._parser is None:
+                    self.root = self._root
+                    if self._close_file:
+                        self._file.close()
+                    raise StopIteration
+                # load event buffer
+                del self._events[:]
+                self._index = 0
+                data = self._file.read(16384)
+                if data:
+                    try:
+                        self._parser.feed(data)
+                    except SyntaxError as exc:
+                        self._error = exc
+                else:
+                    self._root = self._parser.close()
+                    self._parser = None
+
+        def __iter__(self):
+            return self
 
 # compatibility
 XMLTreeBuilder = XMLParser
diff --git a/Lib/xml/etree/cElementTree.py b/Lib/xml/etree/cElementTree.py
index a6f127abd5..368e679189 100644
--- a/Lib/xml/etree/cElementTree.py
+++ b/Lib/xml/etree/cElementTree.py
@@ -1,3 +1,3 @@
-# Wrapper module for _elementtree
+# Deprecated alias for xml.etree.ElementTree
 
-from _elementtree import *
+from xml.etree.ElementTree import *
diff --git a/Lib/xml/parsers/expat.py b/Lib/xml/parsers/expat.py
index a805b828d8..bcbe9fb1f8 100644
--- a/Lib/xml/parsers/expat.py
+++ b/Lib/xml/parsers/expat.py
@@ -1,6 +1,4 @@
 """Interface to the Expat non-validating XML parser."""
-__version__ = '$Revision$'
-
 import sys
 
 from pyexpat import *
diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py
index ec8d8e92d6..d7b2db3725 100644
--- a/Lib/xmlrpc/client.py
+++ b/Lib/xmlrpc/client.py
@@ -128,8 +128,11 @@ Exported functions:
 """
 
 import base64
+import sys
 import time
+from datetime import datetime
 import http.client
+import urllib.parse
 from xml.parsers import expat
 import socket
 import errno
@@ -142,17 +145,13 @@ except ImportError:
 # --------------------------------------------------------------------
 # Internal stuff
 
-try:
-    import datetime
-except ImportError:
-    datetime = None
-
 def escape(s):
     s = s.replace("&", "&amp;")
     s = s.replace("<", "&lt;")
     return s.replace(">", "&gt;",)
 
-__version__ = "1.0.1"
+# used in User-Agent header sent
+__version__ = sys.version[:3]
 
 # xmlrpc integer limits
 MAXINT =  2**31-1
@@ -252,21 +251,33 @@ boolean = Boolean = bool
 # Wrapper for XML-RPC DateTime values.  This converts a time value to
 # the format used by XML-RPC.
 # <p>
-# The value can be given as a string in the format
-# "yyyymmddThh:mm:ss", as a 9-item time tuple (as returned by
+# The value can be given as a datetime object, as a string in the
+# format "yyyymmddThh:mm:ss", as a 9-item time tuple (as returned by
 # time.localtime()), or an integer value (as returned by time.time()).
 # The wrapper uses time.localtime() to convert an integer to a time
 # tuple.
 #
-# @param value The time, given as an ISO 8601 string, a time
-#              tuple, or a integer time value.
+# @param value The time, given as a datetime object, an ISO 8601 string,
+#              a time tuple, or an integer time value.
+
+
+# Issue #13305: different format codes across platforms
+_day0 = datetime(1, 1, 1)
+if _day0.strftime('%Y') == '0001':      # Mac OS X
+    def _iso8601_format(value):
+        return value.strftime("%Y%m%dT%H:%M:%S")
+elif _day0.strftime('%4Y') == '0001':   # Linux
+    def _iso8601_format(value):
+        return value.strftime("%4Y%m%dT%H:%M:%S")
+else:
+    def _iso8601_format(value):
+        return value.strftime("%Y%m%dT%H:%M:%S").zfill(17)
+del _day0
+
 
 def _strftime(value):
-    if datetime:
-        if isinstance(value, datetime.datetime):
-            return "%04d%02d%02dT%02d:%02d:%02d" % (
-                value.year, value.month, value.day,
-                value.hour, value.minute, value.second)
+    if isinstance(value, datetime):
+        return _iso8601_format(value)
 
     if not isinstance(value, (tuple, time.struct_time)):
         if value == 0:
@@ -291,9 +302,9 @@ class DateTime:
         if isinstance(other, DateTime):
             s = self.value
             o = other.value
-        elif datetime and isinstance(other, datetime.datetime):
+        elif isinstance(other, datetime):
             s = self.value
-            o = other.strftime("%Y%m%dT%H:%M:%S")
+            o = _iso8601_format(other)
         elif isinstance(other, str):
             s = self.value
             o = other
@@ -361,8 +372,7 @@ def _datetime(data):
     return value
 
 def _datetime_type(data):
-    t = time.strptime(data, "%Y%m%dT%H:%M:%S")
-    return datetime.datetime(*tuple(t)[:6])
+    return datetime.strptime(data, "%Y%m%dT%H:%M:%S")
 
 ##
 # Wrapper for binary data.  This can be used to transport any kind
@@ -377,8 +387,8 @@ class Binary:
         if data is None:
             data = b""
         else:
-            if not isinstance(data, bytes):
-                raise TypeError("expected bytes, not %s" %
+            if not isinstance(data, (bytes, bytearray)):
+                raise TypeError("expected bytes or bytearray, not %s" %
                                 data.__class__.__name__)
             data = bytes(data)  # Make a copy of the bytes!
         self.data = data
@@ -408,7 +418,6 @@ class Binary:
         out.write("<value><base64>\n")
         encoded = base64.encodebytes(self.data)
         out.write(encoded.decode('ascii'))
-        out.write('\n')
         out.write("</base64></value>\n")
 
 def _binary(data):
@@ -522,15 +531,6 @@ class Marshaller:
         write("<value><nil/></value>")
     dispatch[type(None)] = dump_nil
 
-    def dump_int(self, value, write):
-        # in case ints are > 32 bits
-        if value > MAXINT or value < MININT:
-            raise OverflowError("int exceeds XML-RPC limits")
-        write("<value><int>")
-        write(str(value))
-        write("</int></value>\n")
-    #dispatch[int] = dump_int
-
     def dump_bool(self, value, write):
         write("<value><boolean>")
         write(value and "1" or "0")
@@ -545,6 +545,9 @@ class Marshaller:
         write("</int></value>\n")
     dispatch[int] = dump_long
 
+    # backward compatible
+    dump_int = dump_long
+
     def dump_double(self, value, write):
         write("<value><double>")
         write(repr(value))
@@ -557,6 +560,14 @@ class Marshaller:
         write("</string></value>\n")
     dispatch[str] = dump_unicode
 
+    def dump_bytes(self, value, write):
+        write("<value><base64>\n")
+        encoded = base64.encodebytes(value)
+        write(encoded.decode('ascii'))
+        write("</base64></value>\n")
+    dispatch[bytes] = dump_bytes
+    dispatch[bytearray] = dump_bytes
+
     def dump_array(self, value, write):
         i = id(value)
         if i in self.memo:
@@ -589,12 +600,11 @@ class Marshaller:
         del self.memo[i]
     dispatch[dict] = dump_struct
 
-    if datetime:
-        def dump_datetime(self, value, write):
-            write("<value><dateTime.iso8601>")
-            write(_strftime(value))
-            write("</dateTime.iso8601></value>\n")
-        dispatch[datetime.datetime] = dump_datetime
+    def dump_datetime(self, value, write):
+        write("<value><dateTime.iso8601>")
+        write(_strftime(value))
+        write("</dateTime.iso8601></value>\n")
+    dispatch[datetime] = dump_datetime
 
     def dump_instance(self, value, write):
         # check for special wrappers
@@ -628,7 +638,7 @@ class Unmarshaller:
     # and again, if you don't understand what's going on in here,
     # that's perfectly ok.
 
-    def __init__(self, use_datetime=False):
+    def __init__(self, use_datetime=False, use_builtin_types=False):
         self._type = None
         self._stack = []
         self._marks = []
@@ -636,9 +646,8 @@ class Unmarshaller:
         self._methodname = None
         self._encoding = "utf-8"
         self.append = self._stack.append
-        self._use_datetime = use_datetime
-        if use_datetime and not datetime:
-            raise ValueError("the datetime module is not available")
+        self._use_datetime = use_builtin_types or use_datetime
+        self._use_bytes = use_builtin_types
 
     def close(self):
         # return response tuple and target method
@@ -750,6 +759,8 @@ class Unmarshaller:
     def end_base64(self, data):
         value = Binary()
         value.decode(data.encode("ascii"))
+        if self._use_bytes:
+            value = value.data
         self.append(value)
         self._value = 0
     dispatch["base64"] = end_base64
@@ -861,23 +872,26 @@ FastMarshaller = FastParser = FastUnmarshaller = None
 #
 # return A (parser, unmarshaller) tuple.
 
-def getparser(use_datetime=False):
+def getparser(use_datetime=False, use_builtin_types=False):
     """getparser() -> parser, unmarshaller
 
     Create an instance of the fastest available parser, and attach it
     to an unmarshalling object.  Return both objects.
     """
-    if use_datetime and not datetime:
-        raise ValueError("the datetime module is not available")
     if FastParser and FastUnmarshaller:
-        if use_datetime:
+        if use_builtin_types:
+            mkdatetime = _datetime_type
+            mkbytes = base64.decodebytes
+        elif use_datetime:
             mkdatetime = _datetime_type
+            mkbytes = _binary
         else:
             mkdatetime = _datetime
-        target = FastUnmarshaller(True, False, _binary, mkdatetime, Fault)
+            mkbytes = _binary
+        target = FastUnmarshaller(True, False, mkbytes, mkdatetime, Fault)
         parser = FastParser(target)
     else:
-        target = Unmarshaller(use_datetime=use_datetime)
+        target = Unmarshaller(use_datetime=use_datetime, use_builtin_types=use_builtin_types)
         if FastParser:
             parser = FastParser(target)
         else:
@@ -915,7 +929,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None,
 
         encoding: the packet encoding (default is UTF-8)
 
-    All 8-bit strings in the data structure are assumed to use the
+    All byte strings in the data structure are assumed to use the
     packet encoding.  Unicode strings are automatically converted,
     where necessary.
     """
@@ -974,7 +988,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None,
 #     (None if not present).
 # @see Fault
 
-def loads(data, use_datetime=False):
+def loads(data, use_datetime=False, use_builtin_types=False):
     """data -> unmarshalled data, method name
 
     Convert an XML-RPC packet to unmarshalled data plus a method
@@ -983,7 +997,7 @@ def loads(data, use_datetime=False):
     If the XML-RPC packet represents a fault condition, this function
     raises a Fault exception.
     """
-    p, u = getparser(use_datetime=use_datetime)
+    p, u = getparser(use_datetime=use_datetime, use_builtin_types=use_builtin_types)
     p.feed(data)
     p.close()
     return u.close(), u.getmethodname()
@@ -1085,7 +1099,7 @@ class Transport:
     """Handles an HTTP transaction to an XML-RPC server."""
 
     # client identifier (may be overridden)
-    user_agent = "xmlrpclib.py/%s (by www.pythonware.com)" % __version__
+    user_agent = "Python-xmlrpc/%s" % __version__
 
     #if true, we'll request gzip encoding
     accept_gzip_encoding = True
@@ -1095,8 +1109,9 @@ class Transport:
     # that they can decode such a request
     encode_threshold = None #None = don't encode
 
-    def __init__(self, use_datetime=False):
+    def __init__(self, use_datetime=False, use_builtin_types=False):
         self._use_datetime = use_datetime
+        self._use_builtin_types = use_builtin_types
         self._connection = (None, None)
         self._extra_headers = []
 
@@ -1157,7 +1172,8 @@ class Transport:
 
     def getparser(self):
         # get parser and unmarshaller
-        return getparser(use_datetime=self._use_datetime)
+        return getparser(use_datetime=self._use_datetime,
+                         use_builtin_types=self._use_builtin_types)
 
     ##
     # Get authorization info from host parameter
@@ -1175,7 +1191,6 @@ class Transport:
         if isinstance(host, tuple):
             host, x509 = host
 
-        import urllib.parse
         auth, host = urllib.parse.splituser(host)
 
         if auth:
@@ -1364,11 +1379,10 @@ class ServerProxy:
     """
 
     def __init__(self, uri, transport=None, encoding=None, verbose=False,
-                 allow_none=False, use_datetime=False):
+                 allow_none=False, use_datetime=False, use_builtin_types=False):
         # establish a "logical" server connection
 
         # get the url
-        import urllib.parse
         type, uri = urllib.parse.splittype(uri)
         if type not in ("http", "https"):
             raise IOError("unsupported XML-RPC protocol")
@@ -1378,9 +1392,11 @@ class ServerProxy:
 
         if transport is None:
             if type == "https":
-                transport = SafeTransport(use_datetime=use_datetime)
+                handler = SafeTransport
             else:
-                transport = Transport(use_datetime=use_datetime)
+                handler = Transport
+            transport = handler(use_datetime=use_datetime,
+                                use_builtin_types=use_builtin_types)
         self.__transport = transport
 
         self.__encoding = encoding or 'utf-8'
diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py
index fd17026583..54e172670b 100644
--- a/Lib/xmlrpc/server.py
+++ b/Lib/xmlrpc/server.py
@@ -160,11 +160,13 @@ class SimpleXMLRPCDispatcher:
     can be instanced when used by the MultiPathXMLRPCServer
     """
 
-    def __init__(self, allow_none=False, encoding=None):
+    def __init__(self, allow_none=False, encoding=None,
+                 use_builtin_types=False):
         self.funcs = {}
         self.instance = None
         self.allow_none = allow_none
         self.encoding = encoding or 'utf-8'
+        self.use_builtin_types = use_builtin_types
 
     def register_instance(self, instance, allow_dotted_names=False):
         """Registers an instance to respond to XML-RPC requests.
@@ -245,7 +247,7 @@ class SimpleXMLRPCDispatcher:
         """
 
         try:
-            params, method = loads(data)
+            params, method = loads(data, use_builtin_types=self.use_builtin_types)
 
             # generate response
             if dispatch_method is not None:
@@ -575,10 +577,11 @@ class SimpleXMLRPCServer(socketserver.TCPServer,
     _send_traceback_header = False
 
     def __init__(self, addr, requestHandler=SimpleXMLRPCRequestHandler,
-                 logRequests=True, allow_none=False, encoding=None, bind_and_activate=True):
+                 logRequests=True, allow_none=False, encoding=None,
+                 bind_and_activate=True, use_builtin_types=False):
         self.logRequests = logRequests
 
-        SimpleXMLRPCDispatcher.__init__(self, allow_none, encoding)
+        SimpleXMLRPCDispatcher.__init__(self, allow_none, encoding, use_builtin_types)
         socketserver.TCPServer.__init__(self, addr, requestHandler, bind_and_activate)
 
         # [Bug #1222790] If possible, set close-on-exec flag; if a
@@ -598,10 +601,11 @@ class MultiPathXMLRPCServer(SimpleXMLRPCServer):
     Make sure that the requestHandler accepts the paths in question.
     """
     def __init__(self, addr, requestHandler=SimpleXMLRPCRequestHandler,
-                 logRequests=True, allow_none=False, encoding=None, bind_and_activate=True):
+                 logRequests=True, allow_none=False, encoding=None,
+                 bind_and_activate=True, use_builtin_types=False):
 
         SimpleXMLRPCServer.__init__(self, addr, requestHandler, logRequests, allow_none,
-                                    encoding, bind_and_activate)
+                                    encoding, bind_and_activate, use_builtin_types)
         self.dispatchers = {}
         self.allow_none = allow_none
         self.encoding = encoding or 'utf-8'
@@ -631,8 +635,8 @@ class MultiPathXMLRPCServer(SimpleXMLRPCServer):
 class CGIXMLRPCRequestHandler(SimpleXMLRPCDispatcher):
     """Simple handler for XML-RPC data passed through CGI."""
 
-    def __init__(self, allow_none=False, encoding=None):
-        SimpleXMLRPCDispatcher.__init__(self, allow_none, encoding)
+    def __init__(self, allow_none=False, encoding=None, use_builtin_types=False):
+        SimpleXMLRPCDispatcher.__init__(self, allow_none, encoding, use_builtin_types)
 
     def handle_xmlrpc(self, request_text):
         """Handle a single XML-RPC request"""
@@ -927,9 +931,10 @@ class DocXMLRPCServer(  SimpleXMLRPCServer,
 
     def __init__(self, addr, requestHandler=DocXMLRPCRequestHandler,
                  logRequests=True, allow_none=False, encoding=None,
-                 bind_and_activate=True):
+                 bind_and_activate=True, use_builtin_types=False):
         SimpleXMLRPCServer.__init__(self, addr, requestHandler, logRequests,
-                                    allow_none, encoding, bind_and_activate)
+                                    allow_none, encoding, bind_and_activate,
+                                    use_builtin_types)
         XMLRPCDocGenerator.__init__(self)
 
 class DocCGIXMLRPCRequestHandler(   CGIXMLRPCRequestHandler,
@@ -959,8 +964,13 @@ class DocCGIXMLRPCRequestHandler(   CGIXMLRPCRequestHandler,
 
 
 if __name__ == '__main__':
-    print('Running XML-RPC server on port 8000')
     server = SimpleXMLRPCServer(("localhost", 8000))
     server.register_function(pow)
     server.register_function(lambda x,y: x+y, 'add')
-    server.serve_forever()
+    print('Serving XML-RPC on localhost port 8000')
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print("\nKeyboard interrupt received, exiting.")
+        server.server_close()
+        sys.exit(0)
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index b223b4a4c1..3448c61795 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -22,7 +22,18 @@ except ImportError:
     zlib = None
     crc32 = binascii.crc32
 
-__all__ = ["BadZipFile", "BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED",
+try:
+    import bz2 # We may need its compression method
+except ImportError:
+    bz2 = None
+
+try:
+    import lzma # We may need its compression method
+except ImportError:
+    lzma = None
+
+__all__ = ["BadZipFile", "BadZipfile", "error",
+           "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA",
            "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile"]
 
 class BadZipFile(Exception):
@@ -45,8 +56,17 @@ ZIP_MAX_COMMENT = (1 << 16) - 1
 # constants for Zip file compression methods
 ZIP_STORED = 0
 ZIP_DEFLATED = 8
+ZIP_BZIP2 = 12
+ZIP_LZMA = 14
 # Other ZIP compression methods not supported
 
+DEFAULT_VERSION = 20
+ZIP64_VERSION = 45
+BZIP2_VERSION = 46
+LZMA_VERSION = 63
+# we recognize (but not necessarily support) all features up to that version
+MAX_EXTRACT_VERSION = 63
+
 # Below are some formats and associated data for reading/writing headers using
 # the struct module.  The names and structures of headers/records are those used
 # in the PKWARE description of the ZIP file format:
@@ -322,8 +342,8 @@ class ZipInfo (object):
         else:
             # Assume everything else is unix-y
             self.create_system = 3          # System which created ZIP archive
-        self.create_version = 20        # Version which created ZIP archive
-        self.extract_version = 20       # Version needed to extract archive
+        self.create_version = DEFAULT_VERSION  # Version which created ZIP archive
+        self.extract_version = DEFAULT_VERSION # Version needed to extract archive
         self.reserved = 0               # Must be zero
         self.flag_bits = 0              # ZIP flag bits
         self.volume = 0                 # Volume number of file header
@@ -350,6 +370,7 @@ class ZipInfo (object):
 
         extra = self.extra
 
+        min_version = 0
         if zip64 is None:
             zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT
         if zip64:
@@ -363,9 +384,15 @@ class ZipInfo (object):
             # fall back to the ZIP64 extension
             file_size = 0xffffffff
             compress_size = 0xffffffff
-            self.extract_version = max(45, self.extract_version)
-            self.create_version = max(45, self.extract_version)
+            min_version = ZIP64_VERSION
 
+        if self.compress_type == ZIP_BZIP2:
+            min_version = max(BZIP2_VERSION, min_version)
+        elif self.compress_type == ZIP_LZMA:
+            min_version = max(LZMA_VERSION, min_version)
+
+        self.extract_version = max(min_version, self.extract_version)
+        self.create_version = max(min_version, self.create_version)
         filename, flag_bits = self._encodeFilenameFlags()
         header = struct.pack(structFileHeader, stringFileHeader,
                  self.extract_version, self.reserved, flag_bits,
@@ -476,6 +503,57 @@ class _ZipDecrypter:
         return c
 
 
+class LZMACompressor:
+
+    def __init__(self):
+        self._comp = None
+
+    def _init(self):
+        props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1})
+        self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[
+                lzma._decode_filter_properties(lzma.FILTER_LZMA1, props)
+        ])
+        return struct.pack('<BBH', 9, 4, len(props)) + props
+
+    def compress(self, data):
+        if self._comp is None:
+            return self._init() + self._comp.compress(data)
+        return self._comp.compress(data)
+
+    def flush(self):
+        if self._comp is None:
+            return self._init() + self._comp.flush()
+        return self._comp.flush()
+
+
+class LZMADecompressor:
+
+    def __init__(self):
+        self._decomp = None
+        self._unconsumed = b''
+        self.eof = False
+
+    def decompress(self, data):
+        if self._decomp is None:
+            self._unconsumed += data
+            if len(self._unconsumed) <= 4:
+                return b''
+            psize, = struct.unpack('<H', self._unconsumed[2:4])
+            if len(self._unconsumed) <= 4 + psize:
+                return b''
+
+            self._decomp = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=[
+                    lzma._decode_filter_properties(lzma.FILTER_LZMA1,
+                            self._unconsumed[4:4 + psize])
+            ])
+            data = self._unconsumed[4 + psize:]
+            del self._unconsumed
+
+        result = self._decomp.decompress(data)
+        self.eof = self._decomp.eof
+        return result
+
+
 compressor_names = {
     0: 'store',
     1: 'shrink',
@@ -496,6 +574,53 @@ compressor_names = {
     98: 'ppmd',
 }
 
+def _check_compression(compression):
+    if compression == ZIP_STORED:
+        pass
+    elif compression == ZIP_DEFLATED:
+        if not zlib:
+            raise RuntimeError(
+                    "Compression requires the (missing) zlib module")
+    elif compression == ZIP_BZIP2:
+        if not bz2:
+            raise RuntimeError(
+                    "Compression requires the (missing) bz2 module")
+    elif compression == ZIP_LZMA:
+        if not lzma:
+            raise RuntimeError(
+                    "Compression requires the (missing) lzma module")
+    else:
+        raise RuntimeError("That compression method is not supported")
+
+
+def _get_compressor(compress_type):
+    if compress_type == ZIP_DEFLATED:
+        return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
+             zlib.DEFLATED, -15)
+    elif compress_type == ZIP_BZIP2:
+        return bz2.BZ2Compressor()
+    elif compress_type == ZIP_LZMA:
+        return LZMACompressor()
+    else:
+        return None
+
+
+def _get_decompressor(compress_type):
+    if compress_type == ZIP_STORED:
+        return None
+    elif compress_type == ZIP_DEFLATED:
+        return zlib.decompressobj(-15)
+    elif compress_type == ZIP_BZIP2:
+        return bz2.BZ2Decompressor()
+    elif compress_type == ZIP_LZMA:
+        return LZMADecompressor()
+    else:
+        descr = compressor_names.get(compress_type)
+        if descr:
+            raise NotImplementedError("compression type %d (%s)" % (compress_type, descr))
+        else:
+            raise NotImplementedError("compression type %d" % (compress_type,))
+
 
 class ZipExtFile(io.BufferedIOBase):
     """File-like object for reading an archive member.
@@ -518,19 +643,12 @@ class ZipExtFile(io.BufferedIOBase):
         self._close_fileobj = close_fileobj
 
         self._compress_type = zipinfo.compress_type
-        self._compress_size = zipinfo.compress_size
         self._compress_left = zipinfo.compress_size
+        self._left = zipinfo.file_size
 
-        if self._compress_type == ZIP_DEFLATED:
-            self._decompressor = zlib.decompressobj(-15)
-        elif self._compress_type != ZIP_STORED:
-            descr = compressor_names.get(self._compress_type)
-            if descr:
-                raise NotImplementedError("compression type %d (%s)" % (self._compress_type, descr))
-            else:
-                raise NotImplementedError("compression type %d" % (self._compress_type,))
-        self._unconsumed = b''
+        self._decompressor = _get_decompressor(self._compress_type)
 
+        self._eof = False
         self._readbuffer = b''
         self._offset = 0
 
@@ -605,7 +723,11 @@ class ZipExtFile(io.BufferedIOBase):
         """Returns buffered bytes without advancing the position."""
         if n > len(self._readbuffer) - self._offset:
             chunk = self.read(n)
-            self._offset -= len(chunk)
+            if len(chunk) > self._offset:
+                self._readbuffer = chunk + self._readbuffer[self._offset:]
+                self._offset = 0
+            else:
+                self._offset -= len(chunk)
 
         # Return up to 512 bytes to reduce allocation overhead for tight loops.
         return self._readbuffer[self._offset: self._offset + 512]
@@ -617,80 +739,123 @@ class ZipExtFile(io.BufferedIOBase):
         """Read and return up to n bytes.
         If the argument is omitted, None, or negative, data is read and returned until EOF is reached..
         """
-        buf = b''
-        if n is None:
-            n = -1
-        while True:
-            if n < 0:
-                data = self.read1(n)
-            elif n > len(buf):
-                data = self.read1(n - len(buf))
-            else:
-                return buf
-            if len(data) == 0:
-                return buf
+        if n is None or n < 0:
+            buf = self._readbuffer[self._offset:]
+            self._readbuffer = b''
+            self._offset = 0
+            while not self._eof:
+                buf += self._read1(self.MAX_N)
+            return buf
+
+        end = n + self._offset
+        if end < len(self._readbuffer):
+            buf = self._readbuffer[self._offset:end]
+            self._offset = end
+            return buf
+
+        n = end - len(self._readbuffer)
+        buf = self._readbuffer[self._offset:]
+        self._readbuffer = b''
+        self._offset = 0
+        while n > 0 and not self._eof:
+            data = self._read1(n)
+            if n < len(data):
+                self._readbuffer = data
+                self._offset = n
+                buf += data[:n]
+                break
             buf += data
+            n -= len(data)
+        return buf
 
-    def _update_crc(self, newdata, eof):
+    def _update_crc(self, newdata):
         # Update the CRC using the given data.
         if self._expected_crc is None:
             # No need to compute the CRC if we don't have a reference value
             return
         self._running_crc = crc32(newdata, self._running_crc) & 0xffffffff
         # Check the CRC if we're at the end of the file
-        if eof and self._running_crc != self._expected_crc:
+        if self._eof and self._running_crc != self._expected_crc:
             raise BadZipFile("Bad CRC-32 for file %r" % self.name)
 
     def read1(self, n):
         """Read up to n bytes with at most one read() system call."""
 
-        # Simplify algorithm (branching) by transforming negative n to large n.
-        if n < 0 or n is None:
-            n = self.MAX_N
-
-        # Bytes available in read buffer.
-        len_readbuffer = len(self._readbuffer) - self._offset
+        if n is None or n < 0:
+            buf = self._readbuffer[self._offset:]
+            self._readbuffer = b''
+            self._offset = 0
+            data = self._read1(self.MAX_N)
+            buf += data
+            return buf
 
-        # Read from file.
-        if self._compress_left > 0 and n > len_readbuffer + len(self._unconsumed):
-            nbytes = n - len_readbuffer - len(self._unconsumed)
-            nbytes = max(nbytes, self.MIN_READ_SIZE)
-            nbytes = min(nbytes, self._compress_left)
+        end = n + self._offset
+        if end < len(self._readbuffer):
+            buf = self._readbuffer[self._offset:end]
+            self._offset = end
+            return buf
 
-            data = self._fileobj.read(nbytes)
-            self._compress_left -= len(data)
+        n = end - len(self._readbuffer)
+        buf = self._readbuffer[self._offset:]
+        self._readbuffer = b''
+        self._offset = 0
+        if n > 0:
+            data = self._read1(n)
+            if n < len(data):
+                self._readbuffer = data
+                self._offset = n
+                data = data[:n]
+            buf += data
+        return buf
 
-            if data and self._decrypter is not None:
-                data = bytes(map(self._decrypter, data))
+    def _read1(self, n):
+        # Read up to n compressed bytes with at most one read() system call,
+        # decrypt and decompress them.
+        if self._eof or n <= 0:
+            return b''
 
-            if self._compress_type == ZIP_STORED:
-                self._update_crc(data, eof=(self._compress_left==0))
-                self._readbuffer = self._readbuffer[self._offset:] + data
-                self._offset = 0
-            else:
-                # Prepare deflated bytes for decompression.
-                self._unconsumed += data
-
-        # Handle unconsumed data.
-        if (len(self._unconsumed) > 0 and n > len_readbuffer and
-            self._compress_type == ZIP_DEFLATED):
-            data = self._decompressor.decompress(
-                self._unconsumed,
-                max(n - len_readbuffer, self.MIN_READ_SIZE)
-            )
-
-            self._unconsumed = self._decompressor.unconsumed_tail
-            eof = len(self._unconsumed) == 0 and self._compress_left == 0
-            if eof:
+        # Read from file.
+        if self._compress_type == ZIP_DEFLATED:
+            ## Handle unconsumed data.
+            data = self._decompressor.unconsumed_tail
+            if n > len(data):
+                data += self._read2(n - len(data))
+        else:
+            data = self._read2(n)
+
+        if self._compress_type == ZIP_STORED:
+            self._eof = self._compress_left <= 0
+        elif self._compress_type == ZIP_DEFLATED:
+            n = max(n, self.MIN_READ_SIZE)
+            data = self._decompressor.decompress(data, n)
+            self._eof = (self._decompressor.eof or
+                    self._compress_left <= 0 and
+                    not self._decompressor.unconsumed_tail)
+            if self._eof:
                 data += self._decompressor.flush()
+        else:
+            data = self._decompressor.decompress(data)
+            self._eof = self._decompressor.eof or self._compress_left <= 0
+
+        data = data[:self._left]
+        self._left -= len(data)
+        if self._left <= 0:
+            self._eof = True
+        self._update_crc(data)
+        return data
 
-            self._update_crc(data, eof=eof)
-            self._readbuffer = self._readbuffer[self._offset:] + data
-            self._offset = 0
+    def _read2(self, n):
+        if self._compress_left <= 0:
+            return b''
+
+        n = max(n, self.MIN_READ_SIZE)
+        n = min(n, self._compress_left)
+
+        data = self._fileobj.read(n)
+        self._compress_left -= len(data)
 
-        # Read from buffer.
-        data = self._readbuffer[self._offset: self._offset + n]
-        self._offset += len(data)
+        if self._decrypter is not None:
+            data = bytes(map(self._decrypter, data))
         return data
 
     def close(self):
@@ -709,7 +874,8 @@ class ZipFile:
     file: Either the path to the file, or a file-like object.
           If it is a path, the file will be opened and closed by ZipFile.
     mode: The mode can be either read "r", write "w" or append "a".
-    compression: ZIP_STORED (no compression) or ZIP_DEFLATED (requires zlib).
+    compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib),
+                 ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma).
     allowZip64: if True ZipFile will create files with ZIP64 extensions when
                 needed, otherwise it will raise an exception when this would
                 be necessary.
@@ -717,20 +883,14 @@ class ZipFile:
     """
 
     fp = None                   # Set here since __del__ checks it
+    _windows_illegal_name_trans_table = None
 
     def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=False):
         """Open the ZIP file with mode read "r", write "w" or append "a"."""
         if mode not in ("r", "w", "a"):
             raise RuntimeError('ZipFile() requires mode "r", "w", or "a"')
 
-        if compression == ZIP_STORED:
-            pass
-        elif compression == ZIP_DEFLATED:
-            if not zlib:
-                raise RuntimeError(
-                      "Compression requires the (missing) zlib module")
-        else:
-            raise RuntimeError("That compression method is not supported")
+        _check_compression(compression)
 
         self._allowZip64 = allowZip64
         self._didModify = False
@@ -851,6 +1011,9 @@ class ZipFile:
             (x.create_version, x.create_system, x.extract_version, x.reserved,
                 x.flag_bits, x.compress_type, t, d,
                 x.CRC, x.compress_size, x.file_size) = centdir[1:12]
+            if x.extract_version > MAX_EXTRACT_VERSION:
+                raise NotImplementedError("zip file version %.1f" %
+                                          (x.extract_version / 10))
             x.volume, x.internal_attr, x.external_attr = centdir[15:18]
             # Convert date/time code to (year, month, day, hour, min, sec)
             x._raw_time = t
@@ -873,10 +1036,7 @@ class ZipFile:
 
     def namelist(self):
         """Return a list of file names in the archive."""
-        l = []
-        for data in self.filelist:
-            l.append(data.filename)
-        return l
+        return [data.filename for data in self.filelist]
 
     def infolist(self):
         """Return a list of class ZipInfo instances for files in the
@@ -985,6 +1145,14 @@ class ZipFile:
             if fheader[_FH_EXTRA_FIELD_LENGTH]:
                 zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
 
+            if zinfo.flag_bits & 0x20:
+                # Zip 2.7: compressed patched data
+                raise NotImplementedError("compressed patched data (flag bit 5)")
+
+            if zinfo.flag_bits & 0x40:
+                # strong encryption
+                raise NotImplementedError("strong encryption (flag bit 6)")
+
             if zinfo.flag_bits & 0x800:
                 # UTF-8 filename
                 fname_str = fname.decode("utf-8")
@@ -1056,6 +1224,21 @@ class ZipFile:
         for zipinfo in members:
             self.extract(zipinfo, path, pwd)
 
+    @classmethod
+    def _sanitize_windows_name(cls, arcname, pathsep):
+        """Replace bad characters and remove trailing dots from parts."""
+        table = cls._windows_illegal_name_trans_table
+        if not table:
+            illegal = ':<>|"?*'
+            table = str.maketrans(illegal, '_' * len(illegal))
+            cls._windows_illegal_name_trans_table = table
+        arcname = arcname.translate(table)
+        # remove trailing dots
+        arcname = (x.rstrip('.') for x in arcname.split(pathsep))
+        # rejoin, removing empty parts.
+        arcname = pathsep.join(x for x in arcname if x)
+        return arcname
+
     def _extract_member(self, member, targetpath, pwd):
         """Extract the ZipInfo object 'member' to a physical
            file on the path targetpath.
@@ -1069,16 +1252,12 @@ class ZipFile:
         # interpret absolute pathname as relative, remove drive letter or
         # UNC path, redundant separators, "." and ".." components.
         arcname = os.path.splitdrive(arcname)[1]
+        invalid_path_parts = ('', os.path.curdir, os.path.pardir)
         arcname = os.path.sep.join(x for x in arcname.split(os.path.sep)
-                    if x not in ('', os.path.curdir, os.path.pardir))
+                                   if x not in invalid_path_parts)
         if os.path.sep == '\\':
             # filter illegal characters on Windows
-            illegal = ':<>|"?*'
-            table = str.maketrans(illegal, '_' * len(illegal))
-            arcname = arcname.translate(table)
-            # remove trailing dots
-            arcname = (x.rstrip('.') for x in arcname.split(os.path.sep))
-            arcname = os.path.sep.join(x for x in arcname if x)
+            arcname = self._sanitize_windows_name(arcname, os.path.sep)
 
         targetpath = os.path.join(targetpath, arcname)
         targetpath = os.path.normpath(targetpath)
@@ -1109,11 +1288,7 @@ class ZipFile:
         if not self.fp:
             raise RuntimeError(
                   "Attempt to write ZIP archive that was already closed")
-        if zinfo.compress_type == ZIP_DEFLATED and not zlib:
-            raise RuntimeError(
-                  "Compression requires the (missing) zlib module")
-        if zinfo.compress_type not in (ZIP_STORED, ZIP_DEFLATED):
-            raise RuntimeError("That compression method is not supported")
+        _check_compression(zinfo.compress_type)
         if zinfo.file_size > ZIP64_LIMIT:
             if not self._allowZip64:
                 raise LargeZipFile("Filesize would require ZIP64 extensions")
@@ -1151,6 +1326,9 @@ class ZipFile:
         zinfo.file_size = st.st_size
         zinfo.flag_bits = 0x00
         zinfo.header_offset = self.fp.tell()    # Start of header bytes
+        if zinfo.compress_type == ZIP_LZMA:
+            # Compressed data includes an end-of-stream (EOS) marker
+            zinfo.flag_bits |= 0x02
 
         self._writecheck(zinfo)
         self._didModify = True
@@ -1164,6 +1342,7 @@ class ZipFile:
             self.fp.write(zinfo.FileHeader(False))
             return
 
+        cmpr = _get_compressor(zinfo.compress_type)
         with open(filename, "rb") as fp:
             # Must overwrite CRC and sizes with correct data later
             zinfo.CRC = CRC = 0
@@ -1172,11 +1351,6 @@ class ZipFile:
             zip64 = self._allowZip64 and \
                     zinfo.file_size * 1.05 > ZIP64_LIMIT
             self.fp.write(zinfo.FileHeader(zip64))
-            if zinfo.compress_type == ZIP_DEFLATED:
-                cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
-                     zlib.DEFLATED, -15)
-            else:
-                cmpr = None
             file_size = 0
             while 1:
                 buf = fp.read(1024 * 8)
@@ -1235,13 +1409,15 @@ class ZipFile:
         zinfo.header_offset = self.fp.tell()    # Start of header data
         if compress_type is not None:
             zinfo.compress_type = compress_type
+        if zinfo.compress_type == ZIP_LZMA:
+            # Compressed data includes an end-of-stream (EOS) marker
+            zinfo.flag_bits |= 0x02
 
         self._writecheck(zinfo)
         self._didModify = True
         zinfo.CRC = crc32(data) & 0xffffffff       # CRC-32 checksum
-        if zinfo.compress_type == ZIP_DEFLATED:
-            co = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
-                 zlib.DEFLATED, -15)
+        co = _get_compressor(zinfo.compress_type)
+        if co:
             data = co.compress(data) + co.flush()
             zinfo.compress_size = len(data)    # Compressed size
         else:
@@ -1298,18 +1474,22 @@ class ZipFile:
                         header_offset = zinfo.header_offset
 
                     extra_data = zinfo.extra
+                    min_version = 0
                     if extra:
                         # Append a ZIP64 field to the extra's
                         extra_data = struct.pack(
                                 '<HH' + 'Q'*len(extra),
                                 1, 8*len(extra), *extra) + extra_data
 
-                        extract_version = max(45, zinfo.extract_version)
-                        create_version = max(45, zinfo.create_version)
-                    else:
-                        extract_version = zinfo.extract_version
-                        create_version = zinfo.create_version
+                        min_version = ZIP64_VERSION
+
+                    if zinfo.compress_type == ZIP_BZIP2:
+                        min_version = max(BZIP2_VERSION, min_version)
+                    elif zinfo.compress_type == ZIP_LZMA:
+                        min_version = max(LZMA_VERSION, min_version)
 
+                    extract_version = max(min_version, zinfo.extract_version)
+                    create_version = max(min_version, zinfo.create_version)
                     try:
                         filename, flag_bits = zinfo._encodeFilenameFlags()
                         centdir = struct.pack(structCentralDir,