Merge 3.5

8 files changed, 86 insertions, 57 deletions

diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat
index ff7d167e6a..c01400ce1f 100644
--- a/Tools/buildbot/test.bat
+++ b/Tools/buildbot/test.bat
@@ -16,4 +16,4 @@ if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts
 if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts

 

 echo on

-call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=3600 %regrtest_args%

+call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=900 %regrtest_args%

diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py
index c0758078f8..389fffd54c 100755
--- a/Tools/freeze/freeze.py
+++ b/Tools/freeze/freeze.py
@@ -218,7 +218,7 @@ def main():
     ishome = os.path.exists(os.path.join(prefix, 'Python', 'ceval.c'))
 
     # locations derived from options
-    version = sys.version[:3]
+    version = '%d.%d' % sys.version_info[:2]
     flagged_version = version + sys.abiflags
     if win:
         extensions_c = 'frozen_extensions.c'
diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py
index 3c6c14c836..be941c77f2 100755
--- a/Tools/i18n/pygettext.py
+++ b/Tools/i18n/pygettext.py
@@ -156,7 +156,8 @@ If `inputfile' is -, standard input is read.
 """)
 
 import os
-import imp
+import importlib.machinery
+import importlib.util
 import sys
 import glob
 import time
@@ -263,8 +264,7 @@ def _visit_pyfiles(list, dirname, names):
     # get extension for python source files
     if '_py_ext' not in globals():
         global _py_ext
-        _py_ext = [triple[0] for triple in imp.get_suffixes()
-                   if triple[2] == imp.PY_SOURCE][0]
+        _py_ext = importlib.machinery.SOURCE_SUFFIXES[0]
 
     # don't recurse into CVS directories
     if 'CVS' in names:
@@ -277,45 +277,6 @@ def _visit_pyfiles(list, dirname, names):
         )
 
 
-def _get_modpkg_path(dotted_name, pathlist=None):
-    """Get the filesystem path for a module or a package.
-
-    Return the file system path to a file for a module, and to a directory for
-    a package. Return None if the name is not found, or is a builtin or
-    extension module.
-    """
-    # split off top-most name
-    parts = dotted_name.split('.', 1)
-
-    if len(parts) > 1:
-        # we have a dotted path, import top-level package
-        try:
-            file, pathname, description = imp.find_module(parts[0], pathlist)
-            if file: file.close()
-        except ImportError:
-            return None
-
-        # check if it's indeed a package
-        if description[2] == imp.PKG_DIRECTORY:
-            # recursively handle the remaining name parts
-            pathname = _get_modpkg_path(parts[1], [pathname])
-        else:
-            pathname = None
-    else:
-        # plain name
-        try:
-            file, pathname, description = imp.find_module(
-                dotted_name, pathlist)
-            if file:
-                file.close()
-            if description[2] not in [imp.PY_SOURCE, imp.PKG_DIRECTORY]:
-                pathname = None
-        except ImportError:
-            pathname = None
-
-    return pathname
-
-
 def getFilesForName(name):
     """Get a list of module files for a filename, a module or package name,
     or a directory.
@@ -330,7 +291,11 @@ def getFilesForName(name):
             return list
 
         # try to find module or package
-        name = _get_modpkg_path(name)
+        try:
+            spec = importlib.util.find_spec(name)
+            name = spec.origin
+        except ImportError:
+            name = None
         if not name:
             return []
 
diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs
index fa79a8d692..804ab0146d 100644
--- a/Tools/msi/lib/lib_files.wxs
+++ b/Tools/msi/lib/lib_files.wxs
@@ -64,16 +64,10 @@
                     <RegistryValue Key="PythonPath" Type="string" Value="[Lib];[DLLs]" />
                 </RegistryKey>
             </Component>
-            <Component Id="Lib_site_packages_README" Directory="Lib_site_packages" Guid="*">
-                <File Id="Lib_site_packages_README" Name="README.txt" Source="!(bindpath.src)Lib\site-packages\README" KeyPath="yes" />
-            </Component>
             <Component Id="Lib2to3_pickle_remove" Directory="Lib_lib2to3" Guid="$(var.RemoveLib2to3PickleComponentGuid)">
                 <RemoveFile Id="Lib2to3_pickle_remove_files" Name="*.pickle" On="uninstall" />
                 <RemoveFolder Id="Lib2to3_pickle_remove_folder" On="uninstall" />
             </Component>
         </ComponentGroup>
-        <DirectoryRef Id="Lib">
-            <Directory Id="Lib_site_packages" Name="site-packages" />
-        </DirectoryRef>
     </Fragment>
 </Wix>
diff --git a/Tools/parser/unparse.py b/Tools/parser/unparse.py
index 285030e792..72030576d0 100644
--- a/Tools/parser/unparse.py
+++ b/Tools/parser/unparse.py
@@ -322,9 +322,72 @@ class Unparser:
     def _Str(self, tree):
         self.write(repr(tree.s))
 
+    def _JoinedStr(self, t):
+        self.write("f")
+        string = io.StringIO()
+        self._fstring_JoinedStr(t, string.write)
+        self.write(repr(string.getvalue()))
+
+    def _FormattedValue(self, t):
+        self.write("f")
+        string = io.StringIO()
+        self._fstring_FormattedValue(t, string.write)
+        self.write(repr(string.getvalue()))
+
+    def _fstring_JoinedStr(self, t, write):
+        for value in t.values:
+            meth = getattr(self, "_fstring_" + type(value).__name__)
+            meth(value, write)
+
+    def _fstring_Str(self, t, write):
+        value = t.s.replace("{", "{{").replace("}", "}}")
+        write(value)
+
+    def _fstring_Constant(self, t, write):
+        assert isinstance(t.value, str)
+        value = t.value.replace("{", "{{").replace("}", "}}")
+        write(value)
+
+    def _fstring_FormattedValue(self, t, write):
+        write("{")
+        expr = io.StringIO()
+        Unparser(t.value, expr)
+        expr = expr.getvalue().rstrip("\n")
+        if expr.startswith("{"):
+            write(" ")  # Separate pair of opening brackets as "{ {"
+        write(expr)
+        if t.conversion != -1:
+            conversion = chr(t.conversion)
+            assert conversion in "sra"
+            write(f"!{conversion}")
+        if t.format_spec:
+            write(":")
+            meth = getattr(self, "_fstring_" + type(t.format_spec).__name__)
+            meth(t.format_spec, write)
+        write("}")
+
     def _Name(self, t):
         self.write(t.id)
 
+    def _write_constant(self, value):
+        if isinstance(value, (float, complex)):
+            self.write(repr(value).replace("inf", INFSTR))
+        else:
+            self.write(repr(value))
+
+    def _Constant(self, t):
+        value = t.value
+        if isinstance(value, tuple):
+            self.write("(")
+            if len(value) == 1:
+                self._write_constant(value[0])
+                self.write(",")
+            else:
+                interleave(lambda: self.write(", "), self._write_constant, value)
+            self.write(")")
+        else:
+            self._write_constant(t.value)
+
     def _NameConstant(self, t):
         self.write(repr(t.value))
 
@@ -413,7 +476,7 @@ class Unparser:
     def _Tuple(self, t):
         self.write("(")
         if len(t.elts) == 1:
-            (elt,) = t.elts
+            elt = t.elts[0]
             self.dispatch(elt)
             self.write(",")
         else:
@@ -460,7 +523,8 @@ class Unparser:
         # Special case: 3.__abs__() is a syntax error, so if t.value
         # is an integer literal then we need to either parenthesize
         # it or add an extra space to get 3 .__abs__().
-        if isinstance(t.value, ast.Num) and isinstance(t.value.n, int):
+        if ((isinstance(t.value, ast.Num) and isinstance(t.value.n, int))
+           or (isinstance(t.value, ast.Constant) and isinstance(t.value.value, int))):
             self.write(" ")
         self.write(".")
         self.write(t.attr)
diff --git a/Tools/scripts/combinerefs.py b/Tools/scripts/combinerefs.py
index e10e49ad7c..7ca95267c9 100755
--- a/Tools/scripts/combinerefs.py
+++ b/Tools/scripts/combinerefs.py
@@ -6,7 +6,7 @@ combinerefs path
 A helper for analyzing PYTHONDUMPREFS output.
 
 When the PYTHONDUMPREFS envar is set in a debug build, at Python shutdown
-time Py_Finalize() prints the list of all live objects twice:  first it
+time Py_FinalizeEx() prints the list of all live objects twice:  first it
 prints the repr() of each object while the interpreter is still fully intact.
 After cleaning up everything it can, it prints all remaining live objects
 again, but the second time just prints their addresses, refcounts, and type
@@ -41,7 +41,7 @@ CAUTION:  If object is a container type, it may not actually contain all the
 objects shown in the repr:  the repr was captured from the first output block,
 and some of the containees may have been released since then.  For example,
 it's common for the line showing the dict of interned strings to display
-strings that no longer exist at the end of Py_Finalize; this can be recognized
+strings that no longer exist at the end of Py_FinalizeEx; this can be recognized
 (albeit painfully) because such containees don't have a line of their own.
 
 The objects are listed in allocation order, with most-recently allocated
diff --git a/Tools/scripts/nm2def.py b/Tools/scripts/nm2def.py
index 8f07559e21..83bbcd749f 100755
--- a/Tools/scripts/nm2def.py
+++ b/Tools/scripts/nm2def.py
@@ -36,8 +36,8 @@ option to produce this format (since it is the original v7 Unix format).
 """
 import os, sys
 
-PYTHONLIB = 'libpython'+sys.version[:3]+'.a'
-PC_PYTHONLIB = 'Python'+sys.version[0]+sys.version[2]+'.dll'
+PYTHONLIB = 'libpython%d.%d.a' % sys.version_info[:2]
+PC_PYTHONLIB = 'Python%d%d.dll' % sys.version_info[:2]
 NM = 'nm -p -g %s'                      # For Linux, use "nm -g %s"
 
 def symbols(lib=PYTHONLIB,types=('T','C','D')):
diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv
index 978d691d1e..1fb42c6391 100755
--- a/Tools/scripts/pyvenv
+++ b/Tools/scripts/pyvenv
@@ -1,6 +1,12 @@
 #!/usr/bin/env python3
 if __name__ == '__main__':
     import sys
+    import pathlib
+
+    executable = pathlib.Path(sys.executable or 'python3').name
+    print('WARNING: the pyenv script is deprecated in favour of '
+          f'`{executable} -m venv`', file=sys.stderr)
+
     rc = 1
     try:
         import venv