Merge with 21.2.0

7 files changed, 171 insertions, 84 deletions

diff --git a/.hgtags b/.hgtags
index 79d527ad..257ff80e 100644
--- a/.hgtags
+++ b/.hgtags
@@ -260,3 +260,7 @@ c72faa468919fd2f226c97e94d4e64a6506860e5 v20.10.0
 3b5fdd077c7d83d02c4979ad69cc0bf199b47587 v20.10.1
 ddd3f81eb9e0860bf95c380c50a72c52a215231f v21.0.0
 018e4a727cf691d6404cd24ffb25e8eebea2fad4 v20.6.8
+02643fe9503033edd2fc5a54c8d4361a6c185be4 v21.1.0
+40b8fac6db119aca9c462993d01908492769fc4f v21.2.0
+40b8fac6db119aca9c462993d01908492769fc4f v21.2.0
+9959424676a4aac1c14e430ff6f4210fdb0442d9 v21.2.0
diff --git a/CHANGES.rst b/CHANGES.rst
index eeb9a02c..4ee60250 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -8,6 +8,20 @@ Next
 * Pull Request #174: Add more aggressive support for
   Windows SDK in msvc9compiler patch.
 
+v21.2.0
+-------
+
+* #539: In the easy_install get_site_dirs, honor all
+  paths found in ``site.getsitepackages``.
+
+v21.1.0
+-------
+
+* #572: In build_ext, now always import ``_CONFIG_VARS``
+  from ``distutils`` rather than from ``sysconfig``
+  to allow ``distutils.sysconfig.customize_compiler``
+  configure the OS X compiler for ``-dynamiclib``.
+
 v21.0.0
 -------
 
diff --git a/setup.cfg b/setup.cfg
index d0c301b3..e13f8c64 100755
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 21.0.0
+current_version = 21.2.0
 commit = True
 tag = True
 
diff --git a/setup.py b/setup.py
index 58c70785..c497e6a9 100755
--- a/setup.py
+++ b/setup.py
@@ -68,7 +68,7 @@ wheel = ['wheel'] if needs_wheel else []
 
 setup_params = dict(
     name="setuptools",
-    version="21.0.0",
+    version="21.2.0",
     description="Easily download, build, install, upgrade, and uninstall "
                 "Python packages",
     author="Python Packaging Authority",
diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py
index 92e4a189..1caf8c81 100644
--- a/setuptools/command/build_ext.py
+++ b/setuptools/command/build_ext.py
@@ -16,15 +16,32 @@ try:
 except ImportError:
     _build_ext = _du_build_ext
 
-try:
-    # Python 2.7 or >=3.2
-    from sysconfig import _CONFIG_VARS
-except ImportError:
-    from distutils.sysconfig import get_config_var
+from distutils.sysconfig import get_config_var
+
+get_config_var("LDSHARED")  # make sure _config_vars is initialized
+del get_config_var
+from distutils.sysconfig import _config_vars as _CONFIG_VARS
+
+
+def _customize_compiler_for_shlib(compiler):
+    if sys.platform == "darwin":
+        # building .dylib requires additional compiler flags on OSX; here we
+        # temporarily substitute the pyconfig.h variables so that distutils'
+        # 'customize_compiler' uses them before we build the shared libraries.
+        tmp = _CONFIG_VARS.copy()
+        try:
+            # XXX Help!  I don't have any idea whether these are right...
+            _CONFIG_VARS['LDSHARED'] = (
+                "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup")
+            _CONFIG_VARS['CCSHARED'] = " -dynamiclib"
+            _CONFIG_VARS['SO'] = ".dylib"
+            customize_compiler(compiler)
+        finally:
+            _CONFIG_VARS.clear()
+            _CONFIG_VARS.update(tmp)
+    else:
+        customize_compiler(compiler)
 
-    get_config_var("LDSHARED")  # make sure _config_vars is initialized
-    del get_config_var
-    from distutils.sysconfig import _config_vars as _CONFIG_VARS
 
 have_rtld = False
 use_stubs = False
@@ -124,20 +141,7 @@ class build_ext(_build_ext):
         compiler = self.shlib_compiler = new_compiler(
             compiler=self.compiler, dry_run=self.dry_run, force=self.force
         )
-        if sys.platform == "darwin":
-            tmp = _CONFIG_VARS.copy()
-            try:
-                # XXX Help!  I don't have any idea whether these are right...
-                _CONFIG_VARS['LDSHARED'] = (
-                    "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup")
-                _CONFIG_VARS['CCSHARED'] = " -dynamiclib"
-                _CONFIG_VARS['SO'] = ".dylib"
-                customize_compiler(compiler)
-            finally:
-                _CONFIG_VARS.clear()
-                _CONFIG_VARS.update(tmp)
-        else:
-            customize_compiler(compiler)
+        _customize_compiler_for_shlib(compiler)
 
         if self.include_dirs is not None:
             compiler.set_include_dirs(self.include_dirs)
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index ea5cb028..ccc66cf7 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -15,8 +15,10 @@ __ https://pythonhosted.org/setuptools/easy_install.html
 from glob import glob
 from distutils.util import get_platform
 from distutils.util import convert_path, subst_vars
-from distutils.errors import DistutilsArgError, DistutilsOptionError, \
-    DistutilsError, DistutilsPlatformError
+from distutils.errors import (
+    DistutilsArgError, DistutilsOptionError,
+    DistutilsError, DistutilsPlatformError,
+)
 from distutils.command.install import INSTALL_SCHEMES, SCHEME_KEYS
 from distutils import log, dir_util
 from distutils.command.build_scripts import first_line_re
@@ -74,6 +76,12 @@ def is_64bit():
 
 
 def samefile(p1, p2):
+    """
+    Determine if two paths reference the same file.
+
+    Augments os.path.samefile to work on Windows and
+    suppresses errors if the path doesn't exist.
+    """
     both_exist = os.path.exists(p1) and os.path.exists(p2)
     use_samefile = hasattr(os.path, 'samefile') and both_exist
     if use_samefile:
@@ -105,6 +113,9 @@ else:
             return False
 
 
+_one_liner = lambda text: textwrap.dedent(text).strip().replace('\n', '; ')
+
+
 class easy_install(Command):
     """Manage a download/build/install process"""
     description = "Find/get/install Python packages"
@@ -258,8 +269,10 @@ class easy_install(Command):
         self.expand_basedirs()
         self.expand_dirs()
 
-        self._expand('install_dir', 'script_dir', 'build_directory',
-                     'site_dirs')
+        self._expand(
+            'install_dir', 'script_dir', 'build_directory',
+            'site_dirs',
+        )
         # If a non-default installation directory was specified, default the
         # script directory to match it.
         if self.script_dir is None:
@@ -379,9 +392,15 @@ class easy_install(Command):
 
     def expand_dirs(self):
         """Calls `os.path.expanduser` on install dirs."""
-        self._expand_attrs(['install_purelib', 'install_platlib',
-                            'install_lib', 'install_headers',
-                            'install_scripts', 'install_data', ])
+        dirs = [
+            'install_purelib',
+            'install_platlib',
+            'install_lib',
+            'install_headers',
+            'install_scripts',
+            'install_data',
+        ]
+        self._expand_attrs(dirs)
 
     def run(self):
         if self.verbose != self.distribution.verbose:
@@ -515,6 +534,12 @@ class easy_install(Command):
         pth_file = self.pseudo_tempname() + ".pth"
         ok_file = pth_file + '.ok'
         ok_exists = os.path.exists(ok_file)
+        tmpl = _one_liner("""
+            import os
+            f = open({ok_file!r}, 'w')
+            f.write('OK')
+            f.close()
+            """) + '\n'
         try:
             if ok_exists:
                 os.unlink(ok_file)
@@ -526,16 +551,18 @@ class easy_install(Command):
             self.cant_write_to_target()
         else:
             try:
-                f.write("import os; f = open(%r, 'w'); f.write('OK'); "
-                        "f.close()\n" % (ok_file,))
+                f.write(tmpl.format(**locals()))
                 f.close()
                 f = None
                 executable = sys.executable
                 if os.name == 'nt':
                     dirname, basename = os.path.split(executable)
                     alt = os.path.join(dirname, 'pythonw.exe')
-                    if (basename.lower() == 'python.exe' and
-                            os.path.exists(alt)):
+                    use_alt = (
+                        basename.lower() == 'python.exe' and
+                        os.path.exists(alt)
+                    )
+                    if use_alt:
                         # use pythonw.exe to avoid opening a console window
                         executable = alt
 
@@ -602,7 +629,6 @@ class easy_install(Command):
 
     def easy_install(self, spec, deps=False):
         tmpdir = tempfile.mkdtemp(prefix="easy_install-")
-        download = None
         if not self.editable:
             self.install_site_py()
 
@@ -611,9 +637,8 @@ class easy_install(Command):
                 if URL_SCHEME(spec):
                     # It's a url, download it to tmpdir and process
                     self.not_editable(spec)
-                    download = self.package_index.download(spec, tmpdir)
-                    return self.install_item(None, download, tmpdir, deps,
-                                             True)
+                    dl = self.package_index.download(spec, tmpdir)
+                    return self.install_item(None, dl, tmpdir, deps, True)
 
                 elif os.path.exists(spec):
                     # Existing file or directory, just process it directly
@@ -739,8 +764,9 @@ class easy_install(Command):
     def maybe_move(self, spec, dist_filename, setup_base):
         dst = os.path.join(self.build_directory, spec.key)
         if os.path.exists(dst):
-            msg = ("%r already exists in %s; build directory %s will not be "
-                   "kept")
+            msg = (
+                "%r already exists in %s; build directory %s will not be kept"
+            )
             log.warn(msg, spec.key, self.build_directory, setup_base)
             return setup_base
         if os.path.isdir(dist_filename):
@@ -858,8 +884,10 @@ class easy_install(Command):
         return Distribution.from_filename(egg_path, metadata=metadata)
 
     def install_egg(self, egg_path, tmpdir):
-        destination = os.path.join(self.install_dir,
-                                   os.path.basename(egg_path))
+        destination = os.path.join(
+            self.install_dir,
+            os.path.basename(egg_path),
+        )
         destination = os.path.abspath(destination)
         if not self.dry_run:
             ensure_directory(destination)
@@ -869,8 +897,11 @@ class easy_install(Command):
             if os.path.isdir(destination) and not os.path.islink(destination):
                 dir_util.remove_tree(destination, dry_run=self.dry_run)
             elif os.path.exists(destination):
-                self.execute(os.unlink, (destination,), "Removing " +
-                             destination)
+                self.execute(
+                    os.unlink,
+                    (destination,),
+                    "Removing " + destination,
+                )
             try:
                 new_dist_is_zipped = False
                 if os.path.isdir(egg_path):
@@ -887,12 +918,18 @@ class easy_install(Command):
                         f, m = shutil.move, "Moving"
                     else:
                         f, m = shutil.copy2, "Copying"
-                self.execute(f, (egg_path, destination),
-                             (m + " %s to %s") %
-                             (os.path.basename(egg_path),
-                              os.path.dirname(destination)))
-                update_dist_caches(destination,
-                                   fix_zipimporter_caches=new_dist_is_zipped)
+                self.execute(
+                    f,
+                    (egg_path, destination),
+                    (m + " %s to %s") % (
+                        os.path.basename(egg_path),
+                        os.path.dirname(destination)
+                    ),
+                )
+                update_dist_caches(
+                    destination,
+                    fix_zipimporter_caches=new_dist_is_zipped,
+                )
             except:
                 update_dist_caches(destination, fix_zipimporter_caches=False)
                 raise
@@ -915,8 +952,8 @@ class easy_install(Command):
         )
 
         # Convert the .exe to an unpacked egg
-        egg_path = dist.location = os.path.join(tmpdir, dist.egg_name() +
-                                                '.egg')
+        egg_path = os.path.join(tmpdir, dist.egg_name() + '.egg')
+        dist.location = egg_path
         egg_tmp = egg_path + '.tmp'
         _egg_info = os.path.join(egg_tmp, 'EGG-INFO')
         pkg_inf = os.path.join(_egg_info, 'PKG-INFO')
@@ -934,13 +971,13 @@ class easy_install(Command):
             f.close()
         script_dir = os.path.join(_egg_info, 'scripts')
         # delete entry-point scripts to avoid duping
-        self.delete_blockers(
-            [os.path.join(script_dir, args[0]) for args in
-             ScriptWriter.get_args(dist)]
-        )
+        self.delete_blockers([
+            os.path.join(script_dir, args[0])
+            for args in ScriptWriter.get_args(dist)
+        ])
         # Build .egg file from tmpdir
         bdist_egg.make_zipfile(
-            egg_path, egg_tmp, verbose=self.verbose, dry_run=self.dry_run
+            egg_path, egg_tmp, verbose=self.verbose, dry_run=self.dry_run,
         )
         # install the .egg
         return self.install_egg(egg_path, tmpdir)
@@ -1128,7 +1165,7 @@ class easy_install(Command):
             if dist.location in self.pth_file.paths:
                 log.info(
                     "%s is already the active version in easy-install.pth",
-                    dist
+                    dist,
                 )
             else:
                 log.info("Adding %s to easy-install.pth file", dist)
@@ -1189,7 +1226,7 @@ class easy_install(Command):
             if self.optimize:
                 byte_compile(
                     to_compile, optimize=self.optimize, force=1,
-                    dry_run=self.dry_run
+                    dry_run=self.dry_run,
                 )
         finally:
             log.set_verbosity(self.verbose)  # restore original verbosity
@@ -1317,15 +1354,20 @@ def get_site_dirs():
             if sys.platform in ('os2emx', 'riscos'):
                 sitedirs.append(os.path.join(prefix, "Lib", "site-packages"))
             elif os.sep == '/':
-                sitedirs.extend([os.path.join(prefix,
-                                              "lib",
-                                              "python" + sys.version[:3],
-                                              "site-packages"),
-                                 os.path.join(prefix, "lib", "site-python")])
+                sitedirs.extend([
+                    os.path.join(
+                        prefix,
+                        "lib",
+                        "python" + sys.version[:3],
+                        "site-packages",
+                    ),
+                    os.path.join(prefix, "lib", "site-python"),
+                ])
             else:
-                sitedirs.extend(
-                    [prefix, os.path.join(prefix, "lib", "site-packages")]
-                )
+                sitedirs.extend([
+                    prefix,
+                    os.path.join(prefix, "lib", "site-packages"),
+                ])
             if sys.platform == 'darwin':
                 # for framework builds *only* we add the standard Apple
                 # locations. Currently only per-user, but /Library and
@@ -1333,12 +1375,14 @@ def get_site_dirs():
                 if 'Python.framework' in prefix:
                     home = os.environ.get('HOME')
                     if home:
-                        sitedirs.append(
-                            os.path.join(home,
-                                         'Library',
-                                         'Python',
-                                         sys.version[:3],
-                                         'site-packages'))
+                        home_sp = os.path.join(
+                            home,
+                            'Library',
+                            'Python',
+                            sys.version[:3],
+                            'site-packages',
+                        )
+                        sitedirs.append(home_sp)
     lib_paths = get_path('purelib'), get_path('platlib')
     for site_lib in lib_paths:
         if site_lib not in sitedirs:
@@ -1347,6 +1391,11 @@ def get_site_dirs():
     if site.ENABLE_USER_SITE:
         sitedirs.append(site.USER_SITE)
 
+    try:
+        sitedirs.extend(site.getsitepackages())
+    except AttributeError:
+        pass
+
     sitedirs = list(map(normalize_path, sitedirs))
 
     return sitedirs
@@ -1414,8 +1463,8 @@ def extract_wininst_cfg(dist_filename):
             return None  # not a valid tag
 
         f.seek(prepended - (12 + cfglen))
-        cfg = configparser.RawConfigParser(
-            {'version': '', 'target_version': ''})
+        init = {'version': '', 'target_version': ''}
+        cfg = configparser.RawConfigParser(init)
         try:
             part = f.read(cfglen)
             # Read up to the first null byte.
@@ -1438,7 +1487,8 @@ def get_exe_prefixes(exe_filename):
     """Get exe->egg path translations for a given .exe file"""
 
     prefixes = [
-        ('PURELIB/', ''), ('PLATLIB/pywin32_system32', ''),
+        ('PURELIB/', ''),
+        ('PLATLIB/pywin32_system32', ''),
         ('PLATLIB/', ''),
         ('SCRIPTS/', 'EGG-INFO/scripts/'),
         ('DATA/lib/site-packages', ''),
@@ -1598,12 +1648,11 @@ class RewritePthDistributions(PthDistributions):
             yield line
         yield cls.postlude
 
-    _inline = lambda text: textwrap.dedent(text).strip().replace('\n', '; ')
-    prelude = _inline("""
+    prelude = _one_liner("""
         import sys
         sys.__plen = len(sys.path)
         """)
-    postlude = _inline("""
+    postlude = _one_liner("""
         import sys
         new = sys.path[sys.__plen:]
         del sys.path[sys.__plen:]
@@ -2071,8 +2120,11 @@ class WindowsScriptWriter(ScriptWriter):
         "For Windows, add a .py extension"
         ext = dict(console='.pya', gui='.pyw')[type_]
         if ext not in os.environ['PATHEXT'].lower().split(';'):
-            warnings.warn("%s not listed in PATHEXT; scripts will not be "
-                          "recognized as executables." % ext, UserWarning)
+            msg = (
+                "{ext} not listed in PATHEXT; scripts will not be "
+                "recognized as executables."
+            ).format(**locals())
+            warnings.warn(msg, UserWarning)
         old = ['.pya', '.py', '-script.py', '.pyc', '.pyo', '.pyw', '.exe']
         old.remove(ext)
         header = cls._adjust_header(type_, header)
@@ -2238,7 +2290,8 @@ def main(argv=None, **kw):
         setup(
             script_args=['-q', 'easy_install', '-v'] + argv,
             script_name=sys.argv[0] or 'easy_install',
-            distclass=DistributionWithoutHelpCommands, **kw
+            distclass=DistributionWithoutHelpCommands,
+            **kw
         )
 
 
diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py
index 55b8b05a..fd06b6ef 100644
--- a/setuptools/tests/test_easy_install.py
+++ b/setuptools/tests/test_easy_install.py
@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import sys
 import os
-import shutil
 import tempfile
 import site
 import contextlib
@@ -119,6 +118,19 @@ class TestEasyInstallTest:
         with pytest.raises(distutils.errors.DistutilsError):
             cmd.cant_write_to_target()
 
+    def test_all_site_dirs(self, monkeypatch):
+        """
+        get_site_dirs should always return site dirs reported by
+        site.getsitepackages.
+        """
+        mock_gsp = lambda: ['/setuptools/test/site-packages']
+        monkeypatch.setattr(site, 'getsitepackages', mock_gsp, raising=False)
+        assert '/setuptools/test/site-packages' in ei.get_site_dirs()
+
+    def test_all_site_dirs_works_without_getsitepackages(self, monkeypatch):
+        monkeypatch.delattr(site, 'getsitepackages', raising=False)
+        assert ei.get_site_dirs()
+
 
 class TestPTHFileWriter:
     def test_add_from_cwd_site_sets_dirty(self):