diff options
34 files changed, 215 insertions, 231 deletions
diff --git a/MANIFEST.in b/MANIFEST.in index 8a4523a9..dfea2049 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ recursive-include tests *.py recursive-include setuptools/tests *.html recursive-include docs *.py *.txt *.conf *.css *.css_t Makefile indexsidebar.html recursive-include _markerlib *.py +recursive-include setuptools/_vendor * recursive-include pkg_resources *.py *.txt include *.py include *.txt diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index a55478a2..92d0e49c 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -14,6 +14,8 @@ from pkg_resources import (parse_requirements, VersionConflict, parse_version, packaging = pkg_resources.packaging +import six + def safe_repr(obj, short=False): """ copied from Python2.7""" try: @@ -313,7 +315,7 @@ class TestEntryPoints: def checkSubMap(self, m): assert len(m) == len(self.submap_expect) - for key, ep in pkg_resources.iteritems(self.submap_expect): + for key, ep in six.iteritems(self.submap_expect): assert repr(m.get(key)) == repr(ep) submap_expect = dict( @@ -148,6 +148,9 @@ setup_params = dict( Topic :: System :: Systems Administration Topic :: Utilities """).strip().splitlines(), + install_requires=[ + 'six>=1.5', + ], extras_require={ "ssl:sys_platform=='win32'": "wincertstore==0.2", "certs": "certifi==2015.11.20", diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 712ec082..fffcac76 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,5 +1,7 @@ """Extensions to the 'distutils' for large or complex distributions""" +__import__('setuptools.bootstrap').bootstrap.ensure_deps() + import os import functools import distutils.core @@ -8,11 +10,12 @@ from distutils.core import Command as _Command from distutils.util import convert_path from fnmatch import fnmatchcase +from six.moves import filterfalse + import setuptools.version from setuptools.extension import Extension from setuptools.dist import Distribution, Feature, _get_unpatched from setuptools.depends import Require -from setuptools.compat import filterfalse __all__ = [ 'setup', 'Distribution', 'Feature', 'Command', 'Extension', 'Require', diff --git a/setuptools/_vendor/six-1.7.3.egg b/setuptools/_vendor/six-1.7.3.egg Binary files differnew file mode 100644 index 00000000..fe3e82c6 --- /dev/null +++ b/setuptools/_vendor/six-1.7.3.egg diff --git a/setuptools/bootstrap.py b/setuptools/bootstrap.py new file mode 100644 index 00000000..0cd95778 --- /dev/null +++ b/setuptools/bootstrap.py @@ -0,0 +1,27 @@ +""" +When setuptools is installed in a clean environment, it doesn't have its +dependencies, so it can't run to install its dependencies. This module +checks those dependencies and if one or more are missing, it uses vendored +versions. +""" + +import os +import sys +import glob + +def ensure_deps(): + """ + Detect if dependencies are installed and if not, use vendored versions. + """ + try: + __import__('six') + except ImportError: + use_vendor_deps() + +def use_vendor_deps(): + """ + Use vendored versions + """ + here = os.path.dirname(__file__) + eggs = glob.glob(here + '/_vendor/*.egg') + sys.path.extend(eggs) diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 87dce882..73f8e3f1 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -11,9 +11,10 @@ import os import marshal import textwrap +import six + from pkg_resources import get_build_platform, Distribution, ensure_directory from pkg_resources import EntryPoint -from setuptools.compat import basestring from setuptools.extension import Library from setuptools import Command @@ -413,7 +414,7 @@ def iter_symbols(code): for name in code.co_names: yield name for const in code.co_consts: - if isinstance(const, basestring): + if isinstance(const, six.string_types): yield const elif isinstance(const, CodeType): for name in iter_symbols(const): diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index 07b66ccb..ef9ac22d 100755 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -5,9 +5,10 @@ import os import glob import io +import six + from pkg_resources import Distribution, PathMetadata, normalize_path from setuptools.command.easy_install import easy_install -from setuptools.compat import PY3 import setuptools @@ -87,7 +88,7 @@ class develop(easy_install): " installation directory", p, normalize_path(os.curdir)) def install_for_development(self): - if PY3 and getattr(self.distribution, 'use_2to3', False): + if six.PY3 and getattr(self.distribution, 'use_2to3', False): # If we run 2to3 we can not do this inplace: # Ensure metadata is up-to-date diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 9ca3b515..6aab38c8 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -40,6 +40,9 @@ import subprocess import shlex import io +import six +from six.moves import configparser + from setuptools import Command from setuptools.sandbox import run_setup from setuptools.py31compat import get_path, get_config_vars @@ -48,8 +51,6 @@ from setuptools.archive_util import unpack_archive from setuptools.package_index import PackageIndex from setuptools.package_index import URL_SCHEME from setuptools.command import bdist_egg, egg_info -from setuptools.compat import (iteritems, maxsize, basestring, unicode, - reraise, PY2, PY3) from pkg_resources import ( yield_lines, normalize_path, resource_string, ensure_directory, get_distribution, find_distributions, Environment, Requirement, @@ -82,13 +83,13 @@ def samefile(p1, p2): return norm_p1 == norm_p2 -if PY2: +if six.PY2: def _to_ascii(s): return s def isascii(s): try: - unicode(s, 'ascii') + six.text_type(s, 'ascii') return True except UnicodeError: return False @@ -320,7 +321,7 @@ class easy_install(Command): self.local_index = Environment(self.shadow_path + sys.path) if self.find_links is not None: - if isinstance(self.find_links, basestring): + if isinstance(self.find_links, six.string_types): self.find_links = self.find_links.split() else: self.find_links = [] @@ -413,7 +414,7 @@ class easy_install(Command): try: pid = os.getpid() except: - pid = random.randint(0, maxsize) + pid = random.randint(0, sys.maxsize) return os.path.join(self.install_dir, "test-easy-install-%s" % pid) def warn_deprecated_options(self): @@ -1245,7 +1246,7 @@ class easy_install(Command): f = open(sitepy, 'rb') current = f.read() # we want str, not bytes - if PY3: + if six.PY3: current = current.decode() f.close() @@ -1271,7 +1272,7 @@ class easy_install(Command): if not self.user: return home = convert_path(os.path.expanduser("~")) - for name, path in iteritems(self.config_vars): + for name, path in six.iteritems(self.config_vars): if path.startswith(home) and not os.path.isdir(path): self.debug_print("os.makedirs('%s', 0o700)" % path) os.makedirs(path, 0o700) @@ -1415,9 +1416,6 @@ def extract_wininst_cfg(dist_filename): return None f.seek(prepended - 12) - from setuptools.compat import StringIO, configparser - import struct - tag, cfglen, bmlen = struct.unpack("<iii", f.read(12)) if tag not in (0x1234567A, 0x1234567B): return None # not a valid tag @@ -1432,7 +1430,7 @@ def extract_wininst_cfg(dist_filename): # Now the config is in bytes, but for RawConfigParser, it should # be text, so decode it. config = config.decode(sys.getfilesystemencoding()) - cfg.readfp(StringIO(config)) + cfg.readfp(six.StringIO(config)) except configparser.Error: return None if not cfg.has_section('metadata') or not cfg.has_section('Setup'): @@ -1467,7 +1465,7 @@ def get_exe_prefixes(exe_filename): continue if parts[0].upper() in ('PURELIB', 'PLATLIB'): contents = z.read(name) - if PY3: + if six.PY3: contents = contents.decode() for pth in yield_lines(contents): pth = pth.strip().replace('\\', '/') @@ -1643,7 +1641,7 @@ def auto_chmod(func, arg, exc): chmod(arg, stat.S_IWRITE) return func(arg) et, ev, _ = sys.exc_info() - reraise(et, (ev[0], ev[1] + (" %s %s" % (func, arg)))) + six.reraise(et, (ev[0], ev[1] + (" %s %s" % (func, arg)))) def update_dist_caches(dist_path, fix_zipimporter_caches): @@ -2209,7 +2207,7 @@ def get_win_launcher(type): def load_launcher_manifest(name): manifest = pkg_resources.resource_string(__name__, 'launcher manifest.xml') - if PY2: + if six.PY2: return manifest % vars() else: return manifest.decode('utf-8') % vars() diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 1301bd84..19849e66 100755 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -14,7 +14,7 @@ import io import warnings import time -from setuptools.compat import basestring, PY3, StringIO +import six from setuptools import Command from setuptools.command.sdist import sdist @@ -148,7 +148,7 @@ class egg_info(Command): to the file. """ log.info("writing %s to %s", what, filename) - if PY3: + if six.PY3: data = data.encode("utf-8") if not self.dry_run: f = open(filename, 'wb') @@ -412,7 +412,7 @@ def _write_requirements(stream, reqs): def write_requirements(cmd, basename, filename): dist = cmd.distribution - data = StringIO() + data = six.StringIO() _write_requirements(data, dist.install_requires) extras_require = dist.extras_require or {} for extra in sorted(extras_require): @@ -452,12 +452,12 @@ def write_arg(cmd, basename, filename, force=False): def write_entries(cmd, basename, filename): ep = cmd.distribution.entry_points - if isinstance(ep, basestring) or ep is None: + if isinstance(ep, six.string_types) or ep is None: data = ep elif ep is not None: data = [] for section, contents in sorted(ep.items()): - if not isinstance(contents, basestring): + if not isinstance(contents, six.string_types): contents = EntryPoint.parse_group(section, contents) contents = '\n'.join(sorted(map(str, contents.values()))) data.append('[%s]\n%s\n\n' % (section, contents)) diff --git a/setuptools/command/rotate.py b/setuptools/command/rotate.py index 1b073620..09eac496 100755 --- a/setuptools/command/rotate.py +++ b/setuptools/command/rotate.py @@ -3,8 +3,9 @@ from distutils import log from distutils.errors import DistutilsOptionError import os +import six + from setuptools import Command -from setuptools.compat import basestring class rotate(Command): @@ -36,7 +37,7 @@ class rotate(Command): self.keep = int(self.keep) except ValueError: raise DistutilsOptionError("--keep must be an integer") - if isinstance(self.match, basestring): + if isinstance(self.match, six.string_types): self.match = [ convert_path(p.strip()) for p in self.match.split(',') ] diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 71196512..3b9f7dd5 100755 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -5,7 +5,8 @@ import os import sys import io -from setuptools.compat import PY3 +import six + from setuptools.utils import cs_path_exists import pkg_resources @@ -181,7 +182,7 @@ class sdist(orig.sdist): manifest = open(self.manifest, 'rbU') for line in manifest: # The manifest must contain UTF-8. See #303. - if PY3: + if six.PY3: try: line = line.decode('UTF-8') except UnicodeDecodeError: diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index 74c7cad8..f78e0cd5 100755 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py @@ -4,6 +4,8 @@ from distutils.errors import DistutilsOptionError import distutils import os +from six.moves import configparser + from setuptools import Command @@ -37,8 +39,6 @@ def edit_config(filename, settings, dry_run=False): while a dictionary lists settings to be changed or deleted in that section. A setting of ``None`` means to delete that setting. """ - from setuptools.compat import configparser - log.debug("Reading configuration from %s", filename) opts = configparser.RawConfigParser() opts.read([filename]) diff --git a/setuptools/command/test.py b/setuptools/command/test.py index c26f5fc9..5f2e2299 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -2,11 +2,12 @@ from distutils.errors import DistutilsOptionError from unittest import TestLoader import sys +import six + from pkg_resources import (resource_listdir, resource_exists, normalize_path, working_set, _namespace_packages, add_activation_listener, require, EntryPoint) from setuptools import Command -from setuptools.compat import PY3 from setuptools.py31compat import unittest_main @@ -100,7 +101,7 @@ class test(Command): yield self.test_suite def with_project_on_sys_path(self, func): - with_2to3 = PY3 and getattr(self.distribution, 'use_2to3', False) + with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False) if with_2to3: # If we run 2to3 we can not do this inplace: @@ -160,7 +161,7 @@ class test(Command): # Purge modules under test from sys.modules. The test loader will # re-import them from the build location. Required when 2to3 is used # with namespace packages. - if PY3 and getattr(self.distribution, 'use_2to3', False): + if six.PY3 and getattr(self.distribution, 'use_2to3', False): module = self.test_suite.split('.')[0] if module in _namespace_packages: del_modules = [] diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 001ee936..43b5d76a 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -16,17 +16,19 @@ import tempfile import sys import shutil -from setuptools.compat import httplib, urlparse, unicode, iteritems, PY3 +from six.moves import http_client, urllib +import six + from pkg_resources import iter_entry_points -errors = 'surrogateescape' if PY3 else 'strict' +errors = 'surrogateescape' if six.PY3 else 'strict' # This is not just a replacement for byte literals # but works as a general purpose encoder def b(s, encoding='utf-8'): - if isinstance(s, unicode): + if isinstance(s, six.text_type): return s.encode(encoding, errors) return s @@ -113,7 +115,7 @@ class upload_docs(upload): # set up the authentication credentials = b(self.username + ':' + self.password) credentials = standard_b64encode(credentials) - if PY3: + if six.PY3: credentials = credentials.decode('ascii') auth = "Basic " + credentials @@ -122,7 +124,7 @@ class upload_docs(upload): sep_boundary = b('\n--') + b(boundary) end_boundary = sep_boundary + b('--') body = [] - for key, values in iteritems(data): + for key, values in six.iteritems(data): title = '\nContent-Disposition: form-data; name="%s"' % key # handle multiple entries for the same name if not isinstance(values, list): @@ -150,12 +152,12 @@ class upload_docs(upload): # We can't use urllib2 since we need to send the Basic # auth right with the first request schema, netloc, url, params, query, fragments = \ - urlparse(self.repository) + urllib.parse.urlparse(self.repository) assert not params and not query and not fragments if schema == 'http': - conn = httplib.HTTPConnection(netloc) + conn = http_client.HTTPConnection(netloc) elif schema == 'https': - conn = httplib.HTTPSConnection(netloc) + conn = http_client.HTTPSConnection(netloc) else: raise AssertionError("unsupported schema " + schema) diff --git a/setuptools/compat.py b/setuptools/compat.py deleted file mode 100644 index f0175a5d..00000000 --- a/setuptools/compat.py +++ /dev/null @@ -1,70 +0,0 @@ -import sys -import itertools - -PY3 = sys.version_info >= (3,) -PY2 = not PY3 - -if PY2: - basestring = basestring - import __builtin__ as builtins - import ConfigParser as configparser - from StringIO import StringIO - BytesIO = StringIO - func_code = lambda o: o.func_code - func_globals = lambda o: o.func_globals - im_func = lambda o: o.im_func - from htmlentitydefs import name2codepoint - import httplib - from BaseHTTPServer import HTTPServer - from SimpleHTTPServer import SimpleHTTPRequestHandler - from BaseHTTPServer import BaseHTTPRequestHandler - iteritems = lambda o: o.iteritems() - long_type = long - maxsize = sys.maxint - unichr = unichr - unicode = unicode - bytes = str - from urllib import url2pathname, splittag, pathname2url - import urllib2 - from urllib2 import urlopen, HTTPError, URLError, unquote, splituser - from urlparse import urlparse, urlunparse, urljoin, urlsplit, urlunsplit - filterfalse = itertools.ifilterfalse - filter = itertools.ifilter - map = itertools.imap - - exec("""def reraise(tp, value, tb=None): - raise tp, value, tb""") - -if PY3: - basestring = str - import builtins - import configparser - from io import StringIO, BytesIO - func_code = lambda o: o.__code__ - func_globals = lambda o: o.__globals__ - im_func = lambda o: o.__func__ - from html.entities import name2codepoint - import http.client as httplib - from http.server import HTTPServer, SimpleHTTPRequestHandler - from http.server import BaseHTTPRequestHandler - iteritems = lambda o: o.items() - long_type = int - maxsize = sys.maxsize - unichr = chr - unicode = str - bytes = bytes - from urllib.error import HTTPError, URLError - import urllib.request as urllib2 - from urllib.request import urlopen, url2pathname, pathname2url - from urllib.parse import ( - urlparse, urlunparse, unquote, splituser, urljoin, urlsplit, - urlunsplit, splittag, - ) - filterfalse = itertools.filterfalse - filter = filter - map = map - - def reraise(tp, value, tb=None): - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value diff --git a/setuptools/depends.py b/setuptools/depends.py index e87ef3f3..43617e6d 100644 --- a/setuptools/depends.py +++ b/setuptools/depends.py @@ -3,7 +3,8 @@ import imp import marshal from imp import PKG_DIRECTORY, PY_COMPILED, PY_SOURCE, PY_FROZEN from distutils.version import StrictVersion -from setuptools import compat + +import six __all__ = [ 'Require', 'find_module', 'get_module_constant', 'extract_constant' @@ -99,7 +100,8 @@ def _iter_code(code): ptr += 3 if op==EXTENDED_ARG: - extended_arg = arg * compat.long_type(65536) + long_type = six.integer_types[-1] + extended_arg = arg * long_type(65536) continue else: diff --git a/setuptools/dist.py b/setuptools/dist.py index db15809d..7335c967 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -13,8 +13,9 @@ from distutils.core import Distribution as _Distribution from distutils.errors import (DistutilsOptionError, DistutilsPlatformError, DistutilsSetupError) +import six + from setuptools.depends import Require -from setuptools.compat import basestring, PY2 from setuptools import windows_support import pkg_resources @@ -138,7 +139,7 @@ def check_entry_points(dist, attr, value): raise DistutilsSetupError(e) def check_test_suite(dist, attr, value): - if not isinstance(value,basestring): + if not isinstance(value, six.string_types): raise DistutilsSetupError("test_suite must be a string") def check_package_data(dist, attr, value): @@ -682,7 +683,7 @@ class Distribution(_Distribution): """ import sys - if PY2 or self.help_commands: + if six.PY2 or self.help_commands: return _Distribution.handle_display_options(self, option_order) # Stdout may be StringIO (e.g. in tests) diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 2c565e88..657b467f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -9,6 +9,14 @@ import hashlib import itertools from functools import wraps +try: + from urllib.parse import splituser +except ImportError: + from urllib2 import splituser + +import six +from six.moves import urllib, http_client, configparser + from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, require, Environment, find_distributions, safe_name, safe_version, @@ -17,12 +25,6 @@ from pkg_resources import ( from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError -from setuptools.compat import ( - urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, - splituser, url2pathname, name2codepoint, unichr, urljoin, urlsplit, - urlunsplit, configparser, filter, map, -) -from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers @@ -69,10 +71,11 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): - scheme, server, path, parameters, query, fragment = urlparse(url) - base = unquote(path.split('/')[-1]) + parts = urllib.parse.urlparse(url) + scheme, server, path, parameters, query, fragment = parts + base = urllib.parse.unquote(path.split('/')[-1]) if server=='sourceforge.net' and base=='download': # XXX Yuck - base = unquote(path.split('/')[-2]) + base = urllib.parse.unquote(path.split('/')[-2]) if '#' in base: base, fragment = base.split('#',1) return base,fragment @@ -158,7 +161,7 @@ def unique_everseen(iterable, key=None): seen = set() seen_add = seen.add if key is None: - for element in filterfalse(seen.__contains__, iterable): + for element in six.moves.filterfalse(seen.__contains__, iterable): seen_add(element) yield element else: @@ -190,14 +193,14 @@ def find_external_links(url, page): rels = set(map(str.strip, rel.lower().split(','))) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urljoin(url, htmldecode(match.group(1))) + yield urllib.parse.urljoin(url, htmldecode(match.group(1))) for tag in ("<th>Home Page", "<th>Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urljoin(url, htmldecode(match.group(1))) + yield urllib.parse.urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s setuptools/%s" % ( sys.version[:3], require('setuptools')[0].version @@ -240,7 +243,7 @@ class HashChecker(ContentChecker): @classmethod def from_url(cls, url): "Construct a (possibly null) ContentChecker from a URL" - fragment = urlparse(url)[-1] + fragment = urllib.parse.urlparse(url)[-1] if not fragment: return ContentChecker() match = cls.pattern.search(fragment) @@ -275,7 +278,7 @@ class PackageIndex(Environment): self.to_scan = [] if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): self.opener = ssl_support.opener_for(ca_bundle) - else: self.opener = urllib2.urlopen + else: self.opener = urllib.request.urlopen def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -312,7 +315,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. - if isinstance(f, HTTPError): + if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' else: @@ -320,7 +323,7 @@ class PackageIndex(Environment): page = page.decode(charset, "ignore") f.close() for match in HREF.finditer(page): - link = urljoin(base, htmldecode(match.group(1))) + link = urllib.parse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) @@ -343,7 +346,7 @@ class PackageIndex(Environment): def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]): + if (s and s.group(1).lower()=='file') or self.allows(urllib.parse.urlparse(url)[1]): return True msg = ("\nNote: Bypassing %s (disallowed host; see " "http://bit.ly/1dg9ijs for details).\n") @@ -384,7 +387,7 @@ class PackageIndex(Environment): # Process a URL to see if it's for a package page if link.startswith(self.index_url): parts = list(map( - unquote, link[len(self.index_url):].split('/') + urllib.parse.unquote, link[len(self.index_url):].split('/') )) if len(parts)==2 and '#' not in parts[1]: # it's a package page, sanitize and index it @@ -397,7 +400,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): try: - scan(urljoin(url, htmldecode(match.group(1)))) + scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) except ValueError: pass @@ -673,7 +676,7 @@ class PackageIndex(Environment): try: checker = HashChecker.from_url(url) fp = self.open_url(strip_fragment(url)) - if isinstance(fp, HTTPError): + if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -709,21 +712,21 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, httplib.InvalidURL) as v: + except (ValueError, http_client.InvalidURL) as v: msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib2.HTTPError as v: + except urllib.error.HTTPError as v: return v - except urllib2.URLError as v: + except urllib.error.URLError as v: if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine as v: + except http_client.BadStatusLine as v: if warning: self.warn(warning, v.line) else: @@ -732,7 +735,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) - except httplib.HTTPException as v: + except http_client.HTTPException as v: if warning: self.warn(warning, v) else: @@ -763,7 +766,7 @@ class PackageIndex(Environment): elif scheme.startswith('hg+'): return self._download_hg(url, filename) elif scheme=='file': - return url2pathname(urlparse(url)[2]) + return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) @@ -797,7 +800,7 @@ class PackageIndex(Environment): url = url.split('#',1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: - scheme, netloc, path, p, q, f = urlparse(url) + scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/',1) auth, host = splituser(netloc) @@ -808,14 +811,15 @@ class PackageIndex(Environment): else: creds = " --username="+auth netloc = host - url = urlunparse((scheme, netloc, url, p, q, f)) + parts = scheme, netloc, url, p, q, f + url = urllib.parse.urlunparse(parts) self.info("Doing subversion checkout from %s to %s", url, filename) os.system("svn checkout%s -q %s %s" % (creds, url, filename)) return filename @staticmethod def _vcs_split_rev_from_url(url, pop_prefix=False): - scheme, netloc, path, query, frag = urlsplit(url) + scheme, netloc, path, query, frag = urllib.parse.urlsplit(url) scheme = scheme.split('+', 1)[-1] @@ -827,7 +831,7 @@ class PackageIndex(Environment): path, rev = path.rsplit('@', 1) # Also, discard fragment - url = urlunsplit((scheme, netloc, path, query, '')) + url = urllib.parse.urlunsplit((scheme, netloc, path, query, '')) return url, rev @@ -879,7 +883,7 @@ entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def uchr(c): if not isinstance(c, int): return c - if c>255: return unichr(c) + if c>255: return six.unichr(c) return chr(c) def decode_entity(match): @@ -889,7 +893,7 @@ def decode_entity(match): elif what.startswith('#'): what = int(what[1:]) else: - what = name2codepoint.get(what, match.group(0)) + what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) return uchr(what) def htmldecode(text): @@ -920,7 +924,7 @@ def _encode_auth(auth): >>> chr(10) in str(_encode_auth(long_auth)) False """ - auth_s = unquote(auth) + auth_s = urllib.parse.unquote(auth) # convert to bytes auth_bytes = auth_s.encode() # use the legacy interface for Python 2.3 support @@ -984,15 +988,15 @@ class PyPIConfig(configparser.RawConfigParser): return cred -def open_with_auth(url, opener=urllib2.urlopen): +def open_with_auth(url, opener=urllib.request.urlopen): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urlparse(url) + scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): - raise httplib.InvalidURL("nonnumeric port: ''") + raise http_client.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, host = splituser(netloc) @@ -1008,11 +1012,12 @@ def open_with_auth(url, opener=urllib2.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - new_url = urlunparse((scheme,host,path,params,query,frag)) - request = urllib2.Request(new_url) + parts = scheme, host, path, params, query, frag + new_url = urllib.parse.urlunparse(parts) + request = urllib.request.Request(new_url) request.add_header("Authorization", auth) else: - request = urllib2.Request(url) + request = urllib.request.Request(url) request.add_header('User-Agent', user_agent) fp = opener(request) @@ -1020,9 +1025,10 @@ def open_with_auth(url, opener=urllib2.urlopen): if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = urlparse(fp.url) + s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) if s2==scheme and h2==host: - fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2)) + parts = s2, netloc, path2, param2, query2, frag2 + fp.url = urllib.parse.urlunparse(parts) return fp @@ -1035,10 +1041,10 @@ def fix_sf_url(url): def local_open(url): """Read a local path, with special support for directories""" - scheme, server, path, param, query, frag = urlparse(url) - filename = url2pathname(path) + scheme, server, path, param, query, frag = urllib.parse.urlparse(url) + filename = urllib.request.url2pathname(path) if os.path.isfile(filename): - return urllib2.urlopen(url) + return urllib.request.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): @@ -1059,4 +1065,5 @@ def local_open(url): status, message, body = 404, "Path not found", "Not found" headers = {'content-type': 'text/html'} - return HTTPError(url, status, message, headers, StringIO(body)) + body_stream = six.StringIO(body) + return urllib.error.HTTPError(url, status, message, headers, body_stream) diff --git a/setuptools/py26compat.py b/setuptools/py26compat.py index 738b0cc4..e52bd85b 100644 --- a/setuptools/py26compat.py +++ b/setuptools/py26compat.py @@ -4,7 +4,10 @@ Compatibility Support for Python 2.6 and earlier import sys -from setuptools.compat import splittag +try: + from urllib.parse import splittag +except ImportError: + from urllib import splittag def strip_fragment(url): """ diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 85de85ff..43b84791 100755 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -8,6 +8,9 @@ import re import contextlib import pickle +import six +from six.moves import builtins + import pkg_resources if sys.platform.startswith('java'): @@ -22,9 +25,6 @@ _open = open from distutils.errors import DistutilsError from pkg_resources import working_set -from setuptools import compat -from setuptools.compat import builtins - __all__ = [ "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup", ] @@ -138,7 +138,7 @@ class ExceptionSaver: return type, exc = map(pickle.loads, self._saved) - compat.reraise(type, exc, self._tb) + six.reraise(type, exc, self._tb) @contextlib.contextmanager diff --git a/setuptools/ssl_support.py b/setuptools/ssl_support.py index 7394f4f5..8fd7836b 100644 --- a/setuptools/ssl_support.py +++ b/setuptools/ssl_support.py @@ -3,9 +3,10 @@ import socket import atexit import re +from six.moves import urllib, http_client + import pkg_resources from pkg_resources import ResolutionError, ExtractionError -from setuptools.compat import urllib2 try: import ssl @@ -27,17 +28,11 @@ cert_paths = """ """.strip().split() -HTTPSHandler = HTTPSConnection = object - -for what, where in ( - ('HTTPSHandler', ['urllib2','urllib.request']), - ('HTTPSConnection', ['httplib', 'http.client']), -): - for module in where: - try: - exec("from %s import %s" % (module, what)) - except ImportError: - pass +try: + HTTPSHandler = urllib.request.HTTPSHandler + HTTPSConnection = http_client.HTTPSConnection +except AttributeError: + HTTPSHandler = HTTPSConnection = object is_available = ssl is not None and object not in (HTTPSHandler, HTTPSConnection) @@ -198,7 +193,7 @@ class VerifyingHTTPSConn(HTTPSConnection): def opener_for(ca_bundle=None): """Get a urlopen() replacement that uses ca_bundle for verification""" - return urllib2.build_opener( + return urllib.request.build_opener( VerifyingHTTPSHandler(ca_bundle or find_ca_bundle()) ).open diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index f985a6e4..b2c6894f 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -7,8 +7,8 @@ from distutils.errors import DistutilsOptionError, DistutilsPlatformError from distutils.errors import DistutilsSetupError from distutils.core import Extension from distutils.version import LooseVersion -from setuptools.compat import func_code +import six import pytest import setuptools.dist @@ -52,7 +52,7 @@ class TestDepends: x = "test" y = z - fc = func_code(f1) + fc = six.get_function_code(f1) # unrecognized name assert dep.extract_constant(fc,'q', -1) is None diff --git a/setuptools/tests/contexts.py b/setuptools/tests/contexts.py index 1d29284b..d9dcad84 100644 --- a/setuptools/tests/contexts.py +++ b/setuptools/tests/contexts.py @@ -5,7 +5,7 @@ import sys import contextlib import site -from ..compat import StringIO +import six @contextlib.contextmanager @@ -57,8 +57,8 @@ def quiet(): old_stdout = sys.stdout old_stderr = sys.stderr - new_stdout = sys.stdout = StringIO() - new_stderr = sys.stderr = StringIO() + new_stdout = sys.stdout = six.StringIO() + new_stderr = sys.stderr = six.StringIO() try: yield new_stdout, new_stderr finally: diff --git a/setuptools/tests/server.py b/setuptools/tests/server.py index 6b214279..1fee0563 100644 --- a/setuptools/tests/server.py +++ b/setuptools/tests/server.py @@ -3,10 +3,10 @@ import time import threading -from setuptools.compat import BaseHTTPRequestHandler -from setuptools.compat import HTTPServer, SimpleHTTPRequestHandler -class IndexServer(HTTPServer): +from six.moves import BaseHTTPServer, SimpleHTTPServer + +class IndexServer(BaseHTTPServer.HTTPServer): """Basic single-threaded http server simulating a package index You can use this server in unittest like this:: @@ -18,8 +18,9 @@ class IndexServer(HTTPServer): s.stop() """ def __init__(self, server_address=('', 0), - RequestHandlerClass=SimpleHTTPRequestHandler): - HTTPServer.__init__(self, server_address, RequestHandlerClass) + RequestHandlerClass=SimpleHTTPServer.SimpleHTTPRequestHandler): + BaseHTTPServer.HTTPServer.__init__(self, server_address, + RequestHandlerClass) self._run = True def start(self): @@ -40,19 +41,20 @@ class IndexServer(HTTPServer): port = self.server_port return 'http://127.0.0.1:%s/setuptools/tests/indexes/' % port -class RequestRecorder(BaseHTTPRequestHandler): +class RequestRecorder(BaseHTTPServer.BaseHTTPRequestHandler): def do_GET(self): requests = vars(self.server).setdefault('requests', []) requests.append(self) self.send_response(200, 'OK') -class MockServer(HTTPServer, threading.Thread): +class MockServer(BaseHTTPServer.HTTPServer, threading.Thread): """ A simple HTTP Server that records the requests made to it. """ def __init__(self, server_address=('', 0), RequestHandlerClass=RequestRecorder): - HTTPServer.__init__(self, server_address, RequestHandlerClass) + BaseHTTPServer.HTTPServer.__init__(self, server_address, + RequestHandlerClass) threading.Thread.__init__(self) self.setDaemon(True) self.requests = [] diff --git a/setuptools/tests/test_bdist_egg.py b/setuptools/tests/test_bdist_egg.py index ccfb2ea7..f8a68378 100644 --- a/setuptools/tests/test_bdist_egg.py +++ b/setuptools/tests/test_bdist_egg.py @@ -3,6 +3,7 @@ import os import re +import six import pytest from setuptools.dist import Distribution diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index ab5da00e..236b3aa6 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -5,12 +5,13 @@ import site import sys import io +import six + import pytest from setuptools.command.develop import develop from setuptools.dist import Distribution from . import contexts -from setuptools.compat import PY3 SETUP_PY = """\ @@ -85,7 +86,7 @@ class TestDevelop: with io.open(fn) as init_file: init = init_file.read().strip() - expected = 'print("foo")' if PY3 else 'print "foo"' + expected = 'print("foo")' if six.PY3 else 'print "foo"' assert init == expected def test_console_scripts(self, tmpdir): diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index f0330f17..30220b7f 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -14,7 +14,10 @@ import tarfile import logging import itertools import distutils.errors +import io +import six +from six.moves import urllib import pytest try: from unittest import mock @@ -22,8 +25,6 @@ except ImportError: import mock from setuptools import sandbox -from setuptools import compat -from setuptools.compat import StringIO, BytesIO, urlparse from setuptools.sandbox import run_setup import setuptools.command.easy_install as ei from setuptools.command.easy_install import PthDistributions @@ -272,7 +273,7 @@ class TestSetupRequires: p_index = setuptools.tests.server.MockServer() p_index.start() netloc = 1 - p_index_loc = urlparse(p_index.url)[netloc] + p_index_loc = urllib.parse.urlparse(p_index.url)[netloc] if p_index_loc.endswith(':0'): # Some platforms (Jython) don't find a port to which to bind, # so skip this test for them. @@ -391,12 +392,7 @@ def make_trivial_sdist(dist_path, setup_py): """ setup_py_file = tarfile.TarInfo(name='setup.py') - try: - # Python 3 (StringIO gets converted to io module) - MemFile = BytesIO - except AttributeError: - MemFile = StringIO - setup_py_bytes = MemFile(setup_py.encode('utf-8')) + setup_py_bytes = io.BytesIO(setup_py.encode('utf-8')) setup_py_file.size = len(setup_py_bytes.getvalue()) with tarfile_open(dist_path, 'w:gz') as dist: dist.addfile(setup_py_file, fileobj=setup_py_bytes) @@ -431,7 +427,7 @@ class TestScriptHeader: assert actual == expected @pytest.mark.xfail( - compat.PY3 and is_ascii, + six.PY3 and is_ascii, reason="Test fails in this locale on Python 3" ) @mock.patch.dict(sys.modules, java=mock.Mock(lang=mock.Mock(System= diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 07f06db2..11a6ff5a 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -7,12 +7,13 @@ import glob import os import sys +from six.moves import urllib + import pytest from setuptools.command.easy_install import easy_install from setuptools.command import easy_install as easy_install_pkg from setuptools.dist import Distribution -from setuptools.compat import urlopen def setup_module(module): @@ -26,7 +27,7 @@ def setup_module(module): pass try: - urlopen('https://pypi.python.org/pypi') + urllib.request.urlopen('https://pypi.python.org/pypi') except Exception as exc: pytest.skip(str(exc)) diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 746860d5..dca4c2aa 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -4,9 +4,10 @@ import sys import os import distutils.errors -from setuptools.compat import httplib, HTTPError, unicode, pathname2url -from .textwrap import DALS +import six +from six.moves import urllib, http_client +from .textwrap import DALS import pkg_resources import setuptools.package_index from setuptools.tests.server import IndexServer @@ -22,7 +23,7 @@ class TestPackageIndex: except Exception as v: assert url in str(v) else: - assert isinstance(v, HTTPError) + assert isinstance(v, urllib.error.HTTPError) def test_bad_url_typo(self): # issue 16 @@ -38,7 +39,7 @@ class TestPackageIndex: except Exception as v: assert url in str(v) else: - assert isinstance(v, HTTPError) + assert isinstance(v, urllib.error.HTTPError) def test_bad_url_bad_status_line(self): index = setuptools.package_index.PackageIndex( @@ -46,7 +47,7 @@ class TestPackageIndex: ) def _urlopen(*args): - raise httplib.BadStatusLine('line') + raise http_client.BadStatusLine('line') index.opener = _urlopen url = 'http://example.com' @@ -70,7 +71,7 @@ class TestPackageIndex: try: index.open_url(url) except distutils.errors.DistutilsError as error: - msg = unicode(error) + msg = six.text_type(error) assert 'nonnumeric port' in msg or 'getaddrinfo failed' in msg or 'Name or service not known' in msg return raise RuntimeError("Did not raise") @@ -167,7 +168,7 @@ class TestPackageIndex: index_file = tmpdir / 'index.html' with index_file.open('w') as f: f.write('<div>content</div>') - url = 'file:' + pathname2url(str(tmpdir)) + '/' + url = 'file:' + urllib.request.pathname2url(str(tmpdir)) + '/' res = setuptools.package_index.local_open(url) assert 'content' in res.read() diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index 8ec9a4cb..c173d713 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -9,17 +9,17 @@ import unicodedata import contextlib import io +import six import pytest import pkg_resources -from setuptools.compat import StringIO, unicode, PY3, PY2 from setuptools.command.sdist import sdist from setuptools.command.egg_info import manifest_maker from setuptools.dist import Distribution from setuptools.tests import fail_on_ascii -py3_only = pytest.mark.xfail(PY2, reason="Test runs on Python 3 only") +py3_only = pytest.mark.xfail(six.PY2, reason="Test runs on Python 3 only") SETUP_ATTRS = { @@ -37,7 +37,7 @@ setup(**%r) """ % SETUP_ATTRS -if PY3: +if six.PY3: LATIN1_FILENAME = 'smörbröd.py'.encode('latin-1') else: LATIN1_FILENAME = 'sm\xf6rbr\xf6d.py' @@ -47,7 +47,7 @@ else: @contextlib.contextmanager def quiet(): old_stdout, old_stderr = sys.stdout, sys.stderr - sys.stdout, sys.stderr = StringIO(), StringIO() + sys.stdout, sys.stderr = six.StringIO(), six.StringIO() try: yield finally: @@ -56,14 +56,14 @@ def quiet(): # Fake byte literals for Python <= 2.5 def b(s, encoding='utf-8'): - if PY3: + if six.PY3: return s.encode(encoding) return s # Convert to POSIX path def posix(path): - if PY3 and not isinstance(path, str): + if six.PY3 and not isinstance(path, str): return path.replace(os.sep.encode('ascii'), b('/')) else: return path.replace(os.sep, '/') @@ -71,7 +71,7 @@ def posix(path): # HFS Plus uses decomposed UTF-8 def decompose(path): - if isinstance(path, unicode): + if isinstance(path, six.text_type): return unicodedata.normalize('NFD', path) try: path = path.decode('utf-8') @@ -184,7 +184,7 @@ class TestSdistTest: u_contents = contents.decode('UTF-8') # The manifest should contain the UTF-8 filename - if PY2: + if six.PY2: fs_enc = sys.getfilesystemencoding() filename = filename.decode(fs_enc) @@ -289,7 +289,7 @@ class TestSdistTest: cmd.read_manifest() # The filelist should contain the UTF-8 filename - if PY3: + if six.PY3: filename = filename.decode('utf-8') assert filename in cmd.filelist.files @@ -342,7 +342,7 @@ class TestSdistTest: if sys.platform == 'darwin': filename = decompose(filename) - if PY3: + if six.PY3: fs_enc = sys.getfilesystemencoding() if sys.platform == 'win32': @@ -374,7 +374,7 @@ class TestSdistTest: with quiet(): cmd.run() - if PY3: + if six.PY3: # not all windows systems have a default FS encoding of cp1252 if sys.platform == 'win32': # Latin-1 is similar to Windows-1252 however diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index a66294c9..6587dc40 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -4,7 +4,9 @@ from __future__ import unicode_literals import os import site +from distutils.errors import DistutilsError +import six import pytest from setuptools.command.test import test diff --git a/setuptools/unicode_utils.py b/setuptools/unicode_utils.py index d2de941a..f028589e 100644 --- a/setuptools/unicode_utils.py +++ b/setuptools/unicode_utils.py @@ -1,11 +1,11 @@ import unicodedata import sys -from setuptools.compat import unicode as decoded_string +import six # HFS Plus uses decomposed UTF-8 def decompose(path): - if isinstance(path, decoded_string): + if isinstance(path, six.text_type): return unicodedata.normalize('NFD', path) try: path = path.decode('utf-8') @@ -23,7 +23,7 @@ def filesys_decode(path): """ fs_enc = sys.getfilesystemencoding() - if isinstance(path, decoded_string): + if isinstance(path, six.text_type): return path for enc in (fs_enc, "utf-8"): diff --git a/tests/manual_test.py b/tests/manual_test.py index e6489b1c..af4ec09b 100644 --- a/tests/manual_test.py +++ b/tests/manual_test.py @@ -7,7 +7,8 @@ import tempfile import subprocess from distutils.command.install import INSTALL_SCHEMES from string import Template -from setuptools.compat import urlopen + +from six.moves import urllib def _system_call(*args): assert subprocess.call(args) == 0 @@ -76,7 +77,7 @@ def test_full(): f.write(SIMPLE_BUILDOUT) with open('bootstrap.py', 'w') as f: - f.write(urlopen(BOOTSTRAP).read()) + f.write(urllib.request.urlopen(BOOTSTRAP).read()) _system_call('bin/python', 'bootstrap.py') _system_call('bin/buildout', '-q') |