diff options
31 files changed, 577 insertions, 598 deletions
@@ -26,39 +26,23 @@ # swift can use different tox env names tox_envlist: func -- job: - name: swiftclient-functional-py2 - parent: swiftclient-functional - nodeset: openstack-single-node-bionic - description: | - Run functional tests of python-swiftclient under Python 2 - vars: - devstack_localrc: - # devstack dropped support for bionic, but we want it for easier py2 support. - # Set this so we install anyway. - FORCE: "yes" - tox_envlist: py2func - - project: templates: - check-requirements - lib-forward-testing-python3 - - openstack-python-jobs - - openstack-python3-yoga-jobs + - openstack-python3-zed-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: jobs: - swiftclient-swift-functional - swiftclient-functional - - swiftclient-functional-py2 - openstack-tox-py39: voting: true gate: jobs: - swiftclient-swift-functional - swiftclient-functional - - swiftclient-functional-py2 - openstack-tox-py39: voting: true post: diff --git a/doc/source/conf.py b/doc/source/conf.py index 2673d6c..1c5fc69 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Swiftclient documentation build configuration file, created by # sphinx-quickstart on Tue Apr 17 02:17:37 2012. # diff --git a/examples/copy.py b/examples/copy.py index e928db4..808cbd5 100644 --- a/examples/copy.py +++ b/examples/copy.py @@ -9,17 +9,17 @@ logger = logging.getLogger(__name__) with SwiftService() as swift: try: - obj = SwiftCopyObject("c", {"Destination": "/cont/d"}) + obj = SwiftCopyObject("c", {"destination": "/cont/d"}) for i in swift.copy( "cont", ["a", "b", obj], - {"meta": ["foo:bar"], "Destination": "/cc"}): + {"meta": ["foo:bar"], "destination": "/cc"}): if i["success"]: if i["action"] == "copy_object": print( "object %s copied from /%s/%s" % (i["destination"], i["container"], i["object"]) ) - if i["action"] == "create_container": + elif i["action"] == "create_container": print( "container %s created" % i["container"] ) diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 0945c81..0d20256 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -72,7 +71,7 @@ copyright = '%d, OpenStack Foundation' % datetime.datetime.now().year # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index fb60ee0..f1da9a6 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + yoga xena wallaby victoria diff --git a/releasenotes/source/yoga.rst b/releasenotes/source/yoga.rst new file mode 100644 index 0000000..7cd5e90 --- /dev/null +++ b/releasenotes/source/yoga.rst @@ -0,0 +1,6 @@ +========================= +Yoga Series Release Notes +========================= + +.. release-notes:: + :branch: stable/yoga diff --git a/requirements.txt b/requirements.txt index 4757239..94cc57f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1 @@ -futures>=3.0.0;python_version=='2.7' # BSD -requests>=1.1.0 -six>=1.9.0 +requests>=2.4.0 diff --git a/run_tests.sh b/run_tests.sh index 6991cd1..d1fc50a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,7 +9,7 @@ function usage { echo "" echo "This script is deprecated and currently retained for compatibility." echo 'You can run the full test suite for multiple environments by running "tox".' - echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only' + echo 'You can run tests for only python 3.9 by running "tox -e py39", or run only' echo 'the pep8 tests with "tox -e pep8".' exit } @@ -39,7 +39,7 @@ if [ $just_pep8 -eq 1 ]; then exit fi -tox -e py27 $toxargs 2>&1 | tee run_tests.err.log || exit +tox -e py39 $toxargs 2>&1 | tee run_tests.err.log || exit if [ ${PIPESTATUS[0]} -ne 0 ]; then exit ${PIPESTATUS[0]} fi @@ -3,9 +3,11 @@ name = python-swiftclient summary = OpenStack Object Storage API Client Library description_file = README.rst +license = Apache License, Version 2.0 author = OpenStack author_email = openstack-discuss@lists.openstack.org home_page = https://docs.openstack.org/python-swiftclient/latest/ +python_requires = >=3.6 classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -14,13 +16,12 @@ classifier = Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3 :: Only [files] packages = @@ -41,9 +42,6 @@ console_scripts = keystoneauth1.plugin = v1password = swiftclient.authv1:PasswordLoader -[bdist_wheel] -universal = 1 - [pbr] skip_authors = True skip_changelog = True @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,12 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT -import setuptools, sys - -if sys.version_info < (2, 7): - sys.exit('Sorry, Python < 2.7 is not supported for' - ' python-swiftclient>=3.0') +import setuptools setuptools.setup( setup_requires=['pbr'], diff --git a/swiftclient/__init__.py b/swiftclient/__init__.py index dc192af..38750d1 100644 --- a/swiftclient/__init__.py +++ b/swiftclient/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2012 Rackspace # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/swiftclient/authv1.py b/swiftclient/authv1.py index d70acac..84bc38a 100644 --- a/swiftclient/authv1.py +++ b/swiftclient/authv1.py @@ -40,7 +40,7 @@ import datetime import json import time -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin # Note that while we import keystoneauth1 here, we *don't* need to add it to # requirements.txt -- this entire module only makes sense (and should only be @@ -68,7 +68,7 @@ UTC = _UTC() del _UTC -class ServiceCatalogV1(object): +class ServiceCatalogV1: def __init__(self, auth_url, storage_url, account): self.auth_url = auth_url self._storage_url = storage_url @@ -148,7 +148,7 @@ class ServiceCatalogV1(object): raise exceptions.EndpointNotFound(msg) -class AccessInfoV1(object): +class AccessInfoV1: """An object for encapsulating a raw v1 auth token.""" def __init__(self, auth_url, storage_url, account, username, auth_token, diff --git a/swiftclient/client.py b/swiftclient/client.py index 544247a..e42ac70 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -18,20 +18,18 @@ OpenStack Swift client library used internally """ import socket import re -import requests import logging import warnings -from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError -from six.moves import http_client -from six.moves.urllib.parse import quote as _quote, unquote -from six.moves.urllib.parse import urljoin, urlparse, urlunparse +import http.client as http_client +from urllib.parse import quote, unquote +from urllib.parse import urljoin, urlparse, urlunparse from time import sleep, time -import six from swiftclient import version as swiftclient_version from swiftclient.exceptions import ClientException +from swiftclient.requests_compat import SwiftClientRequestsSession from swiftclient.utils import ( iter_wrapper, LengthWrapper, ReadableToIterable, parse_api_response, get_body) @@ -48,25 +46,11 @@ USER_METADATA_TYPE = tuple('x-%s-meta-' % type_ for type_ in URI_PATTERN_INFO = re.compile(r'/info') URI_PATTERN_VERSION = re.compile(r'\/v\d+\.?\d*(\/.*)?') -try: - from logging import NullHandler -except ImportError: - # Added in Python 2.7 - class NullHandler(logging.Handler): - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - ksexceptions = ksclient_v2 = ksclient_v3 = ksa_v3 = None try: from keystoneclient import exceptions as ksexceptions # prevent keystoneclient warning us that it has no log handlers - logging.getLogger('keystoneclient').addHandler(NullHandler()) + logging.getLogger('keystoneclient').addHandler(logging.NullHandler()) from keystoneclient.v2_0 import client as ksclient_v2 except ImportError: pass @@ -78,22 +62,8 @@ try: except ImportError: pass -# requests version 1.2.3 try to encode headers in ascii, preventing -# utf-8 encoded header to be 'prepared'. This also affects all -# (or at least most) versions of requests on py3 -if StrictVersion(requests.__version__) < StrictVersion('2.0.0') \ - or not six.PY2: - from requests.structures import CaseInsensitiveDict - - def prepare_unicode_headers(self, headers): - if headers: - self.headers = CaseInsensitiveDict(headers) - else: - self.headers = CaseInsensitiveDict() - requests.models.PreparedRequest.prepare_headers = prepare_unicode_headers - logger = logging.getLogger("swiftclient") -logger.addHandler(NullHandler()) +logger.addHandler(logging.NullHandler()) #: Default behaviour is to redact header values known to contain secrets, #: such as ``X-Auth-Key`` and ``X-Auth-Token``. Up to the first 16 chars @@ -194,56 +164,32 @@ def http_log(args, kwargs, resp, body): def parse_header_string(data): - if not isinstance(data, (six.text_type, six.binary_type)): + if not isinstance(data, (str, bytes)): data = str(data) - if six.PY2: - if isinstance(data, six.text_type): - # Under Python2 requests only returns binary_type, but if we get - # some stray text_type input, this should prevent unquote from - # interpreting %-encoded data as raw code-points. - data = data.encode('utf8') + if isinstance(data, bytes): + # Under Python3 requests only returns text_type and tosses (!) the + # rest of the headers. If that ever changes, this should be a sane + # approach. try: - unquoted = unquote(data).decode('utf8') + data = data.decode('ascii') except UnicodeDecodeError: - try: - return data.decode('utf8') - except UnicodeDecodeError: - return quote(data).decode('utf8') - else: - if isinstance(data, six.binary_type): - # Under Python3 requests only returns text_type and tosses (!) the - # rest of the headers. If that ever changes, this should be a sane - # approach. - try: - data = data.decode('ascii') - except UnicodeDecodeError: - data = quote(data) - try: - unquoted = unquote(data, errors='strict') - except UnicodeDecodeError: - return data + data = quote(data) + try: + unquoted = unquote(data, errors='strict') + except UnicodeDecodeError: + return data return unquoted -def quote(value, safe='/'): - """ - Patched version of urllib.quote that encodes utf8 strings before quoting. - On Python 3, call directly urllib.parse.quote(). - """ - if six.PY3: - return _quote(value, safe=safe) - return _quote(encode_utf8(value), safe) - - def encode_utf8(value): - if type(value) in six.integer_types + (float, bool): + if type(value) in (int, float, bool): # As of requests 2.11.0, headers must be byte- or unicode-strings. # Convert some known-good types as a convenience for developers. # Note that we *don't* convert subclasses, as they may have overriddden # __str__ or __repr__. # See https://github.com/kennethreitz/requests/pull/3366 for more info value = str(value) - if isinstance(value, six.text_type): + if isinstance(value, str): value = value.encode('utf8') return value @@ -255,7 +201,7 @@ def encode_meta_headers(headers): value = encode_utf8(value) header = header.lower() - if (isinstance(header, six.string_types) and + if (isinstance(header, str) and header.startswith(USER_METADATA_TYPE)): header = encode_utf8(header) @@ -263,7 +209,7 @@ def encode_meta_headers(headers): return ret -class _ObjectBody(object): +class _ObjectBody: """ Readable and iterable object body response wrapper. """ @@ -377,7 +323,7 @@ class _RetryBody(_ObjectBody): return buf -class HTTPConnection(object): +class HTTPConnection: def __init__(self, url, proxy=None, cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=False, default_user_agent=None, timeout=None): @@ -412,7 +358,7 @@ class HTTPConnection(object): self.host = self.parsed_url.netloc self.port = self.parsed_url.port self.requests_args = {} - self.request_session = requests.Session() + self.request_session = SwiftClientRequestsSession() # Don't use requests's default headers self.request_session.headers = None self.resp = None @@ -486,12 +432,12 @@ class HTTPConnection(object): old_getheader = self.resp.raw.getheader def _decode_header(string): - if string is None or six.PY2: + if string is None: return string return string.encode('iso-8859-1').decode('utf-8') def _encode_header(string): - if string is None or six.PY2: + if string is None: return string return string.encode('utf-8').decode('iso-8859-1') @@ -1448,11 +1394,6 @@ def put_object(url, token=None, container=None, name=None, contents=None, content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type - elif 'Content-Type' not in headers: - if StrictVersion(requests.__version__) < StrictVersion('2.4.0'): - # python-requests sets application/x-www-form-urlencoded otherwise - # if using python3. - headers['Content-Type'] = '' if not contents: headers['Content-Length'] = '0' @@ -1475,7 +1416,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, warnings.warn(warn_msg, stacklevel=2) # Match requests's is_stream test if hasattr(contents, '__iter__') and not isinstance(contents, ( - six.text_type, six.binary_type, list, tuple, dict)): + str, bytes, list, tuple, dict)): contents = iter_wrapper(contents) conn.request('PUT', path, contents, headers) @@ -1685,7 +1626,7 @@ def get_capabilities(http_conn): return parse_api_response(resp_headers, body) -class Connection(object): +class Connection: """ Convenience class to make requests that will also retry the request @@ -1705,7 +1646,7 @@ class Connection(object): starting_backoff=1, max_backoff=64, tenant_name=None, os_options=None, auth_version="1", cacert=None, insecure=False, cert=None, cert_key=None, - ssl_compression=True, retry_on_ratelimit=False, + ssl_compression=True, retry_on_ratelimit=True, timeout=None, session=None, force_auth_retry=False): """ :param authurl: authentication URL @@ -1737,9 +1678,9 @@ class Connection(object): will be made. This may provide a performance increase for https upload/download operations. :param retry_on_ratelimit: by default, a ratelimited connection will - raise an exception to the caller. Setting - this parameter to True will cause a retry - after a backoff. + retry after a backoff. Setting this + parameter to False will cause an exception + to be raised to the caller. :param timeout: The connect timeout for the HTTP connection. :param session: A keystoneauth session object. :param force_auth_retry: reset auth info even if client got unexpected @@ -1884,7 +1825,7 @@ class Connection(object): self.http_conn = None elif 500 <= err.http_status <= 599: pass - elif self.retry_on_ratelimit and err.http_status == 498: + elif self.retry_on_ratelimit and err.http_status in (498, 429): pass else: raise diff --git a/swiftclient/exceptions.py b/swiftclient/exceptions.py index a9b993c..f0d1b5d 100644 --- a/swiftclient/exceptions.py +++ b/swiftclient/exceptions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from six.moves import urllib +import urllib class ClientException(Exception): diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index f128790..eaccec6 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -13,16 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -import six import sys from concurrent.futures import ThreadPoolExecutor -from six.moves.queue import PriorityQueue +from queue import PriorityQueue -class OutputManager(object): +class OutputManager: """ One object to manage and provide helper functions for output. @@ -72,12 +69,8 @@ class OutputManager(object): self.print_pool.submit(self._write, data, self.print_stream) def _write(self, data, stream): - if six.PY3: - stream.buffer.write(data) - stream.flush() - if six.PY2: - stream.write(data) - stream.flush() + stream.buffer.write(data) + stream.flush() def print_msg(self, msg, *fmt_args): if fmt_args: @@ -102,8 +95,6 @@ class OutputManager(object): def _print(self, item, stream=None): if stream is None: stream = self.print_stream - if six.PY2 and isinstance(item, six.text_type): - item = item.encode('utf8') print(item, file=stream) def _print_error(self, item, count=1): @@ -117,7 +108,7 @@ class OutputManager(object): self.error_print_pool.submit(self._print_error, msg, count=0) -class MultiThreadingManager(object): +class MultiThreadingManager: """ One object to manage context for multi-threading. This should make bin/swift less error-prone and allow us to test this code. diff --git a/swiftclient/requests_compat.py b/swiftclient/requests_compat.py new file mode 100644 index 0000000..c2371b7 --- /dev/null +++ b/swiftclient/requests_compat.py @@ -0,0 +1,57 @@ +# Copyright (c) 2010-2022 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +from requests.sessions import merge_setting, merge_hooks +from requests.structures import CaseInsensitiveDict + + +class SwiftClientPreparedRequest(requests.PreparedRequest): + def prepare_headers(self, headers): + try: + return super().prepare_headers(headers) + except UnicodeError: + # If we got an unicode error from the superclass's prepare_headers, + # we had a non-spec-compliant non-ASCII header + # (e.g. an UTF-8 encoded Swift object metadata header). + # In that case, we just pass it through and hope nothing + # bad will happen from not following the HTTP spec. + self.headers = CaseInsensitiveDict(headers or {}) + + +class SwiftClientRequestsSession(requests.Session): + + def prepare_request(self, request): + # Close to the superclass's implementation, + # but no cookies or .netrc authentication overrides here. + p = SwiftClientPreparedRequest() + headers = merge_setting( + request.headers, + self.headers, + dict_class=CaseInsensitiveDict, + ) + p.prepare( + method=request.method.upper(), + url=request.url, + files=request.files, + data=request.data, + json=request.json, + headers=headers, + params=merge_setting(request.params, self.params), + auth=merge_setting(request.auth, self.auth), + cookies=None, + hooks=merge_hooks(request.hooks, self.hooks), + ) + return p diff --git a/swiftclient/service.py b/swiftclient/service.py index 685b748..9a6c7a1 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,9 +12,8 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals -import logging +import logging import os from collections import defaultdict @@ -22,6 +21,7 @@ from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT from hashlib import md5 +from io import StringIO from os import environ, makedirs, stat, utime from os.path import ( basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep @@ -30,10 +30,9 @@ from posixpath import join as urljoin from random import shuffle from time import time from threading import Thread -from six import Iterator, StringIO, string_types, text_type -from six.moves.queue import Queue -from six.moves.queue import Empty as QueueEmpty -from six.moves.urllib.parse import quote +from queue import Queue +from queue import Empty as QueueEmpty +from urllib.parse import quote import json @@ -55,7 +54,7 @@ DISK_BUFFER = 2 ** 16 logger = logging.getLogger("swiftclient.service") -class ResultsIterator(Iterator): +class ResultsIterator: def __init__(self, futures): self.futures = interruptable_as_completed(futures) @@ -156,6 +155,7 @@ def _build_default_global_options(): "user": environ.get('ST_USER'), "key": environ.get('ST_KEY'), "retries": 5, + "retry_on_ratelimit": True, "force_auth_retry": False, "os_username": environ.get('OS_USERNAME'), "os_user_id": environ.get('OS_USER_ID'), @@ -272,9 +272,12 @@ def get_conn(options): """ Return a connection building it from the options. """ + options = dict(_default_global_options, **options) return Connection(options['auth'], options['user'], options['key'], + timeout=options.get('timeout'), + retry_on_ratelimit=options['retry_on_ratelimit'], retries=options['retries'], auth_version=options['auth_version'], os_options=options['os_options'], @@ -317,16 +320,16 @@ def split_headers(options, prefix=''): return headers -class SwiftUploadObject(object): +class SwiftUploadObject: """ Class for specifying an object upload, allowing the object source, name and options to be specified separately for each individual object. """ def __init__(self, source, object_name=None, options=None): - if isinstance(source, string_types): + if isinstance(source, str): self.object_name = object_name or source elif source is None or hasattr(source, 'read'): - if not object_name or not isinstance(object_name, string_types): + if not object_name or not isinstance(object_name, str): raise SwiftError('Object names must be specified as ' 'strings for uploads from None or file ' 'like objects.') @@ -343,13 +346,13 @@ class SwiftUploadObject(object): self.source = source -class SwiftPostObject(object): +class SwiftPostObject: """ Class for specifying an object post, allowing the headers/metadata to be specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -357,13 +360,13 @@ class SwiftPostObject(object): self.options = options -class SwiftDeleteObject(object): +class SwiftDeleteObject: """ Class for specifying an object delete, allowing the headers/metadata to be specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -371,7 +374,7 @@ class SwiftDeleteObject(object): self.options = options -class SwiftCopyObject(object): +class SwiftCopyObject: """ Class for specifying an object copy, allowing the destination/headers/metadata/fresh_metadata to be specified @@ -379,7 +382,7 @@ class SwiftCopyObject(object): destination and fresh_metadata should be set in options """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -407,7 +410,7 @@ class SwiftCopyObject(object): ) -class _SwiftReader(object): +class _SwiftReader: """ Class for downloading objects from swift and raising appropriate errors on failures caused by either invalid md5sum or size of the @@ -472,7 +475,7 @@ class _SwiftReader(object): return self._actual_read -class SwiftService(object): +class SwiftService: """ Service for performing swift operations """ @@ -837,7 +840,7 @@ class SwiftService(object): post_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftPostObject(o) post_objects.append(obj) elif isinstance(o, SwiftPostObject): @@ -1643,7 +1646,7 @@ class SwiftService(object): upload_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftUploadObject(o, urljoin(pseudo_folder, o.lstrip('/'))) upload_objects.append(obj) @@ -2039,11 +2042,6 @@ class SwiftService(object): if headers is None: headers = {} segment_results.sort(key=lambda di: di['segment_index']) - for seg in segment_results: - seg_loc = seg['segment_location'].lstrip('/') - if isinstance(seg_loc, text_type): - seg_loc = seg_loc.encode('utf-8') - manifest_data = json.dumps([ { 'path': d['segment_location'], @@ -2584,7 +2582,7 @@ class SwiftService(object): delete_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftDeleteObject(o) delete_objects.append(obj) elif isinstance(o, SwiftDeleteObject): @@ -2939,7 +2937,7 @@ class SwiftService(object): copy_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftCopyObject(o, options) copy_objects.append(obj) elif isinstance(o, SwiftCopyObject): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 76473fd..4bcb251 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function, unicode_literals - import argparse import getpass import io @@ -27,8 +25,7 @@ import warnings from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join -from six import text_type, PY2 -from six.moves.urllib.parse import unquote, urlparse +from urllib.parse import unquote, urlparse from sys import argv as sys_argv, exit, stderr, stdin from time import gmtime, strftime @@ -193,10 +190,6 @@ def st_delete(parser, args, output_manager, return_parser=False): for o, err in r.get('result', {}).get('Errors', []): # o will be of the form quote("/<cont>/<obj>") o = unquote(o) - if PY2: - # In PY3, unquote(unicode) uses utf-8 like we - # want, but PY2 uses latin-1 - o = o.encode('latin-1').decode('utf-8') output_manager.error('Error Deleting: {0}: {1}' .format(o[1:], err)) try: @@ -1441,6 +1434,8 @@ Optional arguments: ISO 8601 UTC timestamp instead of a Unix timestamp. --ip-range If present, the temporary URL will be restricted to the given ip or ip range. + --digest The digest algorithm to use. Defaults to sha256, but + older clusters may only support sha1. '''.strip('\n') @@ -1470,6 +1465,12 @@ def st_tempurl(parser, args, thread_manager, return_parser=False): help=("If present, the temporary URL will be restricted to the " "given ip or ip range."), ) + parser.add_argument( + '--digest', choices=('sha1', 'sha256', 'sha512'), + default='sha256', + help=("The digest algorithm to use. Defaults to sha256, but " + "older clusters may only support sha1."), + ) # We return the parser to build up the bash_completion if return_parser: @@ -1494,7 +1495,8 @@ def st_tempurl(parser, args, thread_manager, return_parser=False): absolute=options['absolute_expiry'], iso8601=options['iso8601'], prefix=options['prefix_based'], - ip_range=options['ip_range']) + ip_range=options['ip_range'], + digest=options['digest']) except ValueError as err: thread_manager.error(err) return @@ -1742,6 +1744,9 @@ def add_default_args(parser): parser.add_argument('-K', '--key', dest='key', default=environ.get('ST_KEY'), help='Key for obtaining an auth token.') + parser.add_argument('-T', '--timeout', type=int, dest='timeout', + default=None, + help='Timeout in seconds to wait for response.') parser.add_argument('-R', '--retries', type=int, default=5, dest='retries', help='The number of times to retry a failed ' 'connection.') @@ -1940,8 +1945,6 @@ def add_default_args(parser): def main(arguments=None): argv = sys_argv if arguments is None else arguments - argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] - parser = argparse.ArgumentParser( add_help=False, formatter_class=HelpFormatter, usage=''' %(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 656acad..c865d27 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -13,17 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous utility functions for use with Swift.""" + +import base64 from calendar import timegm -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping +from collections.abc import Mapping import gzip import hashlib import hmac +import io import json import logging -import six import time import traceback @@ -42,7 +41,7 @@ def config_true_value(value): This function comes from swift.common.utils.config_true_value() """ return value is True or \ - (isinstance(value, six.string_types) and value.lower() in TRUE_VALUES) + (isinstance(value, str) and value.lower() in TRUE_VALUES) def prt_bytes(num_bytes, human_flag): @@ -72,7 +71,8 @@ def prt_bytes(num_bytes, human_flag): def generate_temp_url(path, seconds, key, method, absolute=False, - prefix=False, iso8601=False, ip_range=None): + prefix=False, iso8601=False, ip_range=None, + digest='sha256'): """Generates a temporary URL that gives unauthenticated access to the Swift object. @@ -97,7 +97,11 @@ def generate_temp_url(path, seconds, key, method, absolute=False, instead of a UNIX timestamp will be created. :param ip_range: if a valid ip range, restricts the temporary URL to the range of ips. - :raises ValueError: if timestamp or path is not in valid format. + :param digest: digest algorithm to use. Must be one of ``sha1``, + ``sha256``, or ``sha512``. + :raises ValueError: if timestamp or path is not in valid format, + or if digest is not one of ``sha1``, ``sha256``, or + ``sha512``. :return: the path portion of a temporary URL """ try: @@ -134,7 +138,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, except ValueError: raise ValueError(TIME_ERRMSG) - if isinstance(path, six.binary_type): + if isinstance(path, bytes): try: path_for_body = path.decode('utf-8') except UnicodeDecodeError: @@ -142,6 +146,11 @@ def generate_temp_url(path, seconds, key, method, absolute=False, else: path_for_body = path + if isinstance(digest, str) and digest in ('sha1', 'sha256', 'sha512'): + digest = getattr(hashlib, digest) + if digest not in (hashlib.sha1, hashlib.sha256, hashlib.sha512): + raise ValueError('digest must be one of sha1, sha256, or sha512') + parts = path_for_body.split('/', 4) if len(parts) != 5 or parts[0] or not all(parts[1:(4 if prefix else 5)]): if prefix: @@ -165,7 +174,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, ('prefix:' if prefix else '') + path_for_body] if ip_range: - if isinstance(ip_range, six.binary_type): + if isinstance(ip_range, bytes): try: ip_range = ip_range.decode('utf-8') except UnicodeDecodeError: @@ -174,27 +183,32 @@ def generate_temp_url(path, seconds, key, method, absolute=False, ) hmac_parts.insert(0, "ip=%s" % ip_range) - hmac_body = u'\n'.join(hmac_parts) + hmac_body = '\n'.join(hmac_parts) # Encode to UTF-8 for py3 compatibility - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') - sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest() + mac = hmac.new(key, hmac_body.encode('utf-8'), digest) + if digest == hashlib.sha512: + sig = 'sha512:' + base64.urlsafe_b64encode( + mac.digest()).decode('ascii').strip('=') + else: + sig = mac.hexdigest() if iso8601: expiration = time.strftime( EXPIRES_ISO8601_FORMAT, time.gmtime(expiration)) - temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( + temp_url = '{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) if ip_range: - temp_url += u'&temp_url_ip_range={}'.format(ip_range) + temp_url += '&temp_url_ip_range={}'.format(ip_range) if prefix: - temp_url += u'&temp_url_prefix={}'.format(parts[4]) + temp_url += '&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller - if isinstance(path, six.binary_type): + if isinstance(path, bytes): return temp_url.encode('utf-8') else: return temp_url @@ -202,7 +216,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, def get_body(headers, body): if headers.get('content-encoding') == 'gzip': - with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz: + with gzip.GzipFile(fileobj=io.BytesIO(body), mode='r') as gz: nbody = gz.read() return nbody return body @@ -224,7 +238,7 @@ def split_request_headers(options, prefix=''): if isinstance(options, Mapping): options = options.items() for item in options: - if isinstance(item, six.string_types): + if isinstance(item, str): if ':' not in item: raise ValueError( "Metadata parameter %s must contain a ':'.\n" @@ -256,7 +270,7 @@ def report_traceback(): return None, None -class NoopMD5(object): +class NoopMD5: def __init__(self, *a, **kw): pass @@ -267,7 +281,7 @@ class NoopMD5(object): return '' -class ReadableToIterable(object): +class ReadableToIterable: """ Wrap a filelike object and act as an iterator. @@ -316,7 +330,7 @@ class ReadableToIterable(object): return self -class LengthWrapper(object): +class LengthWrapper: """ Wrap a filelike object with a maximum length. @@ -401,8 +415,6 @@ def n_groups(seq, n): def normalize_manifest_path(path): - if six.PY2 and isinstance(path, six.text_type): - path = path.encode('utf-8') if path.startswith('/'): return path[1:] return path diff --git a/test-requirements.txt b/test-requirements.txt index c2fb2c6..a4b64ee 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,6 @@ -hacking>=1.1.0,<1.2.0;python_version<'3.0' # Apache-2.0 -hacking>=3.2.0,<3.3.0;python_version>='3.0' # Apache-2.0 +hacking>=3.2.0,<3.3.0 # Apache-2.0 -coverage!=4.4,>=4.0 # Apache-2.0 +coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 -mock>=1.2.0 # BSD stestr>=2.0.0,!=3.0.0 # Apache-2.0 -openstacksdk>=0.11.0 # Apache-2.0 +openstacksdk>=0.11.0 # Apache-2.0 diff --git a/test/functional/__init__.py b/test/functional/__init__.py index f0fea3b..249dafe 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser import os -from six.moves import configparser TEST_CONFIG = None diff --git a/test/functional/test_swiftclient.py b/test/functional/test_swiftclient.py index 8d001d0..a5c1211 100644 --- a/test/functional/test_swiftclient.py +++ b/test/functional/test_swiftclient.py @@ -17,8 +17,6 @@ import unittest import time from io import BytesIO -import six - import swiftclient from . import TEST_CONFIG @@ -334,7 +332,7 @@ class TestFunctional(unittest.TestCase): resp_chunk_size=resp_chunk_size) data = next(body) self.assertEqual(self.test_data[:resp_chunk_size], data) - self.assertTrue(1, self.conn.attempts) + self.assertEqual(1, self.conn.attempts) for chunk in body.resp: # Flush remaining data from underlying response # (simulate a dropped connection) @@ -371,13 +369,13 @@ class TestFunctional(unittest.TestCase): hdrs, body = self.conn.get_object(self.containername, self.objectname) data = body self.assertEqual(self.test_data, data) - self.assertTrue(1, self.conn.attempts) + self.assertEqual(1, self.conn.attempts) hdrs, body = self.conn.get_object(self.containername, self.objectname, resp_chunk_size=0) data = body self.assertEqual(self.test_data, data) - self.assertTrue(1, self.conn.attempts) + self.assertEqual(1, self.conn.attempts) def test_post_account(self): self.conn.post_account({'x-account-meta-data': 'Something'}) @@ -411,18 +409,12 @@ class TestFunctional(unittest.TestCase): def test_post_object_unicode_header_name(self): self.conn.post_object(self.containername, self.objectname, - {u'x-object-meta-\U0001f44d': u'\U0001f44d'}) + {'x-object-meta-\U0001f44d': '\U0001f44d'}) # Note that we can't actually read this header back on py3; see # https://bugs.python.org/issue37093 # We'll have to settle for just testing that the POST doesn't blow up # with a UnicodeDecodeError - if six.PY2: - headers = self.conn.head_object( - self.containername, self.objectname) - self.assertIn(u'x-object-meta-\U0001f44d', headers) - self.assertEqual(u'\U0001f44d', - headers.get(u'x-object-meta-\U0001f44d')) def test_copy_object(self): self.conn.put_object( diff --git a/test/unit/test_authv1.py b/test/unit/test_authv1.py index 2ddf24b..b4de7e0 100644 --- a/test/unit/test_authv1.py +++ b/test/unit/test_authv1.py @@ -14,15 +14,15 @@ import datetime import json -import mock import unittest +from unittest import mock from keystoneauth1 import plugin from keystoneauth1 import loading from keystoneauth1 import exceptions from swiftclient import authv1 -class TestDataNoAccount(object): +class TestDataNoAccount: options = dict( auth_url='http://saio:8080/auth/v1.0', username='test:tester', @@ -32,7 +32,7 @@ class TestDataNoAccount(object): token = 'token' -class TestDataWithAccount(object): +class TestDataWithAccount: options = dict( auth_url='http://saio:8080/auth/v1.0', username='test2:tester2', diff --git a/test/unit/test_command_helpers.py b/test/unit/test_command_helpers.py index 24684ae..3e51aa9 100644 --- a/test/unit/test_command_helpers.py +++ b/test/unit/test_command_helpers.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock -from six import StringIO +from io import StringIO import unittest +from unittest import mock from swiftclient import command_helpers as h from swiftclient.multithreading import OutputManager diff --git a/test/unit/test_multithreading.py b/test/unit/test_multithreading.py index e9732cd..8237d82 100644 --- a/test/unit/test_multithreading.py +++ b/test/unit/test_multithreading.py @@ -12,13 +12,13 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. + +from queue import Queue, Empty import sys import unittest import threading -import six from concurrent.futures import as_completed -from six.moves.queue import Queue, Empty from time import sleep from swiftclient import multithreading as mt @@ -192,18 +192,18 @@ class TestOutputManager(unittest.TestCase): thread_manager.print_msg('one-argument') thread_manager.print_msg('one %s, %d fish', 'fish', 88) thread_manager.error('I have %d problems, but a %s is not one', - 99, u'\u062A\u062A') + 99, '\u062A\u062A') thread_manager.print_msg('some\n%s\nover the %r', 'where', - u'\u062A\u062A') + '\u062A\u062A') thread_manager.error('one-error-argument') thread_manager.error('Sometimes\n%.1f%% just\ndoes not\nwork!', 3.14159) thread_manager.print_raw( - u'some raw bytes: \u062A\u062A'.encode('utf-8')) + 'some raw bytes: \u062A\u062A'.encode('utf-8')) thread_manager.print_items([ ('key', 'value'), - ('object', u'O\u0308bject'), + ('object', 'O\u0308bject'), ]) thread_manager.print_raw(b'\xffugly\xffraw') @@ -216,23 +216,18 @@ class TestOutputManager(unittest.TestCase): # The threads should have been cleaned up self.assertEqual(starting_thread_count, threading.active_count()) - if six.PY3: - over_the = "over the '\u062a\u062a'\n" - else: - over_the = "over the u'\\u062a\\u062a'\n" - # We write to the CaptureStream so no decoding is performed self.assertEqual(''.join([ 'one-argument\n', 'one fish, 88 fish\n', 'some\n', 'where\n', - over_the, - u'some raw bytes: \u062a\u062a', + "over the '\u062a\u062a'\n", + 'some raw bytes: \u062a\u062a', ' key: value\n', - u' object: O\u0308bject\n' + ' object: O\u0308bject\n' ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) self.assertEqual(''.join([ - u'I have 99 problems, but a \u062A\u062A is not one\n', + 'I have 99 problems, but a \u062A\u062A is not one\n', 'one-error-argument\n', 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' ]), err_stream.getvalue().decode('utf8')) diff --git a/test/unit/test_service.py b/test/unit/test_service.py index e86a4ff..1176a1f 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2014 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,21 +12,21 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals + +import builtins import contextlib -import mock +import io import os -import six import tempfile import unittest import time import json +from io import BytesIO +from unittest import mock from concurrent.futures import Future from hashlib import md5 -from mock import Mock, PropertyMock -from six.moves.queue import Queue, Empty as QueueEmptyError -from six import BytesIO +from queue import Queue, Empty as QueueEmptyError from time import sleep import swiftclient @@ -47,12 +46,6 @@ for key in os.environ: clean_os_environ[key] = '' -if six.PY2: - import __builtin__ as builtins -else: - import builtins - - class TestSwiftPostObject(unittest.TestCase): def setUp(self): @@ -221,9 +214,9 @@ class TestSwiftReader(unittest.TestCase): class _TestServiceBase(unittest.TestCase): def _get_mock_connection(self, attempts=2): - m = Mock(spec=Connection) - type(m).attempts = PropertyMock(return_value=attempts) - type(m).auth_end_time = PropertyMock(return_value=4) + m = mock.Mock(spec=Connection) + type(m).attempts = mock.PropertyMock(return_value=attempts) + type(m).auth_end_time = mock.PropertyMock(return_value=4) return m def _get_queue(self, q): @@ -278,7 +271,7 @@ class TestServiceDelete(_TestServiceBase): def test_delete_segment_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.delete_object = Mock(side_effect=self.exc) + mock_conn.delete_object = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_segment', 'object': 'test_s', @@ -304,7 +297,7 @@ class TestServiceDelete(_TestServiceBase): def test_delete_object(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock(return_value={}) + mock_conn.head_object = mock.Mock(return_value={}) expected_r = self._get_expected({ 'action': 'delete_object', 'success': True @@ -352,7 +345,7 @@ class TestServiceDelete(_TestServiceBase): def test_delete_object_with_headers(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock(return_value={}) + mock_conn.head_object = mock.Mock(return_value={}) expected_r = self._get_expected({ 'action': 'delete_object', 'success': True @@ -375,7 +368,7 @@ class TestServiceDelete(_TestServiceBase): def test_delete_object_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.delete_object = Mock(side_effect=self.exc) + mock_conn.delete_object = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_object', 'success': False, @@ -408,7 +401,7 @@ class TestServiceDelete(_TestServiceBase): # additional query string to cause the right delete server side mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock( + mock_conn.head_object = mock.Mock( return_value={'x-static-large-object': True} ) expected_r = self._get_expected({ @@ -441,10 +434,10 @@ class TestServiceDelete(_TestServiceBase): # A DLO object is determined in _delete_object by heading the object # and checking for the existence of a x-object-manifest header. # Mock that here. - mock_conn.head_object = Mock( + mock_conn.head_object = mock.Mock( return_value={'x-object-manifest': 'manifest_c/manifest_p'} ) - mock_conn.get_container = Mock( + mock_conn.get_container = mock.Mock( side_effect=[(None, [{'name': 'test_seg_1'}, {'name': 'test_seg_2'}]), (None, {})] @@ -501,7 +494,7 @@ class TestServiceDelete(_TestServiceBase): def test_delete_empty_container_exception(self): mock_conn = self._get_mock_connection() - mock_conn.delete_container = Mock(side_effect=self.exc) + mock_conn.delete_container = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_container', 'success': False, @@ -570,7 +563,7 @@ class TestServiceDelete(_TestServiceBase): stub_headers, json.dumps(stub_resp).encode('utf8')) obj_list = ['x%02d' % i for i in range(100)] expected = [{ - 'action': u'bulk_delete', + 'action': 'bulk_delete', 'attempts': 0, 'container': 'c', 'objects': list(objs), @@ -600,7 +593,7 @@ class TestServiceDelete(_TestServiceBase): obj_list = [SwiftDeleteObject('x%02d' % i, options={'version_id': i}) for i in range(100)] expected = [{ - 'action': u'delete_object', + 'action': 'delete_object', 'attempts': 0, 'container': 'c', 'object': obj.object_name, @@ -831,7 +824,7 @@ class TestServiceList(_TestServiceBase): (None, [{'name': 'test_c'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) expected_r = self._get_expected({ 'action': 'list_account_part', @@ -847,12 +840,12 @@ class TestServiceList(_TestServiceBase): self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) - mock_conn.head_container = Mock(return_value={'test_m': '1'}) + mock_conn.head_container = mock.Mock(return_value={'test_m': '1'}) get_account_returns = [ (None, [{'name': 'test_c'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) expected_r_long = self._get_expected({ 'action': 'list_account_part', @@ -874,7 +867,7 @@ class TestServiceList(_TestServiceBase): (None, [{'name': 'test_c'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) expected_r = self._get_expected({ 'action': 'list_account_part', @@ -898,7 +891,7 @@ class TestServiceList(_TestServiceBase): def test_list_account_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.get_account = Mock(side_effect=self.exc) + mock_conn.get_account = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'list_account_part', 'success': False, @@ -924,7 +917,7 @@ class TestServiceList(_TestServiceBase): (None, [{'name': 'test_o'}]), (None, []) ] - mock_conn.get_container = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock.Mock(side_effect=get_container_returns) expected_r = self._get_expected({ 'action': 'list_container_part', @@ -941,12 +934,12 @@ class TestServiceList(_TestServiceBase): self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) - mock_conn.head_container = Mock(return_value={'test_m': '1'}) + mock_conn.head_container = mock.Mock(return_value={'test_m': '1'}) get_container_returns = [ (None, [{'name': 'test_o'}]), (None, []) ] - mock_conn.get_container = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock.Mock(side_effect=get_container_returns) expected_r_long = self._get_expected({ 'action': 'list_container_part', @@ -970,7 +963,7 @@ class TestServiceList(_TestServiceBase): (None, [{'name': 'b'}, {'name': 'c'}]), (None, []) ] - mock_get_cont = Mock(side_effect=get_container_returns) + mock_get_cont = mock.Mock(side_effect=get_container_returns) mock_conn.get_container = mock_get_cont expected_r = self._get_expected({ @@ -1004,7 +997,7 @@ class TestServiceList(_TestServiceBase): (None, [{'name': 'test_o'}]), (None, []) ] - mock_conn.get_container = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock.Mock(side_effect=get_container_returns) expected_r = self._get_expected({ 'action': 'list_container_part', @@ -1033,7 +1026,7 @@ class TestServiceList(_TestServiceBase): def test_list_container_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.get_container = Mock(side_effect=self.exc) + mock_conn.get_container = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'list_container_part', 'container': 'test_c', @@ -1126,7 +1119,7 @@ class TestServiceList(_TestServiceBase): (None, [{'name': 'container14'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) mock_get_conn.return_value = mock_conn s = SwiftService(options=self.opts) @@ -1269,7 +1262,7 @@ class TestService(unittest.TestCase): for obj in objects: with mock.patch('swiftclient.service.Connection') as mock_conn, \ mock.patch.object(builtins, 'open') as mock_open: - mock_open.return_value = six.StringIO('asdf') + mock_open.return_value = io.StringIO('asdf') mock_conn.return_value.head_object.side_effect = \ ClientException('Not Found', http_status=404) mock_conn.return_value.put_object.return_value =\ @@ -2224,7 +2217,7 @@ class TestServiceDownload(_TestServiceBase): sub_page.side_effect = fake_sub_page - r = Mock(spec=Future) + r = mock.Mock(spec=Future) r.result.return_value = self._get_expected({ 'success': True, 'start_time': 1, @@ -2263,7 +2256,7 @@ class TestServiceDownload(_TestServiceBase): return repr(self.value) def _make_result(): - r = Mock(spec=Future) + r = mock.Mock(spec=Future) r.result.return_value = self._get_expected({ 'success': True, 'start_time': 1, @@ -2319,7 +2312,7 @@ class TestServiceDownload(_TestServiceBase): def test_download_object_job(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -2335,7 +2328,7 @@ class TestServiceDownload(_TestServiceBase): }) with mock.patch.object(builtins, 'open') as mock_open: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2361,7 +2354,7 @@ class TestServiceDownload(_TestServiceBase): def test_download_object_job_with_mtime(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991', @@ -2379,7 +2372,7 @@ class TestServiceDownload(_TestServiceBase): with mock.patch.object(builtins, 'open') as mock_open, \ mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2407,7 +2400,7 @@ class TestServiceDownload(_TestServiceBase): def test_download_object_job_bad_mtime(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991', @@ -2425,7 +2418,7 @@ class TestServiceDownload(_TestServiceBase): with mock.patch.object(builtins, 'open') as mock_open, \ mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2452,7 +2445,7 @@ class TestServiceDownload(_TestServiceBase): def test_download_object_job_ignore_mtime(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991', @@ -2470,7 +2463,7 @@ class TestServiceDownload(_TestServiceBase): with mock.patch.object(builtins, 'open') as mock_open, \ mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2498,7 +2491,7 @@ class TestServiceDownload(_TestServiceBase): def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() - mock_conn.get_object = Mock(side_effect=self.exc) + mock_conn.get_object = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'success': False, 'error': self.exc, @@ -3032,7 +3025,7 @@ class TestServicePost(_TestServiceBase): Check post method translates strings and objects to _post_object_job calls correctly """ - tm_instance = Mock() + tm_instance = mock.Mock() thread_manager.return_value = tm_instance self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) @@ -3077,7 +3070,7 @@ class TestServiceCopy(_TestServiceBase): Check copy method translates strings and objects to _copy_object_job calls correctly """ - tm_instance = Mock() + tm_instance = mock.Mock() thread_manager.return_value = tm_instance self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 2331eaa..f6af6cb 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -12,23 +12,21 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals +import io import contextlib from genericpath import getmtime import getpass import hashlib import json import logging -import mock import os import tempfile import unittest +from unittest import mock import textwrap from time import localtime, mktime, strftime, strptime -import six - import swiftclient from swiftclient.service import SwiftError import swiftclient.shell @@ -47,10 +45,7 @@ try: except ImportError: InsecureRequestWarning = None -if six.PY2: - BUILTIN_OPEN = '__builtin__.open' -else: - BUILTIN_OPEN = 'builtins.open' +BUILTIN_OPEN = 'builtins.open' mocked_os_environ = { 'ST_AUTH': 'http://localhost:8080/auth/v1.0', @@ -405,7 +400,7 @@ class TestShell(unittest.TestCase): def test_list_json(self, connection): connection.return_value.get_account.side_effect = [ [None, [{'name': 'container'}]], - [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], + [None, [{'name': '\u263A', 'some-custom-key': 'and value'}]], [None, []], ] @@ -417,7 +412,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_account.assert_has_calls(calls) listing = [{'name': 'container'}, - {'name': u'\u263A', 'some-custom-key': 'and value'}] + {'name': '\u263A', 'some-custom-key': 'and value'}] expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' self.assertEqual(output.out, expected) @@ -632,7 +627,7 @@ class TestShell(unittest.TestCase): @mock.patch('swiftclient.service.makedirs') @mock.patch('swiftclient.service.Connection') def test_download(self, connection, makedirs): - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -667,7 +662,7 @@ class TestShell(unittest.TestCase): makedirs.reset_mock() # Test downloading single object - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -683,7 +678,7 @@ class TestShell(unittest.TestCase): self.assertEqual([], makedirs.mock_calls) # Test downloading without md5 checks - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -701,7 +696,7 @@ class TestShell(unittest.TestCase): self.assertEqual([], makedirs.mock_calls) # Test downloading single object to stdout - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -1669,7 +1664,7 @@ class TestShell(unittest.TestCase): with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: with CaptureOutput() as out: mock_func.return_value = [res] - swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + swiftclient.shell.main(base_argv + [container]) mock_func.assert_called_once_with(container=container) self.assertTrue(out.out.find( @@ -1682,7 +1677,7 @@ class TestShell(unittest.TestCase): with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: with CaptureOutput() as out: mock_func.return_value = [res] - swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + swiftclient.shell.main(base_argv + [container]) mock_func.assert_called_once_with(container=container) self.assertTrue(out.out.find( @@ -2082,7 +2077,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range=None) + iso8601=False, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_prefix_based(self, temp_url): @@ -2091,7 +2086,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=True, ip_range=None) + iso8601=False, prefix=True, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_in(self, temp_url): @@ -2103,7 +2098,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range=None) + iso8601=False, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_out(self, temp_url): @@ -2112,7 +2107,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=True, prefix=False, ip_range=None) + iso8601=True, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -2121,7 +2116,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - iso8601=False, prefix=False, ip_range=None) + iso8601=False, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_with_ip_range(self, temp_url): @@ -2130,11 +2125,11 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range='1.2.3.4') + iso8601=False, prefix=False, ip_range='1.2.3.4', digest='sha256') def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" @@ -2142,14 +2137,14 @@ class TestShell(unittest.TestCase): self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "http://saio:8080/v1/a/c/o", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) expected = "http://saio:8080%s" % expected self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "/v1/a/c/", - "secret_key", "--absolute", "--prefix"] + "secret_key", "--absolute", "--prefix", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' @@ -2158,7 +2153,8 @@ class TestShell(unittest.TestCase): self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "/v1/a/c/", - "secret_key", "--absolute", "--prefix", '--iso8601'] + "secret_key", "--absolute", "--prefix", '--iso8601', + "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' @@ -2171,7 +2167,7 @@ class TestShell(unittest.TestCase): strftime(EXPIRES_ISO8601_FORMAT[:-1], localtime(60))) for d in dates: argv = ["", "tempurl", "GET", d, "/v1/a/c/o", - "secret_key"] + "secret_key", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" @@ -2182,19 +2178,20 @@ class TestShell(unittest.TestCase): mktime(strptime('2005-05-01', SHORT_EXPIRES_ISO8601_FORMAT)))) argv = ["", "tempurl", "GET", ts, "/v1/a/c/", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) expected = output.out argv = ["", "tempurl", "GET", '2005-05-01', "/v1/a/c/", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", - "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + "secret_key", "--absolute", "--ip-range", "1.2.3.4", + "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" @@ -2204,6 +2201,39 @@ class TestShell(unittest.TestCase): ) self.assertEqual(expected, output.out) + def test_temp_url_digests_output(self): + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + s = "db04994a589b1a2538bff694f0a4f57c7a397617ac2cb49f924d222bbe2b3e01" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % s + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--digest", "sha256"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + # same signature/expectation + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--digest", "sha1"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--digest", "sha512"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = ("sha512:nMXwEAHu3jzlCZi4wWO1juEq4DikFlX8a729PLJVvUp" + "vg0GpgkJnX5uCG1x-v2KfTrmRtLOcT7KBK2RXLW1uKw") + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', @@ -3289,7 +3319,7 @@ class TestAuth(MockHttpTest): } mock_resp = self.fake_http_connection(200, headers=headers) with mock.patch('swiftclient.client.http_connection', new=mock_resp): - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', @@ -3308,7 +3338,7 @@ class TestAuth(MockHttpTest): def test_auth_verbose(self): with mock.patch('swiftclient.client.http_connection') as mock_conn: - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', @@ -3332,7 +3362,7 @@ class TestAuth(MockHttpTest): os_options = {'tenant_name': 'demo'} with mock.patch('swiftclient.client.get_auth_keystone', new=fake_get_auth_keystone(os_options)): - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', @@ -3353,7 +3383,7 @@ class TestAuth(MockHttpTest): def test_auth_verbose_v2(self): with mock.patch('swiftclient.client.get_auth_keystone') \ as mock_keystone: - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index ea5f502..ae3e76f 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -16,16 +16,15 @@ import gzip import json import logging -import mock -import six +import io import socket import string import unittest +from unittest import mock import warnings import tempfile from hashlib import md5 -from six import binary_type -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from requests.exceptions import RequestException from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, @@ -102,7 +101,7 @@ class TestClientException(unittest.TestCase): self.assertIn('(txn: some-other-id)', str(exc)) -class MockHttpResponse(object): +class MockHttpResponse: def __init__(self, status=0, headers=None, verify=False): self.status = status self.status_code = status @@ -116,7 +115,7 @@ class MockHttpResponse(object): self.headers.update(headers) self.closed = False - class Raw(object): + class Raw: def __init__(self, headers): self.headers = headers @@ -164,34 +163,34 @@ class TestHttpHelpers(MockHttpTest): self.assertEqual('bytes%FF', c.quote(value)) value = 'native string' self.assertEqual('native%20string', c.quote(value)) - value = u'unicode string' + value = 'unicode string' self.assertEqual('unicode%20string', c.quote(value)) - value = u'unicode:\xe9\u20ac' + value = 'unicode:\xe9\u20ac' self.assertEqual('unicode%3A%C3%A9%E2%82%AC', c.quote(value)) def test_parse_header_string(self): value = b'bytes' - self.assertEqual(u'bytes', c.parse_header_string(value)) - value = u'unicode:\xe9\u20ac' - self.assertEqual(u'unicode:\xe9\u20ac', c.parse_header_string(value)) + self.assertEqual('bytes', c.parse_header_string(value)) + value = 'unicode:\xe9\u20ac' + self.assertEqual('unicode:\xe9\u20ac', c.parse_header_string(value)) value = 'native%20string' - self.assertEqual(u'native string', c.parse_header_string(value)) + self.assertEqual('native string', c.parse_header_string(value)) value = b'encoded%20bytes%E2%82%AC' - self.assertEqual(u'encoded bytes\u20ac', c.parse_header_string(value)) + self.assertEqual('encoded bytes\u20ac', c.parse_header_string(value)) value = 'encoded%20unicode%E2%82%AC' - self.assertEqual(u'encoded unicode\u20ac', + self.assertEqual('encoded unicode\u20ac', c.parse_header_string(value)) value = b'bad%20bytes%ff%E2%82%AC' - self.assertEqual(u'bad%20bytes%ff%E2%82%AC', + self.assertEqual('bad%20bytes%ff%E2%82%AC', c.parse_header_string(value)) - value = u'bad%20unicode%ff\u20ac' - self.assertEqual(u'bad%20unicode%ff\u20ac', + value = 'bad%20unicode%ff\u20ac' + self.assertEqual('bad%20unicode%ff\u20ac', c.parse_header_string(value)) value = b'really%20bad\xffbytes' - self.assertEqual(u'really%2520bad%FFbytes', + self.assertEqual('really%2520bad%FFbytes', c.parse_header_string(value)) def test_http_connection(self): @@ -206,20 +205,20 @@ class TestHttpHelpers(MockHttpTest): def test_encode_meta_headers(self): headers = {'abc': '123', - u'x-container-meta-\u0394': 123, - u'x-account-meta-\u0394': 12.3, - u'x-object-meta-\u0394': True} + 'x-container-meta-\u0394': 123, + 'x-account-meta-\u0394': 12.3, + 'x-object-meta-\u0394': True} r = swiftclient.encode_meta_headers(headers) self.assertEqual(len(headers), len(r)) # ensure non meta headers are not encoded - self.assertIs(type(r.get('abc')), binary_type) + self.assertIs(type(r.get('abc')), bytes) del r['abc'] for k, v in r.items(): - self.assertIs(type(k), binary_type) - self.assertIs(type(v), binary_type) + self.assertIs(type(k), bytes) + self.assertIs(type(v), bytes) self.assertIn(v, (b'123', b'12.3', b'True')) def test_set_user_agent_default(self): @@ -587,10 +586,10 @@ class TestGetAuth(MockHttpTest): "application_credential_id": "proejct_id", "application_credential_secret": "secret"} - class FakeEndpointData(object): + class FakeEndpointData: catalog_url = 'http://swift.cluster/v1/KEY_project_id' - class FakeKeystoneuth1v3Session(object): + class FakeKeystoneuth1v3Session: def __init__(self, auth): self.auth = auth @@ -1112,9 +1111,9 @@ class TestGetObject(MockHttpTest): conn = c.http_connection('http://www.test.com') headers, data = c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object', http_conn=conn) - self.assertEqual(u't\xe9st', headers.get('x-utf-8-header', '')) - self.assertEqual(u'%ff', headers.get('x-non-utf-8-header', '')) - self.assertEqual(u'%FF', headers.get('x-binary-header', '')) + self.assertEqual('t\xe9st', headers.get('x-utf-8-header', '')) + self.assertEqual('%ff', headers.get('x-non-utf-8-header', '')) + self.assertEqual('%FF', headers.get('x-binary-header', '')) def test_chunk_size_read_method(self): conn = c.Connection('http://auth.url/', 'some_user', 'some_key') @@ -1325,30 +1324,28 @@ class TestHeadObject(MockHttpTest): class TestPutObject(MockHttpTest): - @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) value = c.put_object(*args) - self.assertIsInstance(value, six.string_types) + self.assertIsInstance(value, str) self.assertEqual(value, EMPTY_ETAG) self.assertRequests([ ('PUT', '/container/obj', 'body', { 'x-auth-token': 'TOKEN', 'content-length': '4', - 'content-type': '' }), ]) def test_unicode_ok(self): - conn = c.http_connection(u'http://www.test.com/') - mock_file = six.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') - args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + conn = c.http_connection('http://www.test.com/') + mock_file = io.StringIO('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + args = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', mock_file) - text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + text = '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} @@ -1356,7 +1353,7 @@ class TestPutObject(MockHttpTest): conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request value = c.put_object(*args, headers=headers, http_conn=conn) - self.assertIsInstance(value, six.string_types) + self.assertIsInstance(value, str) # Test for RFC-2616 encoded symbols self.assertIn(("a-b", b".x:yz mn:fg:lp"), resp.buffer) @@ -1366,7 +1363,7 @@ class TestPutObject(MockHttpTest): def test_chunk_warning(self): conn = c.http_connection('http://www.test.com/') - mock_file = six.StringIO('asdf') + mock_file = io.StringIO('asdf') args = ('asdf', 'asdf', 'asdf', 'asdf', mock_file) resp = MockHttpResponse() conn[1].getresponse = resp.fake_response @@ -1383,7 +1380,6 @@ class TestPutObject(MockHttpTest): self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, UserWarning)) - @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_server_error(self): body = 'c' * 60 headers = {'foo': 'bar'} @@ -1398,8 +1394,7 @@ class TestPutObject(MockHttpTest): self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { - 'x-auth-token': 'asdf', - 'content-type': ''}), + 'x-auth-token': 'asdf'}), ]) def test_query_string(self): @@ -1415,7 +1410,7 @@ class TestPutObject(MockHttpTest): def test_raw_upload(self): # Raw upload happens when content_length is passed to put_object - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1437,7 +1432,7 @@ class TestPutObject(MockHttpTest): def test_chunk_upload(self): # Chunked upload happens when no content_length is passed to put_object - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1462,7 +1457,7 @@ class TestPutObject(MockHttpTest): def data(): for chunk in ('foo', '', 'bar'): yield chunk - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1529,7 +1524,7 @@ class TestPutObject(MockHttpTest): self.assertEqual(etag, contents.get_md5sum()) def test_params(self): - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1540,20 +1535,8 @@ class TestPutObject(MockHttpTest): self.assertEqual(request_header['etag'], b'1234-5678') self.assertEqual(request_header['content-type'], b'text/plain') - @mock.patch('swiftclient.requests.__version__', '2.2.0') - def test_no_content_type_old_requests(self): - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - - c.put_object(url='http://www.test.com', http_conn=conn) - request_header = resp.requests_params['headers'] - self.assertEqual(request_header['content-type'], b'') - - @mock.patch('swiftclient.requests.__version__', '2.4.0') - def test_no_content_type_new_requests(self): - conn = c.http_connection(u'http://www.test.com/') + def test_no_content_type(self): + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1563,7 +1546,7 @@ class TestPutObject(MockHttpTest): self.assertNotIn('content-type', request_header) def test_content_type_in_headers(self): - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1603,12 +1586,12 @@ class TestPostObject(MockHttpTest): }) def test_unicode_ok(self): - conn = c.http_connection(u'http://www.test.com/') - args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') - text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + conn = c.http_connection('http://www.test.com/') + args = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + text = '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, b'X-Header2': 'value', 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', @@ -1907,66 +1890,66 @@ class TestGetCapabilities(MockHttpTest): class TestHTTPConnection(MockHttpTest): def test_bad_url_scheme(self): - url = u'www.test.com' + url = 'www.test.com' with self.assertRaises(c.ClientException) as exc_context: c.http_connection(url) exc = exc_context.exception - expected = u'Unsupported scheme "" in url "www.test.com"' + expected = 'Unsupported scheme "" in url "www.test.com"' self.assertEqual(expected, str(exc)) - url = u'://www.test.com' + url = '://www.test.com' with self.assertRaises(c.ClientException) as exc_context: c.http_connection(url) exc = exc_context.exception - expected = u'Unsupported scheme "" in url "://www.test.com"' + expected = 'Unsupported scheme "" in url "://www.test.com"' self.assertEqual(expected, str(exc)) - url = u'blah://www.test.com' + url = 'blah://www.test.com' with self.assertRaises(c.ClientException) as exc_context: c.http_connection(url) exc = exc_context.exception - expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' + expected = 'Unsupported scheme "blah" in url "blah://www.test.com"' self.assertEqual(expected, str(exc)) def test_ok_url_scheme(self): for scheme in ('http', 'https', 'HTTP', 'HTTPS'): - url = u'%s://www.test.com' % scheme + url = '%s://www.test.com' % scheme parsed_url, conn = c.http_connection(url) self.assertEqual(scheme.lower(), parsed_url.scheme) - self.assertEqual(u'%s://www.test.com' % scheme, conn.url) + self.assertEqual('%s://www.test.com' % scheme, conn.url) def test_ok_proxy(self): - conn = c.http_connection(u'http://www.test.com/', + conn = c.http_connection('http://www.test.com/', proxy='http://localhost:8080') self.assertEqual(conn[1].requests_args['proxies']['http'], 'http://localhost:8080') def test_bad_proxy(self): try: - c.http_connection(u'http://www.test.com/', proxy='localhost:8080') + c.http_connection('http://www.test.com/', proxy='localhost:8080') except c.ClientException as e: self.assertEqual(e.msg, "Proxy's missing scheme") def test_cacert(self): - conn = c.http_connection(u'http://www.test.com/', + conn = c.http_connection('http://www.test.com/', cacert='/dev/urandom') self.assertEqual(conn[1].requests_args['verify'], '/dev/urandom') def test_insecure(self): - conn = c.http_connection(u'http://www.test.com/', insecure=True) + conn = c.http_connection('http://www.test.com/', insecure=True) self.assertEqual(conn[1].requests_args['verify'], False) def test_cert(self): - conn = c.http_connection(u'http://www.test.com/', cert='minnie') + conn = c.http_connection('http://www.test.com/', cert='minnie') self.assertEqual(conn[1].requests_args['cert'], 'minnie') def test_cert_key(self): conn = c.http_connection( - u'http://www.test.com/', cert='minnie', cert_key='mickey') + 'http://www.test.com/', cert='minnie', cert_key='mickey') self.assertEqual(conn[1].requests_args['cert'], ('minnie', 'mickey')) def test_response_connection_released(self): - _parsed_url, conn = c.http_connection(u'http://www.test.com/') + _parsed_url, conn = c.http_connection('http://www.test.com/') conn.resp = MockHttpResponse() conn.resp.raw = mock.Mock() conn.resp.raw.read.side_effect = ["Chunk", ""] @@ -1976,36 +1959,10 @@ class TestHTTPConnection(MockHttpTest): self.assertFalse(resp.read()) self.assertTrue(resp.closed) - @unittest.skipIf(six.PY3, 'python2 specific test') - def test_response_python2_headers(self): - '''Test utf-8 headers in Python 2. - ''' - _, conn = c.http_connection(u'http://www.test.com/') - conn.resp = MockHttpResponse( - status=200, - headers={ - '\xd8\xaa-unicode': '\xd8\xaa-value', - 'empty-header': '' - } - ) - - resp = conn.getresponse() - self.assertEqual( - '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) - self.assertEqual( - '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) - self.assertEqual('', resp.getheader('empty-header')) - self.assertEqual( - dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), - ('empty-header', ''), - ('etag', '"%s"' % EMPTY_ETAG)]), - dict(resp.getheaders())) - - @unittest.skipIf(six.PY2, 'python3 specific test') - def test_response_python3_headers(self): - '''Test latin1-encoded headers in Python 3. + def test_response_headers(self): + '''Test latin1-encoded headers. ''' - _, conn = c.http_connection(u'http://www.test.com/') + _, conn = c.http_connection('http://www.test.com/') conn.resp = MockHttpResponse( status=200, headers={ @@ -2173,30 +2130,37 @@ class TestConnection(MockHttpTest): pass c.sleep = quick_sleep - # test retries - conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', - retry_on_ratelimit=True) - code_iter = [200] + [498] * (conn.retries + 1) - auth_resp_headers = { - 'x-auth-token': 'asdf', - 'x-storage-url': 'http://storage/v1/test', - } - c.http_connection = self.fake_http_connection( - *code_iter, headers=auth_resp_headers) - with self.assertRaises(c.ClientException) as exc_context: - conn.head_account() - self.assertIn('Account HEAD failed', str(exc_context.exception)) - self.assertEqual(conn.attempts, conn.retries + 1) + def test_status_code(code): + # test retries + conn = c.Connection('http://www.test.com/auth/v1.0', + 'asdf', 'asdf', retry_on_ratelimit=True) + code_iter = [200] + [code] * (conn.retries + 1) + auth_resp_headers = { + 'x-auth-token': 'asdf', + 'x-storage-url': 'http://storage/v1/test', + } + c.http_connection = self.fake_http_connection( + *code_iter, headers=auth_resp_headers) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(code, exc_context.exception.http_status) + self.assertEqual(conn.attempts, conn.retries + 1) - # test default no-retry - c.http_connection = self.fake_http_connection( - 200, 498, - headers=auth_resp_headers) - conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') - with self.assertRaises(c.ClientException) as exc_context: - conn.head_account() - self.assertIn('Account HEAD failed', str(exc_context.exception)) - self.assertEqual(conn.attempts, 1) + # test default no-retry + c.http_connection = self.fake_http_connection( + 200, code, + headers=auth_resp_headers) + conn = c.Connection('http://www.test.com/auth/v1.0', + 'asdf', 'asdf', retry_on_ratelimit=False) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(code, exc_context.exception.http_status) + self.assertEqual(conn.attempts, 1) + + test_status_code(498) + test_status_code(429) def test_retry_with_socket_error(self): def quick_sleep(*args): @@ -2586,10 +2550,10 @@ class TestConnection(MockHttpTest): def test_reset_stream(self): - class LocalContents(object): + class LocalContents: def __init__(self, tell_value=0): - self.data = six.BytesIO(string.ascii_letters.encode() * 10) + self.data = io.BytesIO(string.ascii_letters.encode() * 10) self.data.seek(tell_value) self.reads = [] self.seeks = [] @@ -2608,7 +2572,7 @@ class TestConnection(MockHttpTest): self.reads.append((size, read_data)) return read_data - class LocalConnection(object): + class LocalConnection: def __init__(self, parsed_url=None): self.reason = "" @@ -2887,7 +2851,7 @@ class TestLogging(MockHttpTest): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') value = c.put_object(*args) - self.assertIsInstance(value, six.string_types) + self.assertIsInstance(value, str) def test_head_error(self): c.http_connection = self.fake_http_connection(500) @@ -2901,9 +2865,9 @@ class TestLogging(MockHttpTest): self.assertEqual(exc_context.exception.http_status, 404) def test_content_encoding_gzip_body_is_logged_decoded(self): - buf = six.BytesIO() + buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') - data = {"test": u"\u2603"} + data = {"test": "\u2603"} decoded_body = json.dumps(data).encode('utf-8') gz.write(decoded_body) gz.close() @@ -2920,7 +2884,7 @@ class TestLogging(MockHttpTest): self.assertEqual(exc_context.exception.http_status, 500) # it will log the decoded body self.assertEqual([ - mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf ' + mock.call('REQ: %s', 'curl -i http://www.test.com/asdf/asdf ' '-X GET -H "X-Auth-Token: ..."'), mock.call('RESP STATUS: %s %s', 500, 'Fake'), mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}), @@ -2931,9 +2895,9 @@ class TestLogging(MockHttpTest): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_value = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c') unicode_token_encoded = unicode_token_value.encode('utf8') set_cookie_value = 'X-Auth-Token=%s' % token_value set_cookie_encoded = set_cookie_value.encode('utf8') @@ -2957,8 +2921,8 @@ class TestLogging(MockHttpTest): out = [] for _, args, kwargs in mock_log.mock_calls: for arg in args: - out.append(u'%s' % arg) - output = u''.join(out) + out.append('%s' % arg) + output = ''.join(out) self.assertIn('X-Auth-Token', output) self.assertIn(token_value[:16] + '...', output) self.assertIn('X-Storage-Token', output) @@ -2973,9 +2937,9 @@ class TestLogging(MockHttpTest): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_value = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c') c.logger_settings['redact_sensitive_headers'] = False unicode_token_encoded = unicode_token_value.encode('utf8') c.http_log( @@ -2997,8 +2961,8 @@ class TestLogging(MockHttpTest): out = [] for _, args, kwargs in mock_log.mock_calls: for arg in args: - out.append(u'%s' % arg) - output = u''.join(out) + out.append('%s' % arg) + output = ''.join(out) self.assertIn('X-Auth-Token', output) self.assertIn(token_value, output) self.assertIn('X-Storage-Token', output) @@ -3006,12 +2970,12 @@ class TestLogging(MockHttpTest): @mock.patch('swiftclient.client.logger.debug') def test_unicode_path(self, mock_log): - path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') + path = 'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') c.http_log(['GET', path], {}, MockHttpResponse(status=200, headers=[]), '') request_log_line = mock_log.mock_calls[0] self.assertEqual('REQ: %s', request_log_line[1][0]) - self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), + self.assertEqual('curl -i -X GET %s' % path.decode('utf-8'), request_log_line[1][1]) diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index cbee82b..129208d 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -14,13 +14,13 @@ # limitations under the License. import gzip +import io import json import unittest -import mock -import six +from unittest import mock import tempfile from time import gmtime, localtime, mktime, strftime, strptime -from hashlib import md5, sha1 +import hashlib from swiftclient import utils as u @@ -127,27 +127,75 @@ class TestTempURL(unittest.TestCase): seconds = 3600 key = 'correcthorsebatterystaple' method = 'GET' - expected_url = url + ('?temp_url_sig=temp_url_signature' - '&temp_url_expires=1400003600') expected_body = '\n'.join([ method, '1400003600', url, ]).encode('utf-8') + @property + def expected_url(self): + if isinstance(self.url, bytes): + return self.url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + return self.url + (u'?temp_url_sig=temp_url_signature' + u'&temp_url_expires=1400003600') + + @property + def expected_sha512_url(self): + if isinstance(self.url, bytes): + return self.url + (b'?temp_url_sig=sha512:dGVtcF91cmxfc2lnbmF0dXJl' + b'&temp_url_expires=1400003600') + return self.url + (u'?temp_url_sig=sha512:dGVtcF91cmxfc2lnbmF0dXJl' + u'&temp_url_expires=1400003600') + @mock.patch('hmac.HMAC') @mock.patch('time.time', return_value=1400000000) - def test_generate_temp_url(self, time_mock, hmac_mock): + def test_generate_sha1_temp_url(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method, digest='sha1') + key = self.key + if not isinstance(key, bytes): + key = key.encode('utf-8') + self.assertEqual(url, self.expected_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, hashlib.sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_sha512_temp_url(self, time_mock, hmac_mock): + hmac_mock().digest.return_value = b'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method, digest=hashlib.sha512) + key = self.key + if not isinstance(key, bytes): + key = key.encode('utf-8') + self.assertEqual(url, self.expected_sha512_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, hashlib.sha512), + mock.call().digest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_sha256_temp_url_by_default(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, self.seconds, self.key, self.method) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') self.assertEqual(url, self.expected_url) self.assertEqual(hmac_mock.mock_calls, [ mock.call(), - mock.call(key, self.expected_body, sha1), + mock.call(key, self.expected_body, hashlib.sha256), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(self.url)) @@ -170,10 +218,10 @@ class TestTempURL(unittest.TestCase): self.key, self.method, ip_range=ip_range) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') - if isinstance(ip_range, six.binary_type): + if isinstance(ip_range, bytes): ip_range_expected_url = ( expected_url + ip_range.decode('utf-8') ) @@ -195,7 +243,7 @@ class TestTempURL(unittest.TestCase): self.assertEqual(url, ip_range_expected_url) self.assertEqual(hmac_mock.mock_calls, [ - mock.call(key, expected_body, sha1), + mock.call(key, expected_body, hashlib.sha256), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(path)) @@ -215,7 +263,7 @@ class TestTempURL(unittest.TestCase): lt = localtime() expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) - if not isinstance(self.expected_url, six.string_types): + if not isinstance(self.expected_url, str): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: @@ -228,7 +276,7 @@ class TestTempURL(unittest.TestCase): expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) - if not isinstance(self.expected_url, six.string_types): + if not isinstance(self.expected_url, str): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: @@ -246,17 +294,17 @@ class TestTempURL(unittest.TestCase): self.key, self.method, iso8601=True) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600)) - if not isinstance(self.url, six.string_types): + if not isinstance(self.url, str): self.assertTrue(url.endswith(bytes(expires, 'utf-8'))) else: self.assertTrue(url.endswith(expires)) self.assertEqual(hmac_mock.mock_calls, [ mock.call(), - mock.call(key, self.expected_body, sha1), + mock.call(key, self.expected_body, hashlib.sha256), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(self.url)) @@ -280,11 +328,11 @@ class TestTempURL(unittest.TestCase): url = u.generate_temp_url(path, self.seconds, self.key, self.method, prefix=True) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') self.assertEqual(url, expected_url) self.assertEqual(hmac_mock.mock_calls, [ - mock.call(key, expected_body, sha1), + mock.call(key, expected_body, hashlib.sha256), mock.call().hexdigest(), ]) @@ -299,12 +347,12 @@ class TestTempURL(unittest.TestCase): @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature") def test_generate_absolute_expiry_temp_url(self, hmac_mock): - if isinstance(self.expected_url, six.binary_type): + if isinstance(self.expected_url, bytes): expected_url = self.expected_url.replace( b'1400003600', b'2146636800') else: expected_url = self.expected_url.replace( - u'1400003600', u'2146636800') + '1400003600', '2146636800') url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, absolute=True) self.assertEqual(url, expected_url) @@ -372,34 +420,28 @@ class TestTempURL(unittest.TestCase): class TestTempURLUnicodePathAndKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3' - key = u'k\u00e9y' - expected_url = (u'%s?temp_url_sig=temp_url_signature' - u'&temp_url_expires=1400003600') % url - expected_body = u'\n'.join([ - u'GET', - u'1400003600', + url = '/v1/\u00e4/c/\u00f3' + key = 'k\u00e9y' + expected_body = '\n'.join([ + 'GET', + '1400003600', url, ]).encode('utf-8') class TestTempURLUnicodePathBytesKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3' - key = u'k\u00e9y'.encode('utf-8') - expected_url = (u'%s?temp_url_sig=temp_url_signature' - u'&temp_url_expires=1400003600') % url + url = '/v1/\u00e4/c/\u00f3' + key = 'k\u00e9y'.encode('utf-8') expected_body = '\n'.join([ - u'GET', - u'1400003600', + 'GET', + '1400003600', url, ]).encode('utf-8') class TestTempURLBytesPathUnicodeKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') - key = u'k\u00e9y' - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') + url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = 'k\u00e9y' expected_body = b'\n'.join([ b'GET', b'1400003600', @@ -408,10 +450,8 @@ class TestTempURLBytesPathUnicodeKey(TestTempURL): class TestTempURLBytesPathAndKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') - key = u'k\u00e9y'.encode('utf-8') - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') + url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = 'k\u00e9y'.encode('utf-8') expected_body = b'\n'.join([ b'GET', b'1400003600', @@ -420,10 +460,8 @@ class TestTempURLBytesPathAndKey(TestTempURL): class TestTempURLBytesPathAndNonUtf8Key(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') key = b'k\xffy' - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') expected_body = b'\n'.join([ b'GET', b'1400003600', @@ -436,7 +474,7 @@ class TestReadableToIterable(unittest.TestCase): def test_iter(self): chunk_size = 4 write_data = tuple(x.encode() for x in ('a', 'b', 'c', 'd')) - actual_md5sum = md5() + actual_md5sum = hashlib.md5() with tempfile.TemporaryFile() as f: for x in write_data: @@ -454,8 +492,8 @@ class TestReadableToIterable(unittest.TestCase): def test_md5_creation(self): # Check creation with a real and noop md5 class data = u.ReadableToIterable(None, None, md5=True) - self.assertEqual(md5().hexdigest(), data.get_md5sum()) - self.assertIs(type(md5()), type(data.md5sum)) + self.assertEqual(hashlib.md5().hexdigest(), data.get_md5sum()) + self.assertIs(type(hashlib.md5()), type(data.md5sum)) data = u.ReadableToIterable(None, None, md5=False) self.assertEqual('', data.get_md5sum()) @@ -463,8 +501,8 @@ class TestReadableToIterable(unittest.TestCase): def test_unicode(self): # Check no errors are raised if unicode data is feed in. - unicode_data = u'abc' - actual_md5sum = md5(unicode_data.encode()).hexdigest() + unicode_data = 'abc' + actual_md5sum = hashlib.md5(unicode_data.encode()).hexdigest() chunk_size = 2 with tempfile.TemporaryFile(mode='w+') as f: @@ -486,27 +524,29 @@ class TestReadableToIterable(unittest.TestCase): class TestLengthWrapper(unittest.TestCase): def test_stringio(self): - contents = six.StringIO(u'a' * 50 + u'b' * 50) + contents = io.StringIO('a' * 50 + 'b' * 50) contents.seek(22) data = u.LengthWrapper(contents, 42, True) - s = u'a' * 28 + u'b' * 14 - read_data = u''.join(iter(data.read, '')) + s = 'a' * 28 + 'b' * 14 + read_data = ''.join(iter(data.read, '')) self.assertEqual(42, len(data)) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s.encode()).hexdigest(), + data.get_md5sum()) data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5().hexdigest(), data.get_md5sum()) - read_data = u''.join(iter(data.read, '')) + read_data = ''.join(iter(data.read, '')) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s.encode()).hexdigest(), + data.get_md5sum()) def test_bytesio(self): - contents = six.BytesIO(b'a' * 50 + b'b' * 50) + contents = io.BytesIO(b'a' * 50 + b'b' * 50) contents.seek(22) data = u.LengthWrapper(contents, 42, True) s = b'a' * 28 + b'b' * 14 @@ -515,7 +555,7 @@ class TestLengthWrapper(unittest.TestCase): self.assertEqual(42, len(data)) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), data.get_md5sum()) def test_tempfile(self): with tempfile.NamedTemporaryFile(mode='wb') as f: @@ -529,7 +569,7 @@ class TestLengthWrapper(unittest.TestCase): self.assertEqual(42, len(data)) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), data.get_md5sum()) def test_segmented_file(self): with tempfile.NamedTemporaryFile(mode='wb') as f: @@ -548,15 +588,18 @@ class TestLengthWrapper(unittest.TestCase): self.assertEqual(segment_length, len(data)) self.assertEqual(segment_length, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), + data.get_md5sum()) data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5().hexdigest(), + data.get_md5sum()) read_data = b''.join(iter(data.read, '')) self.assertEqual(segment_length, len(data)) self.assertEqual(segment_length, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), + data.get_md5sum()) class TestGroupers(unittest.TestCase): @@ -591,12 +634,12 @@ class TestApiResponeParser(unittest.TestCase): def test_utf8_default(self): result = u.parse_api_response( - {}, u'{"test": "\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) + {}, '{"test": "\u2603"}'.encode('utf8')) + self.assertEqual({'test': '\u2603'}, result) result = u.parse_api_response( - {}, u'{"test": "\\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) + {}, '{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': '\u2603'}, result) def test_bad_json(self): self.assertRaises(ValueError, u.parse_api_response, @@ -610,38 +653,38 @@ class TestApiResponeParser(unittest.TestCase): result = u.parse_api_response( {'content-type': 'application/json; charset=iso8859-1'}, b'{"t\xe9st": "\xff"}') - self.assertEqual({u't\xe9st': u'\xff'}, result) + self.assertEqual({'t\xe9st': '\xff'}, result) def test_gzipped_utf8(self): - buf = six.BytesIO() + buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.write('{"test": "\u2603"}'.encode('utf8')) gz.close() result = u.parse_api_response( {'content-encoding': 'gzip'}, buf.getvalue()) - self.assertEqual({'test': u'\u2603'}, result) + self.assertEqual({'test': '\u2603'}, result) class TestGetBody(unittest.TestCase): def test_not_gzipped(self): result = u.parse_api_response( - {}, u'{"test": "\\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) + {}, '{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': '\u2603'}, result) def test_gzipped_body(self): - buf = six.BytesIO() + buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.write('{"test": "\u2603"}'.encode('utf8')) gz.close() result = u.parse_api_response( {'content-encoding': 'gzip'}, buf.getvalue()) - self.assertEqual({'test': u'\u2603'}, result) + self.assertEqual({'test': '\u2603'}, result) -class JSONTracker(object): +class JSONTracker: def __init__(self, data): self.data = data self.calls = [] diff --git a/test/unit/utils.py b/test/unit/utils.py index 3190e9d..87d3210 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -12,17 +12,19 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. + import functools +import io +import importlib +import os import sys -from requests import RequestException -from requests.structures import CaseInsensitiveDict from time import sleep import unittest -import mock -import six -import os -from six.moves import reload_module -from six.moves.urllib.parse import urlparse, ParseResult +from unittest import mock + +from requests import RequestException +from requests.structures import CaseInsensitiveDict +from urllib.parse import urlparse, ParseResult from swiftclient import client as c from swiftclient import shell as s from swiftclient.utils import EMPTY_ETAG @@ -406,7 +408,7 @@ class MockHttpTest(unittest.TestCase): # un-hygienic mocking on the swiftclient.client module; which may lead # to some unfortunate test order dependency bugs by way of the broken # window theory if any other modules are similarly patched - reload_module(c) + importlib.reload(c) class CaptureStreamPrinter(object): @@ -421,24 +423,20 @@ class CaptureStreamPrinter(object): # No encoding, just convert the raw bytes into a str for testing # The below call also validates that we have a byte string. self._captured_stream.write( - data if isinstance(data, six.binary_type) else data.encode('utf8')) + data if isinstance(data, bytes) else data.encode('utf8')) class CaptureStream(object): def __init__(self, stream): self.stream = stream - self._buffer = six.BytesIO() + self._buffer = io.BytesIO() self._capture = CaptureStreamPrinter(self._buffer) self.streams = [self._capture] @property def buffer(self): - if six.PY3: - return self._buffer - else: - raise AttributeError( - 'Output stream has no attribute "buffer" in Python2') + return self._buffer def flush(self): pass @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3,pep8 +envlist = py3,pep8 minversion = 3.18.0 skipsdist = True @@ -21,16 +21,13 @@ allowlist_externals = sh passenv = SWIFT_* *_proxy [testenv:pep8] -basepython = python3 commands = python -m flake8 swiftclient test [testenv:venv] -basepython = python3 commands = {posargs} [testenv:cover] -basepython = python3 setenv = PYTHON=coverage run --source swiftclient --parallel-mode commands = @@ -41,7 +38,6 @@ commands = coverage report [testenv:func] -basepython = python3 setenv = OS_TEST_PATH=test.functional PYTHON=coverage run --source swiftclient --parallel-mode @@ -56,14 +52,7 @@ commands = coverage report -m rm -f .coverage -[testenv:py2func] -basepython=python2 -setenv = {[testenv:func]setenv} -allowlist_externals = {[testenv:func]allowlist_externals} -commands = {[testenv:func]commands} - [testenv:docs] -basepython = python3 usedevelop = False deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt @@ -88,7 +77,6 @@ show-source = True exclude = .venv,.tox,dist,doc,*egg [testenv:bindep] -basepython = python3 # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system # dependencies are missing! This also means that bindep must be installed @@ -98,14 +86,12 @@ deps = bindep commands = bindep test [testenv:releasenotes] -basepython = python3 usedevelop = False deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:pdf-docs] -basepython = python3 deps = {[testenv:docs]deps} allowlist_externals = make |