diff options
| -rw-r--r-- | .zuul.yaml | 3 | ||||
| -rw-r--r-- | CONTRIBUTING.rst | 25 | ||||
| -rw-r--r-- | README.rst | 6 | ||||
| -rw-r--r-- | doc/requirements.txt | 7 | ||||
| -rw-r--r-- | doc/source/conf.py | 12 | ||||
| -rw-r--r-- | doc/source/contributor/contributing.rst | 14 | ||||
| -rw-r--r-- | doc/source/index.rst | 1 | ||||
| -rw-r--r-- | lower-constraints.txt | 4 | ||||
| -rw-r--r-- | releasenotes/source/conf.py | 8 | ||||
| -rw-r--r-- | releasenotes/source/index.rst | 1 | ||||
| -rw-r--r-- | releasenotes/source/victoria.rst | 6 | ||||
| -rw-r--r-- | requirements.txt | 2 | ||||
| -rw-r--r-- | setup.cfg | 13 | ||||
| -rw-r--r-- | swiftclient/client.py | 101 | ||||
| -rw-r--r-- | swiftclient/exceptions.py | 12 | ||||
| -rw-r--r-- | swiftclient/service.py | 19 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 59 | ||||
| -rw-r--r-- | swiftclient/utils.py | 7 | ||||
| -rw-r--r-- | test-requirements.txt | 3 | ||||
| -rw-r--r-- | test/sample.conf | 10 | ||||
| -rw-r--r-- | test/unit/test_multithreading.py | 2 | ||||
| -rw-r--r-- | test/unit/test_shell.py | 46 | ||||
| -rw-r--r-- | test/unit/test_swiftclient.py | 106 | ||||
| -rw-r--r-- | test/unit/test_utils.py | 52 | ||||
| -rw-r--r-- | tox.ini | 5 |
25 files changed, 406 insertions, 118 deletions
@@ -27,6 +27,7 @@ - job: name: swiftclient-functional-py2 parent: swiftclient-functional + nodeset: openstack-single-node-bionic description: | Run functional tests of python-swiftclient under Python 2 vars: @@ -38,7 +39,7 @@ - lib-forward-testing-python3 - openstack-lower-constraints-jobs - openstack-python-jobs - - openstack-python3-ussuri-jobs + - openstack-python3-wallaby-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0bde968..493a6c5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,18 +1,19 @@ -If you would like to contribute to the development of OpenStack, you -must follow the steps in this page: +The source repository for this project can be found at: - https://docs.openstack.org/infra/manual/developers.html + https://opendev.org/openstack/python-swiftclient -Once those steps have been completed, changes to OpenStack should be -submitted for review via the Gerrit tool, following the workflow -documented at: +Pull requests submitted through GitHub are not monitored. - https://docs.openstack.org/infra/manual/developers.html#development-workflow +To start contributing to OpenStack, follow the steps in the contribution guide +to set up and use Gerrit: -Gerrit is the review system used in the OpenStack projects. We're sorry, -but we won't be able to respond to pull requests submitted through -GitHub. + https://docs.openstack.org/contributors/code-and-documentation/quick-start.html -Bugs should be filed on Launchpad, not Github: +Bugs should be filed on Launchpad: - https://bugs.launchpad.net/python-swiftclient + https://bugs.launchpad.net/python-swiftclient + +For more specific information about contributing to this repository, see the +swiftclient contributor guide: + + https://docs.openstack.org/python-swiftclient/latest/contributor/contributing.html @@ -23,7 +23,7 @@ in the `OpenStack wiki`__. __ https://docs.openstack.org/infra/manual/developers.html This code is based on the original client previously included with -`OpenStack's Swift`__ The python-swiftclient is licensed under the +`OpenStack's Swift`__. The python-swiftclient is licensed under the Apache License like the rest of OpenStack. __ https://github.com/openstack/swift @@ -32,21 +32,17 @@ __ https://github.com/openstack/swift * `PyPI`_ - package installation * `Online Documentation`_ * `Launchpad project`_ - release management -* `Blueprints`_ - feature specifications * `Bugs`_ - issue tracking * `Source`_ -* `Specs`_ * `How to Contribute`_ * `Release Notes`_ .. _PyPI: https://pypi.org/project/python-swiftclient .. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ .. _Launchpad project: https://launchpad.net/python-swiftclient -.. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient .. _Source: https://opendev.org/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html -.. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient .. contents:: Contents: diff --git a/doc/requirements.txt b/doc/requirements.txt index 6cdad2a..6894ce1 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,4 @@ keystoneauth1>=3.4.0 # Apache-2.0 -sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD -sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD -reno>=2.5.0 # Apache-2.0 -openstackdocstheme>=1.20.0 # Apache-2.0 +sphinx>=1.6.2,!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0 # BSD +reno>=3.1.0 # Apache-2.0 +openstackdocstheme>=2.2.1 # Apache-2.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index a8ad3ad..8381605 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -55,6 +55,12 @@ master_doc = 'index' # General information about the project. copyright = u'2013-2016 OpenStack, LLC.' +# -- Options for openstackdocstheme ------------------------------------------- +openstackdocs_repo_name = 'openstack/python-swiftclient' +openstackdocs_bug_project = 'python-swiftclient' +openstackdocs_bug_tag = '' +openstackdocs_pdf_link = True + # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None @@ -88,7 +94,7 @@ exclude_trees = [] # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'native' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -131,10 +137,6 @@ html_theme_options = {'show_other_versions': True} # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True diff --git a/doc/source/contributor/contributing.rst b/doc/source/contributor/contributing.rst new file mode 100644 index 0000000..b9f1e91 --- /dev/null +++ b/doc/source/contributor/contributing.rst @@ -0,0 +1,14 @@ +============================ +So You Want to Contribute... +============================ + +For general information on contributing to OpenStack, please check out the +`contributor guide <https://docs.openstack.org/contributors/>`_ to get started. +It covers all the basics that are common to all OpenStack projects: the +accounts you need, the basics of interacting with our Gerrit review system, how +we communicate as a community, etc. + +The python-swiftclient is maintained by the OpenStack Swift project. +To understand our development process and how you can contribute to it, please +look at the Swift project's general contributor's page: +http://docs.openstack.org/swift/latest/contributor/contributing.html
\ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index ab05c6b..ae30972 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,6 +16,7 @@ Developer Documentation .. toctree:: :maxdepth: 2 + contributor/contributing cli/index service-api client-api diff --git a/lower-constraints.txt b/lower-constraints.txt index ead0279..28a1060 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -20,7 +20,6 @@ MarkupSafe==1.0 mccabe==0.2.1 mock==1.2.0 netaddr==0.7.10 -openstackdocstheme==1.20.0 openstacksdk==0.11.0 oslo.config==1.2.0 pbr==2.0.0 @@ -33,12 +32,9 @@ python-mimeparse==1.6.0 python-subunit==1.0.0 pytz==2013.6 PyYAML==3.12 -reno==2.5.0 requests==1.1.0 six==1.9.0 snowballstemmer==1.2.1 -sphinx==1.6.2 -sphinxcontrib-websupport==1.0.1 stestr==2.0.0 testtools==2.2.0 traceback2==1.4.0 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index c71f41d..a050a54 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -108,7 +108,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'native' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -339,6 +339,6 @@ htmlhelp_basename = 'SwiftClientReleaseNotesdoc' locale_dirs = ['locale/'] # -- Options for openstackdocstheme ------------------------------------------- -repository_name = 'openstack/python-swiftclient' -bug_project = 'python-swiftclient' -bug_tag = '' +openstackdocs_repo_name = 'openstack/python-swiftclient' +openstackdocs_bug_project = 'python-swiftclient' +openstackdocs_bug_tag = '' diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index d46593b..a63715f 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + victoria ussuri train stein diff --git a/releasenotes/source/victoria.rst b/releasenotes/source/victoria.rst new file mode 100644 index 0000000..4efc7b6 --- /dev/null +++ b/releasenotes/source/victoria.rst @@ -0,0 +1,6 @@ +============================= +Victoria Series Release Notes +============================= + +.. release-notes:: + :branch: stable/victoria diff --git a/requirements.txt b/requirements.txt index 1c2ce33..4757239 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD +futures>=3.0.0;python_version=='2.7' # BSD requests>=1.1.0 six>=1.9.0 @@ -19,10 +19,7 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 - -[global] -setup-hooks = - pbr.hooks.setup_hook + Programming Language :: Python :: 3.8 [files] packages = @@ -43,14 +40,6 @@ console_scripts = keystoneauth1.plugin = v1password = swiftclient.authv1:PasswordLoader -[build_sphinx] -source-dir = doc/source -build-dir = doc/build -all_files = 1 - -[upload_sphinx] -upload-dir = doc/build/html - [bdist_wheel] universal = 1 diff --git a/swiftclient/client.py b/swiftclient/client.py index 449b6cd..5c63b60 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -45,6 +45,8 @@ AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) USER_METADATA_TYPE = tuple('x-%s-meta-' % type_ for type_ in ('container', 'account', 'object')) +URI_PATTERN_INFO = re.compile(r'/info') +URI_PATTERN_VERSION = re.compile(r'\/v\d+\.?\d*(\/.*)?') try: from logging import NullHandler @@ -60,7 +62,7 @@ except ImportError: def createLock(self): self.lock = None -ksexceptions = ksclient_v2 = ksclient_v3 = None +ksexceptions = ksclient_v2 = ksclient_v3 = ksa_v3 = None try: from keystoneclient import exceptions as ksexceptions # prevent keystoneclient warning us that it has no log handlers @@ -70,6 +72,9 @@ except ImportError: pass try: from keystoneclient.v3 import client as ksclient_v3 + from keystoneauth1.identity import v3 as ksa_v3 + from keystoneauth1 import session as ksa_session + from keystoneauth1 import exceptions as ksauthexceptions except ImportError: pass @@ -438,14 +443,16 @@ class HTTPConnection(object): if timeout: self.requests_args['timeout'] = timeout - def __del__(self): - """Cleanup resources other than memory""" - if self.request_session: - # The session we create must be closed to free up file descriptors - try: - self.request_session.close() - finally: - self.request_session = None + if not six.PY2: + def __del__(self): + """Cleanup resources other than memory""" + if self.request_session: + # The session we create must be closed to free up + # file descriptors + try: + self.request_session.close() + finally: + self.request_session = None def _request(self, *arg, **kwarg): """Final wrapper before requests call, to be patched in tests""" @@ -615,6 +622,51 @@ Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment variables to be set or overridden with -A, -U, or -K.''') + filter_kwargs = {} + service_type = os_options.get('service_type') or 'object-store' + endpoint_type = os_options.get('endpoint_type') or 'publicURL' + if os_options.get('region_name'): + filter_kwargs['attr'] = 'region' + filter_kwargs['filter_value'] = os_options['region_name'] + + if os_options.get('auth_type') and os_options['auth_type'] not in ( + 'password', 'v2password', 'v3password', + 'v3applicationcredential'): + raise ClientException( + 'Swiftclient currently only supports v3applicationcredential ' + 'for auth_type') + elif os_options.get('auth_type') == 'v3applicationcredential': + if ksa_v3 is None: + raise ClientException('Auth v3applicationcredential requires ' + 'keystoneauth1 package; consider upgrading ' + 'to python-keystoneclient>=2.0.0') + + try: + auth = ksa_v3.ApplicationCredential( + auth_url=auth_url, + application_credential_secret=os_options.get( + 'application_credential_secret'), + application_credential_id=os_options.get( + 'application_credential_id')) + sess = ksa_session.Session(auth=auth) + token = sess.get_token() + except ksauthexceptions.Unauthorized: + msg = 'Unauthorized. Check application credential id and secret.' + raise ClientException(msg) + except ksauthexceptions.AuthorizationFailure as err: + raise ClientException('Authorization Failure. %s' % err) + + try: + endpoint = sess.get_endpoint_data(service_type=service_type, + endpoint_type=endpoint_type, + **filter_kwargs) + + return endpoint.catalog_url, token + except ksauthexceptions.EndpointNotFound: + raise ClientException( + 'Endpoint for %s not found - ' + 'have you specified a region?' % service_type) + try: _ksclient = ksclient.Client( username=user, @@ -642,13 +694,8 @@ variables to be set or overridden with -A, -U, or -K.''') raise ClientException(msg) except ksexceptions.AuthorizationFailure as err: raise ClientException('Authorization Failure. %s' % err) - service_type = os_options.get('service_type') or 'object-store' - endpoint_type = os_options.get('endpoint_type') or 'publicURL' + try: - filter_kwargs = {} - if os_options.get('region_name'): - filter_kwargs['attr'] = 'region' - filter_kwargs['filter_value'] = os_options['region_name'] endpoint = _ksclient.service_catalog.url_for( service_type=service_type, endpoint_type=endpoint_type, @@ -717,9 +764,12 @@ def get_auth(auth_url, user, key, **kwargs): if kwargs.get('tenant_name'): os_options['tenant_name'] = kwargs['tenant_name'] - if not (os_options.get('tenant_name') or os_options.get('tenant_id') or - os_options.get('project_name') or - os_options.get('project_id')): + if os_options.get('auth_type') == 'v3applicationcredential': + pass + elif not (os_options.get('tenant_name') or + os_options.get('tenant_id') or + os_options.get('project_name') or + os_options.get('project_id')): if auth_version in AUTH_VERSIONS_V2: raise ClientException('No tenant specified') raise ClientException('No project name or project id specified.') @@ -1935,11 +1985,22 @@ class Connection(object): response_dict=response_dict, headers=headers) - def get_capabilities(self, url=None): + def _map_url(self, url): url = url or self.url if not url: url, _ = self.get_auth() - parsed = urlparse(urljoin(url, '/info')) + scheme, netloc, path, params, query, fragment = urlparse(url) + if URI_PATTERN_VERSION.search(path): + path = URI_PATTERN_VERSION.sub('/info', path) + elif not URI_PATTERN_INFO.search(path): + if path.endswith('/'): + path += 'info' + else: + path += '/info' + return urlunparse((scheme, netloc, path, params, query, fragment)) + + def get_capabilities(self, url=None): + parsed = urlparse(self._map_url(url)) if not self.http_conn: self.http_conn = self.http_connection(url) return get_capabilities((parsed, self.http_conn[1])) diff --git a/swiftclient/exceptions.py b/swiftclient/exceptions.py index da70379..a9b993c 100644 --- a/swiftclient/exceptions.py +++ b/swiftclient/exceptions.py @@ -35,6 +35,13 @@ class ClientException(Exception): self.http_response_content = http_response_content self.http_response_headers = http_response_headers + self.transaction_id = None + if self.http_response_headers: + for header in ('X-Trans-Id', 'X-Openstack-Request-Id'): + if header in self.http_response_headers: + self.transaction_id = self.http_response_headers[header] + break + @classmethod def from_response(cls, resp, msg=None, body=None): msg = msg or '%s %s' % (resp.status_code, resp.reason) @@ -78,4 +85,7 @@ class ClientException(Exception): else: b += ' [first 60 chars of response] %s' \ % self.http_response_content[:60] - return b and '%s: %s' % (a, b) or a + c = '' + if self.transaction_id: + c = ' (txn: %s)' % self.transaction_id + return b and '%s: %s%s' % (a, b, c) or (a + c) diff --git a/swiftclient/service.py b/swiftclient/service.py index fb334fd..cd96a5b 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -110,6 +110,9 @@ def process_options(options): else: options['auth_version'] = '2.0' + if options.get('os_auth_type', None) == 'v3applicationcredential': + options['auth_version'] == '3' + # Use new-style args if old ones not present if not options['auth'] and options['os_auth_url']: options['auth'] = options['os_auth_url'] @@ -134,6 +137,11 @@ def process_options(options): 'auth_token': options['os_auth_token'], 'object_storage_url': options['os_storage_url'], 'region_name': options['os_region_name'], + 'auth_type': options['os_auth_type'], + 'application_credential_id': + options['os_application_credential_id'], + 'application_credential_secret': + options['os_application_credential_secret'], } @@ -162,6 +170,11 @@ def _build_default_global_options(): "os_project_domain_id": environ.get('OS_PROJECT_DOMAIN_ID'), "os_auth_url": environ.get('OS_AUTH_URL'), "os_auth_token": environ.get('OS_AUTH_TOKEN'), + "os_auth_type": environ.get('OS_AUTH_TYPE'), + "os_application_credential_id": + environ.get('OS_APPLICATION_CREDENTIAL_ID'), + "os_application_credential_secret": + environ.get('OS_APPLICATION_CREDENTIAL_SECRET'), "os_storage_url": environ.get('OS_STORAGE_URL'), "os_region_name": environ.get('OS_REGION_NAME'), "os_service_type": environ.get('OS_SERVICE_TYPE'), @@ -261,7 +274,7 @@ def get_conn(options): return Connection(options['auth'], options['user'], options['key'], - options['retries'], + retries=options['retries'], auth_version=options['auth_version'], os_options=options['os_options'], snet=options['snet'], @@ -270,7 +283,9 @@ def get_conn(options): cert=options['os_cert'], cert_key=options['os_key'], ssl_compression=options['ssl_compression'], - force_auth_retry=options['force_auth_retry']) + force_auth_retry=options['force_auth_retry'], + starting_backoff=options.get('starting_backoff', 1), + max_backoff=options.get('max_backoff', 64)) def mkdirs(path): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 1b34c08..dbcd437 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1651,16 +1651,29 @@ def parse_args(parser, args, enforce_requires=True): return options, args if enforce_requires: - if options['auth_version'] == '3': + if options['os_auth_type'] and options['os_auth_type'] not in ( + 'password', 'v1password', 'v2password', 'v3password', + 'v3applicationcredential'): + exit('Only "v3applicationcredential" is supported for ' + '--os-auth-type') + elif options['os_auth_type'] == 'v3applicationcredential': + if not (options['os_application_credential_id'] and + options['os_application_credential_secret']): + exit('Auth version 3 (application credential) requires ' + 'OS_APPLICATION_CREDENTIAL_ID and ' + 'OS_APPLICATION_CREDENTIAL_SECRET to be set or ' + 'overridden with --os-application-credential-id and ' + '--os-application-credential-secret respectively.') + elif options['auth_version'] == '3': if not options['auth']: - exit('Auth version 3 requires OS_AUTH_URL to be set or ' + + exit('Auth version 3 requires OS_AUTH_URL to be set or ' 'overridden with --os-auth-url') if not (options['user'] or options['os_user_id']): - exit('Auth version 3 requires either OS_USERNAME or ' + - 'OS_USER_ID to be set or overridden with ' + + exit('Auth version 3 requires either OS_USERNAME or ' + 'OS_USER_ID to be set or overridden with ' '--os-username or --os-user-id respectively.') if not options['key']: - exit('Auth version 3 requires OS_PASSWORD to be set or ' + + exit('Auth version 3 requires OS_PASSWORD to be set or ' 'overridden with --os-password') elif not (options['auth'] and options['user'] and options['key']): exit(''' @@ -1831,6 +1844,29 @@ def add_default_args(parser): 'env[OS_AUTH_URL].') os_grp.add_argument('--os_auth_url', help=argparse.SUPPRESS) + os_grp.add_argument('--os-auth-type', + metavar='<auth-type>', + default=environ.get('OS_AUTH_TYPE'), + help='OpenStack auth type for v3. Defaults to ' + 'env[OS_AUTH_TYPE].') + os_grp.add_argument('--os_auth_type', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-application-credential-id', + metavar='<auth-application-credential-id>', + default=environ.get('OS_APPLICATION_CREDENTIAL_ID'), + help='OpenStack appplication credential id. ' + 'Defaults to env[OS_APPLICATION_CREDENTIAL_ID].') + os_grp.add_argument('--os_application_credential_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-application-credential-secret', + metavar='<auth-application-credential-secret>', + default=environ.get( + 'OS_APPLICATION_CREDENTIAL_SECRET'), + help='OpenStack appplication credential secret. ' + 'Defaults to ' + 'env[OS_APPLICATION_CREDENTIAL_SECRET].') + os_grp.add_argument('--os_application_credential_secret', + help=argparse.SUPPRESS) os_grp.add_argument('--os-auth-token', metavar='<auth-token>', default=environ.get('OS_AUTH_TOKEN'), @@ -1915,6 +1951,11 @@ def main(arguments=None): [--os-project-domain-name <auth-project-domain-name>] [--os-auth-url <auth-url>] [--os-auth-token <auth-token>] + [--os-auth-type <os-auth-type>] + [--os-application-credential-id + <auth-application-credential-id>] + [--os-application-credential-secret + <auth-application-credential-secret>] [--os-storage-url <storage-url>] [--os-region-name <region-name>] [--os-service-type <service-type>] @@ -1967,6 +2008,11 @@ Examples: --os-user-id abcdef0123456789abcdef0123456789 \\ --os-password password list + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-application-credential-id d78683c92f0e4f9b9b02a2e208039412 \\ + --os-application-credential-secret APPLICTION_CREDENTIAL_SECRET \\ + --os-auth-type v3applicationcredential list + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ list @@ -2012,8 +2058,9 @@ Examples: try: globals()['st_%s' % args[0]](parser, argv[1:], output) except ClientException as err: + trans_id = err.transaction_id + err.transaction_id = None # clear it so we aren't overly noisy output.error(str(err)) - trans_id = (err.http_response_headers or {}).get('X-Trans-Id') if trans_id: output.error("Failed Transaction ID: %s", parse_header_string(trans_id)) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 9e43237..656acad 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -14,7 +14,10 @@ # limitations under the License. """Miscellaneous utility functions for use with Swift.""" from calendar import timegm -import collections +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping import gzip import hashlib import hmac @@ -218,7 +221,7 @@ def parse_api_response(headers, body): def split_request_headers(options, prefix=''): headers = {} - if isinstance(options, collections.Mapping): + if isinstance(options, Mapping): options = options.items() for item in options: if isinstance(item, six.string_types): diff --git a/test-requirements.txt b/test-requirements.txt index 5dba1a6..c2fb2c6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ -hacking>=1.1.0,<1.2.0 # Apache-2.0 +hacking>=1.1.0,<1.2.0;python_version<'3.0' # Apache-2.0 +hacking>=3.2.0,<3.3.0;python_version>='3.0' # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 diff --git a/test/sample.conf b/test/sample.conf index 2b19de4..04dfdd9 100644 --- a/test/sample.conf +++ b/test/sample.conf @@ -1,16 +1,10 @@ [func_test] # sample config -auth_host = 127.0.0.1 -auth_port = 8080 -auth_ssl = no -auth_prefix = /auth/ +auth_uri = http://127.0.0.1:8080/auth/v1.0/ ## sample config for Swift with Keystone v2 API # For keystone v3 change auth_version to 3 and auth_prefix to /v3/ #auth_version = 2 -#auth_host = localhost -#auth_port = 5000 -#auth_ssl = no -#auth_prefix = /v2.0/ +#auth_uri = http://localhost:5000/v2.0/ # You may want to run tests against endpoints that use development certs # without installing the CA system-wide. Use this to trust an extra set diff --git a/test/unit/test_multithreading.py b/test/unit/test_multithreading.py index 8944d48..e9732cd 100644 --- a/test/unit/test_multithreading.py +++ b/test/unit/test_multithreading.py @@ -37,7 +37,7 @@ class ThreadTestCase(unittest.TestCase): self.got_args_kwargs.put((args, kwargs)) if item == 'sleep': - sleep(1) + sleep(.1) if item == 'go boom': raise Exception('I went boom!') diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index a63d16b..f94e5e2 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -2395,6 +2395,8 @@ class TestParsing(TestBase): 'object_storage_url', 'project_domain_id', 'user_id', 'user_domain_id', 'tenant_id', 'service_type', 'project_id', 'auth_token', + 'auth_type', 'application_credential_id', + 'application_credential_secret', 'project_domain_name'] for key in expected_os_opts_keys: self.assertIn(key, actual_os_opts_dict) @@ -2686,6 +2688,50 @@ class TestParsing(TestBase): swiftclient.shell.main(args) self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + def test_command_args_v3applicationcredential(self): + result = [None, None] + fake_command = self._make_fake_command(result) + opts = {"auth_version": "3"} + os_opts = { + "auth_type": "v3applicationcredential", + "application_credential_id": "proejct_id", + "application_credential_secret": "secret", + "auth_url": "http://example.com:5000/v3"} + + args = _make_args("stat", opts, os_opts) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self.assertEqual(['stat'], result[1]) + with mock.patch('swiftclient.shell.st_stat', fake_command): + args = args + ["container_name"] + swiftclient.shell.main(args) + self.assertEqual(["stat", "container_name"], result[1]) + + def test_insufficient_args_v3applicationcredential(self): + opts = {"auth_version": "3"} + os_opts = { + "auth_type": "v3applicationcredential", + "application_credential_secret": "secret", + "auth_url": "http://example.com:5000/v3"} + + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 (application credential) requires', + str(cm.exception)) + + os_opts = { + "auth_type": "v3oidcpassword", + "application_credential_id": "proejct_id", + "application_credential_secret": "secret", + "auth_url": "http://example.com:5000/v3"} + + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Only "v3applicationcredential" is supported for', + str(cm.exception)) + def test_password_prompt(self): def do_test(opts, os_opts, auth_version): args = _make_args("stat", opts, os_opts) diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 2d45deb..2644e33 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -84,6 +84,23 @@ class TestClientException(unittest.TestCase): self.assertIs(True, hasattr(exc, key)) self.assertEqual(getattr(exc, key), value) + def test_transaction_id_from_headers(self): + exc = c.ClientException('test') + self.assertIsNone(exc.transaction_id) + + exc = c.ClientException('test', http_response_headers={}) + self.assertIsNone(exc.transaction_id) + + exc = c.ClientException('test', http_response_headers={ + 'X-Trans-Id': 'some-id'}) + self.assertEqual(exc.transaction_id, 'some-id') + self.assertIn('(txn: some-id)', str(exc)) + + exc = c.ClientException('test', http_response_headers={ + 'X-Openstack-Request-Id': 'some-other-id'}) + self.assertEqual(exc.transaction_id, 'some-other-id') + self.assertIn('(txn: some-other-id)', str(exc)) + class MockHttpResponse(object): def __init__(self, status=0, headers=None, verify=False): @@ -562,6 +579,63 @@ class TestGetAuth(MockHttpTest): self.assertTrue(url.startswith("http")) self.assertTrue(token) + def test_auth_v3applicationcredential(self): + from keystoneauth1 import exceptions as ksauthexceptions + + os_options = { + "auth_type": "v3applicationcredential", + "application_credential_id": "proejct_id", + "application_credential_secret": "secret"} + + class FakeEndpointData(object): + catalog_url = 'http://swift.cluster/v1/KEY_project_id' + + class FakeKeystoneuth1v3Session(object): + + def __init__(self, auth): + self.auth = auth + self.token = 'token' + + def get_token(self): + if self.auth.auth_url == 'http://keystone:5000/v3': + return self.token + elif self.auth.auth_url == 'http://keystone:9000/v3': + raise ksauthexceptions.AuthorizationFailure + else: + raise ksauthexceptions.Unauthorized + + def get_endpoint_data(self, service_type, endpoint_type, **kwargs): + return FakeEndpointData() + + mock_sess = FakeKeystoneuth1v3Session + with mock.patch('keystoneauth1.session.Session', mock_sess): + url, token = c.get_auth('http://keystone:5000', '', '', + os_options=os_options, + auth_version="3") + + self.assertTrue(url.startswith("http")) + self.assertEqual(url, 'http://swift.cluster/v1/KEY_project_id') + self.assertEqual(token, 'token') + + with mock.patch('keystoneauth1.session.Session', mock_sess): + with self.assertRaises(c.ClientException) as exc_mgr: + url, token = c.get_auth('http://keystone:9000', '', '', + os_options=os_options, + auth_version="3") + + body = 'Unauthorized. Check application credential id and secret.' + body = 'Authorization Failure. Cannot authorize API client.' + self.assertEqual(exc_mgr.exception.__str__()[-89:], body) + + with mock.patch('keystoneauth1.session.Session', mock_sess): + with self.assertRaises(c.ClientException) as exc_mgr: + url, token = c.get_auth('http://keystone:5000', '', '', + os_options=os_options, + auth_version="2") + + body = 'Unauthorized. Check application credential id and secret.' + self.assertEqual(exc_mgr.exception.__str__()[-89:], body) + def test_get_keystone_client_2_0(self): # check the correct auth version is passed to get_auth_keystone os_options = {'tenant_name': 'asdf'} @@ -2035,6 +2109,38 @@ class TestConnection(MockHttpTest): self.assertEqual(request['headers']['x-auth-token'], 'tToken') + def test_url_mapping(self): + conn = c.Connection() + uri_versions = { + 'http://storage.test.com': + 'http://storage.test.com/info', + 'http://storage.test.com/': + 'http://storage.test.com/info', + 'http://storage.test.com/v1': + 'http://storage.test.com/info', + 'http://storage.test.com/v1/': + 'http://storage.test.com/info', + 'http://storage.test.com/swift': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/swift/': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/v1.0': + 'http://storage.test.com/info', + 'http://storage.test.com/swift/v1.0': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/v111': + 'http://storage.test.com/info', + 'http://storage.test.com/v111/test': + 'http://storage.test.com/info', + 'http://storage.test.com/v1/test': + 'http://storage.test.com/info', + 'http://storage.test.com/swift/v1.0/test': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/v1.0/test': + 'http://storage.test.com/info'} + for uri_k, uri_v in uri_versions.items(): + self.assertEqual(conn._map_url(uri_k), uri_v) + def test_get_capabilities(self): conn = c.Connection() with mock.patch('swiftclient.client.get_capabilities') as get_cap: diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 97abc44..cbee82b 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -521,15 +521,15 @@ class TestLengthWrapper(unittest.TestCase): with tempfile.NamedTemporaryFile(mode='wb') as f: f.write(b'a' * 100) f.flush() - contents = open(f.name, 'rb') - data = u.LengthWrapper(contents, 42, True) - s = b'a' * 42 - read_data = b''.join(iter(data.read, '')) + with open(f.name, 'rb') as contents: + data = u.LengthWrapper(contents, 42, True) + s = b'a' * 42 + read_data = b''.join(iter(data.read, '')) - self.assertEqual(42, len(data)) - self.assertEqual(42, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) def test_segmented_file(self): with tempfile.NamedTemporaryFile(mode='wb') as f: @@ -539,24 +539,24 @@ class TestLengthWrapper(unittest.TestCase): f.write((c * segment_length).encode()) f.flush() for i, c in enumerate(segments): - contents = open(f.name, 'rb') - contents.seek(i * segment_length) - data = u.LengthWrapper(contents, segment_length, True) - read_data = b''.join(iter(data.read, '')) - s = (c * segment_length).encode() - - self.assertEqual(segment_length, len(data)) - self.assertEqual(segment_length, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) - - data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) - read_data = b''.join(iter(data.read, '')) - self.assertEqual(segment_length, len(data)) - self.assertEqual(segment_length, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + with open(f.name, 'rb') as contents: + contents.seek(i * segment_length) + data = u.LengthWrapper(contents, segment_length, True) + read_data = b''.join(iter(data.read, '')) + s = (c * segment_length).encode() + + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + read_data = b''.join(iter(data.read, '')) + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) class TestGroupers(unittest.TestCase): @@ -1,11 +1,10 @@ [tox] -envlist = py27,py37,pep8 +envlist = py27,py38,pep8 minversion = 2.0 skipsdist = True [testenv] usedevelop = True -install_command = python -m pip install -U {opts} {packages} list_dependencies_command = python -m pip freeze setenv = LANG=en_US.utf-8 @@ -68,7 +67,7 @@ basepython = python3 usedevelop = False deps = -r{toxinidir}/doc/requirements.txt commands= - python setup.py build_sphinx -W + sphinx-build -W -b html doc/source doc/build/html -W [flake8] # it's not a bug that we aren't using all of hacking, ignore: |