diff options
author | Matthew Oliver <matt@oliver.net.au> | 2022-02-03 16:18:39 +1100 |
---|---|---|
committer | Alistair Coles <alistairncoles@gmail.com> | 2022-02-03 14:41:13 +0000 |
commit | 589ac355f3ddf505983925d80b81c20107ceace0 (patch) | |
tree | 08fdb536bc61addb5fe39ffce7edc0ce1cdb1851 | |
parent | 8ac63b760928548cd5cc1b3565ca14de3d36ce7f (diff) | |
download | swift-589ac355f3ddf505983925d80b81c20107ceace0.tar.gz |
Move *_swift_info functions into a new registry module
The *_swift_info functions use in module global dicts to provide a
registry mechanism for registering and getting swift info.
This is an abnormal pattern and doesn't quite fit into utils. Further
we looking at following this pattern for sensitive info to trim in the
future.
So this patch does some house cleaning and moves this registry to a new
module swift.common.registry. And updates all the references to it.
For backwards compat we still import the *_swift_info methods into utils
for any 3rd party tools or middleware.
Change-Id: I71fd7f50d1aafc001d6905438f42de4e58af8421
44 files changed, 454 insertions, 391 deletions
diff --git a/swift/common/middleware/account_quotas.py b/swift/common/middleware/account_quotas.py index 5b7731b81..cd41e34da 100644 --- a/swift/common/middleware/account_quotas.py +++ b/swift/common/middleware/account_quotas.py @@ -54,7 +54,7 @@ account size has been updated. from swift.common.swob import HTTPForbidden, HTTPBadRequest, \ HTTPRequestEntityTooLarge, wsgify -from swift.common.utils import register_swift_info +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_account_info diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 86798d755..df0c881a7 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -206,8 +206,8 @@ from swift.common.swob import Request, HTTPBadGateway, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPNotAcceptable, \ HTTPLengthRequired, HTTPException, HTTPServerError, wsgify, \ bytes_to_wsgi, str_to_wsgi, wsgi_unquote, wsgi_quote, wsgi_to_str -from swift.common.utils import get_logger, register_swift_info, \ - StreamingPile +from swift.common.utils import get_logger, StreamingPile +from swift.common.registry import register_swift_info from swift.common import constraints from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT from swift.common.request_helpers import is_user_meta diff --git a/swift/common/middleware/cname_lookup.py b/swift/common/middleware/cname_lookup.py index a1a3b4d76..2578c08fc 100644 --- a/swift/common/middleware/cname_lookup.py +++ b/swift/common/middleware/cname_lookup.py @@ -44,7 +44,8 @@ from swift.common.middleware import RewriteContext from swift.common.swob import Request, HTTPBadRequest, \ str_to_wsgi, wsgi_to_str from swift.common.utils import cache_from_env, get_logger, is_valid_ip, \ - list_from_csv, parse_socket_string, register_swift_info + list_from_csv, parse_socket_string +from swift.common.registry import register_swift_info def lookup_cname(domain, resolver): # pragma: no cover diff --git a/swift/common/middleware/container_quotas.py b/swift/common/middleware/container_quotas.py index a78876aca..2a10c8d2c 100644 --- a/swift/common/middleware/container_quotas.py +++ b/swift/common/middleware/container_quotas.py @@ -54,7 +54,7 @@ For example:: from swift.common.http import is_success from swift.common.swob import HTTPRequestEntityTooLarge, HTTPBadRequest, \ wsgify -from swift.common.utils import register_swift_info +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_container_info diff --git a/swift/common/middleware/container_sync.py b/swift/common/middleware/container_sync.py index 74c63afa4..cc6b16aee 100644 --- a/swift/common/middleware/container_sync.py +++ b/swift/common/middleware/container_sync.py @@ -19,8 +19,9 @@ from swift.common.constraints import valid_api_version from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify from swift.common.utils import ( - config_true_value, get_logger, register_swift_info, streq_const_time) + config_true_value, get_logger, streq_const_time) from swift.proxy.controllers.base import get_container_info +from swift.common.registry import register_swift_info class ContainerSync(object): diff --git a/swift/common/middleware/crossdomain.py b/swift/common/middleware/crossdomain.py index 13a45a316..ffe73d43f 100644 --- a/swift/common/middleware/crossdomain.py +++ b/swift/common/middleware/crossdomain.py @@ -14,7 +14,7 @@ # limitations under the License. from swift.common.swob import Request, Response -from swift.common.utils import register_swift_info +from swift.common.registry import register_swift_info class CrossDomainMiddleware(object): diff --git a/swift/common/middleware/crypto/__init__.py b/swift/common/middleware/crypto/__init__.py index b526fcbaa..a70e7b860 100644 --- a/swift/common/middleware/crypto/__init__.py +++ b/swift/common/middleware/crypto/__init__.py @@ -20,7 +20,8 @@ instance of an :class:`~swift.common.middleware.crypto.encrypter.Encrypter`. from swift.common.middleware.crypto.decrypter import Decrypter from swift.common.middleware.crypto.encrypter import Encrypter -from swift.common.utils import config_true_value, register_swift_info +from swift.common.utils import config_true_value +from swift.common.registry import register_swift_info def filter_factory(global_conf, **local_conf): diff --git a/swift/common/middleware/domain_remap.py b/swift/common/middleware/domain_remap.py index f42e65b88..aec8e90e9 100644 --- a/swift/common/middleware/domain_remap.py +++ b/swift/common/middleware/domain_remap.py @@ -100,8 +100,8 @@ storage end points as sync destinations. from swift.common.middleware import RewriteContext from swift.common.swob import Request, HTTPBadRequest, wsgi_quote -from swift.common.utils import config_true_value, list_from_csv, \ - register_swift_info +from swift.common.utils import config_true_value, list_from_csv +from swift.common.registry import register_swift_info class _DomainRemapContext(RewriteContext): diff --git a/swift/common/middleware/etag_quoter.py b/swift/common/middleware/etag_quoter.py index ea3cbb30d..d67c1b48b 100644 --- a/swift/common/middleware/etag_quoter.py +++ b/swift/common/middleware/etag_quoter.py @@ -42,7 +42,8 @@ useful if some subset of applications expect Etags to be bare MD5s. from swift.common.constraints import valid_api_version from swift.common.http import is_success from swift.common.swob import Request -from swift.common.utils import config_true_value, register_swift_info +from swift.common.utils import config_true_value +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_account_info, get_container_info diff --git a/swift/common/middleware/formpost.py b/swift/common/middleware/formpost.py index af5a5cf5d..84a8ee09b 100644 --- a/swift/common/middleware/formpost.py +++ b/swift/common/middleware/formpost.py @@ -132,9 +132,10 @@ from six.moves.urllib.parse import quote from swift.common.constraints import valid_api_version from swift.common.exceptions import MimeInvalid from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata -from swift.common.utils import streq_const_time, register_swift_info, \ - parse_content_disposition, parse_mime_headers, \ - iter_multipart_mime_documents, reiterate, close_if_possible +from swift.common.utils import streq_const_time, parse_content_disposition, \ + parse_mime_headers, iter_multipart_mime_documents, reiterate, \ + close_if_possible +from swift.common.registry import register_swift_info from swift.common.wsgi import make_pre_authed_env from swift.common.swob import HTTPUnauthorized, wsgi_to_str, str_to_wsgi from swift.proxy.controllers.base import get_account_info, get_container_info diff --git a/swift/common/middleware/name_check.py b/swift/common/middleware/name_check.py index 9fea31649..b13c5a76b 100644 --- a/swift/common/middleware/name_check.py +++ b/swift/common/middleware/name_check.py @@ -41,7 +41,8 @@ The filter returns HTTPBadRequest if path is invalid. ''' import re -from swift.common.utils import get_logger, register_swift_info +from swift.common.utils import get_logger +from swift.common.registry import register_swift_info from swift.common.swob import Request, HTTPBadRequest diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 96ab03694..801cb7a84 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -17,7 +17,8 @@ from swift import gettext_ as _ import eventlet -from swift.common.utils import cache_from_env, get_logger, register_swift_info +from swift.common.utils import cache_from_env, get_logger +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_account_info, get_container_info from swift.common.constraints import valid_api_version from swift.common.memcached import MemcacheConnectionError diff --git a/swift/common/middleware/read_only.py b/swift/common/middleware/read_only.py index c62959c8e..3d54fdc5b 100644 --- a/swift/common/middleware/read_only.py +++ b/swift/common/middleware/read_only.py @@ -14,8 +14,8 @@ # limitations under the License. from swift.common.constraints import check_account_format from swift.common.swob import HTTPMethodNotAllowed, Request -from swift.common.utils import get_logger, config_true_value, \ - register_swift_info +from swift.common.utils import get_logger, config_true_value +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_info """ diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index 30c9ec28e..bd71bf058 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -23,8 +23,8 @@ from swift.common import swob from swift.common.http import HTTP_OK from swift.common.middleware.versioned_writes.object_versioning import \ DELETE_MARKER_CONTENT_TYPE -from swift.common.utils import json, public, config_true_value, Timestamp, \ - get_swift_info +from swift.common.utils import json, public, config_true_value, Timestamp +from swift.common.registry import get_swift_info from swift.common.middleware.s3api.controllers.base import Controller from swift.common.middleware.s3api.etree import Element, SubElement, \ diff --git a/swift/common/middleware/s3api/controllers/multi_delete.py b/swift/common/middleware/s3api/controllers/multi_delete.py index 6d1c522b5..49a51f663 100644 --- a/swift/common/middleware/s3api/controllers/multi_delete.py +++ b/swift/common/middleware/s3api/controllers/multi_delete.py @@ -18,7 +18,8 @@ import json from swift.common.constraints import MAX_OBJECT_NAME_LENGTH from swift.common.http import HTTP_NO_CONTENT -from swift.common.utils import public, StreamingPile, get_swift_info +from swift.common.utils import public, StreamingPile +from swift.common.registry import get_swift_info from swift.common.middleware.s3api.controllers.base import Controller, \ bucket_operation diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py index cae97b5ae..b1f310e6e 100644 --- a/swift/common/middleware/s3api/controllers/obj.py +++ b/swift/common/middleware/s3api/controllers/obj.py @@ -20,7 +20,8 @@ from swift.common.http import HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_NO_CONTENT from swift.common.request_helpers import update_etag_is_at_header from swift.common.swob import Range, content_range_header_value, \ normalize_etag -from swift.common.utils import public, list_from_csv, get_swift_info +from swift.common.utils import public, list_from_csv +from swift.common.registry import get_swift_info from swift.common.middleware.versioned_writes.object_versioning import \ DELETE_MARKER_CONTENT_TYPE diff --git a/swift/common/middleware/s3api/controllers/versioning.py b/swift/common/middleware/s3api/controllers/versioning.py index 9c835f3c2..2d31d2af5 100644 --- a/swift/common/middleware/s3api/controllers/versioning.py +++ b/swift/common/middleware/s3api/controllers/versioning.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.common.utils import public, get_swift_info, config_true_value +from swift.common.utils import public, config_true_value +from swift.common.registry import get_swift_info from swift.common.middleware.s3api.controllers.base import Controller, \ bucket_operation diff --git a/swift/common/middleware/s3api/s3api.py b/swift/common/middleware/s3api/s3api.py index 9c225c53b..56b6e997f 100644 --- a/swift/common/middleware/s3api/s3api.py +++ b/swift/common/middleware/s3api/s3api.py @@ -156,11 +156,11 @@ from swift.common.middleware.s3api.exception import NotS3Request, \ from swift.common.middleware.s3api.s3request import get_request_class from swift.common.middleware.s3api.s3response import ErrorResponse, \ InternalError, MethodNotAllowed, S3ResponseBase, S3NotImplemented -from swift.common.utils import get_logger, register_swift_info, \ - config_true_value, config_positive_int_value, split_path, \ - closing_if_possible, list_from_csv +from swift.common.utils import get_logger, config_true_value, \ + config_positive_int_value, split_path, closing_if_possible, list_from_csv from swift.common.middleware.s3api.utils import Config from swift.common.middleware.s3api.acl_handlers import get_acl_handler +from swift.common.registry import register_swift_info class ListingEtagMiddleware(object): diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index d4c33512d..152285340 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -25,8 +25,8 @@ import six from six.moves.urllib.parse import quote, unquote, parse_qsl import string -from swift.common.utils import split_path, json, get_swift_info, \ - close_if_possible, md5 +from swift.common.utils import split_path, json, close_if_possible, md5 +from swift.common.registry import get_swift_info from swift.common import swob from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \ HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \ diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 88554e1cb..857a52f22 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -345,9 +345,10 @@ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ RESPONSE_REASONS, str_to_wsgi, bytes_to_wsgi, wsgi_to_str, wsgi_quote from swift.common.utils import get_logger, config_true_value, \ get_valid_utf8_str, override_bytes_from_content_type, split_path, \ - register_swift_info, RateLimitedIterator, quote, close_if_possible, \ - closing_if_possible, LRUCache, StreamingPile, strict_b64decode, \ - Timestamp, drain_and_close, get_expirer_container, md5 + RateLimitedIterator, quote, close_if_possible, closing_if_possible, \ + LRUCache, StreamingPile, strict_b64decode, Timestamp, drain_and_close, \ + get_expirer_container, md5 +from swift.common.registry import register_swift_info from swift.common.request_helpers import SegmentedIterable, \ get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header, \ get_container_update_override_key, update_ignore_range_header diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 7e25e6167..23377dfae 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -131,7 +131,8 @@ from six.moves.urllib.parse import urlparse from swift.common.request_helpers import html_escape from swift.common.utils import human_readable, split_path, config_true_value, \ - quote, register_swift_info, get_logger + quote, get_logger +from swift.common.registry import register_swift_info from swift.common.wsgi import make_env, WSGIContext from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound, \ diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py index 2aac5d1eb..d951577d9 100644 --- a/swift/common/middleware/symlink.py +++ b/swift/common/middleware/symlink.py @@ -201,9 +201,10 @@ import json import os from cgi import parse_header -from swift.common.utils import get_logger, register_swift_info, split_path, \ +from swift.common.utils import get_logger, split_path, \ MD5_OF_EMPTY_STRING, close_if_possible, closing_if_possible, \ config_true_value, drain_and_close +from swift.common.registry import register_swift_info from swift.common.constraints import check_account_format from swift.common.wsgi import WSGIContext, make_subrequest, \ make_pre_authed_request diff --git a/swift/common/middleware/tempauth.py b/swift/common/middleware/tempauth.py index be1ff0b6c..c7242faa5 100644 --- a/swift/common/middleware/tempauth.py +++ b/swift/common/middleware/tempauth.py @@ -191,7 +191,8 @@ from swift.common.request_helpers import get_sys_meta_prefix from swift.common.middleware.acl import ( clean_acl, parse_acl, referrer_allowed, acls_from_account_info) from swift.common.utils import cache_from_env, get_logger, \ - split_path, config_true_value, register_swift_info + split_path, config_true_value +from swift.common.registry import register_swift_info from swift.common.utils import config_read_reseller_options, quote from swift.proxy.controllers.base import get_account_info diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index 1fcd5bcdd..c7169eba0 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -314,8 +314,8 @@ from swift.common.header_key_dict import HeaderKeyDict from swift.common.swob import header_to_environ_key, HTTPUnauthorized, \ HTTPBadRequest, wsgi_to_str from swift.common.utils import split_path, get_valid_utf8_str, \ - register_swift_info, get_hmac, streq_const_time, quote, get_logger, \ - strict_b64decode + get_hmac, streq_const_time, quote, get_logger, strict_b64decode +from swift.common.registry import register_swift_info DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target' @@ -498,7 +498,7 @@ class TempURL(object): if env['REQUEST_METHOD'] == 'OPTIONS': return self.app(env, start_response) info = self._get_temp_url_info(env) - temp_url_sig, temp_url_expires, temp_url_prefix, filename,\ + temp_url_sig, temp_url_expires, temp_url_prefix, filename, \ inline_disposition, temp_url_ip_range = info if temp_url_sig is None and temp_url_expires is None: return self.app(env, start_response) diff --git a/swift/common/middleware/versioned_writes/__init__.py b/swift/common/middleware/versioned_writes/__init__.py index 1bc41c3e2..368cb9b71 100644 --- a/swift/common/middleware/versioned_writes/__init__.py +++ b/swift/common/middleware/versioned_writes/__init__.py @@ -25,8 +25,8 @@ from swift.common.middleware.versioned_writes. \ from swift.common.middleware.versioned_writes. \ object_versioning import ObjectVersioningMiddleware -from swift.common.utils import config_true_value, register_swift_info, \ - get_swift_info +from swift.common.utils import config_true_value +from swift.common.registry import register_swift_info, get_swift_info def filter_factory(global_conf, **local_conf): diff --git a/swift/common/registry.py b/swift/common/registry.py new file mode 100644 index 000000000..9e515ae23 --- /dev/null +++ b/swift/common/registry.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Used by get_swift_info and register_swift_info to store information about +# the swift cluster. +from copy import deepcopy + +_swift_info = {} +_swift_admin_info = {} + + +def get_swift_info(admin=False, disallowed_sections=None): + """ + Returns information about the swift cluster that has been previously + registered with the register_swift_info call. + + :param admin: boolean value, if True will additionally return an 'admin' + section with information previously registered as admin + info. + :param disallowed_sections: list of section names to be withheld from the + information returned. + :returns: dictionary of information about the swift cluster. + """ + disallowed_sections = disallowed_sections or [] + info = deepcopy(_swift_info) + for section in disallowed_sections: + key_to_pop = None + sub_section_dict = info + for sub_section in section.split('.'): + if key_to_pop: + sub_section_dict = sub_section_dict.get(key_to_pop, {}) + if not isinstance(sub_section_dict, dict): + sub_section_dict = {} + break + key_to_pop = sub_section + sub_section_dict.pop(key_to_pop, None) + + if admin: + info['admin'] = dict(_swift_admin_info) + info['admin']['disallowed_sections'] = list(disallowed_sections) + return info + + +def register_swift_info(name='swift', admin=False, **kwargs): + """ + Registers information about the swift cluster to be retrieved with calls + to get_swift_info. + + NOTE: Do not use "." in the param: name or any keys in kwargs. "." is used + in the disallowed_sections to remove unwanted keys from /info. + + :param name: string, the section name to place the information under. + :param admin: boolean, if True, information will be registered to an + admin section which can optionally be withheld when + requesting the information. + :param kwargs: key value arguments representing the information to be + added. + :raises ValueError: if name or any of the keys in kwargs has "." in it + """ + if name == 'admin' or name == 'disallowed_sections': + raise ValueError('\'{0}\' is reserved name.'.format(name)) + + if admin: + dict_to_use = _swift_admin_info + else: + dict_to_use = _swift_info + if name not in dict_to_use: + if "." in name: + raise ValueError('Cannot use "." in a swift_info key: %s' % name) + dict_to_use[name] = {} + for key, val in kwargs.items(): + if "." in key: + raise ValueError('Cannot use "." in a swift_info key: %s' % key) + dict_to_use[name][key] = val diff --git a/swift/common/utils.py b/swift/common/utils.py index 624e29963..2981e7f7d 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -45,7 +45,6 @@ from random import random, shuffle from contextlib import contextmanager, closing import ctypes import ctypes.util -from copy import deepcopy from optparse import OptionParser import traceback import warnings @@ -90,6 +89,9 @@ from swift.common.http import is_server_error from swift.common.header_key_dict import HeaderKeyDict from swift.common.linkat import linkat +# For backwards compatability with 3rd party middlewares +from swift.common.registry import register_swift_info, get_swift_info # noqa + # logging doesn't import patched as cleanly as one would like from logging.handlers import SysLogHandler import logging @@ -318,77 +320,6 @@ def get_hmac(request_method, path, expires, key, digest=sha1, return hmac.new(key, message, digest).hexdigest() -# Used by get_swift_info and register_swift_info to store information about -# the swift cluster. -_swift_info = {} -_swift_admin_info = {} - - -def get_swift_info(admin=False, disallowed_sections=None): - """ - Returns information about the swift cluster that has been previously - registered with the register_swift_info call. - - :param admin: boolean value, if True will additionally return an 'admin' - section with information previously registered as admin - info. - :param disallowed_sections: list of section names to be withheld from the - information returned. - :returns: dictionary of information about the swift cluster. - """ - disallowed_sections = disallowed_sections or [] - info = deepcopy(_swift_info) - for section in disallowed_sections: - key_to_pop = None - sub_section_dict = info - for sub_section in section.split('.'): - if key_to_pop: - sub_section_dict = sub_section_dict.get(key_to_pop, {}) - if not isinstance(sub_section_dict, dict): - sub_section_dict = {} - break - key_to_pop = sub_section - sub_section_dict.pop(key_to_pop, None) - - if admin: - info['admin'] = dict(_swift_admin_info) - info['admin']['disallowed_sections'] = list(disallowed_sections) - return info - - -def register_swift_info(name='swift', admin=False, **kwargs): - """ - Registers information about the swift cluster to be retrieved with calls - to get_swift_info. - - NOTE: Do not use "." in the param: name or any keys in kwargs. "." is used - in the disallowed_sections to remove unwanted keys from /info. - - :param name: string, the section name to place the information under. - :param admin: boolean, if True, information will be registered to an - admin section which can optionally be withheld when - requesting the information. - :param kwargs: key value arguments representing the information to be - added. - :raises ValueError: if name or any of the keys in kwargs has "." in it - """ - if name == 'admin' or name == 'disallowed_sections': - raise ValueError('\'{0}\' is reserved name.'.format(name)) - - if admin: - dict_to_use = _swift_admin_info - else: - dict_to_use = _swift_info - if name not in dict_to_use: - if "." in name: - raise ValueError('Cannot use "." in a swift_info key: %s' % name) - dict_to_use[name] = {} - for key, val in kwargs.items(): - if "." in key: - raise ValueError('Cannot use "." in a swift_info key: %s' % key) - dict_to_use[name][key] = val - - def backward(f, blocksize=4096): """ A generator returning lines from a file starting with the last line, diff --git a/swift/proxy/controllers/info.py b/swift/proxy/controllers/info.py index 1d148e998..05bc1a998 100644 --- a/swift/proxy/controllers/info.py +++ b/swift/proxy/controllers/info.py @@ -16,8 +16,8 @@ import json from time import time -from swift.common.utils import public, get_hmac, get_swift_info, \ - streq_const_time +from swift.common.utils import public, get_hmac, streq_const_time +from swift.common.registry import get_swift_info from swift.proxy.controllers.base import Controller, delay_denial from swift.common.swob import HTTPOk, HTTPForbidden, HTTPUnauthorized diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1e1388c41..547410213 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -34,8 +34,9 @@ from swift.common.ring import Ring from swift.common.utils import Watchdog, get_logger, \ get_remote_client, split_path, config_true_value, generate_trans_id, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \ - register_swift_info, parse_prefixed_conf, config_auto_int_value, \ + parse_prefixed_conf, config_auto_int_value, \ config_request_node_count_value, config_percent_value +from swift.common.registry import register_swift_info from swift.common.constraints import check_utf8, valid_api_version from swift.proxy.controllers import AccountController, ContainerController, \ ObjectControllerRouter, InfoController diff --git a/test/unit/common/middleware/crypto/test_crypto.py b/test/unit/common/middleware/crypto/test_crypto.py index 882e959de..2daef42bd 100644 --- a/test/unit/common/middleware/crypto/test_crypto.py +++ b/test/unit/common/middleware/crypto/test_crypto.py @@ -15,7 +15,7 @@ import unittest import mock -from swift.common import utils +from swift.common import registry from swift.common.middleware import crypto @@ -24,36 +24,38 @@ class TestCrypto(unittest.TestCase): def do_test(conf, expect_enabled): fake_app = object() - with mock.patch.dict('swift.common.utils._swift_admin_info', + with mock.patch.dict('swift.common.registry._swift_admin_info', clear=True): # we're not expecting utils._swift_info to be modified but mock # it anyway just in case it is - with mock.patch.dict('swift.common.utils._swift_info', + with mock.patch.dict('swift.common.registry._swift_info', clear=True): # Sanity checks... - self.assertNotIn('encryption', utils._swift_admin_info) + self.assertNotIn('encryption', registry._swift_admin_info) self.assertNotIn('encryption', - utils.get_swift_info(admin=True)) - self.assertNotIn('encryption', - utils.get_swift_info(admin=True)['admin']) + registry.get_swift_info(admin=True)) + self.assertNotIn( + 'encryption', + registry.get_swift_info(admin=True)['admin']) factory = crypto.filter_factory(conf) self.assertTrue(callable(factory)) filtered_app = factory(fake_app) - self.assertNotIn('encryption', utils._swift_info) - self.assertNotIn('encryption', utils.get_swift_info()) + self.assertNotIn('encryption', registry._swift_info) + self.assertNotIn('encryption', registry.get_swift_info()) self.assertNotIn('encryption', - utils.get_swift_info(admin=True)) + registry.get_swift_info(admin=True)) - self.assertIn('encryption', utils._swift_admin_info) - self.assertDictEqual({'enabled': expect_enabled}, - utils._swift_admin_info['encryption']) + self.assertIn('encryption', registry._swift_admin_info) + self.assertDictEqual( + {'enabled': expect_enabled}, + registry._swift_admin_info['encryption']) self.assertIn('encryption', - utils.get_swift_info(admin=True)['admin']) + registry.get_swift_info(admin=True)['admin']) self.assertDictEqual( {'enabled': expect_enabled}, - utils.get_swift_info( + registry.get_swift_info( admin=True)['admin']['encryption']) self.assertIsInstance(filtered_app, crypto.decrypter.Decrypter) diff --git a/test/unit/common/middleware/s3api/test_s3api.py b/test/unit/common/middleware/s3api/test_s3api.py index 57a2cde6e..a0f37f24e 100644 --- a/test/unit/common/middleware/s3api/test_s3api.py +++ b/test/unit/common/middleware/s3api/test_s3api.py @@ -28,7 +28,7 @@ from six.moves.urllib.parse import unquote, quote import swift.common.middleware.s3api from swift.common.middleware.s3api.utils import Config from swift.common.middleware.keystoneauth import KeystoneAuth -from swift.common import swob, utils +from swift.common import swob, registry from swift.common.swob import Request from swift.common.utils import md5 @@ -759,15 +759,15 @@ class TestS3ApiMiddleware(S3ApiTestCase): def test_mfa(self): self._test_unsupported_header('x-amz-mfa') - @mock.patch.object(utils, '_swift_admin_info', new_callable=dict) + @mock.patch.object(registry, '_swift_admin_info', new_callable=dict) def test_server_side_encryption(self, mock_info): sse_header = 'x-amz-server-side-encryption' self._test_unsupported_header(sse_header, 'AES256') self._test_unsupported_header(sse_header, 'aws:kms') - utils.register_swift_info('encryption', admin=True, enabled=False) + registry.register_swift_info('encryption', admin=True, enabled=False) self._test_unsupported_header(sse_header, 'AES256') self._test_unsupported_header(sse_header, 'aws:kms') - utils.register_swift_info('encryption', admin=True, enabled=True) + registry.register_swift_info('encryption', admin=True, enabled=True) # AES256 now works self.swift.register('PUT', '/v1/AUTH_X/bucket/object', swob.HTTPCreated, {}, None) @@ -870,7 +870,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): def test_registered_defaults(self): conf_from_file = {k: str(v) for k, v in self.conf.items()} filter_factory(conf_from_file) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('s3api' in swift_info) self.assertEqual(swift_info['s3api'].get('max_bucket_listing'), self.conf['max_bucket_listing']) diff --git a/test/unit/common/middleware/test_bulk.py b/test/unit/common/middleware/test_bulk.py index 166abe8ef..5b3a324ba 100644 --- a/test/unit/common/middleware/test_bulk.py +++ b/test/unit/common/middleware/test_bulk.py @@ -29,7 +29,7 @@ from eventlet import sleep from mock import patch, call from test.debug_logger import debug_logger from test.unit.common.middleware.helpers import FakeSwift -from swift.common import utils, constraints +from swift.common import utils, constraints, registry from swift.common.header_key_dict import HeaderKeyDict from swift.common.middleware import bulk from swift.common.swob import Request, Response, HTTPException, \ @@ -1035,12 +1035,12 @@ class TestConfig(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): bulk.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('bulk_upload' in swift_info) self.assertTrue(isinstance( swift_info['bulk_upload'].get('max_containers_per_extraction'), diff --git a/test/unit/common/middleware/test_cname_lookup.py b/test/unit/common/middleware/test_cname_lookup.py index 2dbea6654..600228e55 100644 --- a/test/unit/common/middleware/test_cname_lookup.py +++ b/test/unit/common/middleware/test_cname_lookup.py @@ -24,7 +24,7 @@ except ImportError: skip = True else: # executed if the try has no errors skip = False -from swift.common import utils +from swift.common import registry from swift.common.middleware import cname_lookup from swift.common.swob import Request, HTTPMovedPermanently @@ -437,17 +437,17 @@ class TestCNAMELookup(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): cname_lookup.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('cname_lookup', swift_info) self.assertEqual(swift_info['cname_lookup'].get('lookup_depth'), 1) def test_registered_nondefaults(self): cname_lookup.filter_factory({'lookup_depth': '2'}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('cname_lookup', swift_info) self.assertEqual(swift_info['cname_lookup'].get('lookup_depth'), 2) diff --git a/test/unit/common/middleware/test_domain_remap.py b/test/unit/common/middleware/test_domain_remap.py index e89db9a73..317a20a44 100644 --- a/test/unit/common/middleware/test_domain_remap.py +++ b/test/unit/common/middleware/test_domain_remap.py @@ -18,7 +18,7 @@ import unittest from swift.common.swob import Request, HTTPMovedPermanently from swift.common.middleware import domain_remap -from swift.common import utils +from swift.common import registry class FakeApp(object): @@ -321,12 +321,12 @@ class TestDomainRemapClientMangling(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): domain_remap.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('domain_remap', swift_info) self.assertEqual(swift_info['domain_remap'], { 'default_reseller_prefix': None}) @@ -334,7 +334,7 @@ class TestSwiftInfo(unittest.TestCase): def test_registered_nondefaults(self): domain_remap.filter_factory({'default_reseller_prefix': 'cupcake', 'mangle_client_paths': 'yes'}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('domain_remap', swift_info) self.assertEqual(swift_info['domain_remap'], { 'default_reseller_prefix': 'cupcake'}) diff --git a/test/unit/common/middleware/test_name_check.py b/test/unit/common/middleware/test_name_check.py index a9d56f117..09c97f39e 100644 --- a/test/unit/common/middleware/test_name_check.py +++ b/test/unit/common/middleware/test_name_check.py @@ -26,7 +26,7 @@ import unittest from swift.common.swob import Request, Response from swift.common.middleware import name_check -from swift.common import utils +from swift.common import registry MAX_LENGTH = 255 FORBIDDEN_CHARS = '\'\"<>`' @@ -117,12 +117,12 @@ class TestNameCheckMiddleware(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): name_check.filter_factory({})(FakeApp()) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('name_check' in swift_info) self.assertTrue(isinstance( swift_info['name_check'].get('maximum_length'), @@ -139,7 +139,7 @@ class TestSwiftInfo(unittest.TestCase): 'forbidden_chars': '\'\"`', 'forbidden_regexp': r"/\./|/\.\./|/\.$"} name_check.filter_factory(conf)(FakeApp()) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('name_check' in swift_info) self.assertEqual(swift_info['name_check'].get('maximum_length'), 512) self.assertEqual(set(swift_info['name_check'].get('forbidden_chars')), diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 8af95edaa..7b137f9ba 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -24,7 +24,7 @@ from swift.common.middleware import ratelimit from swift.proxy.controllers.base import get_cache_key, \ headers_to_container_info from swift.common.swob import Request -from swift.common import utils +from swift.common import registry threading = eventlet.patcher.original('threading') @@ -546,8 +546,8 @@ class TestRateLimit(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): @@ -568,7 +568,7 @@ class TestSwiftInfo(unittest.TestCase): 'container_listing_ratelimit_50': 50} ratelimit.filter_factory(test_limits)('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('ratelimit', swift_info) self.assertEqual(swift_info['ratelimit'] ['account_ratelimit'], 1.0) diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index f4c45423f..045778a16 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -24,7 +24,7 @@ from mock import patch import six from io import BytesIO -from swift.common import swob, utils +from swift.common import swob, registry from swift.common.header_key_dict import HeaderKeyDict from swift.common.middleware import slo from swift.common.swob import Request, HTTPException, str_to_wsgi, \ @@ -4534,12 +4534,12 @@ class TestSloBulkDeleter(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): mware = slo.filter_factory({})('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), mware.max_manifest_segments) @@ -4564,7 +4564,7 @@ class TestSwiftInfo(unittest.TestCase): rate_limit_segments_per_sec=2, yield_frequency=5, concurrency=1, delete_concurrency=3, allow_async_delete='y') mware = slo.filter_factory(conf)('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), 500) self.assertEqual(swift_info['slo'].get('min_segment_size'), 1) diff --git a/test/unit/common/middleware/test_symlink.py b/test/unit/common/middleware/test_symlink.py index 3687305bf..9d8838284 100644 --- a/test/unit/common/middleware/test_symlink.py +++ b/test/unit/common/middleware/test_symlink.py @@ -25,7 +25,8 @@ from swift.common.middleware import symlink, copy, versioned_writes, \ listing_formats from swift.common.swob import Request from swift.common.request_helpers import get_reserved_name -from swift.common.utils import MD5_OF_EMPTY_STRING, get_swift_info +from swift.common.utils import MD5_OF_EMPTY_STRING +from swift.common.registry import get_swift_info from test.unit.common.middleware.helpers import FakeSwift from test.unit.common.middleware.test_versioned_writes import FakeCache diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 0ec82615c..00546d7c3 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -41,7 +41,7 @@ from time import time, strftime, gmtime from swift.common.middleware import tempauth, tempurl from swift.common.header_key_dict import HeaderKeyDict from swift.common.swob import Request, Response -from swift.common import utils +from swift.common import utils, registry class FakeApp(object): @@ -1571,12 +1571,12 @@ class TestTempURL(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): tempurl.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('tempurl', swift_info) info = swift_info['tempurl'] self.assertEqual(set(info['methods']), @@ -1599,7 +1599,7 @@ class TestSwiftInfo(unittest.TestCase): 'outgoing_allow_headers': 'x-object-meta-* content-type', 'allowed_digests': 'sha512 md5 not-a-valid-digest', }) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('tempurl', swift_info) info = swift_info['tempurl'] self.assertEqual(set(info['methods']), diff --git a/test/unit/common/middleware/test_versioned_writes.py b/test/unit/common/middleware/test_versioned_writes.py index 2f997940a..ff6ec59da 100644 --- a/test/unit/common/middleware/test_versioned_writes.py +++ b/test/unit/common/middleware/test_versioned_writes.py @@ -19,7 +19,7 @@ import os import time import mock import unittest -from swift.common import swob, utils +from swift.common import swob, utils, registry from swift.common.middleware import versioned_writes, copy from swift.common.swob import Request from test.unit.common.middleware import helpers @@ -1515,19 +1515,19 @@ class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): versioned_writes.filter_factory({})('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() # in default, versioned_writes is not in swift_info self.assertNotIn('versioned_writes', swift_info) def test_registered_explicitly_set(self): versioned_writes.filter_factory( {'allow_versioned_writes': 'true'})('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('versioned_writes', swift_info) self.assertEqual( swift_info['versioned_writes'].get('allowed_flags'), diff --git a/test/unit/common/test_registry.py b/test/unit/common/test_registry.py new file mode 100644 index 000000000..6fc17b3ba --- /dev/null +++ b/test/unit/common/test_registry.py @@ -0,0 +1,213 @@ +# Copyright (c) 2022 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.common import registry, utils + +import unittest + + +class TestSwiftInfo(unittest.TestCase): + + def tearDown(self): + registry._swift_info = {} + registry._swift_admin_info = {} + + def test_register_swift_info(self): + registry.register_swift_info(foo='bar') + registry.register_swift_info(lorem='ipsum') + registry.register_swift_info('cap1', cap1_foo='cap1_bar') + registry.register_swift_info('cap1', cap1_lorem='cap1_ipsum') + + self.assertTrue('swift' in registry._swift_info) + self.assertTrue('foo' in registry._swift_info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + self.assertTrue('lorem' in registry._swift_info['swift']) + self.assertEqual(registry._swift_info['swift']['lorem'], 'ipsum') + + self.assertTrue('cap1' in registry._swift_info) + self.assertTrue('cap1_foo' in registry._swift_info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_foo'], 'cap1_bar') + self.assertTrue('cap1_lorem' in registry._swift_info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_lorem'], + 'cap1_ipsum') + + self.assertRaises(ValueError, + registry.register_swift_info, 'admin', foo='bar') + + self.assertRaises(ValueError, + registry.register_swift_info, 'disallowed_sections', + disallowed_sections=None) + + registry.register_swift_info('goodkey', foo='5.6') + self.assertRaises(ValueError, + registry.register_swift_info, 'bad.key', foo='5.6') + data = {'bad.key': '5.6'} + self.assertRaises(ValueError, + registry.register_swift_info, 'goodkey', **data) + + def test_get_swift_info(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info() + + self.assertNotIn('admin', info) + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + + self.assertIn('cap1', info) + self.assertIn('cap1_foo', info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_foo'], 'cap1_bar') + + def test_get_swift_info_with_disallowed_sections(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}, + 'cap2': {'cap2_foo': 'cap2_bar'}, + 'cap3': {'cap3_foo': 'cap3_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info(disallowed_sections=['cap1', 'cap3']) + + self.assertNotIn('admin', info) + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(info['swift']['foo'], 'bar') + + self.assertNotIn('cap1', info) + + self.assertIn('cap2', info) + self.assertIn('cap2_foo', info['cap2']) + self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') + + self.assertNotIn('cap3', info) + + def test_register_swift_admin_info(self): + registry.register_swift_info(admin=True, admin_foo='admin_bar') + registry.register_swift_info(admin=True, admin_lorem='admin_ipsum') + registry.register_swift_info('cap1', admin=True, ac1_foo='ac1_bar') + registry.register_swift_info('cap1', admin=True, ac1_lorem='ac1_ipsum') + + self.assertIn('swift', registry._swift_admin_info) + self.assertIn('admin_foo', registry._swift_admin_info['swift']) + self.assertEqual( + registry._swift_admin_info['swift']['admin_foo'], 'admin_bar') + self.assertIn('admin_lorem', registry._swift_admin_info['swift']) + self.assertEqual( + registry._swift_admin_info['swift']['admin_lorem'], 'admin_ipsum') + + self.assertIn('cap1', registry._swift_admin_info) + self.assertIn('ac1_foo', registry._swift_admin_info['cap1']) + self.assertEqual( + registry._swift_admin_info['cap1']['ac1_foo'], 'ac1_bar') + self.assertIn('ac1_lorem', registry._swift_admin_info['cap1']) + self.assertEqual( + registry._swift_admin_info['cap1']['ac1_lorem'], 'ac1_ipsum') + + self.assertNotIn('swift', registry._swift_info) + self.assertNotIn('cap1', registry._swift_info) + + def test_get_swift_admin_info(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info(admin=True) + + self.assertIn('admin', info) + self.assertIn('admin_cap1', info['admin']) + self.assertIn('ac1_foo', info['admin']['admin_cap1']) + self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + + self.assertIn('cap1', info) + self.assertIn('cap1_foo', info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_foo'], 'cap1_bar') + + def test_get_swift_admin_info_with_disallowed_sections(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}, + 'cap2': {'cap2_foo': 'cap2_bar'}, + 'cap3': {'cap3_foo': 'cap3_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info( + admin=True, disallowed_sections=['cap1', 'cap3']) + + self.assertIn('admin', info) + self.assertIn('admin_cap1', info['admin']) + self.assertIn('ac1_foo', info['admin']['admin_cap1']) + self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') + self.assertIn('disallowed_sections', info['admin']) + self.assertIn('cap1', info['admin']['disallowed_sections']) + self.assertNotIn('cap2', info['admin']['disallowed_sections']) + self.assertIn('cap3', info['admin']['disallowed_sections']) + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(info['swift']['foo'], 'bar') + + self.assertNotIn('cap1', info) + + self.assertIn('cap2', info) + self.assertIn('cap2_foo', info['cap2']) + self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') + + self.assertNotIn('cap3', info) + + def test_get_swift_admin_info_with_disallowed_sub_sections(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar', + 'cap1_moo': 'cap1_baa'}, + 'cap2': {'cap2_foo': 'cap2_bar'}, + 'cap3': {'cap2_foo': 'cap2_bar'}, + 'cap4': {'a': {'b': {'c': 'c'}, + 'b.c': 'b.c'}}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info( + admin=True, disallowed_sections=['cap1.cap1_foo', 'cap3', + 'cap4.a.b.c']) + self.assertNotIn('cap3', info) + self.assertEqual(info['cap1']['cap1_moo'], 'cap1_baa') + self.assertNotIn('cap1_foo', info['cap1']) + self.assertNotIn('c', info['cap4']['a']['b']) + self.assertEqual(info['cap4']['a']['b.c'], 'b.c') + + def test_get_swift_info_with_unmatched_disallowed_sections(self): + cap1 = {'cap1_foo': 'cap1_bar', + 'cap1_moo': 'cap1_baa'} + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': cap1} + # expect no exceptions + info = registry.get_swift_info( + disallowed_sections=['cap2.cap1_foo', 'cap1.no_match', + 'cap1.cap1_foo.no_match.no_match']) + self.assertEqual(info['cap1'], cap1) + + def test_register_swift_info_import_from_utils(self): + # verify that the functions are available to import from utils + utils.register_swift_info(foo='bar') + self.assertTrue('swift' in registry._swift_info) + self.assertTrue('foo' in registry._swift_info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + self.assertEqual(registry.get_swift_info(admin=True), + utils.get_swift_info(admin=True)) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index a5272a8b8..ba2ff6df2 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -5152,191 +5152,6 @@ class TestUnlinkOlder(unittest.TestCase): utils.unlink_paths_older_than([path], next(self.ts)) -class TestSwiftInfo(unittest.TestCase): - - def tearDown(self): - utils._swift_info = {} - utils._swift_admin_info = {} - - def test_register_swift_info(self): - utils.register_swift_info(foo='bar') - utils.register_swift_info(lorem='ipsum') - utils.register_swift_info('cap1', cap1_foo='cap1_bar') - utils.register_swift_info('cap1', cap1_lorem='cap1_ipsum') - - self.assertTrue('swift' in utils._swift_info) - self.assertTrue('foo' in utils._swift_info['swift']) - self.assertEqual(utils._swift_info['swift']['foo'], 'bar') - self.assertTrue('lorem' in utils._swift_info['swift']) - self.assertEqual(utils._swift_info['swift']['lorem'], 'ipsum') - - self.assertTrue('cap1' in utils._swift_info) - self.assertTrue('cap1_foo' in utils._swift_info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') - self.assertTrue('cap1_lorem' in utils._swift_info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_lorem'], 'cap1_ipsum') - - self.assertRaises(ValueError, - utils.register_swift_info, 'admin', foo='bar') - - self.assertRaises(ValueError, - utils.register_swift_info, 'disallowed_sections', - disallowed_sections=None) - - utils.register_swift_info('goodkey', foo='5.6') - self.assertRaises(ValueError, - utils.register_swift_info, 'bad.key', foo='5.6') - data = {'bad.key': '5.6'} - self.assertRaises(ValueError, - utils.register_swift_info, 'goodkey', **data) - - def test_get_swift_info(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info() - - self.assertNotIn('admin', info) - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(utils._swift_info['swift']['foo'], 'bar') - - self.assertIn('cap1', info) - self.assertIn('cap1_foo', info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') - - def test_get_swift_info_with_disallowed_sections(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}, - 'cap2': {'cap2_foo': 'cap2_bar'}, - 'cap3': {'cap3_foo': 'cap3_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info(disallowed_sections=['cap1', 'cap3']) - - self.assertNotIn('admin', info) - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(info['swift']['foo'], 'bar') - - self.assertNotIn('cap1', info) - - self.assertIn('cap2', info) - self.assertIn('cap2_foo', info['cap2']) - self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') - - self.assertNotIn('cap3', info) - - def test_register_swift_admin_info(self): - utils.register_swift_info(admin=True, admin_foo='admin_bar') - utils.register_swift_info(admin=True, admin_lorem='admin_ipsum') - utils.register_swift_info('cap1', admin=True, ac1_foo='ac1_bar') - utils.register_swift_info('cap1', admin=True, ac1_lorem='ac1_ipsum') - - self.assertIn('swift', utils._swift_admin_info) - self.assertIn('admin_foo', utils._swift_admin_info['swift']) - self.assertEqual( - utils._swift_admin_info['swift']['admin_foo'], 'admin_bar') - self.assertIn('admin_lorem', utils._swift_admin_info['swift']) - self.assertEqual( - utils._swift_admin_info['swift']['admin_lorem'], 'admin_ipsum') - - self.assertIn('cap1', utils._swift_admin_info) - self.assertIn('ac1_foo', utils._swift_admin_info['cap1']) - self.assertEqual( - utils._swift_admin_info['cap1']['ac1_foo'], 'ac1_bar') - self.assertIn('ac1_lorem', utils._swift_admin_info['cap1']) - self.assertEqual( - utils._swift_admin_info['cap1']['ac1_lorem'], 'ac1_ipsum') - - self.assertNotIn('swift', utils._swift_info) - self.assertNotIn('cap1', utils._swift_info) - - def test_get_swift_admin_info(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info(admin=True) - - self.assertIn('admin', info) - self.assertIn('admin_cap1', info['admin']) - self.assertIn('ac1_foo', info['admin']['admin_cap1']) - self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(utils._swift_info['swift']['foo'], 'bar') - - self.assertIn('cap1', info) - self.assertIn('cap1_foo', info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') - - def test_get_swift_admin_info_with_disallowed_sections(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}, - 'cap2': {'cap2_foo': 'cap2_bar'}, - 'cap3': {'cap3_foo': 'cap3_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info( - admin=True, disallowed_sections=['cap1', 'cap3']) - - self.assertIn('admin', info) - self.assertIn('admin_cap1', info['admin']) - self.assertIn('ac1_foo', info['admin']['admin_cap1']) - self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') - self.assertIn('disallowed_sections', info['admin']) - self.assertIn('cap1', info['admin']['disallowed_sections']) - self.assertNotIn('cap2', info['admin']['disallowed_sections']) - self.assertIn('cap3', info['admin']['disallowed_sections']) - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(info['swift']['foo'], 'bar') - - self.assertNotIn('cap1', info) - - self.assertIn('cap2', info) - self.assertIn('cap2_foo', info['cap2']) - self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') - - self.assertNotIn('cap3', info) - - def test_get_swift_admin_info_with_disallowed_sub_sections(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar', - 'cap1_moo': 'cap1_baa'}, - 'cap2': {'cap2_foo': 'cap2_bar'}, - 'cap3': {'cap2_foo': 'cap2_bar'}, - 'cap4': {'a': {'b': {'c': 'c'}, - 'b.c': 'b.c'}}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info( - admin=True, disallowed_sections=['cap1.cap1_foo', 'cap3', - 'cap4.a.b.c']) - self.assertNotIn('cap3', info) - self.assertEqual(info['cap1']['cap1_moo'], 'cap1_baa') - self.assertNotIn('cap1_foo', info['cap1']) - self.assertNotIn('c', info['cap4']['a']['b']) - self.assertEqual(info['cap4']['a']['b.c'], 'b.c') - - def test_get_swift_info_with_unmatched_disallowed_sections(self): - cap1 = {'cap1_foo': 'cap1_bar', - 'cap1_moo': 'cap1_baa'} - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': cap1} - # expect no exceptions - info = utils.get_swift_info( - disallowed_sections=['cap2.cap1_foo', 'cap1.no_match', - 'cap1.cap1_foo.no_match.no_match']) - self.assertEqual(info['cap1'], cap1) - - class TestFileLikeIter(unittest.TestCase): def test_iter_file_iter(self): diff --git a/test/unit/proxy/controllers/test_info.py b/test/unit/proxy/controllers/test_info.py index 2317acfbe..7109ffa8a 100644 --- a/test/unit/proxy/controllers/test_info.py +++ b/test/unit/proxy/controllers/test_info.py @@ -20,15 +20,15 @@ from mock import Mock from swift.proxy.controllers import InfoController from swift.proxy.server import Application as ProxyApp -from swift.common import utils +from swift.common import utils, registry from swift.common.swob import Request, HTTPException class TestInfoController(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def get_controller(self, expose_info=None, disallowed_sections=None, admin_key=None): @@ -54,8 +54,8 @@ class TestInfoController(unittest.TestCase): def test_get_info(self): controller = self.get_controller(expose_info=True) - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) @@ -80,8 +80,8 @@ class TestInfoController(unittest.TestCase): def test_get_info_cors(self): controller = self.get_controller(expose_info=True) - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}, @@ -99,8 +99,8 @@ class TestInfoController(unittest.TestCase): def test_head_info(self): controller = self.get_controller(expose_info=True) - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'HEAD'}) @@ -111,9 +111,9 @@ class TestInfoController(unittest.TestCase): def test_disallow_info(self): controller = self.get_controller(expose_info=True, disallowed_sections=['foo2']) - utils._swift_info = {'foo': {'bar': 'baz'}, - 'foo2': {'bar2': 'baz2'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}, + 'foo2': {'bar2': 'baz2'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) @@ -128,8 +128,8 @@ class TestInfoController(unittest.TestCase): def test_disabled_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, '') @@ -144,8 +144,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') @@ -165,8 +165,8 @@ class TestInfoController(unittest.TestCase): def test_head_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') @@ -191,8 +191,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_method(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('HEAD', '/info', expires, 'secret-admin-key') @@ -207,8 +207,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_expires(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = 1 sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') @@ -233,8 +233,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_path(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/foo', expires, 'secret-admin-key') @@ -249,8 +249,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_key(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/foo', expires, 'invalid-admin-key') @@ -266,9 +266,9 @@ class TestInfoController(unittest.TestCase): controller = self.get_controller(expose_info=True, disallowed_sections=['foo2'], admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}, - 'foo2': {'bar2': 'baz2'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}, + 'foo2': {'bar2': 'baz2'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 1550ba55b..7486e9fae 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -67,7 +67,7 @@ from swift.common.middleware import proxy_logging, versioned_writes, \ from swift.common.middleware.acl import parse_acl, format_acl from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist, \ APIVersionError, ChunkReadError -from swift.common import utils, constraints +from swift.common import utils, constraints, registry from swift.common.utils import hash_path, storage_directory, \ parse_content_type, parse_mime_headers, \ iter_multipart_mime_documents, public, mkdirs, NullLogger, md5 @@ -11274,8 +11274,8 @@ class TestProxyObjectPerformance(unittest.TestCase): StoragePolicy(3, 'bert', object_ring=FakeRing())]) class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): app = proxy_server.Application({}, @@ -11312,7 +11312,7 @@ class TestSwiftInfo(unittest.TestCase): # other items are added to swift info self.assertEqual(len(si), 17) - si = utils.get_swift_info()['swift'] + si = registry.get_swift_info()['swift'] # Tehse settings is by default excluded by disallowed_sections self.assertEqual(si['valid_api_versions'], constraints.VALID_API_VERSIONS) |