diff options
| -rw-r--r-- | doc/manpages/proxy-server.conf.5 | 29 | ||||
| -rw-r--r-- | etc/proxy-server.conf-sample | 11 | ||||
| -rw-r--r-- | setup.py | 1 | ||||
| -rw-r--r-- | swift/common/middleware/cname_lookup.py | 152 | ||||
| -rw-r--r-- | swift/common/middleware/domain_remap.py | 130 | ||||
| -rw-r--r-- | test/unit/common/middleware/test_cname_lookup.py | 166 | ||||
| -rw-r--r-- | test/unit/common/middleware/test_domain_remap.py | 126 |
7 files changed, 615 insertions, 0 deletions
diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index 324e7af53..af2f648ea 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -255,6 +255,35 @@ to r. Will limit PUT, DELETE, and POST requests to /a/c/o. The default is ''. .RE +.RS 0 +.IP "\fB[filter:domain_remap]\fR" +.RE + +Middleware that translates container and account parts of a domain to path parameters that the proxy server understands. The container.account.storageurl/object gets translated to container.account.storageurl/path_root/account/container/object and account.storageurl/path_root/container/object gets translated to account.storageurl/path_root/account/container/object + +.RS 3 +.IP \fBuse\fR +Entry point for paste.deploy for the domain_remap middleware. This is the reference to the installed python egg. +The default is \fBegg:swift#domain_remap\fR. +.IP "\fBset log_name\fR" +Label used when logging. The default is domain_remap. +.IP "\fBset log_headers\fR" +Enables the ability to log request headers. The default is False. +.IP \fBstorage_domain\fR +The domain to be used by the middleware. +.IP \fBpath_root\fR +The path root value for the storage URL. The default is v1. +.IP \fBreseller_prefixes\fR +Browsers can convert a host header to lowercase, so check that reseller +prefix on the account is the correct case. This is done by comparing the +items in the reseller_prefixes config option to the found prefix. If they +match except for case, the item from reseller_prefixes will be used +instead of the found reseller prefix. The reseller_prefixes list is exclusive. +If defined, any request with an account prefix not in that list will be ignored +by this middleware. Defaults to 'AUTH'. +.RE + + .RS 0 .IP "\fB[filter:catch_errors]\fR" diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index cc3ae4c7b..83fc5a44d 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -168,6 +168,17 @@ use = egg:swift#ratelimit # container_ratelimit_10 = 50 # container_ratelimit_50 = 20 +[filter:domain_remap] +use = egg:swift#domain_remap +# You can override the default log routing for this filter here: +# set log_name = domain_remap +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = False +# storage_domain = example.com +# path_root = v1 +# reseller_prefixes = AUTH + [filter:catch_errors] use = egg:swift#catch_errors # You can override the default log routing for this filter here: @@ -86,6 +86,7 @@ setup( 'ratelimit=swift.common.middleware.ratelimit:filter_factory', 'cname_lookup=swift.common.middleware.cname_lookup:filter_factory', 'catch_errors=swift.common.middleware.catch_errors:filter_factory', + 'domain_remap=swift.common.middleware.domain_remap:filter_factory', 'swift3=swift.common.middleware.swift3:filter_factory', 'staticweb=swift.common.middleware.staticweb:filter_factory', 'tempauth=swift.common.middleware.tempauth:filter_factory', diff --git a/swift/common/middleware/cname_lookup.py b/swift/common/middleware/cname_lookup.py new file mode 100644 index 000000000..ae55f6056 --- /dev/null +++ b/swift/common/middleware/cname_lookup.py @@ -0,0 +1,152 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +CNAME Lookup Middleware + +Middleware that translates an unknown domain in the host header to +something that ends with the configured storage_domain by looking up +the given domain's CNAME record in DNS. + +This middleware will continue to follow a CNAME chain in DNS until it finds +a record ending in the configured storage domain or it reaches the configured +maximum lookup depth. If a match is found, the environment's Host header is +rewritten and the request is passed further down the WSGI chain. +""" + +from webob import Request +from webob.exc import HTTPBadRequest +try: + import dns.resolver + from dns.exception import DNSException + from dns.resolver import NXDOMAIN, NoAnswer +except ImportError: + # catch this to allow docs to be built without the dependency + MODULE_DEPENDENCY_MET = False +else: # executed if the try block finishes with no errors + MODULE_DEPENDENCY_MET = True + +from swift.common.utils import cache_from_env, get_logger + + +def lookup_cname(domain): # pragma: no cover + """ + Given a domain, returns its DNS CNAME mapping and DNS ttl. + + :param domain: domain to query on + :returns: (ttl, result) + """ + try: + answer = dns.resolver.query(domain, 'CNAME').rrset + ttl = answer.ttl + result = answer.items[0].to_text() + result = result.rstrip('.') + return ttl, result + except (DNSException, NXDOMAIN, NoAnswer): + return 0, None + + +class CNAMELookupMiddleware(object): + """ + CNAME Lookup Middleware + + See above for a full description. + + :param app: The next WSGI filter or app in the paste.deploy + chain. + :param conf: The configuration dict for the middleware. + """ + + def __init__(self, app, conf): + if not MODULE_DEPENDENCY_MET: + # reraise the exception if the dependency wasn't met + raise ImportError('dnspython is required for this module') + self.app = app + self.storage_domain = conf.get('storage_domain', 'example.com') + if self.storage_domain and self.storage_domain[0] != '.': + self.storage_domain = '.' + self.storage_domain + self.lookup_depth = int(conf.get('lookup_depth', '1')) + self.memcache = None + self.logger = get_logger(conf, log_route='cname-lookup') + + def __call__(self, env, start_response): + if not self.storage_domain: + return self.app(env, start_response) + given_domain = env['HTTP_HOST'] + port = '' + if ':' in given_domain: + given_domain, port = given_domain.rsplit(':', 1) + if given_domain == self.storage_domain[1:]: # strip initial '.' + return self.app(env, start_response) + a_domain = given_domain + if not a_domain.endswith(self.storage_domain): + if self.memcache is None: + self.memcache = cache_from_env(env) + error = True + for tries in xrange(self.lookup_depth): + found_domain = None + if self.memcache: + memcache_key = ''.join(['cname-', a_domain]) + found_domain = self.memcache.get(memcache_key) + if not found_domain: + ttl, found_domain = lookup_cname(a_domain) + if self.memcache: + memcache_key = ''.join(['cname-', given_domain]) + self.memcache.set(memcache_key, found_domain, + timeout=ttl) + if found_domain is None or found_domain == a_domain: + # no CNAME records or we're at the last lookup + error = True + found_domain = None + break + elif found_domain.endswith(self.storage_domain): + # Found it! + self.logger.info( + _('Mapped %(given_domain)s to %(found_domain)s') % + {'given_domain': given_domain, + 'found_domain': found_domain}) + if port: + env['HTTP_HOST'] = ':'.join([found_domain, port]) + else: + env['HTTP_HOST'] = found_domain + error = False + break + else: + # try one more deep in the chain + self.logger.debug(_('Following CNAME chain for ' \ + '%(given_domain)s to %(found_domain)s') % + {'given_domain': given_domain, + 'found_domain': found_domain}) + a_domain = found_domain + if error: + if found_domain: + msg = 'CNAME lookup failed after %d tries' % \ + self.lookup_depth + else: + msg = 'CNAME lookup failed to resolve to a valid domain' + resp = HTTPBadRequest(request=Request(env), body=msg, + content_type='text/plain') + return resp(env, start_response) + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): # pragma: no cover + conf = global_conf.copy() + conf.update(local_conf) + + def cname_filter(app): + return CNAMELookupMiddleware(app, conf) + return cname_filter diff --git a/swift/common/middleware/domain_remap.py b/swift/common/middleware/domain_remap.py new file mode 100644 index 000000000..a1cc3016b --- /dev/null +++ b/swift/common/middleware/domain_remap.py @@ -0,0 +1,130 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Domain Remap Middleware + +Middleware that translates container and account parts of a domain to +path parameters that the proxy server understands. + +container.account.storageurl/object gets translated to +container.account.storageurl/path_root/account/container/object + +account.storageurl/path_root/container/object gets translated to +account.storageurl/path_root/account/container/object + +Browsers can convert a host header to lowercase, so check that reseller +prefix on the account is the correct case. This is done by comparing the +items in the reseller_prefixes config option to the found prefix. If they +match except for case, the item from reseller_prefixes will be used +instead of the found reseller prefix. The reseller_prefixes list is +exclusive. If defined, any request with an account prefix not in that list +will be ignored by this middleware. reseller_prefixes defaults to 'AUTH'. + +Note that this middleware requires that container names and account names +(except as described above) must be DNS-compatible. This means that the +account name created in the system and the containers created by users +cannot exceed 63 characters or have UTF-8 characters. These are +restrictions over and above what swift requires and are not explicitly +checked. Simply put, the this middleware will do a best-effort attempt to +derive account and container names from elements in the domain name and +put those derived values into the URL path (leaving the Host header +unchanged). + +Also note that using container sync with remapped domain names is not +advised. With container sync, you should use the true storage end points as +sync destinations. +""" + +from webob import Request +from webob.exc import HTTPBadRequest + + +class DomainRemapMiddleware(object): + """ + Domain Remap Middleware + + See above for a full description. + + :param app: The next WSGI filter or app in the paste.deploy + chain. + :param conf: The configuration dict for the middleware. + """ + + def __init__(self, app, conf): + self.app = app + self.storage_domain = conf.get('storage_domain', 'example.com') + if self.storage_domain and self.storage_domain[0] != '.': + self.storage_domain = '.' + self.storage_domain + self.path_root = conf.get('path_root', 'v1').strip('/') + prefixes = conf.get('reseller_prefixes', 'AUTH') + self.reseller_prefixes = [x.strip() for x in prefixes.split(',') + if x.strip()] + self.reseller_prefixes_lower = [x.lower() + for x in self.reseller_prefixes] + + def __call__(self, env, start_response): + if not self.storage_domain: + return self.app(env, start_response) + given_domain = env['HTTP_HOST'] + port = '' + if ':' in given_domain: + given_domain, port = given_domain.rsplit(':', 1) + if given_domain.endswith(self.storage_domain): + parts_to_parse = given_domain[:-len(self.storage_domain)] + parts_to_parse = parts_to_parse.strip('.').split('.') + len_parts_to_parse = len(parts_to_parse) + if len_parts_to_parse == 2: + container, account = parts_to_parse + elif len_parts_to_parse == 1: + container, account = None, parts_to_parse[0] + else: + resp = HTTPBadRequest(request=Request(env), + body='Bad domain in host header', + content_type='text/plain') + return resp(env, start_response) + if '_' not in account and '-' in account: + account = account.replace('-', '_', 1) + account_reseller_prefix = account.split('_', 1)[0].lower() + if account_reseller_prefix not in self.reseller_prefixes_lower: + # account prefix is not in config list. bail. + return self.app(env, start_response) + prefix_index = self.reseller_prefixes_lower.index( + account_reseller_prefix) + real_prefix = self.reseller_prefixes[prefix_index] + if not account.startswith(real_prefix): + account_suffix = account[len(real_prefix):] + account = real_prefix + account_suffix + path = env['PATH_INFO'].strip('/') + new_path_parts = ['', self.path_root, account] + if container: + new_path_parts.append(container) + if path.startswith(self.path_root): + path = path[len(self.path_root):].lstrip('/') + if path: + new_path_parts.append(path) + new_path = '/'.join(new_path_parts) + env['PATH_INFO'] = new_path + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def domain_filter(app): + return DomainRemapMiddleware(app, conf) + return domain_filter diff --git a/test/unit/common/middleware/test_cname_lookup.py b/test/unit/common/middleware/test_cname_lookup.py new file mode 100644 index 000000000..12ab7ad1f --- /dev/null +++ b/test/unit/common/middleware/test_cname_lookup.py @@ -0,0 +1,166 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from nose import SkipTest + +from webob import Request + +try: + # this test requires the dnspython package to be installed + import dns.resolver +except ImportError: + skip = True +else: # executed if the try has no errors + skip = False +from swift.common.middleware import cname_lookup + +class FakeApp(object): + + def __call__(self, env, start_response): + return "FAKE APP" + + +def start_response(*args): + pass + + +class TestCNAMELookup(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.app = cname_lookup.CNAMELookupMiddleware(FakeApp(), + {'lookup_depth': 2}) + + def test_passthrough(self): + + def my_lookup(d): + return 0, d + cname_lookup.lookup_cname = my_lookup + + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'foo.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'foo.example.com:8080'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') + + def test_good_lookup(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'mysite.com'}) + + def my_lookup(d): + return 0, '%s.example.com' % d + cname_lookup.lookup_cname = my_lookup + + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'mysite.com:8080'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') + + def test_lookup_chain_too_long(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'mysite.com'}) + + def my_lookup(d): + if d == 'mysite.com': + site = 'level1.foo.com' + elif d == 'level1.foo.com': + site = 'level2.foo.com' + elif d == 'level2.foo.com': + site = 'bar.example.com' + return 0, site + cname_lookup.lookup_cname = my_lookup + + resp = self.app(req.environ, start_response) + self.assertEquals(resp, ['CNAME lookup failed after 2 tries']) + + def test_lookup_chain_bad_target(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'mysite.com'}) + + def my_lookup(d): + return 0, 'some.invalid.site.com' + cname_lookup.lookup_cname = my_lookup + + resp = self.app(req.environ, start_response) + self.assertEquals(resp, + ['CNAME lookup failed to resolve to a valid domain']) + + def test_something_weird(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'mysite.com'}) + + def my_lookup(d): + return 0, None + cname_lookup.lookup_cname = my_lookup + + resp = self.app(req.environ, start_response) + self.assertEquals(resp, + ['CNAME lookup failed to resolve to a valid domain']) + + def test_with_memcache(self): + def my_lookup(d): + return 0, '%s.example.com' % d + cname_lookup.lookup_cname = my_lookup + class memcache_stub(object): + def __init__(self): + self.cache = {} + def get(self, key): + return self.cache.get(key, None) + def set(self, key, value, *a, **kw): + self.cache[key] = value + memcache = memcache_stub() + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', + 'swift.cache': memcache}, + headers={'Host': 'mysite.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', + 'swift.cache': memcache}, + headers={'Host': 'mysite.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') + + def test_cname_matching_ending_not_domain(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'foo.com'}) + + def my_lookup(d): + return 0, 'c.aexample.com' + cname_lookup.lookup_cname = my_lookup + + resp = self.app(req.environ, start_response) + self.assertEquals(resp, + ['CNAME lookup failed to resolve to a valid domain']) + + def test_cname_configured_with_empty_storage_domain(self): + app = cname_lookup.CNAMELookupMiddleware(FakeApp(), + {'storage_domain': '', + 'lookup_depth': 2}) + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.a.example.com'}) + + def my_lookup(d): + return 0, None + cname_lookup.lookup_cname = my_lookup + + resp = app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') diff --git a/test/unit/common/middleware/test_domain_remap.py b/test/unit/common/middleware/test_domain_remap.py new file mode 100644 index 000000000..7139ad776 --- /dev/null +++ b/test/unit/common/middleware/test_domain_remap.py @@ -0,0 +1,126 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from webob import Request + +from swift.common.middleware import domain_remap + + +class FakeApp(object): + + def __call__(self, env, start_response): + return env['PATH_INFO'] + + +def start_response(*args): + pass + + +class TestDomainRemap(unittest.TestCase): + + def setUp(self): + self.app = domain_remap.DomainRemapMiddleware(FakeApp(), {}) + + def test_domain_remap_passthrough(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/') + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'example.com:8080'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/') + + def test_domain_remap_account(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/AUTH_a') + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'AUTH-uuid.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/AUTH_uuid') + + def test_domain_remap_account_container(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/AUTH_a/c') + + def test_domain_remap_extra_subdomains(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'x.y.c.AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, ['Bad domain in host header']) + + def test_domain_remap_account_with_path_root(self): + req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/AUTH_a') + + def test_domain_remap_account_container_with_path_root(self): + req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/AUTH_a/c') + + def test_domain_remap_account_container_with_path(self): + req = Request.blank('/obj', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/AUTH_a/c/obj') + + def test_domain_remap_account_container_with_path_root_and_path(self): + req = Request.blank('/v1/obj', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/AUTH_a/c/obj') + + def test_domain_remap_account_matching_ending_not_domain(self): + req = Request.blank('/dontchange', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.aexample.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/dontchange') + + def test_domain_remap_configured_with_empty_storage_domain(self): + self.app = domain_remap.DomainRemapMiddleware(FakeApp(), + {'storage_domain': ''}) + req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.AUTH_a.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/test') + + def test_domain_remap_configured_with_prefixes(self): + conf = {'reseller_prefixes': 'PREFIX'} + self.app = domain_remap.DomainRemapMiddleware(FakeApp(), conf) + req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.prefix_uuid.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/v1/PREFIX_uuid/c/test') + + def test_domain_remap_configured_with_bad_prefixes(self): + conf = {'reseller_prefixes': 'UNKNOWN'} + self.app = domain_remap.DomainRemapMiddleware(FakeApp(), conf) + req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'}, + headers={'Host': 'c.prefix_uuid.example.com'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, '/test') + + +if __name__ == '__main__': + unittest.main() |