diff options
32 files changed, 829 insertions, 265 deletions
diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index 305a6293a..385b27af4 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -394,7 +394,7 @@ Logging level. The default is INFO. .IP "\fB[filter:tempurl]\fR" .RE -Note: Put tempurl just before your auth filter(s) in the pipeline +Note: Put tempurl before slo, dlo, and your auth filter(s) in the pipeline .RS 3 .IP \fBincoming_remove_headers\fR diff --git a/doc/saio/swift/proxy-server.conf b/doc/saio/swift/proxy-server.conf index 0b409c3d4..4ed132197 100644 --- a/doc/saio/swift/proxy-server.conf +++ b/doc/saio/swift/proxy-server.conf @@ -8,7 +8,7 @@ eventlet_debug = true [pipeline:main] # Yes, proxy-logging appears twice. This is so that # middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk slo dlo ratelimit crossdomain tempurl tempauth staticweb container-quotas account-quotas proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl slo dlo ratelimit crossdomain tempauth staticweb container-quotas account-quotas proxy-logging proxy-server [filter:catch_errors] use = egg:swift#catch_errors diff --git a/doc/source/cors.rst b/doc/source/cors.rst index 15bdf10ce..c17ccd7f8 100644 --- a/doc/source/cors.rst +++ b/doc/source/cors.rst @@ -134,7 +134,10 @@ Test CORS Page } request.open(method, url); - request.setRequestHeader('X-Auth-Token', token); + if (token != '') { + // custom headers always trigger a pre-flight request + request.setRequestHeader('X-Auth-Token', token); + } request.send(null); } </script> diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index afa24e4cf..6444f98bf 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -70,7 +70,7 @@ # eventlet_debug = false [pipeline:main] -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk slo dlo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl slo dlo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -406,7 +406,7 @@ use = egg:swift#cname_lookup [filter:staticweb] use = egg:swift#staticweb -# Note: Put tempurl just before your auth filter(s) in the pipeline +# Note: Put tempurl before dlo, slo and your auth filter(s) in the pipeline [filter:tempurl] use = egg:swift#tempurl # The methods allowed with Temp URLs. diff --git a/swift/common/constraints.py b/swift/common/constraints.py index dd5c5c410..7a480eaec 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -18,46 +18,69 @@ import urllib from urllib import unquote from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.utils import ismount, split_path +from swift.common.utils import ismount, split_path, SWIFT_CONF_FILE from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \ HTTPRequestEntityTooLarge, HTTPPreconditionFailed -constraints_conf = ConfigParser() -constraints_conf.read('/etc/swift/swift.conf') +MAX_FILE_SIZE = 5368709122 +MAX_META_NAME_LENGTH = 128 +MAX_META_VALUE_LENGTH = 256 +MAX_META_COUNT = 90 +MAX_META_OVERALL_SIZE = 4096 +MAX_HEADER_SIZE = 8192 +MAX_OBJECT_NAME_LENGTH = 1024 +CONTAINER_LISTING_LIMIT = 10000 +ACCOUNT_LISTING_LIMIT = 10000 +MAX_ACCOUNT_NAME_LENGTH = 256 +MAX_CONTAINER_NAME_LENGTH = 256 + +DEFAULT_CONSTRAINTS = { + 'max_file_size': MAX_FILE_SIZE, + 'max_meta_name_length': MAX_META_NAME_LENGTH, + 'max_meta_value_length': MAX_META_VALUE_LENGTH, + 'max_meta_count': MAX_META_COUNT, + 'max_meta_overall_size': MAX_META_OVERALL_SIZE, + 'max_header_size': MAX_HEADER_SIZE, + 'max_object_name_length': MAX_OBJECT_NAME_LENGTH, + 'container_listing_limit': CONTAINER_LISTING_LIMIT, + 'account_listing_limit': ACCOUNT_LISTING_LIMIT, + 'max_account_name_length': MAX_ACCOUNT_NAME_LENGTH, + 'max_container_name_length': MAX_CONTAINER_NAME_LENGTH, +} + +SWIFT_CONSTRAINTS_LOADED = False +OVERRIDE_CONSTRAINTS = {} # any constraints overridden by SWIFT_CONF_FILE +EFFECTIVE_CONSTRAINTS = {} # populated by reload_constraints + + +def reload_constraints(): + """ + Parse SWIFT_CONF_FILE and reset module level global contraint attrs, + populating OVERRIDE_CONSTRAINTS AND EFFECTIVE_CONSTRAINTS along the way. + """ + global SWIFT_CONSTRAINTS_LOADED, OVERRIDE_CONSTRAINTS + SWIFT_CONSTRAINTS_LOADED = False + OVERRIDE_CONSTRAINTS = {} + constraints_conf = ConfigParser() + if constraints_conf.read(SWIFT_CONF_FILE): + SWIFT_CONSTRAINTS_LOADED = True + for name in DEFAULT_CONSTRAINTS: + try: + value = int(constraints_conf.get('swift-constraints', name)) + except (NoSectionError, NoOptionError): + pass + else: + OVERRIDE_CONSTRAINTS[name] = value + for name, default in DEFAULT_CONSTRAINTS.items(): + value = OVERRIDE_CONSTRAINTS.get(name, default) + EFFECTIVE_CONSTRAINTS[name] = value + # "globals" in this context is module level globals, always. + globals()[name.upper()] = value + + +reload_constraints() -def constraints_conf_int(name, default): - try: - return int(constraints_conf.get('swift-constraints', name)) - except (NoSectionError, NoOptionError): - return default - - -#: Max file size allowed for objects -MAX_FILE_SIZE = constraints_conf_int('max_file_size', - 5368709122) # 5 * 1024 * 1024 * 1024 + 2 -#: Max length of the name of a key for metadata -MAX_META_NAME_LENGTH = constraints_conf_int('max_meta_name_length', 128) -#: Max length of the value of a key for metadata -MAX_META_VALUE_LENGTH = constraints_conf_int('max_meta_value_length', 256) -#: Max number of metadata items -MAX_META_COUNT = constraints_conf_int('max_meta_count', 90) -#: Max overall size of metadata -MAX_META_OVERALL_SIZE = constraints_conf_int('max_meta_overall_size', 4096) -#: Max size of any header -MAX_HEADER_SIZE = constraints_conf_int('max_header_size', 8192) -#: Max object name length -MAX_OBJECT_NAME_LENGTH = constraints_conf_int('max_object_name_length', 1024) -#: Max object list length of a get request for a container -CONTAINER_LISTING_LIMIT = constraints_conf_int('container_listing_limit', - 10000) -#: Max container list length of a get request for an account -ACCOUNT_LISTING_LIMIT = constraints_conf_int('account_listing_limit', 10000) -#: Max account name length -MAX_ACCOUNT_NAME_LENGTH = constraints_conf_int('max_account_name_length', 256) -#: Max container name length -MAX_CONTAINER_NAME_LENGTH = constraints_conf_int('max_container_name_length', - 256) # Maximum slo segments in buffer MAX_BUFFERED_SLO_SEGMENTS = 10000 diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index d23e76026..b0fda5f2b 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -178,7 +178,9 @@ class InternalClient(object): return resp except (Exception, Timeout): exc_type, exc_value, exc_traceback = exc_info() - sleep(2 ** (attempt + 1)) + # sleep only between tries, not after each one + if attempt < self.request_tries - 1: + sleep(2 ** (attempt + 1)) if resp: raise UnexpectedResponse( _('Unexpected response: %s') % resp.status, resp) diff --git a/swift/common/manager.py b/swift/common/manager.py index 47b275e6e..28e928337 100644 --- a/swift/common/manager.py +++ b/swift/common/manager.py @@ -163,6 +163,9 @@ class Manager(object): for name in server_names: self.servers.add(Server(name, run_dir)) + def __iter__(self): + return iter(self.servers) + @command def status(self, **kwargs): """display status of tracked pids for server @@ -251,6 +254,17 @@ class Manager(object): return 1 @command + def kill(self, **kwargs): + """stop a server (no error if not running) + """ + status = self.stop(**kwargs) + kwargs['quiet'] = True + if status and not self.status(**kwargs): + # only exit error if the server is still running + return status + return 0 + + @command def shutdown(self, **kwargs): """allow current requests to finish on supporting servers """ @@ -523,7 +537,7 @@ class Server(object): :param conf_file: path to conf_file to use as first arg :param once: boolean, add once argument to command :param wait: boolean, if true capture stdout with a pipe - :param daemon: boolean, if true ask server to log to console + :param daemon: boolean, if false ask server to log to console :returns : the pid of the spawned process """ @@ -560,6 +574,11 @@ class Server(object): for proc in self.procs: # wait for process to close its stdout output = proc.stdout.read() + if kwargs.get('once', False): + # if you don't want once to wait you can send it to the + # background on the command line, I generally just run with + # no-daemon anyway, but this is quieter + proc.wait() if output: print output start = time.time() diff --git a/swift/common/middleware/dlo.py b/swift/common/middleware/dlo.py index a08b81816..69ab7be50 100644 --- a/swift/common/middleware/dlo.py +++ b/swift/common/middleware/dlo.py @@ -24,7 +24,7 @@ from swift.common.swob import Request, Response, \ from swift.common.utils import get_logger, json, \ RateLimitedIterator, read_conf_dir, quote from swift.common.request_helpers import SegmentedIterable -from swift.common.wsgi import WSGIContext, make_request +from swift.common.wsgi import WSGIContext, make_subrequest from urllib import unquote @@ -36,7 +36,7 @@ class GetContext(WSGIContext): def _get_container_listing(self, req, version, account, container, prefix, marker=''): - con_req = make_request( + con_req = make_subrequest( req.environ, path='/'.join(['', version, account, container]), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index cf23a25fe..1dcdf07d0 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -151,7 +151,7 @@ from swift.common.request_helpers import SegmentedIterable, \ closing_if_possible, close_if_possible from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success -from swift.common.wsgi import WSGIContext, make_request +from swift.common.wsgi import WSGIContext, make_subrequest from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk @@ -216,7 +216,7 @@ class SloGetContext(WSGIContext): Fetch the submanifest, parse it, and return it. Raise exception on failures. """ - sub_req = make_request( + sub_req = make_subrequest( req.environ, path='/'.join(['', version, acc, con, obj]), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, @@ -385,7 +385,7 @@ class SloGetContext(WSGIContext): close_if_possible(resp_iter) del req.environ['swift.non_client_disconnect'] - get_req = make_request( + get_req = make_subrequest( req.environ, method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO') diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index a75c3452a..a6f0c985e 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -29,7 +29,7 @@ from swift.common.exceptions import ListingIterError, SegmentError from swift.common.http import is_success, HTTP_SERVICE_UNAVAILABLE from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable from swift.common.utils import split_path, validate_device_partition -from swift.common.wsgi import make_request +from swift.common.wsgi import make_subrequest def get_param(req, name, default=None): @@ -281,7 +281,7 @@ class SegmentedIterable(object): 'ERROR: While processing manifest %s, ' 'max LO GET time of %ds exceeded' % (self.name, self.max_get_time)) - seg_req = make_request( + seg_req = make_subrequest( self.req.environ, path=seg_path, method='GET', headers={'x-auth-token': self.req.headers.get( 'x-auth-token')}, diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py index ae72e9cf9..978e6c51f 100644 --- a/swift/common/ring/builder.py +++ b/swift/common/ring/builder.py @@ -754,6 +754,7 @@ class RingBuilder(object): """ for dev in self._iter_devs(): dev['sort_key'] = self._sort_key_for(dev) + dev['tiers'] = tiers_for_dev(dev) available_devs = \ sorted((d for d in self._iter_devs() if d['weight']), @@ -764,7 +765,6 @@ class RingBuilder(object): tier2dev_sort_key = defaultdict(list) max_tier_depth = 0 for dev in available_devs: - dev['tiers'] = tiers_for_dev(dev) for tier in dev['tiers']: tier2devs[tier].append(dev) # <-- starts out sorted! tier2dev_sort_key[tier].append(dev['sort_key']) @@ -889,7 +889,7 @@ class RingBuilder(object): # Just to save memory and keep from accidental reuse. for dev in self._iter_devs(): del dev['sort_key'] - dev.pop('tiers', None) # May be absent for devices w/o weight + del dev['tiers'] def _sort_key_for(self, dev): return (dev['parts_wanted'], random.randint(0, 0xFFFF), dev['id']) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 041117163..417b161b0 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -573,9 +573,11 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None, newenv = {} for name in ('eventlet.posthooks', 'HTTP_USER_AGENT', 'HTTP_HOST', 'PATH_INFO', 'QUERY_STRING', 'REMOTE_USER', 'REQUEST_METHOD', - 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT', 'HTTP_ORIGIN', + 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT', + 'HTTP_ORIGIN', 'HTTP_ACCESS_CONTROL_REQUEST_METHOD', 'SERVER_PROTOCOL', 'swift.cache', 'swift.source', - 'swift.trans_id'): + 'swift.trans_id', 'swift.authorize_override', + 'swift.authorize'): if name in env: newenv[name] = env[name] if method: @@ -598,8 +600,8 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None, return newenv -def make_request(env, method=None, path=None, body=None, headers=None, - agent='Swift', swift_source=None, make_env=make_env): +def make_subrequest(env, method=None, path=None, body=None, headers=None, + agent='Swift', swift_source=None, make_env=make_env): """ Makes a new swob.Request based on the current env but with the parameters specified. @@ -623,7 +625,7 @@ def make_request(env, method=None, path=None, body=None, headers=None, have no HTTP_USER_AGENT. :param swift_source: Used to mark the request as originating out of middleware. Will be logged in proxy logs. - :param make_env: make_request calls this make_env to help build the + :param make_env: make_subrequest calls this make_env to help build the swob.Request. :returns: Fresh swob.Request object. """ @@ -655,7 +657,7 @@ def make_pre_authed_env(env, method=None, path=None, agent='Swift', def make_pre_authed_request(env, method=None, path=None, body=None, headers=None, agent='Swift', swift_source=None): - """Same as :py:func:`make_request` but with preauthorization.""" - return make_request( + """Same as :py:func:`make_subrequest` but with preauthorization.""" + return make_subrequest( env, method=method, path=path, body=body, headers=headers, agent=agent, swift_source=swift_source, make_env=make_pre_authed_env) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 7aaf2d5f8..139d6609f 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -212,6 +212,10 @@ def cors_validation(func): # Call through to the decorated method resp = func(*a, **kw) + if controller.app.strict_cors_mode and \ + not controller.is_origin_allowed(cors_info, req_origin): + return resp + # Expose, # - simple response headers, # http://www.w3.org/TR/cors/#simple-response-header @@ -219,24 +223,32 @@ def cors_validation(func): # - user metadata headers # - headers provided by the user in # x-container-meta-access-control-expose-headers - expose_headers = ['cache-control', 'content-language', - 'content-type', 'expires', 'last-modified', - 'pragma', 'etag', 'x-timestamp', 'x-trans-id'] - for header in resp.headers: - if header.startswith('X-Container-Meta') or \ - header.startswith('X-Object-Meta'): - expose_headers.append(header.lower()) - if cors_info.get('expose_headers'): - expose_headers.extend( - [header_line.strip() - for header_line in cors_info['expose_headers'].split(' ') - if header_line.strip()]) - resp.headers['Access-Control-Expose-Headers'] = \ - ', '.join(expose_headers) + if 'Access-Control-Expose-Headers' not in resp.headers: + expose_headers = [ + 'cache-control', 'content-language', 'content-type', + 'expires', 'last-modified', 'pragma', 'etag', + 'x-timestamp', 'x-trans-id'] + for header in resp.headers: + if header.startswith('X-Container-Meta') or \ + header.startswith('X-Object-Meta'): + expose_headers.append(header.lower()) + if cors_info.get('expose_headers'): + expose_headers.extend( + [header_line.strip() + for header_line in + cors_info['expose_headers'].split(' ') + if header_line.strip()]) + resp.headers['Access-Control-Expose-Headers'] = \ + ', '.join(expose_headers) # The user agent won't process the response if the Allow-Origin # header isn't included - resp.headers['Access-Control-Allow-Origin'] = req_origin + if 'Access-Control-Allow-Origin' not in resp.headers: + if cors_info['allow_origin'] and \ + cors_info['allow_origin'].strip() == '*': + resp.headers['Access-Control-Allow-Origin'] = '*' + else: + resp.headers['Access-Control-Allow-Origin'] = req_origin return resp else: @@ -1256,7 +1268,10 @@ class Controller(object): list_from_csv(req.headers['Access-Control-Request-Headers'])) # Populate the response with the CORS preflight headers - headers['access-control-allow-origin'] = req_origin_value + if cors.get('allow_origin', '').strip() == '*': + headers['access-control-allow-origin'] = '*' + else: + headers['access-control-allow-origin'] = req_origin_value if cors.get('max_age') is not None: headers['access-control-max-age'] = cors.get('max_age') headers['access-control-allow-methods'] = \ diff --git a/swift/proxy/server.py b/swift/proxy/server.py index d42688b42..5b4a5b7b2 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -130,6 +130,8 @@ class Application(object): a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip()] + self.strict_cors_mode = config_true_value( + conf.get('strict_cors_mode', 't')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) self.sorting_method = conf.get('sorting_method', 'shuffle').lower() @@ -210,7 +212,8 @@ class Application(object): container_listing_limit=constraints.CONTAINER_LISTING_LIMIT, max_account_name_length=constraints.MAX_ACCOUNT_NAME_LENGTH, max_container_name_length=constraints.MAX_CONTAINER_NAME_LENGTH, - max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH) + max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH, + strict_cors_mode=self.strict_cors_mode) def check_config(self): """ diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 277973442..b4dcb56cf 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -178,6 +178,18 @@ class Connection(object): self.http_connect() return self.storage_url, self.storage_token + def cluster_info(self): + """ + Retrieve the data in /info, or {} on 404 + """ + status = self.make_request('GET', '/info', + cfg={'absolute_path': True}) + if status == 404: + return {} + if not 200 <= status <= 299: + raise ResponseError(self.response, 'GET', '/info') + return json.loads(self.response.read()) + def http_connect(self): self.connection = self.conn_class(self.storage_host, port=self.storage_port) @@ -208,8 +220,8 @@ class Connection(object): def make_request(self, method, path=[], data='', hdrs={}, parms={}, cfg={}): - if not cfg.get('verbatim_path'): - # Set verbatim_path=True to make a request to exactly the given + if not cfg.get('absolute_path'): + # Set absolute_path=True to make a request to exactly the given # path, not storage path + given path. Useful for # non-account/container/object requests. path = self.make_path(path, cfg=cfg) @@ -340,6 +352,16 @@ class Account(Base): self.conn = conn self.name = str(name) + def update_metadata(self, metadata={}, cfg={}): + headers = dict(("X-Account-Meta-%s" % k, v) + for k, v in metadata.items()) + + self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) + if not 200 <= self.conn.response.status <= 299: + raise ResponseError(self.conn.response, 'POST', + self.conn.make_path(self.path)) + return True + def container(self, container_name): return Container(self.conn, self.name, container_name) diff --git a/test/functional/test_object.py b/test/functional/test_object.py index dad8635da..9b999b6b9 100755 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -21,6 +21,7 @@ from uuid import uuid4 from swift_testing import check_response, retry, skip, skip3, \ swift_test_perm, web_front_end +from swift.common.utils import json class TestObject(unittest.TestCase): @@ -619,6 +620,117 @@ class TestObject(unittest.TestCase): self.assertEquals(resp.read(), 'Invalid UTF8 or contains NULL') self.assertEquals(resp.status, 412) + def test_cors(self): + if skip: + raise SkipTest + + def is_strict_mode(url, token, parsed, conn): + conn.request('GET', '/info') + resp = conn.getresponse() + if resp.status // 100 == 2: + info = json.loads(resp.read()) + return info.get('swift', {}).get('strict_cors_mode', False) + return False + + def put_cors_cont(url, token, parsed, conn, orig): + conn.request( + 'PUT', '%s/%s' % (parsed.path, self.container), + '', {'X-Auth-Token': token, + 'X-Container-Meta-Access-Control-Allow-Origin': orig}) + return check_response(conn) + + def put_obj(url, token, parsed, conn, obj): + conn.request( + 'PUT', '%s/%s/%s' % (parsed.path, self.container, obj), + 'test', {'X-Auth-Token': token}) + return check_response(conn) + + def check_cors(url, token, parsed, conn, + method, obj, headers): + if method != 'OPTIONS': + headers['X-Auth-Token'] = token + conn.request( + method, '%s/%s/%s' % (parsed.path, self.container, obj), + '', headers) + return conn.getresponse() + + strict_cors = retry(is_strict_mode) + + resp = retry(put_cors_cont, '*') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(put_obj, 'cat') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(check_cors, + 'OPTIONS', 'cat', {'Origin': 'http://m.com'}) + self.assertEquals(resp.status, 401) + + resp = retry(check_cors, + 'OPTIONS', 'cat', + {'Origin': 'http://m.com', + 'Access-Control-Request-Method': 'GET'}) + + self.assertEquals(resp.status, 200) + resp.read() + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com', + 'X-Web-Mode': 'True'}) + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + '*') + + #################### + + resp = retry(put_cors_cont, 'http://secret.com') + resp.read() + self.assertEquals(resp.status // 100, 2) + + resp = retry(check_cors, + 'OPTIONS', 'cat', + {'Origin': 'http://m.com', + 'Access-Control-Request-Method': 'GET'}) + resp.read() + self.assertEquals(resp.status, 401) + + if strict_cors: + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertTrue('access-control-allow-origin' not in headers) + + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://secret.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + 'http://secret.com') + else: + resp = retry(check_cors, + 'GET', 'cat', {'Origin': 'http://m.com'}) + resp.read() + self.assertEquals(resp.status, 200) + headers = dict((k.lower(), v) for k, v in resp.getheaders()) + self.assertEquals(headers.get('access-control-allow-origin'), + 'http://m.com') + if __name__ == '__main__': unittest.main() diff --git a/test/functional/tests.py b/test/functional/tests.py index f222d4e9b..97e45a627 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -16,53 +16,34 @@ from datetime import datetime import hashlib +import hmac import json import locale import random import StringIO import time import threading -import uuid import unittest +import urllib +import uuid from nose import SkipTest -from ConfigParser import ConfigParser from test import get_config from test.functional.swift_test_client import Account, Connection, File, \ ResponseError -from swift.common.constraints import MAX_FILE_SIZE, MAX_META_NAME_LENGTH, \ - MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ - MAX_OBJECT_NAME_LENGTH, CONTAINER_LISTING_LIMIT, ACCOUNT_LISTING_LIMIT, \ - MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH - -default_constraints = dict(( - ('max_file_size', MAX_FILE_SIZE), - ('max_meta_name_length', MAX_META_NAME_LENGTH), - ('max_meta_value_length', MAX_META_VALUE_LENGTH), - ('max_meta_count', MAX_META_COUNT), - ('max_meta_overall_size', MAX_META_OVERALL_SIZE), - ('max_object_name_length', MAX_OBJECT_NAME_LENGTH), - ('container_listing_limit', CONTAINER_LISTING_LIMIT), - ('account_listing_limit', ACCOUNT_LISTING_LIMIT), - ('max_account_name_length', MAX_ACCOUNT_NAME_LENGTH), - ('max_container_name_length', MAX_CONTAINER_NAME_LENGTH))) -constraints_conf = ConfigParser() -conf_exists = constraints_conf.read('/etc/swift/swift.conf') -# Constraints are set first from the test config, then from -# /etc/swift/swift.conf if it exists. If swift.conf doesn't exist, -# then limit test coverage. This allows SAIO tests to work fine but -# requires remote functional testing to know something about the cluster -# that is being tested. +from swift.common import constraints + + config = get_config('func_test') -for k in default_constraints: +for k in constraints.DEFAULT_CONSTRAINTS: if k in config: # prefer what's in test.conf config[k] = int(config[k]) - elif conf_exists: + elif constraints.SWIFT_CONSTRAINTS_LOADED: # swift.conf exists, so use what's defined there (or swift defaults) # This normally happens when the test is running locally to the cluster # as in a SAIO. - config[k] = default_constraints[k] + config[k] = constraints.EFFECTIVE_CONSTRAINTS[k] else: # .functests don't know what the constraints of the tested cluster are, # so the tests can't reliably pass or fail. Therefore, skip those @@ -1837,19 +1818,8 @@ class TestSloEnv(object): cls.conn.authenticate() if cls.slo_enabled is None: - status = cls.conn.make_request('GET', '/info', - cfg={'verbatim_path': True}) - if not (200 <= status <= 299): - # Can't tell if SLO is enabled or not since we're running - # against an old cluster, so let's skip the tests instead of - # possibly having spurious failures. - cls.slo_enabled = False - else: - # Don't bother looking for ValueError here. If something is - # responding to a GET /info request with invalid JSON, then - # the cluster is broken and a test failure will let us know. - cluster_info = json.loads(cls.conn.response.read()) - cls.slo_enabled = 'slo' in cluster_info + cluster_info = cls.conn.cluster_info() + cls.slo_enabled = 'slo' in cluster_info if not cls.slo_enabled: return @@ -2187,5 +2157,268 @@ class TestObjectVersioningUTF8(Base2, TestObjectVersioning): set_up = False +class TestTempurlEnv(object): + tempurl_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + if cls.tempurl_enabled is None: + cluster_info = cls.conn.cluster_info() + cls.tempurl_enabled = 'tempurl' in cluster_info + if not cls.tempurl_enabled: + return + cls.tempurl_methods = cluster_info['tempurl']['methods'] + + cls.tempurl_key = Utils.create_name() + cls.tempurl_key2 = Utils.create_name() + + cls.account = Account( + cls.conn, config.get('account', config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({ + 'temp-url-key': cls.tempurl_key, + 'temp-url-key-2': cls.tempurl_key2 + }) + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.obj = cls.container.file(Utils.create_name()) + cls.obj.write("obj contents") + cls.other_obj = cls.container.file(Utils.create_name()) + cls.other_obj.write("other obj contents") + + +class TestTempurl(Base): + env = TestTempurlEnv + set_up = False + + def setUp(self): + super(TestTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + self.obj_tempurl_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + # GET tempurls also allow HEAD requests + self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) + + def test_GET_with_key_2(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key2) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + def test_PUT(self): + new_obj = self.env.container.file(Utils.create_name()) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + new_obj.write('new obj contents', + parms=put_parms, cfg={'no_auth_token': True}) + self.assertEqual(new_obj.read(), "new obj contents") + + # PUT tempurls also allow HEAD requests + self.assert_(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) + + def test_HEAD(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + head_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + self.assert_(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) + # HEAD tempurls don't allow PUT or GET requests, despite the fact that + # PUT and GET tempurls both allow HEAD requests + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + self.assertRaises(ResponseError, self.env.other_obj.write, + 'new contents', + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_different_object(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_changing_sig(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_sig'][0] == 'a': + parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] + else: + parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + def test_changing_expires(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_expires'][-1] == '0': + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' + else: + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + +class TestTempurlUTF8(Base2, TestTempurl): + set_up = False + + +class TestSloTempurlEnv(object): + enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(config) + cls.conn.authenticate() + + if cls.enabled is None: + cluster_info = cls.conn.cluster_info() + cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info + + cls.tempurl_key = Utils.create_name() + + cls.account = Account( + cls.conn, config.get('account', config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) + + cls.manifest_container = cls.account.container(Utils.create_name()) + cls.segments_container = cls.account.container(Utils.create_name()) + if not cls.manifest_container.create(): + raise ResponseError(cls.conn.response) + if not cls.segments_container.create(): + raise ResponseError(cls.conn.response) + + seg1 = cls.segments_container.file(Utils.create_name()) + seg1.write('1' * 1024 * 1024) + + seg2 = cls.segments_container.file(Utils.create_name()) + seg2.write('2' * 1024 * 1024) + + cls.manifest_data = [{'size_bytes': 1024 * 1024, + 'etag': seg1.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg1.name)}, + {'size_bytes': 1024 * 1024, + 'etag': seg2.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg2.name)}] + + cls.manifest = cls.manifest_container.file(Utils.create_name()) + cls.manifest.write( + json.dumps(cls.manifest_data), + parms={'multipart-manifest': 'put'}) + + +class TestSloTempurl(Base): + env = TestSloTempurlEnv + set_up = False + + def setUp(self): + super(TestSloTempurl, self).setUp() + if self.env.enabled is False: + raise SkipTest("TempURL and SLO not both enabled") + elif self.env.enabled is not True: + # just some sanity checking + raise Exception( + "Expected enabled to be True/False, got %r" % + (self.env.enabled,)) + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} + + contents = self.env.manifest.read( + parms=parms, + cfg={'no_auth_token': True}) + self.assertEqual(len(contents), 2 * 1024 * 1024) + + # GET tempurls also allow HEAD requests + self.assert_(self.env.manifest.info( + parms=parms, cfg={'no_auth_token': True})) + + +class TestSloTempurlUTF8(Base2, TestSloTempurl): + set_up = False + + if __name__ == '__main__': unittest.main() diff --git a/test/probe/common.py b/test/probe/common.py index d49a4e8f1..3c9192f61 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -15,11 +15,10 @@ from httplib import HTTPConnection import os -from os import kill, path -from signal import SIGTERM from subprocess import Popen, PIPE import sys from time import sleep, time +from collections import defaultdict from swiftclient import get_auth, head_account @@ -30,22 +29,25 @@ from swift.common.manager import Manager from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC +def get_server_number(port, port2server): + server_number = port2server[port] + server, number = server_number[:-1], server_number[-1:] + try: + number = int(number) + except ValueError: + # probably the proxy + return server_number, None + return server, number + + def start_server(port, port2server, pids, check=True): - server = port2server[port] - if server[:-1] in ('account', 'container', 'object'): - if not path.exists('/etc/swift/%s-server/%s.conf' % - (server[:-1], server[-1])): - return None - pids[server] = Popen([ - 'swift-%s-server' % server[:-1], - '/etc/swift/%s-server/%s.conf' % (server[:-1], server[-1])]).pid - if check: - return check_server(port, port2server, pids) - else: - pids[server] = Popen(['swift-%s-server' % server, - '/etc/swift/%s-server.conf' % server]).pid - if check: - return check_server(port, port2server, pids) + server, number = get_server_number(port, port2server) + err = Manager([server]).start(number=number, wait=False) + if err: + raise Exception('unable to start %s' % ( + server if not number else '%s%s' % (server, number))) + if check: + return check_server(port, port2server, pids) return None @@ -97,10 +99,11 @@ def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT): def kill_server(port, port2server, pids): - try: - kill(pids[port2server[port]], SIGTERM) - except Exception as err: - print err + server, number = get_server_number(port, port2server) + err = Manager([server]).kill(number=number) + if err: + raise Exception('unable to kill %s' % (server if not number else + '%s%s' % (server, number))) try_until = time() + 30 while True: try: @@ -116,8 +119,7 @@ def kill_server(port, port2server, pids): def kill_servers(port2server, pids): - for port in port2server: - kill_server(port, port2server, pids) + Manager(['all']).kill() def kill_nonprimary_server(primary_nodes, port2server, pids): @@ -145,18 +147,19 @@ def get_ring(server, force_validate=None): ring.serialized_path, len(ring.devs)) # map server to config by port port_to_config = {} - for node_id in range(1, 5): - conf = readconf('/etc/swift/%s-server/%d.conf' % (server, node_id), - section_name='%s-replicator' % server) - port_to_config[int(conf['bind_port'])] = conf + for server_ in Manager([server]): + for config_path in server_.conf_files(): + conf = readconf(config_path, + section_name='%s-replicator' % server_.type) + port_to_config[int(conf['bind_port'])] = conf for dev in ring.devs: # verify server is exposing mounted device conf = port_to_config[dev['port']] for device in os.listdir(conf['devices']): if device == dev['device']: - dev_path = path.join(conf['devices'], device) - full_path = path.realpath(dev_path) - assert path.exists(full_path), \ + dev_path = os.path.join(conf['devices'], device) + full_path = os.path.realpath(dev_path) + assert os.path.exists(full_path), \ 'device %s in %s was not found (%s)' % ( device, conf['devices'], full_path) break @@ -195,21 +198,22 @@ def reset_environment(): account_ring = get_ring('account') container_ring = get_ring('container') object_ring = get_ring('object') + Manager(['main']).start(wait=False) port2server = {} - config_dict = {} for server, port in [('account', 6002), ('container', 6001), ('object', 6000)]: for number in xrange(1, 9): port2server[port + (number * 10)] = '%s%d' % (server, number) for port in port2server: - start_server(port, port2server, pids, check=False) - for port in port2server: check_server(port, port2server, pids) port2server[8080] = 'proxy' - url, token, account = start_server(8080, port2server, pids) + url, token, account = check_server(8080, port2server, pids) + config_dict = defaultdict(dict) for name in ('account', 'container', 'object'): - for server in (name, '%s-replicator' % name): - config_dict[server] = '/etc/swift/%s-server/%%d.conf' % name + for server_name in (name, '%s-replicator' % name): + for server in Manager([server_name]): + for i, conf in enumerate(server.conf_files(), 1): + config_dict[server.server][i] = conf except BaseException: try: raise @@ -226,41 +230,15 @@ def reset_environment(): def get_to_final_state(): - processes = [] - for job in ('account-replicator', 'container-replicator', - 'object-replicator'): - for number in xrange(1, 9): - if not path.exists('/etc/swift/%s-server/%d.conf' % - (job.split('-')[0], number)): - continue - processes.append(Popen([ - 'swift-%s' % job, - '/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number), - 'once'])) - for process in processes: - process.wait() - processes = [] - for job in ('container-updater', 'object-updater'): - for number in xrange(1, 5): - processes.append(Popen([ - 'swift-%s' % job, - '/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number), - 'once'])) - for process in processes: - process.wait() - processes = [] - for job in ('account-replicator', 'container-replicator', - 'object-replicator'): - for number in xrange(1, 9): - if not path.exists('/etc/swift/%s-server/%d.conf' % - (job.split('-')[0], number)): - continue - processes.append(Popen([ - 'swift-%s' % job, - '/etc/swift/%s-server/%d.conf' % (job.split('-')[0], number), - 'once'])) - for process in processes: - process.wait() + replicators = Manager(['account-replicator', 'container-replicator', + 'object-replicator']) + replicators.stop() + updaters = Manager(['container-updater', 'object-updater']) + updaters.stop() + + replicators.once() + updaters.once() + replicators.once() if __name__ == "__main__": diff --git a/test/probe/test_account_failures.py b/test/probe/test_account_failures.py index facbacc92..aa7713f14 100755 --- a/test/probe/test_account_failures.py +++ b/test/probe/test_account_failures.py @@ -14,12 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from subprocess import Popen from unittest import main, TestCase from swiftclient import client from swift.common import direct_client +from swift.common.manager import Manager from test.probe.common import get_to_final_state, kill_nonprimary_server, \ kill_server, kill_servers, reset_environment, start_server @@ -136,14 +136,7 @@ class TestAccountFailures(TestCase): self.assert_(not found1) self.assert_(found2) - processes = [] - for node in xrange(1, 5): - processes.append(Popen([ - 'swift-container-updater', - self.configs['container'] % node, - 'once'])) - for process in processes: - process.wait() + Manager(['container-updater']).once() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index fb1b68b69..f42dd3cee 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -124,7 +124,7 @@ class TestContainerFailures(TestCase): node_id = (onode['port'] - 6000) / 10 device = onode['device'] hash_str = hash_path(self.account, container) - server_conf = readconf(self.configs['container'] % node_id) + server_conf = readconf(self.configs['container-server'][node_id]) devices = server_conf['app:container-server']['devices'] obj_dir = '%s/%s/containers/%s/%s/%s/' % (devices, device, opart, diff --git a/test/probe/test_empty_device_handoff.py b/test/probe/test_empty_device_handoff.py index e42a0fddb..2841bcd42 100755 --- a/test/probe/test_empty_device_handoff.py +++ b/test/probe/test_empty_device_handoff.py @@ -18,7 +18,6 @@ import os import shutil import time -from subprocess import call from unittest import main, TestCase from uuid import uuid4 @@ -29,6 +28,7 @@ from swift.common.exceptions import ClientException from test.probe.common import kill_server, kill_servers, reset_environment,\ start_server from swift.common.utils import readconf +from swift.common.manager import Manager class TestEmptyDevice(TestCase): @@ -44,7 +44,7 @@ class TestEmptyDevice(TestCase): def _get_objects_dir(self, onode): device = onode['device'] node_id = (onode['port'] - 6000) / 10 - obj_server_conf = readconf(self.configs['object'] % node_id) + obj_server_conf = readconf(self.configs['object-server'][node_id]) devices = obj_server_conf['app:object-server']['devices'] obj_dir = '%s/%s' % (devices, device) return obj_dir @@ -143,12 +143,12 @@ class TestEmptyDevice(TestCase): another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] - call(['swift-object-replicator', - self.configs['object-replicator'] % - ((port_num - 6000) / 10), 'once']) - call(['swift-object-replicator', - self.configs['object-replicator'] % - ((another_port_num - 6000) / 10), 'once']) + + num = (port_num - 6000) / 10 + Manager(['object-replicator']).once(number=num) + + another_num = (another_port_num - 6000) / 10 + Manager(['object-replicator']).once(number=another_num) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index 4b6c91b65..e48dd91ad 100755 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -14,13 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from subprocess import Popen from unittest import main, TestCase from uuid import uuid4 from swiftclient import client from swift.common import direct_client +from swift.common.manager import Manager from test.probe.common import kill_nonprimary_server, kill_server, \ kill_servers, reset_environment, start_server @@ -54,13 +54,7 @@ class TestObjectAsyncUpdate(TestCase): start_server(cnode['port'], self.port2server, self.pids) self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) - processes = [] - for node in xrange(1, 5): - processes.append(Popen(['swift-object-updater', - self.configs['object'] % node, - 'once'])) - for process in processes: - process.wait() + Manager(['object-updater']).once() objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] self.assert_(obj in objs) diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index 339648ea4..f8fc119fa 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -70,7 +70,7 @@ class TestObjectFailures(TestCase): node_id = (onode['port'] - 6000) / 10 device = onode['device'] hash_str = hash_path(self.account, container, obj) - obj_server_conf = readconf(self.configs['object'] % node_id) + obj_server_conf = readconf(self.configs['object-server'][node_id]) devices = obj_server_conf['app:object-server']['devices'] obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices, device, opart, diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index dd7b91cda..565b2ff7c 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from subprocess import call, Popen from unittest import main, TestCase from uuid import uuid4 @@ -22,6 +21,7 @@ from swiftclient import client from swift.common import direct_client from swift.common.exceptions import ClientException +from swift.common.manager import Manager from test.probe.common import kill_server, kill_servers, reset_environment, \ start_server @@ -115,25 +115,19 @@ class TestObjectHandoff(TestCase): exc = err self.assertEquals(exc.http_status, 404) # Run the extra server last so it'll remove its extra partition - processes = [] for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] - processes.append(Popen(['swift-object-replicator', - self.configs['object-replicator'] % - ((port_num - 6000) / 10), - 'once'])) - for process in processes: - process.wait() + node_id = (port_num - 6000) / 10 + Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] - call(['swift-object-replicator', - self.configs['object-replicator'] % - ((another_port_num - 6000) / 10), 'once']) + another_num = (another_port_num - 6000) / 10 + Manager(['object-replicator']).once(number=another_num) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': @@ -171,21 +165,15 @@ class TestObjectHandoff(TestCase): direct_client.direct_get_object(onode, opart, self.account, container, obj) # Run the extra server last so it'll remove its extra partition - processes = [] for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] - processes.append(Popen(['swift-object-replicator', - self.configs['object-replicator'] % - ((port_num - 6000) / 10), - 'once'])) - for process in processes: - process.wait() - call(['swift-object-replicator', - self.configs['object-replicator'] % - ((another_port_num - 6000) / 10), 'once']) + node_id = (port_num - 6000) / 10 + Manager(['object-replicator']).once(number=node_id) + another_node_id = (another_port_num - 6000) / 10 + Manager(['object-replicator']).once(number=another_node_id) exc = None try: direct_client.direct_get_object(another_onode, opart, self.account, diff --git a/test/probe/test_replication_servers_working.py b/test/probe/test_replication_servers_working.py index 96f0de1c9..dae315ac2 100644 --- a/test/probe/test_replication_servers_working.py +++ b/test/probe/test_replication_servers_working.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from subprocess import Popen from unittest import main, TestCase from uuid import uuid4 import os @@ -25,6 +24,7 @@ from swiftclient import client from test.probe.common import kill_servers, reset_environment from swift.common.utils import readconf +from swift.common.manager import Manager def collect_info(path_list): @@ -102,7 +102,7 @@ class TestReplicatorFunctions(TestCase): path_list = [] # Figure out where the devices are for node_id in range(1, 5): - conf = readconf(self.configs['object'] % node_id) + conf = readconf(self.configs['object-server'][node_id]) device_path = conf['app:object-server']['devices'] for dev in self.object_ring.devs: if dev['port'] == int(conf['app:object-server']['bind_port']): @@ -126,18 +126,9 @@ class TestReplicatorFunctions(TestCase): test_node_files_list.append(files) test_node_dir_list = dir_list[num] # Run all replicators - processes = [] - try: - for num in xrange(1, 9): - for server in ['object-replicator', - 'container-replicator', - 'account-replicator']: - if not os.path.exists(self.configs[server] % (num)): - continue - processes.append(Popen(['swift-%s' % (server), - self.configs[server] % (num), - 'forever'])) + Manager(['object-replicator', 'container-replicator', + 'account-replicator']).start() # Delete some files for directory in os.listdir(test_node): @@ -211,8 +202,8 @@ class TestReplicatorFunctions(TestCase): raise time.sleep(1) finally: - for process in processes: - process.kill() + Manager(['object-replicator', 'container-replicator', + 'account-replicator']).stop() if __name__ == '__main__': diff --git a/test/sample.conf b/test/sample.conf index 32d984e0d..161a588aa 100644 --- a/test/sample.conf +++ b/test/sample.conf @@ -25,15 +25,17 @@ password2 = testing2 username3 = tester3 password3 = testing3 -# Default constraints if not defined here, the test runner will try -# to set them from /etc/swift/swift.conf. If that file isn't found, -# the test runner will skip tests that depend on these values. +# If not defined here, the test runner will try to use the default constraint +# values as constructed by the constraints module, which will attempt to get +# them from /etc/swift/swift.conf, if possible. Then, if the swift.conf file +# isn't found, the test runner will skip tests that depend on those values. # Note that the cluster must have "sane" values for the test suite to pass. #max_file_size = 5368709122 #max_meta_name_length = 128 #max_meta_value_length = 256 #max_meta_count = 90 #max_meta_overall_size = 4096 +#max_header_size = 8192 #max_object_name_length = 1024 #container_listing_limit = 10000 #account_listing_limit = 10000 diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index 0ea957b53..52cc624e2 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -42,6 +42,11 @@ class FakeSwift(object): if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] + if 'swift.authorize' in env: + resp = env['swift.authorize']() + if resp: + return resp(env, start_response) + headers = swob.Request(env).headers self._calls.append((method, path, headers)) self.swift_sources.append(env.get('swift.source')) diff --git a/test/unit/common/middleware/test_dlo.py b/test/unit/common/middleware/test_dlo.py index de495f1bc..3f01eecfe 100644 --- a/test/unit/common/middleware/test_dlo.py +++ b/test/unit/common/middleware/test_dlo.py @@ -758,6 +758,19 @@ class TestDloGetManifest(DloTestCase): self.assertEqual(body, 'aaaaabbbbbcccc') self.assertTrue(isinstance(exc, exceptions.SegmentError)) + def test_get_with_auth_overridden(self): + auth_got_called = [0] + + def my_auth(): + auth_got_called[0] += 1 + return None + + req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', + environ={'REQUEST_METHOD': 'GET', + 'swift.authorize': my_auth}) + status, headers, body = self.call_dlo(req) + self.assertTrue(auth_got_called[0] > 1) + def fake_start_response(*args, **kwargs): pass diff --git a/test/unit/common/ring/test_builder.py b/test/unit/common/ring/test_builder.py index 1cdb93910..8564008e7 100644 --- a/test/unit/common/ring/test_builder.py +++ b/test/unit/common/ring/test_builder.py @@ -107,6 +107,33 @@ class TestRingBuilder(unittest.TestCase): self.assertNotEquals(r0.to_dict(), r1.to_dict()) self.assertEquals(r1.to_dict(), r2.to_dict()) + def test_rebalance_part_on_deleted_other_part_on_drained(self): + rb = ring.RingBuilder(8, 3, 1) + rb.add_dev({'id': 0, 'region': 1, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 1, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 1, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) + rb.add_dev({'id': 4, 'region': 1, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) + rb.add_dev({'id': 5, 'region': 1, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10005, 'device': 'sda1'}) + + rb.rebalance(seed=1) + # We want a partition where 1 replica is on a removed device, 1 + # replica is on a 0-weight device, and 1 on a normal device. To + # guarantee we have one, we see where partition 123 is, then + # manipulate its devices accordingly. + zero_weight_dev_id = rb._replica2part2dev[1][123] + delete_dev_id = rb._replica2part2dev[2][123] + + rb.set_dev_weight(zero_weight_dev_id, 0.0) + rb.remove_dev(delete_dev_id) + rb.rebalance() + def test_set_replicas(self): rb = ring.RingBuilder(8, 3.2, 1) rb.devs_changed = False @@ -125,7 +152,7 @@ class TestRingBuilder(unittest.TestCase): self.assertRaises(exceptions.DuplicateDeviceError, rb.add_dev, dev) self.assertEqual(dev_id, 0) rb = ring.RingBuilder(8, 3, 1) - #test add new dev with no id + # test add new dev with no id dev_id = rb.add_dev({'zone': 0, 'region': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 6000}) self.assertEquals(rb.devs[0]['id'], 0) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index aaf5269ac..9abf153db 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -15,6 +15,7 @@ import unittest import mock +import tempfile from test import safe_repr from test.unit import MockTrue @@ -245,5 +246,85 @@ class TestConstraints(unittest.TestCase): constraints.check_copy_from_header, req) +class TestConstraintsConfig(unittest.TestCase): + + def test_default_constraints(self): + for key in constraints.DEFAULT_CONSTRAINTS: + # if there is local over-rides in swift.conf we just continue on + if key in constraints.OVERRIDE_CONSTRAINTS: + continue + # module level attrs (that aren't in OVERRIDE) should have the + # same value as the DEFAULT map + module_level_value = getattr(constraints, key.upper()) + self.assertEquals(constraints.DEFAULT_CONSTRAINTS[key], + module_level_value) + + def test_effective_constraints(self): + for key in constraints.DEFAULT_CONSTRAINTS: + # module level attrs should always mirror the same value as the + # EFFECTIVE map + module_level_value = getattr(constraints, key.upper()) + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], + module_level_value) + # if there are local over-rides in swift.conf those should be + # reflected in the EFFECTIVE, otherwise we expect the DEFAULTs + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], + constraints.OVERRIDE_CONSTRAINTS.get( + key, constraints.DEFAULT_CONSTRAINTS[key])) + + def test_override_constraints(self): + try: + with tempfile.NamedTemporaryFile() as f: + f.write('[swift-constraints]\n') + # set everything to 1 + for key in constraints.DEFAULT_CONSTRAINTS: + f.write('%s = 1\n' % key) + f.flush() + with mock.patch.object(constraints, 'SWIFT_CONF_FILE', + f.name): + constraints.reload_constraints() + for key in constraints.DEFAULT_CONSTRAINTS: + # module level attrs should all be 1 + module_level_value = getattr(constraints, key.upper()) + self.assertEquals(module_level_value, 1) + # all keys should be in OVERRIDE + self.assertEquals(constraints.OVERRIDE_CONSTRAINTS[key], + module_level_value) + # module level attrs should always mirror the same value as + # the EFFECTIVE map + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], + module_level_value) + finally: + constraints.reload_constraints() + + def test_reload_reset(self): + try: + with tempfile.NamedTemporaryFile() as f: + f.write('[swift-constraints]\n') + # set everything to 1 + for key in constraints.DEFAULT_CONSTRAINTS: + f.write('%s = 1\n' % key) + f.flush() + with mock.patch.object(constraints, 'SWIFT_CONF_FILE', + f.name): + constraints.reload_constraints() + self.assertTrue(constraints.SWIFT_CONSTRAINTS_LOADED) + self.assertEquals(sorted(constraints.DEFAULT_CONSTRAINTS.keys()), + sorted(constraints.OVERRIDE_CONSTRAINTS.keys())) + # file is now deleted... + with mock.patch.object(constraints, 'SWIFT_CONF_FILE', + f.name): + constraints.reload_constraints() + # no constraints have been loaded from non-existant swift.conf + self.assertFalse(constraints.SWIFT_CONSTRAINTS_LOADED) + # no constraints are in OVERRIDE + self.assertEquals([], constraints.OVERRIDE_CONSTRAINTS.keys()) + # the EFFECTIVE constraints mirror DEFAULT + self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS, + constraints.DEFAULT_CONSTRAINTS) + finally: + constraints.reload_constraints() + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index 586749744..987782d30 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -1331,6 +1331,12 @@ class TestManager(unittest.TestCase): for s in m.servers: self.assert_(str(s) in replicators) + def test_iter(self): + m = manager.Manager(['all']) + self.assertEquals(len(list(m)), len(manager.ALL_SERVERS)) + for server in m: + self.assert_(server.server in manager.ALL_SERVERS) + def test_status(self): class MockServer(object): @@ -1560,6 +1566,9 @@ class TestManager(unittest.TestCase): def stop(self, **kwargs): return self.pids + def status(self, **kwargs): + return not self.pids + def __init__(self, server_pids, run_dir=manager.RUN_DIR): self.server_pids = server_pids @@ -1593,6 +1602,14 @@ class TestManager(unittest.TestCase): m = manager.Manager(['test']) status = m.stop() self.assertEquals(status, 1) + # test kill not running + server_pids = { + 'test': [] + } + manager.Server = MockServerFactory(server_pids) + m = manager.Manager(['test']) + status = m.kill() + self.assertEquals(status, 0) # test won't die server_pids = { 'test': [None] diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 8ff92eaa3..24807a5f7 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -2241,7 +2241,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') - def test_basic_put_with_x_copy_from_accross_container(self): + def test_basic_put_with_x_copy_from_across_container(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': 'c2/o'}) @@ -2407,7 +2407,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') - def test_COPY_accross_containers(self): + def test_COPY_across_containers(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c2/o'}) @@ -3822,9 +3822,7 @@ class TestObjectController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) - self.assertEquals( - 'https://bar.baz', - resp.headers['access-control-allow-origin']) + self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) @@ -3840,10 +3838,11 @@ class TestObjectController(unittest.TestCase): def stubContainerInfo(*args): return { 'cors': { - 'allow_origin': 'http://foo.bar' + 'allow_origin': 'http://not.foo.bar' } } controller.container_info = stubContainerInfo + controller.app.strict_cors_mode = False def objectGET(controller, req): return Response(headers={ @@ -3874,6 +3873,50 @@ class TestObjectController(unittest.TestCase): 'x-trans-id', 'x-object-meta-color']) self.assertEquals(expected_exposed, exposed) + controller.app.strict_cors_mode = True + req = Request.blank( + '/v1/a/c/o.jpg', + {'REQUEST_METHOD': 'GET'}, + headers={'Origin': 'http://foo.bar'}) + + resp = cors_validation(objectGET)(controller, req) + + self.assertEquals(200, resp.status_int) + self.assertTrue('access-control-allow-origin' not in resp.headers) + + def test_CORS_valid_with_obj_headers(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + + def stubContainerInfo(*args): + return { + 'cors': { + 'allow_origin': 'http://foo.bar' + } + } + controller.container_info = stubContainerInfo + + def objectGET(controller, req): + return Response(headers={ + 'X-Object-Meta-Color': 'red', + 'X-Super-Secret': 'hush', + 'Access-Control-Allow-Origin': 'http://obj.origin', + 'Access-Control-Expose-Headers': 'x-trans-id' + }) + + req = Request.blank( + '/v1/a/c/o.jpg', + {'REQUEST_METHOD': 'GET'}, + headers={'Origin': 'http://foo.bar'}) + + resp = cors_validation(objectGET)(controller, req) + + self.assertEquals(200, resp.status_int) + self.assertEquals('http://obj.origin', + resp.headers['access-control-allow-origin']) + self.assertEquals('x-trans-id', + resp.headers['access-control-expose-headers']) + def _gather_x_container_headers(self, controller_call, req, *connect_args, **kwargs): header_list = kwargs.pop('header_list', ['X-Container-Device', @@ -4841,9 +4884,7 @@ class TestContainerController(unittest.TestCase): req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) - self.assertEquals( - 'https://bar.baz', - resp.headers['access-control-allow-origin']) + self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) |