diff options
42 files changed, 1152 insertions, 516 deletions
diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index 5ebeef43a..1c03197ea 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -415,21 +415,6 @@ read from /etc/swift/memcache.conf (see memcache.conf-sample) or lacking that file, it will default to 127.0.0.1:11211. You can specify multiple servers separated with commas, as in: 10.1.2.3:11211,10.1.2.4:11211. (IPv6 addresses must follow rfc3986 section-3.2.2, i.e. [::1]:11211) -.IP \fBmemcache_serialization_support\fR -This sets how memcache values are serialized and deserialized: -.RE - -.PD 0 -.RS 10 -.IP "0 = older, insecure pickle serialization" -.IP "1 = json serialization but pickles can still be read (still insecure)" -.IP "2 = json serialization only (secure and the default)" -.RE - -.RS 10 -To avoid an instant full cache flush, existing installations should upgrade with 0, then set to 1 and reload, then after some time (24 hours) set to 2 and reload. In the future, the ability to use pickle serialization will be removed. - -If not set in the configuration file, the value for memcache_serialization_support will be read from /etc/swift/memcache.conf if it exists (see memcache.conf-sample). Otherwise, the default value as indicated above will be used. .RE .PD diff --git a/etc/memcache.conf-sample b/etc/memcache.conf-sample index b375eb402..f85e49edc 100644 --- a/etc/memcache.conf-sample +++ b/etc/memcache.conf-sample @@ -5,16 +5,6 @@ # (IPv6 addresses must follow rfc3986 section-3.2.2, i.e. [::1]:11211) # memcache_servers = 127.0.0.1:11211 # -# Sets how memcache values are serialized and deserialized: -# 0 = older, insecure pickle serialization -# 1 = json serialization but pickles can still be read (still insecure) -# 2 = json serialization only (secure and the default) -# To avoid an instant full cache flush, existing installations should -# upgrade with 0, then set to 1 and reload, then after some time (24 hours) -# set to 2 and reload. -# In the future, the ability to use pickle serialization will be removed. -# memcache_serialization_support = 2 -# # Sets the maximum number of connections to each memcached server per worker # memcache_max_connections = 2 # diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index ef49c430f..44a456219 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -740,18 +740,6 @@ use = egg:swift#memcache # follow rfc3986 section-3.2.2, i.e. [::1]:11211) # memcache_servers = 127.0.0.1:11211 # -# Sets how memcache values are serialized and deserialized: -# 0 = older, insecure pickle serialization -# 1 = json serialization but pickles can still be read (still insecure) -# 2 = json serialization only (secure and the default) -# If not set here, the value for memcache_serialization_support will be read -# from /etc/swift/memcache.conf (see memcache.conf-sample). -# To avoid an instant full cache flush, existing installations should -# upgrade with 0, then set to 1 and reload, then after some time (24 hours) -# set to 2 and reload. -# In the future, the ability to use pickle serialization will be removed. -# memcache_serialization_support = 2 -# # Sets the maximum number of connections to each memcached server per worker # memcache_max_connections = 2 # diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 7136fb332..59a661189 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -138,7 +138,7 @@ class DaemonStrategy(object): def kill_children(*args): self.running = False - self.logger.info('SIGTERM received') + self.logger.notice('SIGTERM received (%s)', os.getpid()) signal.signal(signal.SIGTERM, signal.SIG_IGN) os.killpg(0, signal.SIGTERM) os._exit(0) diff --git a/swift/common/db_auditor.py b/swift/common/db_auditor.py index 635fcc6ec..229cfbdc4 100644 --- a/swift/common/db_auditor.py +++ b/swift/common/db_auditor.py @@ -22,7 +22,7 @@ from eventlet import Timeout import swift.common.db from swift.common.utils import get_logger, audit_location_generator, \ - config_true_value, dump_recon_cache, ratelimit_sleep + config_true_value, dump_recon_cache, EventletRateLimiter from swift.common.daemon import Daemon from swift.common.exceptions import DatabaseAuditorException from swift.common.recon import DEFAULT_RECON_CACHE_PATH, \ @@ -56,9 +56,9 @@ class DatabaseAuditor(Daemon): self.logging_interval = 3600 # once an hour self.passes = 0 self.failures = 0 - self.running_time = 0 self.max_dbs_per_second = \ float(conf.get('{}s_per_second'.format(self.server_type), 200)) + self.rate_limiter = EventletRateLimiter(self.max_dbs_per_second) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.recon_cache_path = conf.get('recon_cache_path', @@ -88,8 +88,7 @@ class DatabaseAuditor(Daemon): reported = time.time() self.passes = 0 self.failures = 0 - self.running_time = ratelimit_sleep( - self.running_time, self.max_dbs_per_second) + self.rate_limiter.wait() return reported def run_forever(self, *args, **kwargs): diff --git a/swift/common/memcached.py b/swift/common/memcached.py index 1a371e58b..199e73066 100644 --- a/swift/common/memcached.py +++ b/swift/common/memcached.py @@ -45,7 +45,6 @@ http://github.com/memcached/memcached/blob/1.4.2/doc/protocol.txt """ import six -import six.moves.cPickle as pickle import json import logging import time @@ -66,7 +65,6 @@ IO_TIMEOUT = 2.0 PICKLE_FLAG = 1 JSON_FLAG = 2 NODE_WEIGHT = 50 -PICKLE_PROTOCOL = 2 TRY_COUNT = 3 # if ERROR_LIMIT_COUNT errors occur in ERROR_LIMIT_TIME seconds, the server @@ -176,7 +174,7 @@ class MemcacheRing(object): def __init__( self, servers, connect_timeout=CONN_TIMEOUT, io_timeout=IO_TIMEOUT, pool_timeout=POOL_TIMEOUT, - tries=TRY_COUNT, allow_pickle=False, allow_unpickle=False, + tries=TRY_COUNT, max_conns=2, tls_context=None, logger=None, error_limit_count=ERROR_LIMIT_COUNT, error_limit_time=ERROR_LIMIT_TIME, @@ -200,8 +198,6 @@ class MemcacheRing(object): self._connect_timeout = connect_timeout self._io_timeout = io_timeout self._pool_timeout = pool_timeout - self._allow_pickle = allow_pickle - self._allow_unpickle = allow_unpickle or allow_pickle if logger is None: self.logger = logging.getLogger() else: @@ -258,6 +254,7 @@ class MemcacheRing(object): """ pos = bisect(self._sorted, key) served = [] + any_yielded = False while len(served) < self._tries: pos = (pos + 1) % len(self._sorted) server = self._ring[self._sorted[pos]] @@ -270,6 +267,7 @@ class MemcacheRing(object): try: with MemcachePoolTimeout(self._pool_timeout): fp, sock = self._client_cache[server].get() + any_yielded = True yield server, fp, sock except MemcachePoolTimeout as e: self._exception_occurred( @@ -281,34 +279,34 @@ class MemcacheRing(object): # object. self._exception_occurred( server, e, action='connecting', sock=sock) + if not any_yielded: + self.logger.error('All memcached servers error-limited') def _return_conn(self, server, fp, sock): """Returns a server connection to the pool.""" self._client_cache[server].put((fp, sock)) def set(self, key, value, serialize=True, time=0, - min_compress_len=0): + min_compress_len=0, raise_on_error=False): """ Set a key/value pair in memcache :param key: key :param value: value :param serialize: if True, value is serialized with JSON before sending - to memcache, or with pickle if configured to use - pickle instead of JSON (to avoid cache poisoning) + to memcache :param time: the time to live :param min_compress_len: minimum compress length, this parameter was added to keep the signature compatible with python-memcached interface. This implementation ignores it. + :param raise_on_error: if True, propagate Timeouts and other errors. + By default, errors are ignored. """ key = md5hash(key) timeout = sanitize_timeout(time) flags = 0 - if serialize and self._allow_pickle: - value = pickle.dumps(value, PICKLE_PROTOCOL) - flags |= PICKLE_FLAG - elif serialize: + if serialize: if isinstance(value, bytes): value = value.decode('utf8') value = json.dumps(value).encode('ascii') @@ -340,14 +338,18 @@ class MemcacheRing(object): return except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) + if raise_on_error: + raise MemcacheConnectionError( + "No memcached connections succeeded.") - def get(self, key): + def get(self, key, raise_on_error=False): """ Gets the object specified by key. It will also unserialize the object - before returning if it is serialized in memcache with JSON, or if it - is pickled and unpickling is allowed. + before returning if it is serialized in memcache with JSON. :param key: key + :param raise_on_error: if True, propagate Timeouts and other errors. + By default, errors are treated as cache misses. :returns: value of the key in memcache """ key = md5hash(key) @@ -366,11 +368,8 @@ class MemcacheRing(object): size = int(line[3]) value = fp.read(size) if int(line[2]) & PICKLE_FLAG: - if self._allow_unpickle: - value = pickle.loads(value) - else: - value = None - elif int(line[2]) & JSON_FLAG: + value = None + if int(line[2]) & JSON_FLAG: value = json.loads(value) fp.readline() line = fp.readline().strip().split() @@ -378,6 +377,9 @@ class MemcacheRing(object): return value except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) + if raise_on_error: + raise MemcacheConnectionError( + "No memcached connections succeeded.") def incr(self, key, delta=1, time=0): """ @@ -479,8 +481,7 @@ class MemcacheRing(object): :param server_key: key to use in determining which server in the ring is used :param serialize: if True, value is serialized with JSON before sending - to memcache, or with pickle if configured to use - pickle instead of JSON (to avoid cache poisoning) + to memcache. :param time: the time to live :min_compress_len: minimum compress length, this parameter was added to keep the signature compatible with @@ -493,10 +494,7 @@ class MemcacheRing(object): for key, value in mapping.items(): key = md5hash(key) flags = 0 - if serialize and self._allow_pickle: - value = pickle.dumps(value, PICKLE_PROTOCOL) - flags |= PICKLE_FLAG - elif serialize: + if serialize: if isinstance(value, bytes): value = value.decode('utf8') value = json.dumps(value).encode('ascii') @@ -540,10 +538,7 @@ class MemcacheRing(object): size = int(line[3]) value = fp.read(size) if int(line[2]) & PICKLE_FLAG: - if self._allow_unpickle: - value = pickle.loads(value) - else: - value = None + value = None elif int(line[2]) & JSON_FLAG: value = json.loads(value) responses[line[1]] = value diff --git a/swift/common/middleware/memcache.py b/swift/common/middleware/memcache.py index 4fa2b551c..562b0b9d8 100644 --- a/swift/common/middleware/memcache.py +++ b/swift/common/middleware/memcache.py @@ -33,7 +33,6 @@ class MemcacheMiddleware(object): self.app = app self.logger = get_logger(conf, log_route='memcache') self.memcache_servers = conf.get('memcache_servers') - serialization_format = conf.get('memcache_serialization_support') try: # Originally, while we documented using memcache_max_connections # we only accepted max_connections @@ -44,7 +43,6 @@ class MemcacheMiddleware(object): memcache_options = {} if (not self.memcache_servers - or serialization_format is None or max_conns <= 0): path = os.path.join(conf.get('swift_dir', '/etc/swift'), 'memcache.conf') @@ -62,13 +60,6 @@ class MemcacheMiddleware(object): memcache_conf.get('memcache', 'memcache_servers') except (NoSectionError, NoOptionError): pass - if serialization_format is None: - try: - serialization_format = \ - memcache_conf.get('memcache', - 'memcache_serialization_support') - except (NoSectionError, NoOptionError): - pass if max_conns <= 0: try: new_max_conns = \ @@ -111,10 +102,6 @@ class MemcacheMiddleware(object): self.memcache_servers = '127.0.0.1:11211' if max_conns <= 0: max_conns = 2 - if serialization_format is None: - serialization_format = 2 - else: - serialization_format = int(serialization_format) self.memcache = MemcacheRing( [s.strip() for s in self.memcache_servers.split(',') if s.strip()], @@ -122,8 +109,6 @@ class MemcacheMiddleware(object): pool_timeout=pool_timeout, tries=tries, io_timeout=io_timeout, - allow_pickle=(serialization_format == 0), - allow_unpickle=(serialization_format <= 1), max_conns=max_conns, tls_context=self.tls_context, logger=self.logger, diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index e9b0fdf54..c9443fa97 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -34,7 +34,7 @@ from swift.common.middleware.s3api.s3response import \ MalformedXML, InvalidLocationConstraint, NoSuchBucket, \ BucketNotEmpty, VersionedBucketNotEmpty, InternalError, \ ServiceUnavailable, NoSuchKey -from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX +from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, S3Timestamp MAX_PUT_BUCKET_BODY_SIZE = 10240 @@ -291,7 +291,7 @@ class BucketController(Controller): contents = SubElement(elem, 'Contents') SubElement(contents, 'Key').text = name SubElement(contents, 'LastModified').text = \ - o['last_modified'][:-3] + 'Z' + S3Timestamp.from_isoformat(o['last_modified']).s3xmlformat if contents.tag != 'DeleteMarker': if 's3_etag' in o: # New-enough MUs are already in the right format diff --git a/swift/common/middleware/s3api/controllers/multi_upload.py b/swift/common/middleware/s3api/controllers/multi_upload.py index 3ce12c5f7..3f23f25a1 100644 --- a/swift/common/middleware/s3api/controllers/multi_upload.py +++ b/swift/common/middleware/s3api/controllers/multi_upload.py @@ -397,7 +397,7 @@ class UploadsController(Controller): SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(upload_elem, 'StorageClass').text = 'STANDARD' SubElement(upload_elem, 'Initiated').text = \ - u['last_modified'][:-3] + 'Z' + S3Timestamp.from_isoformat(u['last_modified']).s3xmlformat for p in prefixes: elem = SubElement(result_elem, 'CommonPrefixes') @@ -582,7 +582,7 @@ class UploadController(Controller): part_elem = SubElement(result_elem, 'Part') SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1] SubElement(part_elem, 'LastModified').text = \ - i['last_modified'][:-3] + 'Z' + S3Timestamp.from_isoformat(i['last_modified']).s3xmlformat SubElement(part_elem, 'ETag').text = '"%s"' % i['hash'] SubElement(part_elem, 'Size').text = str(i['bytes']) diff --git a/swift/common/middleware/s3api/utils.py b/swift/common/middleware/s3api/utils.py index 4c8a4fd67..40ff9388f 100644 --- a/swift/common/middleware/s3api/utils.py +++ b/swift/common/middleware/s3api/utils.py @@ -15,6 +15,7 @@ import base64 import calendar +import datetime import email.utils import re import six @@ -108,9 +109,19 @@ def validate_bucket_name(name, dns_compliant_bucket_names): class S3Timestamp(utils.Timestamp): + S3_XML_FORMAT = "%Y-%m-%dT%H:%M:%S.000Z" + @property def s3xmlformat(self): - return self.isoformat[:-7] + '.000Z' + dt = datetime.datetime.utcfromtimestamp(self.ceil()) + return dt.strftime(self.S3_XML_FORMAT) + + @classmethod + def from_s3xmlformat(cls, date_string): + dt = datetime.datetime.strptime(date_string, cls.S3_XML_FORMAT) + dt = dt.replace(tzinfo=utils.UTC) + seconds = calendar.timegm(dt.timetuple()) + return cls(seconds) @property def amz_date_format(self): diff --git a/swift/common/utils.py b/swift/common/utils.py index 06c9ded30..2b7a75b7a 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1324,6 +1324,15 @@ class Timestamp(object): @property def isoformat(self): + """ + Get an isoformat string representation of the 'normal' part of the + Timestamp with microsecond precision and no trailing timezone, for + example: + + 1970-01-01T00:00:00.000000 + + :return: an isoformat string + """ t = float(self.normal) if six.PY3: # On Python 3, round manually using ROUND_HALF_EVEN rounding @@ -1350,6 +1359,33 @@ class Timestamp(object): isoformat += ".000000" return isoformat + @classmethod + def from_isoformat(cls, date_string): + """ + Parse an isoformat string representation of time to a Timestamp object. + + :param date_string: a string formatted as per an Timestamp.isoformat + property. + :return: an instance of this class. + """ + start = datetime.datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%f") + delta = start - EPOCH + # This calculation is based on Python 2.7's Modules/datetimemodule.c, + # function delta_to_microseconds(), but written in Python. + return cls(delta.total_seconds()) + + def ceil(self): + """ + Return the 'normal' part of the timestamp rounded up to the nearest + integer number of seconds. + + This value should be used whenever the second-precision Last-Modified + time of a resource is required. + + :return: a float value with second precision. + """ + return math.ceil(float(self)) + def __eq__(self, other): if other is None: return False @@ -1494,13 +1530,7 @@ def last_modified_date_to_timestamp(last_modified_date_str): Convert a last modified date (like you'd get from a container listing, e.g. 2014-02-28T23:22:36.698390) to a float. """ - start = datetime.datetime.strptime(last_modified_date_str, - '%Y-%m-%dT%H:%M:%S.%f') - delta = start - EPOCH - - # This calculation is based on Python 2.7's Modules/datetimemodule.c, - # function delta_to_microseconds(), but written in Python. - return Timestamp(delta.total_seconds()) + return Timestamp.from_isoformat(last_modified_date_str) def normalize_delete_at_timestamp(timestamp, high_precision=False): @@ -1723,7 +1753,7 @@ class RateLimitedIterator(object): self.iterator = iter(iterable) self.elements_per_second = elements_per_second self.limit_after = limit_after - self.running_time = 0 + self.rate_limiter = EventletRateLimiter(elements_per_second) self.ratelimit_if = ratelimit_if def __iter__(self): @@ -1736,8 +1766,7 @@ class RateLimitedIterator(object): if self.limit_after > 0: self.limit_after -= 1 else: - self.running_time = ratelimit_sleep(self.running_time, - self.elements_per_second) + self.rate_limiter.wait() return next_value __next__ = next @@ -3451,6 +3480,105 @@ def audit_location_generator(devices, datadir, suffix='', hook_post_device(os.path.join(devices, device)) +class AbstractRateLimiter(object): + # 1,000 milliseconds = 1 second + clock_accuracy = 1000.0 + + def __init__(self, max_rate, rate_buffer=5, burst_after_idle=False, + running_time=0): + """ + :param max_rate: The maximum rate per second allowed for the process. + Must be > 0 to engage rate-limiting behavior. + :param rate_buffer: Number of seconds the rate counter can drop and be + allowed to catch up (at a faster than listed rate). A larger number + will result in larger spikes in rate but better average accuracy. + :param burst_after_idle: If False (the default) then the rate_buffer + allowance is lost after the rate limiter has not been called for + more than rate_buffer seconds. If True then the rate_buffer + allowance is preserved during idle periods which means that a burst + of requests may be granted immediately after the idle period. + :param running_time: The running time in milliseconds of the next + allowable request. Setting this to any time in the past will cause + the rate limiter to immediately allow requests; setting this to a + future time will cause the rate limiter to deny requests until that + time. If ``burst_after_idle`` is True then this can + be set to current time (ms) to avoid an initial burst, or set to + running_time < (current time - rate_buffer ms) to allow an initial + burst. + """ + self.max_rate = max_rate + self.rate_buffer_ms = rate_buffer * self.clock_accuracy + self.burst_after_idle = burst_after_idle + self.running_time = running_time + self.time_per_incr = (self.clock_accuracy / self.max_rate + if self.max_rate else 0) + + def _sleep(self, seconds): + # subclasses should override to implement a sleep + raise NotImplementedError + + def is_allowed(self, incr_by=1, now=None, block=False): + """ + Check if the calling process is allowed to proceed according to the + rate limit. + + :param incr_by: How much to increment the counter. Useful if you want + to ratelimit 1024 bytes/sec and have differing sizes + of requests. Must be > 0 to engage rate-limiting + behavior. + :param now: The time in seconds; defaults to time.time() + :param block: if True, the call will sleep until the calling process + is allowed to proceed; otherwise the call returns immediately. + :return: True if the the calling process is allowed to proceed, False + otherwise. + """ + if self.max_rate <= 0 or incr_by <= 0: + return True + + now = now or time.time() + # Convert seconds to milliseconds + now = now * self.clock_accuracy + + # Calculate time per request in milliseconds + time_per_request = self.time_per_incr * float(incr_by) + + # Convert rate_buffer to milliseconds and compare + if now - self.running_time > self.rate_buffer_ms: + self.running_time = now + if self.burst_after_idle: + self.running_time -= self.rate_buffer_ms + + if now >= self.running_time: + self.running_time += time_per_request + allowed = True + elif block: + sleep_time = (self.running_time - now) / self.clock_accuracy + # increment running time before sleeping in case the sleep allows + # another thread to inspect the rate limiter state + self.running_time += time_per_request + # Convert diff to a floating point number of seconds and sleep + self._sleep(sleep_time) + allowed = True + else: + allowed = False + + return allowed + + def wait(self, incr_by=1, now=None): + self.is_allowed(incr_by=incr_by, now=now, block=True) + + +class EventletRateLimiter(AbstractRateLimiter): + def __init__(self, max_rate, rate_buffer=5, running_time=0, + burst_after_idle=False): + super(EventletRateLimiter, self).__init__( + max_rate, rate_buffer=rate_buffer, running_time=running_time, + burst_after_idle=burst_after_idle) + + def _sleep(self, seconds): + eventlet.sleep(seconds) + + def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): """ Will eventlet.sleep() for the appropriate time so that the max_rate @@ -3471,30 +3599,18 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): A larger number will result in larger spikes in rate but better average accuracy. Must be > 0 to engage rate-limiting behavior. - """ - if max_rate <= 0 or incr_by <= 0: - return running_time - - # 1,000 milliseconds = 1 second - clock_accuracy = 1000.0 - - # Convert seconds to milliseconds - now = time.time() * clock_accuracy - - # Calculate time per request in milliseconds - time_per_request = clock_accuracy * (float(incr_by) / max_rate) - - # Convert rate_buffer to milliseconds and compare - if now - running_time > rate_buffer * clock_accuracy: - running_time = now - elif running_time - now > time_per_request: - # Convert diff back to a floating point number of seconds and sleep - eventlet.sleep((running_time - now) / clock_accuracy) - - # Return the absolute time for the next interval in milliseconds; note - # that time could have passed well beyond that point, but the next call - # will catch that and skip the sleep. - return running_time + time_per_request + :return: The absolute time for the next interval in milliseconds; note + that time could have passed well beyond that point, but the next call + will catch that and skip the sleep. + """ + warnings.warn( + 'ratelimit_sleep() is deprecated; use the ``EventletRateLimiter`` ' + 'class instead.', DeprecationWarning + ) + rate_limit = EventletRateLimiter(max_rate, rate_buffer=rate_buffer, + running_time=running_time) + rate_limit.wait(incr_by=incr_by) + return rate_limit.running_time class ContextPool(GreenPool): diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index bce576fa5..eb5c0ffe2 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -1163,7 +1163,7 @@ def run_wsgi(conf_path, app_section, *args, **kwargs): logger.error('Stopping with unexpected signal %r' % running_context[1]) else: - logger.error('%s received (%s)', signame, os.getpid()) + logger.notice('%s received (%s)', signame, os.getpid()) if running_context[1] == signal.SIGTERM: os.killpg(0, signal.SIGTERM) elif running_context[1] == signal.SIGUSR1: diff --git a/swift/container/backend.py b/swift/container/backend.py index 6c0e55278..17546a887 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -1665,9 +1665,9 @@ class ContainerBroker(DatabaseBroker): if ('no such table: %s' % SHARD_RANGE_TABLE) not in str(err): raise - def _get_shard_range_rows(self, connection=None, include_deleted=False, - states=None, include_own=False, - exclude_others=False): + def _get_shard_range_rows(self, connection=None, includes=None, + include_deleted=False, states=None, + include_own=False, exclude_others=False): """ Returns a list of shard range rows. @@ -1676,6 +1676,8 @@ class ContainerBroker(DatabaseBroker): ``exclude_others=True``. :param connection: db connection + :param includes: restricts the returned list to the shard range that + includes the given value :param include_deleted: include rows marked as deleted :param states: include only rows matching the given state(s); can be an int or a list of ints. @@ -1719,6 +1721,9 @@ class ContainerBroker(DatabaseBroker): if exclude_others: conditions.append('name = ?') params.append(self.path) + if includes is not None: + conditions.extend(('lower < ?', "(upper = '' OR upper >= ?)")) + params.extend((includes, includes)) if conditions: condition = ' WHERE ' + ' AND '.join(conditions) columns = SHARD_RANGE_KEYS[:-2] @@ -1833,16 +1838,18 @@ class ContainerBroker(DatabaseBroker): shard_ranges = [ ShardRange(*row) for row in self._get_shard_range_rows( - include_deleted=include_deleted, states=states, - include_own=include_own, + includes=includes, include_deleted=include_deleted, + states=states, include_own=include_own, exclude_others=exclude_others)] shard_ranges.sort(key=ShardRange.sort_key) + if includes: + return shard_ranges[:1] if shard_ranges else [] shard_ranges = filter_shard_ranges(shard_ranges, includes, marker, end_marker) - if not includes and fill_gaps: + if fill_gaps: own_shard_range = self._own_shard_range() if shard_ranges: last_upper = shard_ranges[-1].upper diff --git a/swift/container/server.py b/swift/container/server.py index 5f8d748a8..8afc04750 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -17,7 +17,6 @@ import json import os import time import traceback -import math from eventlet import Timeout @@ -598,7 +597,7 @@ class ContainerController(BaseStorageServer): is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type resp = HTTPNoContent(request=req, headers=headers, charset='utf-8') - resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp'])) + resp.last_modified = Timestamp(headers['X-PUT-Timestamp']).ceil() return resp def update_data_record(self, record): @@ -804,7 +803,7 @@ class ContainerController(BaseStorageServer): ret = Response(request=req, headers=resp_headers, body=body, content_type=out_content_type, charset='utf-8') - ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp'])) + ret.last_modified = Timestamp(resp_headers['X-PUT-Timestamp']).ceil() if not ret.body: ret.status_int = HTTP_NO_CONTENT return ret diff --git a/swift/container/sharder.py b/swift/container/sharder.py index e933b7cb9..166fe91d2 100644 --- a/swift/container/sharder.py +++ b/swift/container/sharder.py @@ -1493,6 +1493,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): num_found += part_num_found if num_found: + # the found stat records the number of DBs in which any misplaced + # rows were found, not the total number of misplaced rows self._increment_stat('misplaced', 'found', statsd=True) self.logger.debug('Moved %s misplaced objects' % num_found) self._increment_stat('misplaced', 'success' if success else 'failure') diff --git a/swift/container/updater.py b/swift/container/updater.py index a22bf0b71..fc7b60eaf 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -31,7 +31,7 @@ from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout, LockTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, \ - dump_recon_cache, majority_size, Timestamp, ratelimit_sleep, \ + dump_recon_cache, majority_size, Timestamp, EventletRateLimiter, \ eventlet_monkey_patch from swift.common.daemon import Daemon from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR @@ -59,10 +59,10 @@ class ContainerUpdater(Daemon): float(conf.get('slowdown', '0.01')) + 0.01) else: containers_per_second = 50 - self.containers_running_time = 0 self.max_containers_per_second = \ float(conf.get('containers_per_second', containers_per_second)) + self.rate_limiter = EventletRateLimiter(self.max_containers_per_second) self.node_timeout = float(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.no_changes = 0 @@ -226,9 +226,7 @@ class ContainerUpdater(Daemon): self.logger.exception( "Error processing container %s: %s", dbfile, e) - self.containers_running_time = ratelimit_sleep( - self.containers_running_time, - self.max_containers_per_second) + self.rate_limiter.wait() def process_container(self, dbfile): """ diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index e0d95bb4d..f9013748a 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -30,7 +30,7 @@ from swift.common.daemon import Daemon from swift.common.storage_policy import POLICIES from swift.common.utils import ( config_auto_int_value, dump_recon_cache, get_logger, list_from_csv, - listdir, load_pkg_resource, parse_prefixed_conf, ratelimit_sleep, + listdir, load_pkg_resource, parse_prefixed_conf, EventletRateLimiter, readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter) from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH @@ -85,8 +85,10 @@ class AuditorWorker(object): self.auditor_type = 'ZBF' self.log_time = int(conf.get('log_time', 3600)) self.last_logged = 0 - self.files_running_time = 0 - self.bytes_running_time = 0 + self.files_rate_limiter = EventletRateLimiter( + self.max_files_per_second) + self.bytes_rate_limiter = EventletRateLimiter( + self.max_bytes_per_second) self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_files_processed = 0 @@ -146,8 +148,7 @@ class AuditorWorker(object): loop_time = time.time() self.failsafe_object_audit(location) self.logger.timing_since('timing', loop_time) - self.files_running_time = ratelimit_sleep( - self.files_running_time, self.max_files_per_second) + self.files_rate_limiter.wait() self.total_files_processed += 1 now = time.time() if now - self.last_logged >= self.log_time: @@ -266,10 +267,7 @@ class AuditorWorker(object): with closing(reader): for chunk in reader: chunk_len = len(chunk) - self.bytes_running_time = ratelimit_sleep( - self.bytes_running_time, - self.max_bytes_per_second, - incr_by=chunk_len) + self.bytes_rate_limiter.wait(incr_by=chunk_len) self.bytes_processed += chunk_len self.total_bytes_processed += chunk_len for watcher in self.watchers: diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 68ea00b75..588b087a0 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -330,7 +330,11 @@ def quarantine_renamer(device_path, corrupted_file_path): to_dir = join(device_path, 'quarantined', get_data_dir(policy), basename(from_dir)) - invalidate_hash(dirname(from_dir)) + if len(basename(from_dir)) == 3: + # quarantining whole suffix + invalidate_hash(from_dir) + else: + invalidate_hash(dirname(from_dir)) try: renamer(from_dir, to_dir, fsync=False) except OSError as e: @@ -1173,10 +1177,10 @@ class BaseDiskFileManager(object): ondisk_info = self.cleanup_ondisk_files( hsh_path, policy=policy) except OSError as err: + partition_path = dirname(path) + objects_path = dirname(partition_path) + device_path = dirname(objects_path) if err.errno == errno.ENOTDIR: - partition_path = dirname(path) - objects_path = dirname(partition_path) - device_path = dirname(objects_path) # The made-up filename is so that the eventual dirpath() # will result in this object directory that we care about. # Some failures will result in an object directory @@ -1190,6 +1194,24 @@ class BaseDiskFileManager(object): 'it is not a directory', {'hsh_path': hsh_path, 'quar_path': quar_path}) continue + elif err.errno == errno.ENODATA: + try: + # We've seen cases where bad sectors lead to ENODATA + # here; use a similar hack as above + quar_path = quarantine_renamer( + device_path, + join(hsh_path, "made-up-filename")) + orig_path = hsh_path + except (OSError, IOError): + # We've *also* seen the bad sectors lead to us needing + # to quarantine the whole suffix + quar_path = quarantine_renamer(device_path, hsh_path) + orig_path = path + logging.exception( + 'Quarantined %(orig_path)s to %(quar_path)s because ' + 'it could not be listed', {'orig_path': orig_path, + 'quar_path': quar_path}) + continue raise if not ondisk_info['files']: continue @@ -1527,6 +1549,24 @@ class BaseDiskFileManager(object): 'it is not a directory', {'object_path': object_path, 'quar_path': quar_path}) raise DiskFileNotExist() + elif err.errno == errno.ENODATA: + try: + # We've seen cases where bad sectors lead to ENODATA here; + # use a similar hack as above + quar_path = self.quarantine_renamer( + dev_path, + join(object_path, "made-up-filename")) + orig_path = object_path + except (OSError, IOError): + # We've *also* seen the bad sectors lead to us needing to + # quarantine the whole suffix, not just the hash dir + quar_path = self.quarantine_renamer(dev_path, object_path) + orig_path = os.path.dirname(object_path) + logging.exception( + 'Quarantined %(orig_path)s to %(quar_path)s because ' + 'it could not be listed', {'orig_path': orig_path, + 'quar_path': quar_path}) + raise DiskFileNotExist() if err.errno != errno.ENOENT: raise raise DiskFileNotExist() @@ -2528,6 +2568,20 @@ class BaseDiskFile(object): # want this one file and not its parent. os.path.join(self._datadir, "made-up-filename"), "Expected directory, found file at %s" % self._datadir) + elif err.errno == errno.ENODATA: + try: + # We've seen cases where bad sectors lead to ENODATA here + raise self._quarantine( + # similar hack to above + os.path.join(self._datadir, "made-up-filename"), + "Failed to list directory at %s" % self._datadir) + except (OSError, IOError): + # We've *also* seen the bad sectors lead to us needing to + # quarantine the whole suffix, not just the hash dir + raise self._quarantine( + # skip the above hack to rename the suffix + self._datadir, + "Failed to list directory at %s" % self._datadir) elif err.errno != errno.ENOENT: raise DiskFileError( "Error listing directory %s: %s" % (self._datadir, err)) diff --git a/swift/obj/server.py b/swift/obj/server.py index 35e7d4600..e157323bb 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -24,7 +24,6 @@ import multiprocessing import time import traceback import socket -import math from eventlet import sleep, wsgi, Timeout, tpool from eventlet.greenthread import spawn @@ -1113,7 +1112,7 @@ class ObjectController(BaseStorageServer): key.lower() in self.allowed_headers): response.headers[key] = value response.etag = metadata['ETag'] - response.last_modified = math.ceil(float(file_x_ts)) + response.last_modified = file_x_ts.ceil() response.content_length = obj_size try: response.content_encoding = metadata[ @@ -1181,7 +1180,7 @@ class ObjectController(BaseStorageServer): response.headers[key] = value response.etag = metadata['ETag'] ts = Timestamp(metadata['X-Timestamp']) - response.last_modified = math.ceil(float(ts)) + response.last_modified = ts.ceil() # Needed for container sync feature response.headers['X-Timestamp'] = ts.normal response.headers['X-Backend-Timestamp'] = ts.internal diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 01c609f13..2ee7c35fa 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -33,7 +33,8 @@ from swift.common.ring import Ring from swift.common.utils import get_logger, renamer, write_pickle, \ dump_recon_cache, config_true_value, RateLimitedIterator, split_path, \ eventlet_monkey_patch, get_redirect_data, ContextPool, hash_path, \ - non_negative_float, config_positive_int_value, non_negative_int + non_negative_float, config_positive_int_value, non_negative_int, \ + EventletRateLimiter from swift.common.daemon import Daemon from swift.common.header_key_dict import HeaderKeyDict from swift.common.storage_policy import split_policy_string, PolicyError @@ -43,16 +44,17 @@ from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR, \ HTTP_MOVED_PERMANENTLY -class RateLimiterBucket(object): - def __init__(self, update_delta): - self.update_delta = update_delta - self.last_time = 0 +class RateLimiterBucket(EventletRateLimiter): + """ + Extends EventletRateLimiter to also maintain a deque of items that have + been deferred due to rate-limiting, and to provide a comparator for sorting + instanced by readiness. + """ + def __init__(self, max_updates_per_second): + super(RateLimiterBucket, self).__init__(max_updates_per_second, + rate_buffer=0) self.deque = deque() - @property - def wait_until(self): - return self.last_time + self.update_delta - def __len__(self): return len(self.deque) @@ -62,10 +64,10 @@ class RateLimiterBucket(object): __nonzero__ = __bool__ # py2 def __lt__(self, other): - # used to sort buckets by readiness + # used to sort RateLimiterBuckets by readiness if isinstance(other, RateLimiterBucket): - return self.wait_until < other.wait_until - return self.wait_until < other + return self.running_time < other.running_time + return self.running_time < other class BucketizedUpdateSkippingLimiter(object): @@ -124,15 +126,11 @@ class BucketizedUpdateSkippingLimiter(object): self.stats = stats # if we want a smaller "blast radius" we could make this number bigger self.num_buckets = max(num_buckets, 1) - try: - self.bucket_update_delta = 1.0 / max_elements_per_group_per_second - except ZeroDivisionError: - self.bucket_update_delta = -1 self.max_deferred_elements = max_deferred_elements self.deferred_buckets = deque() self.drain_until = drain_until self.salt = str(uuid.uuid4()) - self.buckets = [RateLimiterBucket(self.bucket_update_delta) + self.buckets = [RateLimiterBucket(max_elements_per_group_per_second) for _ in range(self.num_buckets)] self.buckets_ordered_by_readiness = None @@ -151,9 +149,8 @@ class BucketizedUpdateSkippingLimiter(object): for update_ctx in self.iterator: bucket = self.buckets[self._bucket_key(update_ctx['update'])] now = self._get_time() - if now >= bucket.wait_until: + if bucket.is_allowed(now=now): # no need to ratelimit, just return next update - bucket.last_time = now return update_ctx self.stats.deferrals += 1 @@ -194,13 +191,12 @@ class BucketizedUpdateSkippingLimiter(object): bucket = self.buckets_ordered_by_readiness.get_nowait() if now < self.drain_until: # wait for next element to be ready - time.sleep(max(0, bucket.wait_until - now)) + bucket.wait(now=now) # drain the most recently deferred element item = bucket.deque.pop() if bucket: # bucket has more deferred elements, re-insert in queue in # correct chronological position - bucket.last_time = self._get_time() self.buckets_ordered_by_readiness.put(bucket) self.stats.drains += 1 self.logger.increment("drains") diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 825fbb7c2..e0b95107e 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -40,6 +40,7 @@ from eventlet import sleep from eventlet.timeout import Timeout import six +from swift.common.memcached import MemcacheConnectionError from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \ @@ -2400,9 +2401,13 @@ class Controller(object): if skip_chance and random.random() < skip_chance: self.logger.increment('shard_updating.cache.skip') else: - cached_ranges = memcache.get(cache_key) - self.logger.increment('shard_updating.cache.%s' % ( - 'hit' if cached_ranges else 'miss')) + try: + cached_ranges = memcache.get( + cache_key, raise_on_error=True) + cache_state = 'hit' if cached_ranges else 'miss' + except MemcacheConnectionError: + cache_state = 'error' + self.logger.increment('shard_updating.cache.%s' % cache_state) if cached_ranges: shard_ranges = [ diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index 4ac00d010..562f9afa5 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -14,12 +14,12 @@ # limitations under the License. import json -import math import random import six from six.moves.urllib.parse import unquote +from swift.common.memcached import MemcacheConnectionError from swift.common.utils import public, private, csv_append, Timestamp, \ config_true_value, ShardRange, cache_from_env, filter_shard_ranges from swift.common.constraints import check_metadata, CONTAINER_LISTING_LIMIT @@ -152,9 +152,14 @@ class ContainerController(Controller): if skip_chance and random.random() < skip_chance: self.logger.increment('shard_listing.cache.skip') else: - cached_ranges = memcache.get(cache_key) - self.logger.increment('shard_listing.cache.%s' % ( - 'hit' if cached_ranges else 'miss')) + try: + cached_ranges = memcache.get( + cache_key, raise_on_error=True) + cache_state = 'hit' if cached_ranges else 'miss' + except MemcacheConnectionError: + cache_state = 'error' + self.logger.increment( + 'shard_listing.cache.%s' % cache_state) if cached_ranges is not None: infocache[cache_key] = tuple(cached_ranges) @@ -170,8 +175,8 @@ class ContainerController(Controller): # GETorHEAD_base does not, so don't set it here either resp = Response(request=req, body=shard_range_body) update_headers(resp, headers) - resp.last_modified = math.ceil( - float(headers['x-put-timestamp'])) + resp.last_modified = Timestamp( + headers['x-put-timestamp']).ceil() resp.environ['swift_x_timestamp'] = headers.get( 'x-timestamp') resp.accept_ranges = 'bytes' diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index c3989bcea..04050ae6e 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -1009,8 +1009,7 @@ class ReplicatedObjectController(BaseObjectController): etag = etags.pop() if len(etags) else None resp = self.best_response(req, statuses, reasons, bodies, 'Object PUT', etag=etag) - resp.last_modified = math.ceil( - float(Timestamp(req.headers['X-Timestamp']))) + resp.last_modified = Timestamp(req.headers['X-Timestamp']).ceil() return resp @@ -3498,6 +3497,5 @@ class ECObjectController(BaseObjectController): resp = self.best_response(req, statuses, reasons, bodies, 'Object PUT', etag=etag, quorum_size=min_conns) - resp.last_modified = math.ceil( - float(Timestamp(req.headers['X-Timestamp']))) + resp.last_modified = Timestamp(req.headers['X-Timestamp']).ceil() return resp diff --git a/test/functional/s3api/test_multi_upload.py b/test/functional/s3api/test_multi_upload.py index 3ac8d668d..1ff0b5e8b 100644 --- a/test/functional/s3api/test_multi_upload.py +++ b/test/functional/s3api/test_multi_upload.py @@ -29,7 +29,8 @@ from six.moves import urllib, zip, zip_longest import test.functional as tf from swift.common.middleware.s3api.etree import fromstring, tostring, \ Element, SubElement -from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, mktime +from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, mktime, \ + S3Timestamp from swift.common.utils import md5 from test.functional.s3api import S3ApiBase @@ -213,7 +214,8 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], '0') - expected_parts_list = [(headers['etag'], mktime(headers['date']))] + expected_parts_list = [(headers['etag'], + mktime(headers['last-modified']))] # Upload Part Copy key, upload_id = uploads[1] @@ -242,8 +244,8 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') - last_modified = elem.find('LastModified').text - self.assertTrue(last_modified is not None) + copy_resp_last_modified = elem.find('LastModified').text + self.assertIsNotNone(copy_resp_last_modified) self.assertEqual(resp_etag, etag) @@ -256,15 +258,10 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertEqual(200, status) elem = fromstring(body, 'ListPartsResult') - # FIXME: COPY result drops milli/microseconds but GET doesn't - last_modified_gets = [p.find('LastModified').text - for p in elem.iterfind('Part')] - self.assertEqual( - last_modified_gets[0].rsplit('.', 1)[0], - last_modified.rsplit('.', 1)[0], - '%r != %r' % (last_modified_gets[0], last_modified)) - # There should be *exactly* two parts in the result - self.assertEqual(1, len(last_modified_gets)) + listing_last_modified = [p.find('LastModified').text + for p in elem.iterfind('Part')] + # There should be *exactly* one parts in the result + self.assertEqual(listing_last_modified, [copy_resp_last_modified]) # List Parts key, upload_id = uploads[0] @@ -299,15 +296,10 @@ class TestS3ApiMultiUpload(S3ApiBase): for (expected_etag, expected_date), p in \ zip(expected_parts_list, elem.findall('Part')): last_modified = p.find('LastModified').text - self.assertTrue(last_modified is not None) - # TODO: sanity check - # (kota_) How do we check the sanity? - # the last-modified header drops milli-seconds info - # by the constraint of the format. - # For now, we can do either the format check or round check - # last_modified_from_xml = mktime(last_modified) - # self.assertEqual(expected_date, - # last_modified_from_xml) + self.assertIsNotNone(last_modified) + last_modified_from_xml = S3Timestamp.from_s3xmlformat( + last_modified) + self.assertEqual(expected_date, float(last_modified_from_xml)) self.assertEqual(expected_etag, p.find('ETag').text) self.assertEqual(self.min_segment_size, int(p.find('Size').text)) etags.append(p.find('ETag').text) @@ -496,7 +488,7 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertIsNotNone(o.find('LastModified').text) self.assertRegex( o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') + r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000Z$') self.assertEqual(o.find('ETag').text, exp_etag) self.assertEqual(o.find('Size').text, str(exp_size)) self.assertIsNotNone(o.find('StorageClass').text) @@ -932,8 +924,8 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') - last_modified = elem.find('LastModified').text - self.assertTrue(last_modified is not None) + copy_resp_last_modified = elem.find('LastModified').text + self.assertIsNotNone(copy_resp_last_modified) self.assertEqual(resp_etag, etag) @@ -945,16 +937,10 @@ class TestS3ApiMultiUpload(S3ApiBase): elem = fromstring(body, 'ListPartsResult') - # FIXME: COPY result drops milli/microseconds but GET doesn't - last_modified_gets = [p.find('LastModified').text - for p in elem.iterfind('Part')] - self.assertEqual( - last_modified_gets[0].rsplit('.', 1)[0], - last_modified.rsplit('.', 1)[0], - '%r != %r' % (last_modified_gets[0], last_modified)) - + listing_last_modified = [p.find('LastModified').text + for p in elem.iterfind('Part')] # There should be *exactly* one parts in the result - self.assertEqual(1, len(last_modified_gets)) + self.assertEqual(listing_last_modified, [copy_resp_last_modified]) # Abort Multipart Upload key, upload_id = uploads[0] @@ -1044,8 +1030,8 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') - last_modifieds = [elem.find('LastModified').text] - self.assertTrue(last_modifieds[0] is not None) + copy_resp_last_modifieds = [elem.find('LastModified').text] + self.assertTrue(copy_resp_last_modifieds[0] is not None) self.assertEqual(resp_etag, etags[0]) @@ -1062,8 +1048,8 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') - last_modifieds.append(elem.find('LastModified').text) - self.assertTrue(last_modifieds[1] is not None) + copy_resp_last_modifieds.append(elem.find('LastModified').text) + self.assertTrue(copy_resp_last_modifieds[1] is not None) self.assertEqual(resp_etag, etags[1]) @@ -1075,15 +1061,9 @@ class TestS3ApiMultiUpload(S3ApiBase): elem = fromstring(body, 'ListPartsResult') - # FIXME: COPY result drops milli/microseconds but GET doesn't - last_modified_gets = [p.find('LastModified').text - for p in elem.iterfind('Part')] - self.assertEqual( - [lm.rsplit('.', 1)[0] for lm in last_modified_gets], - [lm.rsplit('.', 1)[0] for lm in last_modifieds]) - - # There should be *exactly* two parts in the result - self.assertEqual(2, len(last_modified_gets)) + listing_last_modified = [p.find('LastModified').text + for p in elem.iterfind('Part')] + self.assertEqual(listing_last_modified, copy_resp_last_modifieds) # Abort Multipart Upload key, upload_id = uploads[0] diff --git a/test/functional/s3api/test_object.py b/test/functional/s3api/test_object.py index f8245318e..5c0d84753 100644 --- a/test/functional/s3api/test_object.py +++ b/test/functional/s3api/test_object.py @@ -22,6 +22,7 @@ import boto # pylint: disable-msg=E0611,F0401 from distutils.version import StrictVersion +import calendar import email.parser from email.utils import formatdate, parsedate from time import mktime @@ -30,6 +31,7 @@ import six import test.functional as tf from swift.common.middleware.s3api.etree import fromstring +from swift.common.middleware.s3api.utils import S3Timestamp from swift.common.utils import md5, quote from test.functional.s3api import S3ApiBase @@ -98,21 +100,32 @@ class TestS3ApiObject(S3ApiBase): elem = fromstring(body, 'CopyObjectResult') self.assertTrue(elem.find('LastModified').text is not None) - last_modified_xml = elem.find('LastModified').text + copy_resp_last_modified_xml = elem.find('LastModified').text self.assertTrue(elem.find('ETag').text is not None) self.assertEqual(etag, elem.find('ETag').text.strip('"')) self._assertObjectEtag(dst_bucket, dst_obj, etag) - # Check timestamp on Copy: + # Check timestamp on Copy in listing: status, headers, body = \ self.conn.make_request('GET', dst_bucket) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') + self.assertEqual( + elem.find('Contents').find("LastModified").text, + copy_resp_last_modified_xml) + + # GET Object copy + status, headers, body = \ + self.conn.make_request('GET', dst_bucket, dst_obj) + self.assertEqual(status, 200) - # FIXME: COPY result drops milli/microseconds but GET doesn't + self.assertCommonResponseHeaders(headers, etag) + self.assertTrue(headers['last-modified'] is not None) self.assertEqual( - elem.find('Contents').find("LastModified").text.rsplit('.', 1)[0], - last_modified_xml.rsplit('.', 1)[0]) + float(S3Timestamp.from_s3xmlformat(copy_resp_last_modified_xml)), + calendar.timegm(parsedate(headers['last-modified']))) + self.assertTrue(headers['content-type'] is not None) + self.assertEqual(headers['content-length'], str(len(content))) # GET Object status, headers, body = \ @@ -770,6 +783,26 @@ class TestS3ApiObject(S3ApiBase): self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) + # check we can use the last modified time from the listing... + status, headers, body = \ + self.conn.make_request('GET', self.bucket) + elem = fromstring(body, 'ListBucketResult') + last_modified = elem.find('./Contents/LastModified').text + listing_datetime = S3Timestamp.from_s3xmlformat(last_modified) + headers = \ + {'If-Unmodified-Since': formatdate(listing_datetime)} + status, headers, body = \ + self.conn.make_request('GET', self.bucket, obj, headers=headers) + self.assertEqual(status, 200) + self.assertCommonResponseHeaders(headers) + + headers = \ + {'If-Modified-Since': formatdate(listing_datetime)} + status, headers, body = \ + self.conn.make_request('GET', self.bucket, obj, headers=headers) + self.assertEqual(status, 304) + self.assertCommonResponseHeaders(headers) + def test_get_object_if_match(self): obj = 'object' self.conn.make_request('PUT', self.bucket, obj) diff --git a/test/functional/s3api/test_presigned.py b/test/functional/s3api/test_presigned.py index a06206b8b..4ee115660 100644 --- a/test/functional/s3api/test_presigned.py +++ b/test/functional/s3api/test_presigned.py @@ -92,7 +92,7 @@ class TestS3ApiPresignedUrls(S3ApiBase): self.assertIsNotNone(o.find('LastModified').text) self.assertRegex( o.find('LastModified').text, - r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') + r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000Z$') self.assertIsNotNone(o.find('ETag').text) self.assertEqual(o.find('Size').text, '0') self.assertIsNotNone(o.find('StorageClass').text is not None) diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 40d606933..266763e0a 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -410,17 +410,22 @@ def track(f): class FakeMemcache(object): - def __init__(self): + def __init__(self, error_on_set=None, error_on_get=None): self.store = {} self.calls = [] self.error_on_incr = False + self.error_on_get = error_on_get or [] + self.error_on_set = error_on_set or [] self.init_incr_return_neg = False def clear_calls(self): del self.calls[:] @track - def get(self, key): + def get(self, key, raise_on_error=False): + if self.error_on_get and self.error_on_get.pop(0): + if raise_on_error: + raise MemcacheConnectionError() return self.store.get(key) @property @@ -428,7 +433,10 @@ class FakeMemcache(object): return self.store.keys @track - def set(self, key, value, serialize=True, time=0): + def set(self, key, value, serialize=True, time=0, raise_on_error=False): + if self.error_on_set and self.error_on_set.pop(0): + if raise_on_error: + raise MemcacheConnectionError() if serialize: value = json.loads(json.dumps(value)) else: @@ -1042,9 +1050,9 @@ def make_timestamp_iter(offset=0): @contextmanager -def mock_timestamp_now(now=None): +def mock_timestamp_now(now=None, klass=Timestamp): if now is None: - now = Timestamp.now() + now = klass.now() with mocklib.patch('swift.common.utils.Timestamp.now', classmethod(lambda c: now)): yield now diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index 6b6b1dc73..2eae7faa9 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -201,7 +201,7 @@ class TestS3ApiBucket(S3ApiTestCase): items = [] for o in objects: items.append((o.find('./Key').text, o.find('./ETag').text)) - self.assertEqual('2011-01-05T02:19:14.275Z', + self.assertEqual('2011-01-05T02:19:15.000Z', o.find('./LastModified').text) expected = [ (i[0].encode('utf-8') if six.PY2 else i[0], @@ -211,6 +211,37 @@ class TestS3ApiBucket(S3ApiTestCase): ] self.assertEqual(items, expected) + def test_bucket_GET_last_modified_rounding(self): + objects_list = [ + {'name': 'a', 'last_modified': '2011-01-05T02:19:59.275290', + 'content_type': 'application/octet-stream', + 'hash': 'ahash', 'bytes': '12345'}, + {'name': 'b', 'last_modified': '2011-01-05T02:19:59.000000', + 'content_type': 'application/octet-stream', + 'hash': 'ahash', 'bytes': '12345'}, + ] + self.swift.register( + 'GET', '/v1/AUTH_test/junk', + swob.HTTPOk, {'Content-Type': 'application/json'}, + json.dumps(objects_list)) + req = Request.blank('/junk', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + + elem = fromstring(body, 'ListBucketResult') + name = elem.find('./Name').text + self.assertEqual(name, 'junk') + objects = elem.iterchildren('Contents') + actual = [(obj.find('./Key').text, obj.find('./LastModified').text) + for obj in objects] + self.assertEqual( + [('a', '2011-01-05T02:20:00.000Z'), + ('b', '2011-01-05T02:19:59.000Z')], + actual) + def test_bucket_GET_url_encoded(self): bucket_name = 'junk' req = Request.blank('/%s?encoding-type=url' % bucket_name, @@ -229,7 +260,7 @@ class TestS3ApiBucket(S3ApiTestCase): items = [] for o in objects: items.append((o.find('./Key').text, o.find('./ETag').text)) - self.assertEqual('2011-01-05T02:19:14.275Z', + self.assertEqual('2011-01-05T02:19:15.000Z', o.find('./LastModified').text) self.assertEqual(items, [ @@ -673,9 +704,9 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual([v.find('./VersionId').text for v in versions], ['null' for v in objects]) # Last modified in self.objects is 2011-01-05T02:19:14.275290 but - # the returned value is 2011-01-05T02:19:14.275Z + # the returned value is rounded up to 2011-01-05T02:19:15Z self.assertEqual([v.find('./LastModified').text for v in versions], - [v[1][:-3] + 'Z' for v in objects]) + ['2011-01-05T02:19:15.000Z'] * len(objects)) self.assertEqual([v.find('./ETag').text for v in versions], [PFS_ETAG if v[0] == 'pfs-obj' else '"0-N"' if v[0] == 'slo' else '"0"' diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py index 63ab8e81f..4eff8015a 100644 --- a/test/unit/common/middleware/s3api/test_multi_upload.py +++ b/test/unit/common/middleware/s3api/test_multi_upload.py @@ -51,29 +51,41 @@ XML = '<CompleteMultipartUpload>' \ '</CompleteMultipartUpload>' OBJECTS_TEMPLATE = \ - (('object/X/1', '2014-05-07T19:47:51.592270', '0123456789abcdef', 100), - ('object/X/2', '2014-05-07T19:47:52.592270', 'fedcba9876543210', 200)) + (('object/X/1', '2014-05-07T19:47:51.592270', '0123456789abcdef', 100, + '2014-05-07T19:47:52.000Z'), + ('object/X/2', '2014-05-07T19:47:52.592270', 'fedcba9876543210', 200, + '2014-05-07T19:47:53.000Z')) MULTIPARTS_TEMPLATE = \ - (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), - ('object/X/1', '2014-05-07T19:47:51.592270', '0123456789abcdef', 11), - ('object/X/2', '2014-05-07T19:47:52.592270', 'fedcba9876543210', 21), - ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2), - ('object/Y/1', '2014-05-07T19:47:54.592270', '0123456789abcdef', 12), - ('object/Y/2', '2014-05-07T19:47:55.592270', 'fedcba9876543210', 22), - ('object/Z', '2014-05-07T19:47:56.592270', 'HASH', 3), - ('object/Z/1', '2014-05-07T19:47:57.592270', '0123456789abcdef', 13), - ('object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210', 23), - ('subdir/object/Z', '2014-05-07T19:47:58.592270', 'HASH', 4), + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1, + '2014-05-07T19:47:51.000Z'), + ('object/X/1', '2014-05-07T19:47:51.592270', '0123456789abcdef', 11, + '2014-05-07T19:47:52.000Z'), + ('object/X/2', '2014-05-07T19:47:52.592270', 'fedcba9876543210', 21, + '2014-05-07T19:47:53.000Z'), + ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2, + '2014-05-07T19:47:54.000Z'), + ('object/Y/1', '2014-05-07T19:47:54.592270', '0123456789abcdef', 12, + '2014-05-07T19:47:55.000Z'), + ('object/Y/2', '2014-05-07T19:47:55.592270', 'fedcba9876543210', 22, + '2014-05-07T19:47:56.000Z'), + ('object/Z', '2014-05-07T19:47:56.592270', 'HASH', 3, + '2014-05-07T19:47:57.000Z'), + ('object/Z/1', '2014-05-07T19:47:57.592270', '0123456789abcdef', 13, + '2014-05-07T19:47:58.000Z'), + ('object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210', 23, + '2014-05-07T19:47:59.000Z'), + ('subdir/object/Z', '2014-05-07T19:47:58.592270', 'HASH', 4, + '2014-05-07T19:47:59.000Z'), ('subdir/object/Z/1', '2014-05-07T19:47:58.592270', '0123456789abcdef', - 41), + 41, '2014-05-07T19:47:59.000Z'), ('subdir/object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210', - 41), + 41, '2014-05-07T19:47:59.000Z'), # NB: wsgi strings ('subdir/object/completed\xe2\x98\x83/W/1', '2014-05-07T19:47:58.592270', - '0123456789abcdef', 41), + '0123456789abcdef', 41, '2014-05-07T19:47:59.000Z'), ('subdir/object/completed\xe2\x98\x83/W/2', '2014-05-07T19:47:58.592270', - 'fedcba9876543210', 41)) + 'fedcba9876543210', 41, '2014-05-07T19:47:59')) S3_ETAG = '"%s-2"' % md5(binascii.a2b_hex( '0123456789abcdef0123456789abcdef' @@ -285,7 +297,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(elem.find('MaxUploads').text, '1000') self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(len(elem.findall('Upload')), len(uploads)) - expected_uploads = [(upload[0], '2014-05-07T19:47:50.592Z') + expected_uploads = [(upload[0], '2014-05-07T19:47:51.000Z') for upload in uploads] for u in elem.findall('Upload'): name = u.find('Key').text + '/' + u.find('UploadId').text @@ -310,7 +322,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(elem.find('MaxUploads').text, '1000') self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(len(elem.findall('Upload')), 4) - objects = [(o[0], o[1][:-3] + 'Z') for o in MULTIPARTS_TEMPLATE] + objects = [(o[0], o[4]) for o in MULTIPARTS_TEMPLATE] for u in elem.findall('Upload'): name = u.find('Key').text + '/' + u.find('UploadId').text initiated = u.find('Initiated').text @@ -417,9 +429,12 @@ class TestS3ApiMultiUpload(S3ApiTestCase): def test_bucket_multipart_uploads_GET_with_id_and_key_marker(self): query = 'upload-id-marker=Y&key-marker=object' multiparts = \ - (('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2), - ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12), - ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22)) + (('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2, + '2014-05-07T19:47:54.000Z'), + ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12, + '2014-05-07T19:47:55.000Z'), + ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22, + '2014-05-07T19:47:56.000Z')) status, headers, body = \ self._test_bucket_multipart_uploads_GET(query, multiparts) @@ -427,7 +442,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(elem.find('KeyMarker').text, 'object') self.assertEqual(elem.find('UploadIdMarker').text, 'Y') self.assertEqual(len(elem.findall('Upload')), 1) - objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts] + objects = [(o[0], o[4]) for o in multiparts] for u in elem.findall('Upload'): name = u.find('Key').text + '/' + u.find('UploadId').text initiated = u.find('Initiated').text @@ -447,12 +462,18 @@ class TestS3ApiMultiUpload(S3ApiTestCase): def test_bucket_multipart_uploads_GET_with_key_marker(self): query = 'key-marker=object' multiparts = \ - (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), - ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), - ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21), - ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2), - ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12), - ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22)) + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1, + '2014-05-07T19:47:51.000Z'), + ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11, + '2014-05-07T19:47:52.000Z'), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21, + '2014-05-07T19:47:53.000Z'), + ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2, + '2014-05-07T19:47:54.000Z'), + ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12, + '2014-05-07T19:47:55.000Z'), + ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22, + '2014-05-07T19:47:56.000Z')) status, headers, body = \ self._test_bucket_multipart_uploads_GET(query, multiparts) elem = fromstring(body, 'ListMultipartUploadsResult') @@ -460,11 +481,11 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(elem.find('NextKeyMarker').text, 'object') self.assertEqual(elem.find('NextUploadIdMarker').text, 'Y') self.assertEqual(len(elem.findall('Upload')), 2) - objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts] + objects = [(o[0], o[4]) for o in multiparts] for u in elem.findall('Upload'): name = u.find('Key').text + '/' + u.find('UploadId').text initiated = u.find('Initiated').text - self.assertTrue((name, initiated) in objects) + self.assertIn((name, initiated), objects) self.assertEqual(status.split()[0], '200') _, path, _ = self.swift.calls_with_headers[-1] @@ -480,14 +501,17 @@ class TestS3ApiMultiUpload(S3ApiTestCase): def test_bucket_multipart_uploads_GET_with_prefix(self): query = 'prefix=X' multiparts = \ - (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), - ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), - ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21)) + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1, + '2014-05-07T19:47:51.000Z'), + ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11, + '2014-05-07T19:47:52.000Z'), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21, + '2014-05-07T19:47:53.000Z')) status, headers, body = \ self._test_bucket_multipart_uploads_GET(query, multiparts) elem = fromstring(body, 'ListMultipartUploadsResult') self.assertEqual(len(elem.findall('Upload')), 1) - objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts] + objects = [(o[0], o[4]) for o in multiparts] for u in elem.findall('Upload'): name = u.find('Key').text + '/' + u.find('UploadId').text initiated = u.find('Initiated').text @@ -507,38 +531,56 @@ class TestS3ApiMultiUpload(S3ApiTestCase): def test_bucket_multipart_uploads_GET_with_delimiter(self): query = 'delimiter=/' multiparts = \ - (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), - ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), - ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21), - ('object/Y', '2014-05-07T19:47:50.592270', 'HASH', 2), - ('object/Y/1', '2014-05-07T19:47:51.592270', 'HASH', 21), - ('object/Y/2', '2014-05-07T19:47:52.592270', 'HASH', 22), - ('object/Z', '2014-05-07T19:47:50.592270', 'HASH', 3), - ('object/Z/1', '2014-05-07T19:47:51.592270', 'HASH', 31), - ('object/Z/2', '2014-05-07T19:47:52.592270', 'HASH', 32), - ('subdir/object/X', '2014-05-07T19:47:50.592270', 'HASH', 4), - ('subdir/object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 41), - ('subdir/object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 42), - ('subdir/object/Y', '2014-05-07T19:47:50.592270', 'HASH', 5), - ('subdir/object/Y/1', '2014-05-07T19:47:51.592270', 'HASH', 51), - ('subdir/object/Y/2', '2014-05-07T19:47:52.592270', 'HASH', 52), - ('subdir2/object/Z', '2014-05-07T19:47:50.592270', 'HASH', 6), - ('subdir2/object/Z/1', '2014-05-07T19:47:51.592270', 'HASH', 61), - ('subdir2/object/Z/2', '2014-05-07T19:47:52.592270', 'HASH', 62)) + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1, + '2014-05-07T19:47:51.000Z'), + ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11, + '2014-05-07T19:47:52.000Z'), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21, + '2014-05-07T19:47:53.000Z'), + ('object/Y', '2014-05-07T19:47:50.592270', 'HASH', 2, + '2014-05-07T19:47:51.000Z'), + ('object/Y/1', '2014-05-07T19:47:51.592270', 'HASH', 21, + '2014-05-07T19:47:52.000Z'), + ('object/Y/2', '2014-05-07T19:47:52.592270', 'HASH', 22, + '2014-05-07T19:47:53.000Z'), + ('object/Z', '2014-05-07T19:47:50.592270', 'HASH', 3, + '2014-05-07T19:47:51.000Z'), + ('object/Z/1', '2014-05-07T19:47:51.592270', 'HASH', 31, + '2014-05-07T19:47:52.000Z'), + ('object/Z/2', '2014-05-07T19:47:52.592270', 'HASH', 32, + '2014-05-07T19:47:53.000Z'), + ('subdir/object/X', '2014-05-07T19:47:50.592270', 'HASH', 4, + '2014-05-07T19:47:51.000Z'), + ('subdir/object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 41, + '2014-05-07T19:47:52.000Z'), + ('subdir/object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 42, + '2014-05-07T19:47:53.000Z'), + ('subdir/object/Y', '2014-05-07T19:47:50.592270', 'HASH', 5, + '2014-05-07T19:47:51.000Z'), + ('subdir/object/Y/1', '2014-05-07T19:47:51.592270', 'HASH', 51, + '2014-05-07T19:47:52.000Z'), + ('subdir/object/Y/2', '2014-05-07T19:47:52.592270', 'HASH', 52, + '2014-05-07T19:47:53.000Z'), + ('subdir2/object/Z', '2014-05-07T19:47:50.592270', 'HASH', 6, + '2014-05-07T19:47:51.000Z'), + ('subdir2/object/Z/1', '2014-05-07T19:47:51.592270', 'HASH', 61, + '2014-05-07T19:47:52.000Z'), + ('subdir2/object/Z/2', '2014-05-07T19:47:52.592270', 'HASH', 62, + '2014-05-07T19:47:53.000Z')) status, headers, body = \ self._test_bucket_multipart_uploads_GET(query, multiparts) elem = fromstring(body, 'ListMultipartUploadsResult') self.assertEqual(len(elem.findall('Upload')), 3) self.assertEqual(len(elem.findall('CommonPrefixes')), 2) - objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts + objects = [(o[0], o[4]) for o in multiparts if o[0].startswith('o')] prefixes = set([o[0].split('/')[0] + '/' for o in multiparts if o[0].startswith('s')]) for u in elem.findall('Upload'): name = u.find('Key').text + '/' + u.find('UploadId').text initiated = u.find('Initiated').text - self.assertTrue((name, initiated) in objects) + self.assertIn((name, initiated), objects) for p in elem.findall('CommonPrefixes'): prefix = p.find('Prefix').text self.assertTrue(prefix in prefixes) @@ -557,31 +599,43 @@ class TestS3ApiMultiUpload(S3ApiTestCase): def test_bucket_multipart_uploads_GET_with_multi_chars_delimiter(self): query = 'delimiter=subdir' multiparts = \ - (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), - ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), - ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21), + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1, + '2014-05-07T19:47:51.000Z'), + ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11, + '2014-05-07T19:47:52.000Z'), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21, + '2014-05-07T19:47:53.000Z'), ('dir/subdir/object/X', '2014-05-07T19:47:50.592270', - 'HASH', 3), + 'HASH', 3, '2014-05-07T19:47:51.000Z'), ('dir/subdir/object/X/1', '2014-05-07T19:47:51.592270', - 'HASH', 31), + 'HASH', 31, '2014-05-07T19:47:52.000Z'), ('dir/subdir/object/X/2', '2014-05-07T19:47:52.592270', - 'HASH', 32), - ('subdir/object/X', '2014-05-07T19:47:50.592270', 'HASH', 4), - ('subdir/object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 41), - ('subdir/object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 42), - ('subdir/object/Y', '2014-05-07T19:47:50.592270', 'HASH', 5), - ('subdir/object/Y/1', '2014-05-07T19:47:51.592270', 'HASH', 51), - ('subdir/object/Y/2', '2014-05-07T19:47:52.592270', 'HASH', 52), - ('subdir2/object/Z', '2014-05-07T19:47:50.592270', 'HASH', 6), - ('subdir2/object/Z/1', '2014-05-07T19:47:51.592270', 'HASH', 61), - ('subdir2/object/Z/2', '2014-05-07T19:47:52.592270', 'HASH', 62)) + 'HASH', 32, '2014-05-07T19:47:53.000Z'), + ('subdir/object/X', '2014-05-07T19:47:50.592270', 'HASH', 4, + '2014-05-07T19:47:51.000Z'), + ('subdir/object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 41, + '2014-05-07T19:47:52.000Z'), + ('subdir/object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 42, + '2014-05-07T19:47:53.000Z'), + ('subdir/object/Y', '2014-05-07T19:47:50.592270', 'HASH', 5, + '2014-05-07T19:47:51.000Z'), + ('subdir/object/Y/1', '2014-05-07T19:47:51.592270', 'HASH', 51, + '2014-05-07T19:47:52.000Z'), + ('subdir/object/Y/2', '2014-05-07T19:47:52.592270', 'HASH', 52, + '2014-05-07T19:47:53.000Z'), + ('subdir2/object/Z', '2014-05-07T19:47:50.592270', 'HASH', 6, + '2014-05-07T19:47:51.000Z'), + ('subdir2/object/Z/1', '2014-05-07T19:47:51.592270', 'HASH', 61, + '2014-05-07T19:47:52.000Z'), + ('subdir2/object/Z/2', '2014-05-07T19:47:52.592270', 'HASH', 62, + '2014-05-07T19:47:53.000Z')) status, headers, body = \ self._test_bucket_multipart_uploads_GET(query, multiparts) elem = fromstring(body, 'ListMultipartUploadsResult') self.assertEqual(len(elem.findall('Upload')), 1) self.assertEqual(len(elem.findall('CommonPrefixes')), 2) - objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts + objects = [(o[0], o[4]) for o in multiparts if o[0].startswith('object')] prefixes = ('dir/subdir', 'subdir') for u in elem.findall('Upload'): @@ -607,27 +661,30 @@ class TestS3ApiMultiUpload(S3ApiTestCase): query = 'prefix=dir/&delimiter=/' multiparts = \ (('dir/subdir/object/X', '2014-05-07T19:47:50.592270', - 'HASH', 4), + 'HASH', 4, '2014-05-07T19:47:51.000Z'), ('dir/subdir/object/X/1', '2014-05-07T19:47:51.592270', - 'HASH', 41), + 'HASH', 41, '2014-05-07T19:47:52.000Z'), ('dir/subdir/object/X/2', '2014-05-07T19:47:52.592270', - 'HASH', 42), - ('dir/object/X', '2014-05-07T19:47:50.592270', 'HASH', 5), - ('dir/object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 51), - ('dir/object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 52)) + 'HASH', 42, '2014-05-07T19:47:53.000Z'), + ('dir/object/X', '2014-05-07T19:47:50.592270', 'HASH', 5, + '2014-05-07T19:47:51.000Z'), + ('dir/object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 51, + '2014-05-07T19:47:52.000Z'), + ('dir/object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 52, + '2014-05-07T19:47:53.000Z')) status, headers, body = \ self._test_bucket_multipart_uploads_GET(query, multiparts) elem = fromstring(body, 'ListMultipartUploadsResult') self.assertEqual(len(elem.findall('Upload')), 1) self.assertEqual(len(elem.findall('CommonPrefixes')), 1) - objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts + objects = [(o[0], o[4]) for o in multiparts if o[0].startswith('dir/o')] prefixes = ['dir/subdir/'] for u in elem.findall('Upload'): name = u.find('Key').text + '/' + u.find('UploadId').text initiated = u.find('Initiated').text - self.assertTrue((name, initiated) in objects) + self.assertIn((name, initiated), objects) for p in elem.findall('CommonPrefixes'): prefix = p.find('Prefix').text self.assertTrue(prefix in prefixes) @@ -1838,6 +1895,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase): 'hash': hex(i), 'bytes': 100 * i} for i in range(1, 2000)] + ceil_last_modified = ['2014-05-07T19:%02d:%02d.000Z' + % (47 if (i + 1) % 60 else 48, (i + 1) % 60) + for i in range(1, 2000)] swift_sorted = sorted(swift_parts, key=lambda part: part['name']) self.swift.register('GET', "%s?delimiter=/&format=json&marker=&" @@ -1872,7 +1932,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): s3_parts.append(partnum) self.assertEqual( p.find('LastModified').text, - swift_parts[partnum - 1]['last_modified'][:-3] + 'Z') + ceil_last_modified[partnum - 1]) self.assertEqual(p.find('ETag').text.strip(), '"%s"' % swift_parts[partnum - 1]['hash']) self.assertEqual(p.find('Size').text, @@ -1970,7 +2030,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): for p in elem.findall('Part'): partnum = int(p.find('PartNumber').text) self.assertEqual(p.find('LastModified').text, - OBJECTS_TEMPLATE[partnum - 1][1][:-3] + 'Z') + OBJECTS_TEMPLATE[partnum - 1][4]) self.assertEqual(p.find('ETag').text, '"%s"' % OBJECTS_TEMPLATE[partnum - 1][2]) self.assertEqual(p.find('Size').text, diff --git a/test/unit/common/middleware/s3api/test_obj.py b/test/unit/common/middleware/s3api/test_obj.py index cd7c06d2d..870bf7acf 100644 --- a/test/unit/common/middleware/s3api/test_obj.py +++ b/test/unit/common/middleware/s3api/test_obj.py @@ -28,6 +28,7 @@ import json from swift.common import swob from swift.common.swob import Request from swift.common.middleware.proxy_logging import ProxyLoggingMiddleware +from test.unit import mock_timestamp_now from test.unit.common.middleware.s3api import S3ApiTestCase from test.unit.common.middleware.s3api.test_s3_acl import s3acl @@ -872,33 +873,29 @@ class TestS3ApiObj(S3ApiTestCase): @s3acl def test_object_PUT_copy_metadata_replace(self): - date_header = self.get_date_header() - timestamp = mktime(date_header) - allowed_last_modified = [S3Timestamp(timestamp).s3xmlformat] - status, headers, body = \ - self._test_object_PUT_copy( - swob.HTTPOk, - {'X-Amz-Metadata-Directive': 'REPLACE', - 'X-Amz-Meta-Something': 'oh hai', - 'X-Amz-Meta-Unreadable-Prefix': '\x04w', - 'X-Amz-Meta-Unreadable-Suffix': 'h\x04', - 'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04', - 'Cache-Control': 'hello', - 'content-disposition': 'how are you', - 'content-encoding': 'good and you', - 'content-language': 'great', - 'content-type': 'so', - 'expires': 'yeah', - 'x-robots-tag': 'bye'}) - date_header = self.get_date_header() - timestamp = mktime(date_header) - allowed_last_modified.append(S3Timestamp(timestamp).s3xmlformat) + with mock_timestamp_now(klass=S3Timestamp) as now: + status, headers, body = \ + self._test_object_PUT_copy( + swob.HTTPOk, + {'X-Amz-Metadata-Directive': 'REPLACE', + 'X-Amz-Meta-Something': 'oh hai', + 'X-Amz-Meta-Unreadable-Prefix': '\x04w', + 'X-Amz-Meta-Unreadable-Suffix': 'h\x04', + 'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04', + 'Cache-Control': 'hello', + 'content-disposition': 'how are you', + 'content-encoding': 'good and you', + 'content-language': 'great', + 'content-type': 'so', + 'expires': 'yeah', + 'x-robots-tag': 'bye'}) self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Content-Type'], 'application/xml') self.assertIsNone(headers.get('etag')) elem = fromstring(body, 'CopyObjectResult') - self.assertIn(elem.find('LastModified').text, allowed_last_modified) + self.assertEqual(S3Timestamp(now.ceil()).s3xmlformat, + elem.find('LastModified').text) self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) _, _, headers = self.swift.calls_with_headers[-1] @@ -926,34 +923,30 @@ class TestS3ApiObj(S3ApiTestCase): @s3acl def test_object_PUT_copy_metadata_copy(self): - date_header = self.get_date_header() - timestamp = mktime(date_header) - allowed_last_modified = [S3Timestamp(timestamp).s3xmlformat] - status, headers, body = \ - self._test_object_PUT_copy( - swob.HTTPOk, - {'X-Amz-Metadata-Directive': 'COPY', - 'X-Amz-Meta-Something': 'oh hai', - 'X-Amz-Meta-Unreadable-Prefix': '\x04w', - 'X-Amz-Meta-Unreadable-Suffix': 'h\x04', - 'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04', - 'Cache-Control': 'hello', - 'content-disposition': 'how are you', - 'content-encoding': 'good and you', - 'content-language': 'great', - 'content-type': 'so', - 'expires': 'yeah', - 'x-robots-tag': 'bye'}) - date_header = self.get_date_header() - timestamp = mktime(date_header) - allowed_last_modified.append(S3Timestamp(timestamp).s3xmlformat) + with mock_timestamp_now(klass=S3Timestamp) as now: + status, headers, body = \ + self._test_object_PUT_copy( + swob.HTTPOk, + {'X-Amz-Metadata-Directive': 'COPY', + 'X-Amz-Meta-Something': 'oh hai', + 'X-Amz-Meta-Unreadable-Prefix': '\x04w', + 'X-Amz-Meta-Unreadable-Suffix': 'h\x04', + 'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04', + 'Cache-Control': 'hello', + 'content-disposition': 'how are you', + 'content-encoding': 'good and you', + 'content-language': 'great', + 'content-type': 'so', + 'expires': 'yeah', + 'x-robots-tag': 'bye'}) self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Content-Type'], 'application/xml') self.assertIsNone(headers.get('etag')) elem = fromstring(body, 'CopyObjectResult') - self.assertIn(elem.find('LastModified').text, allowed_last_modified) + self.assertEqual(S3Timestamp(now.ceil()).s3xmlformat, + elem.find('LastModified').text) self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) _, _, headers = self.swift.calls_with_headers[-1] diff --git a/test/unit/common/middleware/s3api/test_utils.py b/test/unit/common/middleware/s3api/test_utils.py index ad6fc119f..9fc0854f5 100644 --- a/test/unit/common/middleware/s3api/test_utils.py +++ b/test/unit/common/middleware/s3api/test_utils.py @@ -81,21 +81,6 @@ class TestS3ApiUtils(unittest.TestCase): self.assertFalse(utils.validate_bucket_name('bucket.', False)) self.assertFalse(utils.validate_bucket_name('a' * 256, False)) - def test_s3timestamp(self): - expected = '1970-01-01T00:00:01.000Z' - # integer - ts = utils.S3Timestamp(1) - self.assertEqual(expected, ts.s3xmlformat) - # milliseconds unit should be floored - ts = utils.S3Timestamp(1.1) - self.assertEqual(expected, ts.s3xmlformat) - # float (microseconds) should be floored too - ts = utils.S3Timestamp(1.000001) - self.assertEqual(expected, ts.s3xmlformat) - # Bigger float (milliseconds) should be floored too - ts = utils.S3Timestamp(1.9) - self.assertEqual(expected, ts.s3xmlformat) - def test_mktime(self): date_headers = [ 'Thu, 01 Jan 1970 00:00:00 -0000', @@ -130,6 +115,48 @@ class TestS3ApiUtils(unittest.TestCase): time.tzset() +class TestS3Timestamp(unittest.TestCase): + def test_s3xmlformat(self): + expected = '1970-01-01T00:00:01.000Z' + # integer + ts = utils.S3Timestamp(1) + self.assertEqual(expected, ts.s3xmlformat) + # milliseconds unit should be rounded up + expected = '1970-01-01T00:00:02.000Z' + ts = utils.S3Timestamp(1.1) + self.assertEqual(expected, ts.s3xmlformat) + # float (microseconds) should be floored too + ts = utils.S3Timestamp(1.000001) + self.assertEqual(expected, ts.s3xmlformat) + # Bigger float (milliseconds) should be floored too + ts = utils.S3Timestamp(1.9) + self.assertEqual(expected, ts.s3xmlformat) + + def test_from_s3xmlformat(self): + ts = utils.S3Timestamp.from_s3xmlformat('2014-06-10T22:47:32.000Z') + self.assertIsInstance(ts, utils.S3Timestamp) + self.assertEqual(1402440452, float(ts)) + self.assertEqual('2014-06-10T22:47:32.000000', ts.isoformat) + + ts = utils.S3Timestamp.from_s3xmlformat('1970-01-01T00:00:00.000Z') + self.assertIsInstance(ts, utils.S3Timestamp) + self.assertEqual(0.0, float(ts)) + self.assertEqual('1970-01-01T00:00:00.000000', ts.isoformat) + + ts = utils.S3Timestamp(1402440452.0) + self.assertIsInstance(ts, utils.S3Timestamp) + ts1 = utils.S3Timestamp.from_s3xmlformat(ts.s3xmlformat) + self.assertIsInstance(ts1, utils.S3Timestamp) + self.assertEqual(ts, ts1) + + def test_from_isoformat(self): + ts = utils.S3Timestamp.from_isoformat('2014-06-10T22:47:32.054580') + self.assertIsInstance(ts, utils.S3Timestamp) + self.assertEqual(1402440452.05458, float(ts)) + self.assertEqual('2014-06-10T22:47:32.054580', ts.isoformat) + self.assertEqual('2014-06-10T22:47:33.000Z', ts.s3xmlformat) + + class TestConfig(unittest.TestCase): def _assert_defaults(self, conf): diff --git a/test/unit/common/middleware/test_memcache.py b/test/unit/common/middleware/test_memcache.py index a5fe65fe6..8ba0c7e15 100644 --- a/test/unit/common/middleware/test_memcache.py +++ b/test/unit/common/middleware/test_memcache.py @@ -37,7 +37,7 @@ class FakeApp(object): class ExcConfigParser(object): def read(self, path): - raise Exception('read called with %r' % path) + raise RuntimeError('read called with %r' % path) class EmptyConfigParser(object): @@ -47,12 +47,10 @@ class EmptyConfigParser(object): def get_config_parser(memcache_servers='1.2.3.4:5', - memcache_serialization_support='1', memcache_max_connections='4', section='memcache', item_size_warning_threshold='75'): _srvs = memcache_servers - _sers = memcache_serialization_support _maxc = memcache_max_connections _section = section _warn_threshold = item_size_warning_threshold @@ -64,8 +62,6 @@ def get_config_parser(memcache_servers='1.2.3.4:5', raise NoSectionError(section_name) return { 'memcache_servers': memcache_servers, - 'memcache_serialization_support': - memcache_serialization_support, 'memcache_max_connections': memcache_max_connections } @@ -78,10 +74,6 @@ def get_config_parser(memcache_servers='1.2.3.4:5', if _srvs == 'error': raise NoOptionError(option, section) return _srvs - elif option == 'memcache_serialization_support': - if _sers == 'error': - raise NoOptionError(option, section) - return _sers elif option in ('memcache_max_connections', 'max_connections'): if _maxc == 'error': @@ -118,23 +110,14 @@ class TestCacheMiddleware(unittest.TestCase): with mock.patch.object(memcache, 'ConfigParser', ExcConfigParser): for d in ({}, {'memcache_servers': '6.7.8.9:10'}, - {'memcache_serialization_support': '0'}, {'memcache_max_connections': '30'}, {'item_size_warning_threshold': 75}, {'memcache_servers': '6.7.8.9:10', - 'memcache_serialization_support': '0'}, - {'memcache_servers': '6.7.8.9:10', - 'memcache_max_connections': '30'}, - {'memcache_serialization_support': '0', - 'memcache_max_connections': '30'}, - {'memcache_servers': '6.7.8.9:10', - 'item_size_warning_threshold': '75'}, - {'memcache_serialization_support': '0', 'item_size_warning_threshold': '75'}, {'item_size_warning_threshold': '75', 'memcache_max_connections': '30'}, ): - with self.assertRaises(Exception) as catcher: + with self.assertRaises(RuntimeError) as catcher: memcache.MemcacheMiddleware(FakeApp(), d) self.assertEqual( str(catcher.exception), @@ -142,23 +125,15 @@ class TestCacheMiddleware(unittest.TestCase): def test_conf_set_no_read(self): with mock.patch.object(memcache, 'ConfigParser', ExcConfigParser): - exc = None - try: - memcache.MemcacheMiddleware( - FakeApp(), {'memcache_servers': '1.2.3.4:5', - 'memcache_serialization_support': '2', - 'memcache_max_connections': '30', - 'item_size_warning_threshold': '80'}) - except Exception as err: - exc = err - self.assertIsNone(exc) + memcache.MemcacheMiddleware( + FakeApp(), {'memcache_servers': '1.2.3.4:5', + 'memcache_max_connections': '30', + 'item_size_warning_threshold': '80'}) def test_conf_default(self): with mock.patch.object(memcache, 'ConfigParser', EmptyConfigParser): app = memcache.MemcacheMiddleware(FakeApp(), {}) self.assertEqual(app.memcache_servers, '127.0.0.1:11211') - self.assertEqual(app.memcache._allow_pickle, False) - self.assertEqual(app.memcache._allow_unpickle, False) self.assertEqual( app.memcache._client_cache['127.0.0.1:11211'].max_size, 2) self.assertEqual(app.memcache.item_size_warning_threshold, -1) @@ -168,12 +143,9 @@ class TestCacheMiddleware(unittest.TestCase): app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', - 'memcache_serialization_support': '0', 'memcache_max_connections': '5', 'item_size_warning_threshold': '75'}) self.assertEqual(app.memcache_servers, '6.7.8.9:10') - self.assertEqual(app.memcache._allow_pickle, True) - self.assertEqual(app.memcache._allow_unpickle, True) self.assertEqual( app.memcache._client_cache['6.7.8.9:10'].max_size, 5) self.assertEqual(app.memcache.item_size_warning_threshold, 75) @@ -209,20 +181,16 @@ class TestCacheMiddleware(unittest.TestCase): get_config_parser(section='foobar')): app = memcache.MemcacheMiddleware(FakeApp(), {}) self.assertEqual(app.memcache_servers, '127.0.0.1:11211') - self.assertEqual(app.memcache._allow_pickle, False) - self.assertEqual(app.memcache._allow_unpickle, False) self.assertEqual( app.memcache._client_cache['127.0.0.1:11211'].max_size, 2) def test_conf_extra_no_option(self): replacement_parser = get_config_parser( - memcache_servers='error', memcache_serialization_support='error', + memcache_servers='error', memcache_max_connections='error') with mock.patch.object(memcache, 'ConfigParser', replacement_parser): app = memcache.MemcacheMiddleware(FakeApp(), {}) self.assertEqual(app.memcache_servers, '127.0.0.1:11211') - self.assertEqual(app.memcache._allow_pickle, False) - self.assertEqual(app.memcache._allow_unpickle, False) self.assertEqual( app.memcache._client_cache['127.0.0.1:11211'].max_size, 2) @@ -231,11 +199,8 @@ class TestCacheMiddleware(unittest.TestCase): app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', - 'memcache_serialization_support': '0', 'max_connections': '5'}) self.assertEqual(app.memcache_servers, '6.7.8.9:10') - self.assertEqual(app.memcache._allow_pickle, True) - self.assertEqual(app.memcache._allow_unpickle, True) self.assertEqual( app.memcache._client_cache['6.7.8.9:10'].max_size, 5) @@ -244,11 +209,8 @@ class TestCacheMiddleware(unittest.TestCase): app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', - 'memcache_serialization_support': '0', 'max_connections': 'bad42'}) self.assertEqual(app.memcache_servers, '6.7.8.9:10') - self.assertEqual(app.memcache._allow_pickle, True) - self.assertEqual(app.memcache._allow_unpickle, True) self.assertEqual( app.memcache._client_cache['6.7.8.9:10'].max_size, 4) @@ -267,8 +229,6 @@ class TestCacheMiddleware(unittest.TestCase): with mock.patch.object(memcache, 'ConfigParser', get_config_parser()): app = memcache.MemcacheMiddleware(FakeApp(), {}) self.assertEqual(app.memcache_servers, '1.2.3.4:5') - self.assertEqual(app.memcache._allow_pickle, False) - self.assertEqual(app.memcache._allow_unpickle, True) self.assertEqual( app.memcache._client_cache['1.2.3.4:5'].max_size, 4) @@ -277,8 +237,6 @@ class TestCacheMiddleware(unittest.TestCase): memcache_max_connections='bad42')): app = memcache.MemcacheMiddleware(FakeApp(), {}) self.assertEqual(app.memcache_servers, '1.2.3.4:5') - self.assertEqual(app.memcache._allow_pickle, False) - self.assertEqual(app.memcache._allow_unpickle, True) self.assertEqual( app.memcache._client_cache['1.2.3.4:5'].max_size, 2) @@ -286,11 +244,8 @@ class TestCacheMiddleware(unittest.TestCase): with mock.patch.object(memcache, 'ConfigParser', get_config_parser()): app = memcache.MemcacheMiddleware( FakeApp(), - {'memcache_servers': '6.7.8.9:10', - 'memcache_serialization_support': '0'}) + {'memcache_servers': '6.7.8.9:10'}) self.assertEqual(app.memcache_servers, '6.7.8.9:10') - self.assertEqual(app.memcache._allow_pickle, True) - self.assertEqual(app.memcache._allow_unpickle, True) self.assertEqual( app.memcache._client_cache['6.7.8.9:10'].max_size, 4) @@ -301,20 +256,15 @@ class TestCacheMiddleware(unittest.TestCase): {'memcache_servers': '6.7.8.9:10', 'memcache_max_connections': '42'}) self.assertEqual(app.memcache_servers, '6.7.8.9:10') - self.assertEqual(app.memcache._allow_pickle, False) - self.assertEqual(app.memcache._allow_unpickle, True) self.assertEqual( app.memcache._client_cache['6.7.8.9:10'].max_size, 42) def test_filter_factory(self): factory = memcache.filter_factory({'max_connections': '3'}, - memcache_servers='10.10.10.10:10', - memcache_serialization_support='1') + memcache_servers='10.10.10.10:10') thefilter = factory('myapp') self.assertEqual(thefilter.app, 'myapp') self.assertEqual(thefilter.memcache_servers, '10.10.10.10:10') - self.assertEqual(thefilter.memcache._allow_pickle, False) - self.assertEqual(thefilter.memcache._allow_unpickle, True) self.assertEqual( thefilter.memcache._client_cache['10.10.10.10:10'].max_size, 3) diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 045778a16..b7b1365e8 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -2423,7 +2423,7 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') # sanity check - self.assertEqual(sleeps, [2.0, 2.0, 2.0, 2.0, 2.0]) + self.assertEqual(sleeps, [1.0] * 11) # give the client the first 4 segments without ratelimiting; we'll # sleep less @@ -2435,7 +2435,7 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') # sanity check - self.assertEqual(sleeps, [2.0, 2.0, 2.0]) + self.assertEqual(sleeps, [1.0] * 7) # ratelimit segments under 35 bytes; this affects a-f del sleeps[:] @@ -2446,7 +2446,7 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') # sanity check - self.assertEqual(sleeps, [2.0, 2.0]) + self.assertEqual(sleeps, [1.0] * 5) # ratelimit segments under 36 bytes; this now affects a-g, netting # us one more sleep than before @@ -2458,7 +2458,7 @@ class TestSloGetManifest(SloTestCase): status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') # sanity check - self.assertEqual(sleeps, [2.0, 2.0, 2.0]) + self.assertEqual(sleeps, [1.0] * 6) def test_get_manifest_with_submanifest(self): req = Request.blank( diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index 2f0549abb..d53d15f10 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -32,6 +32,7 @@ from test.debug_logger import debug_logger class MyDaemon(daemon.Daemon): + WORKERS_HEALTHCHECK_INTERVAL = 0 def __init__(self, conf): self.conf = conf @@ -139,6 +140,7 @@ class TestRunDaemon(unittest.TestCase): with mock.patch('swift.common.daemon.os') as mock_os: func() self.assertEqual(mock_os.method_calls, [ + mock.call.getpid(), mock.call.killpg(0, signal.SIGTERM), # hard exit because bare except handlers can trap SystemExit mock.call._exit(0) diff --git a/test/unit/common/test_memcached.py b/test/unit/common/test_memcached.py index 06e567d86..42c3cab83 100644 --- a/test/unit/common/test_memcached.py +++ b/test/unit/common/test_memcached.py @@ -32,6 +32,7 @@ from eventlet import GreenPool, sleep, Queue from eventlet.pools import Pool from swift.common import memcached +from swift.common.memcached import MemcacheConnectionError from swift.common.utils import md5, human_readable from mock import patch, MagicMock from test.debug_logger import debug_logger @@ -581,19 +582,27 @@ class TestMemcached(unittest.TestCase): self.assertEqual(self.logger.get_lines_for_level('error'), [ 'Error talking to memcached: 1.2.3.4:11211: ' '[Errno 32] Broken pipe', - ] * 11 + [ - 'Error limiting server 1.2.3.4:11211' + ] * 10 + [ + 'Error talking to memcached: 1.2.3.4:11211: ' + '[Errno 32] Broken pipe', + 'Error limiting server 1.2.3.4:11211', + 'All memcached servers error-limited', ]) self.logger.clear() # continued requests just keep bypassing memcache for _ in range(12): memcache_client.set('some_key', [1, 2, 3]) - self.assertEqual(self.logger.get_lines_for_level('error'), []) + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'All memcached servers error-limited', + ] * 12) + self.logger.clear() # and get()s are all a "cache miss" self.assertIsNone(memcache_client.get('some_key')) - self.assertEqual(self.logger.get_lines_for_level('error'), []) + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'All memcached servers error-limited', + ]) def test_error_disabled(self): memcache_client = memcached.MemcacheRing( @@ -611,6 +620,44 @@ class TestMemcached(unittest.TestCase): '[Errno 32] Broken pipe', ] * 20) + def test_error_raising(self): + memcache_client = memcached.MemcacheRing( + ['1.2.3.4:11211'], logger=self.logger, error_limit_time=0) + mock1 = ExplodingMockMemcached() + memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( + [(mock1, mock1)] * 20) + + # expect exception when requested... + with self.assertRaises(MemcacheConnectionError): + memcache_client.set('some_key', [1, 2, 3], raise_on_error=True) + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'Error talking to memcached: 1.2.3.4:11211: ' + '[Errno 32] Broken pipe', + ]) + self.logger.clear() + + with self.assertRaises(MemcacheConnectionError): + memcache_client.get('some_key', raise_on_error=True) + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'Error talking to memcached: 1.2.3.4:11211: ' + '[Errno 32] Broken pipe', + ]) + self.logger.clear() + + # ...but default is no exception + memcache_client.set('some_key', [1, 2, 3]) + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'Error talking to memcached: 1.2.3.4:11211: ' + '[Errno 32] Broken pipe', + ]) + self.logger.clear() + + memcache_client.get('some_key') + self.assertEqual(self.logger.get_lines_for_level('error'), [ + 'Error talking to memcached: 1.2.3.4:11211: ' + '[Errno 32] Broken pipe', + ]) + def test_error_limiting_custom_config(self): def do_calls(time_step, num_calls, **memcache_kwargs): self.logger.clear() @@ -632,8 +679,11 @@ class TestMemcached(unittest.TestCase): self.assertEqual(self.logger.get_lines_for_level('error'), [ 'Error talking to memcached: 1.2.3.5:11211: ' '[Errno 32] Broken pipe', - ] * 11 + [ - 'Error limiting server 1.2.3.5:11211' + ] * 10 + [ + 'Error talking to memcached: 1.2.3.5:11211: ' + '[Errno 32] Broken pipe', + 'Error limiting server 1.2.3.5:11211', + 'All memcached servers error-limited', ]) # with default error_limit_time of 60, one call per 6 secs, error limit @@ -650,8 +700,11 @@ class TestMemcached(unittest.TestCase): self.assertEqual(self.logger.get_lines_for_level('error'), [ 'Error talking to memcached: 1.2.3.5:11211: ' '[Errno 32] Broken pipe', - ] * 11 + [ - 'Error limiting server 1.2.3.5:11211' + ] * 10 + [ + 'Error talking to memcached: 1.2.3.5:11211: ' + '[Errno 32] Broken pipe', + 'Error limiting server 1.2.3.5:11211', + 'All memcached servers error-limited', ]) # with error_limit_time of 70, one call per 6 secs, error_limit_count @@ -660,8 +713,11 @@ class TestMemcached(unittest.TestCase): self.assertEqual(self.logger.get_lines_for_level('error'), [ 'Error talking to memcached: 1.2.3.5:11211: ' '[Errno 32] Broken pipe', - ] * 12 + [ - 'Error limiting server 1.2.3.5:11211' + ] * 11 + [ + 'Error talking to memcached: 1.2.3.5:11211: ' + '[Errno 32] Broken pipe', + 'Error limiting server 1.2.3.5:11211', + 'All memcached servers error-limited', ]) def test_delete(self): @@ -767,24 +823,18 @@ class TestMemcached(unittest.TestCase): def test_serialization(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'], - allow_pickle=True, logger=self.logger) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) memcache_client.set('some_key', [1, 2, 3]) self.assertEqual(memcache_client.get('some_key'), [1, 2, 3]) - memcache_client._allow_pickle = False - memcache_client._allow_unpickle = True - self.assertEqual(memcache_client.get('some_key'), [1, 2, 3]) - memcache_client._allow_unpickle = False + self.assertEqual(len(mock.cache), 1) + key = next(iter(mock.cache)) + self.assertEqual(mock.cache[key][0], b'2') # JSON_FLAG + # Pretend we've got some really old pickle data in there + mock.cache[key] = (b'1',) + mock.cache[key][1:] self.assertIsNone(memcache_client.get('some_key')) - memcache_client.set('some_key', [1, 2, 3]) - self.assertEqual(memcache_client.get('some_key'), [1, 2, 3]) - memcache_client._allow_unpickle = True - self.assertEqual(memcache_client.get('some_key'), [1, 2, 3]) - memcache_client._allow_pickle = True - self.assertEqual(memcache_client.get('some_key'), [1, 2, 3]) def test_connection_pooling(self): with patch('swift.common.memcached.socket') as mock_module: diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 3f14fdcdd..40a34fe8d 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -305,6 +305,28 @@ class TestTimestamp(unittest.TestCase): for value in test_values: self.assertEqual(utils.Timestamp(value).isoformat, expected) + def test_from_isoformat(self): + ts = utils.Timestamp.from_isoformat('2014-06-10T22:47:32.054580') + self.assertIsInstance(ts, utils.Timestamp) + self.assertEqual(1402440452.05458, float(ts)) + self.assertEqual('2014-06-10T22:47:32.054580', ts.isoformat) + + ts = utils.Timestamp.from_isoformat('1970-01-01T00:00:00.000000') + self.assertIsInstance(ts, utils.Timestamp) + self.assertEqual(0.0, float(ts)) + self.assertEqual('1970-01-01T00:00:00.000000', ts.isoformat) + + ts = utils.Timestamp(1402440452.05458) + self.assertIsInstance(ts, utils.Timestamp) + self.assertEqual(ts, utils.Timestamp.from_isoformat(ts.isoformat)) + + def test_ceil(self): + self.assertEqual(0.0, utils.Timestamp(0).ceil()) + self.assertEqual(1.0, utils.Timestamp(0.00001).ceil()) + self.assertEqual(1.0, utils.Timestamp(0.000001).ceil()) + self.assertEqual(12345678.0, utils.Timestamp(12345678.0).ceil()) + self.assertEqual(12345679.0, utils.Timestamp(12345678.000001).ceil()) + def test_not_equal(self): ts = '1402436408.91203_0000000000000001' test_values = ( @@ -5832,6 +5854,140 @@ class TestAffinityLocalityPredicate(unittest.TestCase): utils.affinity_locality_predicate, 'r1z1=1') +class TestEventletRateLimiter(unittest.TestCase): + def test_init(self): + rl = utils.EventletRateLimiter(0.1) + self.assertEqual(0.1, rl.max_rate) + self.assertEqual(0.0, rl.running_time) + self.assertEqual(5000, rl.rate_buffer_ms) + + rl = utils.EventletRateLimiter( + 0.2, rate_buffer=2, running_time=1234567.8) + self.assertEqual(0.2, rl.max_rate) + self.assertEqual(1234567.8, rl.running_time) + self.assertEqual(2000, rl.rate_buffer_ms) + + def test_non_blocking(self): + rate_limiter = utils.EventletRateLimiter(0.1, rate_buffer=0) + with patch('time.time',) as mock_time: + with patch('eventlet.sleep') as mock_sleep: + mock_time.return_value = 0 + self.assertTrue(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + self.assertFalse(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + + mock_time.return_value = 9.99 + self.assertFalse(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + mock_time.return_value = 10.0 + self.assertTrue(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + self.assertFalse(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + + rate_limiter = utils.EventletRateLimiter(0.1, rate_buffer=20) + with patch('time.time',) as mock_time: + with patch('eventlet.sleep') as mock_sleep: + mock_time.return_value = 20.0 + self.assertTrue(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + self.assertTrue(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + self.assertTrue(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + self.assertFalse(rate_limiter.is_allowed()) + mock_sleep.assert_not_called() + + def _do_test(self, max_rate, running_time, start_time, rate_buffer, + burst_after_idle=False, incr_by=1.0): + rate_limiter = utils.EventletRateLimiter( + max_rate, + running_time=1000 * running_time, # msecs + rate_buffer=rate_buffer, + burst_after_idle=burst_after_idle) + grant_times = [] + current_time = [start_time] + + def mock_time(): + return current_time[0] + + def mock_sleep(duration): + current_time[0] += duration + + with patch('time.time', mock_time): + with patch('eventlet.sleep', mock_sleep): + for i in range(5): + rate_limiter.wait(incr_by=incr_by) + grant_times.append(current_time[0]) + return [round(t, 6) for t in grant_times] + + def test_ratelimit(self): + grant_times = self._do_test(1, 0, 1, 0) + self.assertEqual([1, 2, 3, 4, 5], grant_times) + + grant_times = self._do_test(10, 0, 1, 0) + self.assertEqual([1, 1.1, 1.2, 1.3, 1.4], grant_times) + + grant_times = self._do_test(.1, 0, 1, 0) + self.assertEqual([1, 11, 21, 31, 41], grant_times) + + grant_times = self._do_test(.1, 11, 1, 0) + self.assertEqual([11, 21, 31, 41, 51], grant_times) + + def test_incr_by(self): + grant_times = self._do_test(1, 0, 1, 0, incr_by=2.5) + self.assertEqual([1, 3.5, 6, 8.5, 11], grant_times) + + def test_burst(self): + grant_times = self._do_test(1, 1, 4, 0) + self.assertEqual([4, 5, 6, 7, 8], grant_times) + + grant_times = self._do_test(1, 1, 4, 1) + self.assertEqual([4, 5, 6, 7, 8], grant_times) + + grant_times = self._do_test(1, 1, 4, 2) + self.assertEqual([4, 5, 6, 7, 8], grant_times) + + grant_times = self._do_test(1, 1, 4, 3) + self.assertEqual([4, 4, 4, 4, 5], grant_times) + + grant_times = self._do_test(1, 1, 4, 4) + self.assertEqual([4, 4, 4, 4, 5], grant_times) + + grant_times = self._do_test(1, 1, 3, 3) + self.assertEqual([3, 3, 3, 4, 5], grant_times) + + grant_times = self._do_test(1, 0, 2, 3) + self.assertEqual([2, 2, 2, 3, 4], grant_times) + + grant_times = self._do_test(1, 1, 3, 3) + self.assertEqual([3, 3, 3, 4, 5], grant_times) + + grant_times = self._do_test(1, 0, 3, 3) + self.assertEqual([3, 3, 3, 3, 4], grant_times) + + grant_times = self._do_test(1, 1, 3, 3) + self.assertEqual([3, 3, 3, 4, 5], grant_times) + + grant_times = self._do_test(1, 0, 4, 3) + self.assertEqual([4, 5, 6, 7, 8], grant_times) + + def test_burst_after_idle(self): + grant_times = self._do_test(1, 1, 4, 1, burst_after_idle=True) + self.assertEqual([4, 4, 5, 6, 7], grant_times) + + grant_times = self._do_test(1, 1, 4, 2, burst_after_idle=True) + self.assertEqual([4, 4, 4, 5, 6], grant_times) + + grant_times = self._do_test(1, 0, 4, 3, burst_after_idle=True) + self.assertEqual([4, 4, 4, 4, 5], grant_times) + + # running_time = start_time prevents burst on start-up + grant_times = self._do_test(1, 4, 4, 3, burst_after_idle=True) + self.assertEqual([4, 5, 6, 7, 8], grant_times) + + class TestRateLimitedIterator(unittest.TestCase): def run_under_pseudo_time( diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index dd9b92817..c1ed3ca6d 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -4083,9 +4083,22 @@ class TestContainerBroker(unittest.TestCase): actual = broker.get_shard_ranges(marker='e', end_marker='e') self.assertFalse([dict(sr) for sr in actual]) - actual = broker.get_shard_ranges(includes='f') + orig_execute = GreenDBConnection.execute + mock_call_args = [] + + def mock_execute(*args, **kwargs): + mock_call_args.append(args) + return orig_execute(*args, **kwargs) + + with mock.patch('swift.common.db.GreenDBConnection.execute', + mock_execute): + actual = broker.get_shard_ranges(includes='f') self.assertEqual([dict(sr) for sr in shard_ranges[2:3]], [dict(sr) for sr in actual]) + self.assertEqual(1, len(mock_call_args)) + # verify that includes keyword plumbs through to an SQL condition + self.assertIn("WHERE deleted=0 AND name != ? AND lower < ? AND " + "(upper = '' OR upper >= ?)", mock_call_args[0][1]) actual = broker.get_shard_ranges(includes='i') self.assertFalse(actual) @@ -4143,6 +4156,61 @@ class TestContainerBroker(unittest.TestCase): self.assertFalse(actual) @with_tempdir + def test_get_shard_ranges_includes(self, tempdir): + ts = next(self.ts) + start = ShardRange('a/-a', ts, '', 'a') + atof = ShardRange('a/a-f', ts, 'a', 'f') + ftol = ShardRange('a/f-l', ts, 'f', 'l') + ltor = ShardRange('a/l-r', ts, 'l', 'r') + rtoz = ShardRange('a/r-z', ts, 'r', 'z') + end = ShardRange('a/z-', ts, 'z', '') + ranges = [start, atof, ftol, ltor, rtoz, end] + db_path = os.path.join(tempdir, 'container.db') + broker = ContainerBroker(db_path, account='a', container='c') + broker.initialize(next(self.ts).internal, 0) + broker.merge_shard_ranges(ranges) + + actual = broker.get_shard_ranges(includes='') + self.assertEqual(actual, []) + actual = broker.get_shard_ranges(includes=' ') + self.assertEqual(actual, [start]) + actual = broker.get_shard_ranges(includes='b') + self.assertEqual(actual, [atof]) + actual = broker.get_shard_ranges(includes='f') + self.assertEqual(actual, [atof]) + actual = broker.get_shard_ranges(includes='f\x00') + self.assertEqual(actual, [ftol]) + actual = broker.get_shard_ranges(includes='x') + self.assertEqual(actual, [rtoz]) + actual = broker.get_shard_ranges(includes='r') + self.assertEqual(actual, [ltor]) + actual = broker.get_shard_ranges(includes='}') + self.assertEqual(actual, [end]) + + # add some overlapping sub-shards + ftoh = ShardRange('a/f-h', ts, 'f', 'h') + htok = ShardRange('a/h-k', ts, 'h', 'k') + + broker.merge_shard_ranges([ftoh, htok]) + actual = broker.get_shard_ranges(includes='g') + self.assertEqual(actual, [ftoh]) + actual = broker.get_shard_ranges(includes='h') + self.assertEqual(actual, [ftoh]) + actual = broker.get_shard_ranges(includes='k') + self.assertEqual(actual, [htok]) + actual = broker.get_shard_ranges(includes='l') + self.assertEqual(actual, [ftol]) + actual = broker.get_shard_ranges(includes='m') + self.assertEqual(actual, [ltor]) + + # remove l-r from shard ranges and try and find a shard range for an + # item in that range. + ltor.set_deleted(next(self.ts)) + broker.merge_shard_ranges([ltor]) + actual = broker.get_shard_ranges(includes='p') + self.assertEqual(actual, []) + + @with_tempdir def test_overlap_shard_range_order(self, tempdir): db_path = os.path.join(tempdir, 'container.db') broker = ContainerBroker(db_path, account='a', container='c') @@ -4211,9 +4279,25 @@ class TestContainerBroker(unittest.TestCase): [dict(sr) for sr in shard_ranges[:3] + shard_ranges[4:]], [dict(sr) for sr in actual]) - actual = broker.get_shard_ranges(states=SHARD_UPDATE_STATES, - includes='e') - self.assertEqual([shard_ranges[1]], actual) + orig_execute = GreenDBConnection.execute + mock_call_args = [] + + def mock_execute(*args, **kwargs): + mock_call_args.append(args) + return orig_execute(*args, **kwargs) + + with mock.patch('swift.common.db.GreenDBConnection.execute', + mock_execute): + actual = broker.get_shard_ranges(states=SHARD_UPDATE_STATES, + includes='e') + self.assertEqual([dict(shard_ranges[1])], + [dict(sr) for sr in actual]) + self.assertEqual(1, len(mock_call_args)) + # verify that includes keyword plumbs through to an SQL condition + self.assertIn("WHERE deleted=0 AND state in (?,?,?,?) AND name != ? " + "AND lower < ? AND (upper = '' OR upper >= ?)", + mock_call_args[0][1]) + actual = broker.get_shard_ranges(states=SHARD_UPDATE_STATES, includes='j') self.assertEqual([shard_ranges[2]], actual) diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 288864e5e..029197f4f 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -1589,6 +1589,27 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin): ('/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900/' + 'made-up-filename')) + def test_get_diskfile_from_hash_no_data(self): + self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') + with mock.patch(self._manager_mock('diskfile_cls')), \ + mock.patch(self._manager_mock( + 'cleanup_ondisk_files')) as cleanup, \ + mock.patch('swift.obj.diskfile.read_metadata') as readmeta, \ + mock.patch(self._manager_mock( + 'quarantine_renamer')) as quarantine_renamer: + osexc = OSError() + osexc.errno = errno.ENODATA + cleanup.side_effect = osexc + readmeta.return_value = {'name': '/a/c/o'} + self.assertRaises( + DiskFileNotExist, + self.df_mgr.get_diskfile_from_hash, + 'dev', '9', '9a7175077c01a23ade5956b8a2bba900', POLICIES[0]) + quarantine_renamer.assert_called_once_with( + '/srv/dev/', + ('/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900/' + + 'made-up-filename')) + def test_get_diskfile_from_hash_no_dir(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with mock.patch(self._manager_mock('diskfile_cls')), \ @@ -4681,6 +4702,21 @@ class DiskFileMixin(BaseDiskFileTestMixin): self.assertFalse(os.path.exists(hashdir)) self.assertTrue(os.path.exists(os.path.dirname(hashdir))) + def test_quarantine_hashdir_not_listable(self): + df, df_data = self._create_test_file(b'1234567890', account="abc", + container='123', obj='xyz') + hashdir = df._datadir + df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', + 'xyz', policy=POLICIES.legacy) + with mock.patch('os.listdir', + side_effect=OSError(errno.ENODATA, 'nope')): + self.assertRaises(DiskFileQuarantined, df.open) + + # make sure the right thing got quarantined; the suffix dir should not + # have moved, as that could have many objects in it + self.assertFalse(os.path.exists(hashdir)) + self.assertTrue(os.path.exists(os.path.dirname(hashdir))) + def test_create_prealloc(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', 'xyz', policy=POLICIES.legacy) @@ -8524,6 +8560,38 @@ class TestSuffixHashes(unittest.TestCase): ) self.assertTrue(os.path.exists(quarantine_path)) + def test_hash_suffix_cleanup_ondisk_files_enodata_quarantined(self): + for policy in self.iter_policies(): + df = self.df_router[policy].get_diskfile( + self.existing_device, '0', 'a', 'c', 'o', policy=policy) + # make everything down to the hash directory + os.makedirs(df._datadir) + suffix = os.path.basename(os.path.dirname(df._datadir)) + orig_listdir = os.listdir + + def fake_listdir(path): + if path == df._datadir: + raise OSError(errno.ENODATA, 'nope') + return orig_listdir(path) + + df_mgr = self.df_router[policy] + with mock.patch('os.listdir', side_effect=fake_listdir): + hashes = df_mgr.get_hashes(self.existing_device, '0', [suffix], + policy) + self.assertEqual(hashes, {}) + # and hash path is quarantined + self.assertFalse(os.path.exists(df._datadir)) + # each device a quarantined directory + quarantine_base = os.path.join(self.devices, + self.existing_device, 'quarantined') + # the quarantine path is... + quarantine_path = os.path.join( + quarantine_base, # quarantine root + diskfile.get_data_dir(policy), # per-policy data dir + os.path.basename(df._datadir) # name of quarantined file + ) + self.assertTrue(os.path.exists(quarantine_path)) + def test_hash_suffix_cleanup_ondisk_files_other_oserror(self): for policy in self.iter_policies(): timestamp = self.ts() diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index fa3fde32f..941ec3dc3 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -1652,7 +1652,7 @@ class TestObjectUpdater(unittest.TestCase): len(self._find_async_pending_files())) # indexes 0, 2 succeed; 1, 3, 4 deferred but 1 is bumped from deferral # queue by 4; 4, 3 are then drained - latencies = [0, 0.05, .051, 0, 0, 0, .11, .01] + latencies = [0, 0.05, .051, 0, 0, 0, .11] expected_success = 4 contexts_fed_in = [] @@ -1693,7 +1693,7 @@ class TestObjectUpdater(unittest.TestCase): fake_object_update), \ mock.patch('swift.obj.updater.RateLimitedIterator', fake_rate_limited_iterator), \ - mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: daemon.run_once() self.assertEqual(expected_success, daemon.stats.successes) expected_skipped = expected_total - expected_success @@ -1719,7 +1719,7 @@ class TestObjectUpdater(unittest.TestCase): self.assertEqual([aorder[o] for o in expected_updates_sent], [aorder[o] for o in actual_updates_sent]) - self.assertEqual([0, 0, 0, 0, 0, 1, 1, 1], captured_skips_stats) + self.assertEqual([0, 0, 0, 0, 0, 1, 1], captured_skips_stats) expected_deferrals = [ [], @@ -1729,7 +1729,6 @@ class TestObjectUpdater(unittest.TestCase): [objs_fed_in[1], objs_fed_in[3]], [objs_fed_in[3], objs_fed_in[4]], [objs_fed_in[3]], # note: rightmost element is drained - [objs_fed_in[3]], ] self.assertEqual( expected_deferrals, @@ -1776,7 +1775,7 @@ class TestObjectUpdater(unittest.TestCase): # first pass: 0, 2 and 5 succeed, 1, 3, 4, 6 deferred # last 2 deferred items sent before interval elapses latencies = [0, .05, 0.051, 0, 0, .11, 0, 0, - 0.1, 0, 0.1, 0] # total 0.42 + 0.1, 0.1, 0] # total 0.411 expected_success = 5 contexts_fed_in = [] @@ -1820,7 +1819,7 @@ class TestObjectUpdater(unittest.TestCase): fake_object_update), \ mock.patch('swift.obj.updater.RateLimitedIterator', fake_rate_limited_iterator), \ - mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: daemon.run_once() self.assertEqual(expected_success, daemon.stats.successes) expected_skipped = expected_total - expected_success @@ -1840,7 +1839,7 @@ class TestObjectUpdater(unittest.TestCase): self.assertEqual(expected_updates_sent, actual_updates_sent) # skips (un-drained deferrals) not reported until end of cycle - self.assertEqual([0] * 12, captured_skips_stats) + self.assertEqual([0] * 10, captured_skips_stats) objs_fed_in = [ctx['update']['obj'] for ctx in contexts_fed_in] expected_deferrals = [ @@ -1856,8 +1855,6 @@ class TestObjectUpdater(unittest.TestCase): # note: rightmost element is drained [objs_fed_in[1], objs_fed_in[3], objs_fed_in[4], objs_fed_in[6]], [objs_fed_in[1], objs_fed_in[3], objs_fed_in[4]], - [objs_fed_in[1], objs_fed_in[3], objs_fed_in[4]], - [objs_fed_in[1], objs_fed_in[3]], [objs_fed_in[1], objs_fed_in[3]], ] self.assertEqual( @@ -1911,21 +1908,21 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): it = object_updater.BucketizedUpdateSkippingLimiter( [3, 1], self.logger, self.stats, 1000, 10) self.assertEqual(1000, it.num_buckets) - self.assertEqual(0.1, it.bucket_update_delta) + self.assertEqual([10] * 1000, [b.max_rate for b in it.buckets]) self.assertEqual([3, 1], [x for x in it.iterator]) # rate of 0 implies unlimited it = object_updater.BucketizedUpdateSkippingLimiter( iter([3, 1]), self.logger, self.stats, 9, 0) self.assertEqual(9, it.num_buckets) - self.assertEqual(-1, it.bucket_update_delta) + self.assertEqual([0] * 9, [b.max_rate for b in it.buckets]) self.assertEqual([3, 1], [x for x in it.iterator]) # num_buckets is collared at 1 it = object_updater.BucketizedUpdateSkippingLimiter( iter([3, 1]), self.logger, self.stats, 0, 1) self.assertEqual(1, it.num_buckets) - self.assertEqual(1, it.bucket_update_delta) + self.assertEqual([1], [b.max_rate for b in it.buckets]) self.assertEqual([3, 1], [x for x in it.iterator]) def test_iteration_unlimited(self): @@ -1963,7 +1960,7 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): # enough capacity for all deferrals with mock.patch('swift.obj.updater.time.time', side_effect=[now, now, now, now, now, now]): - with mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + with mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: it = object_updater.BucketizedUpdateSkippingLimiter( iter(update_ctxs[:3]), self.logger, self.stats, 1, 10, max_deferred_elements=2, @@ -1982,7 +1979,7 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): # only space for one deferral with mock.patch('swift.obj.updater.time.time', side_effect=[now, now, now, now, now]): - with mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + with mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: it = object_updater.BucketizedUpdateSkippingLimiter( iter(update_ctxs[:3]), self.logger, self.stats, 1, 10, max_deferred_elements=1, @@ -2000,7 +1997,7 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): # only time for one deferral with mock.patch('swift.obj.updater.time.time', side_effect=[now, now, now, now, now + 20, now + 20]): - with mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + with mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: it = object_updater.BucketizedUpdateSkippingLimiter( iter(update_ctxs[:3]), self.logger, self.stats, 1, 10, max_deferred_elements=2, @@ -2019,7 +2016,7 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): with mock.patch('swift.obj.updater.time.time', side_effect=[now, now, now, now, now, now + 20, now + 20]): - with mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + with mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: it = object_updater.BucketizedUpdateSkippingLimiter( iter(update_ctxs), self.logger, self.stats, 1, 10, max_deferred_elements=2, @@ -2048,7 +2045,7 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): # deferrals stick in both buckets with mock.patch('swift.obj.updater.time.time', side_effect=[next(time_iter) for _ in range(12)]): - with mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + with mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: it = object_updater.BucketizedUpdateSkippingLimiter( iter(update_ctxs_1 + update_ctxs_2), self.logger, self.stats, 4, 10, @@ -2073,7 +2070,7 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): # oldest deferral bumped from one bucket due to max_deferrals == 3 with mock.patch('swift.obj.updater.time.time', side_effect=[next(time_iter) for _ in range(10)]): - with mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + with mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: it = object_updater.BucketizedUpdateSkippingLimiter( iter(update_ctxs_1 + update_ctxs_2), self.logger, self.stats, 4, 10, @@ -2097,7 +2094,7 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): # older deferrals bumped from one bucket due to max_deferrals == 2 with mock.patch('swift.obj.updater.time.time', side_effect=[next(time_iter) for _ in range(10)]): - with mock.patch('swift.obj.updater.time.sleep') as mock_sleep: + with mock.patch('swift.common.utils.eventlet.sleep') as mock_sleep: it = object_updater.BucketizedUpdateSkippingLimiter( iter(update_ctxs_1 + update_ctxs_2), self.logger, self.stats, 4, 10, @@ -2119,16 +2116,8 @@ class TestBucketizedUpdateSkippingLimiter(unittest.TestCase): class TestRateLimiterBucket(unittest.TestCase): - def test_wait_until(self): - b1 = object_updater.RateLimiterBucket(10) - self.assertEqual(10, b1.wait_until) - b1.last_time = b1.wait_until - self.assertEqual(20, b1.wait_until) - b1.last_time = 12345.678 - self.assertEqual(12355.678, b1.wait_until) - def test_len(self): - b1 = object_updater.RateLimiterBucket(10) + b1 = object_updater.RateLimiterBucket(0.1) b1.deque.append(1) b1.deque.append(2) self.assertEqual(2, len(b1)) @@ -2136,7 +2125,7 @@ class TestRateLimiterBucket(unittest.TestCase): self.assertEqual(1, len(b1)) def test_bool(self): - b1 = object_updater.RateLimiterBucket(10) + b1 = object_updater.RateLimiterBucket(0.1) self.assertFalse(b1) b1.deque.append(1) self.assertTrue(b1) @@ -2148,13 +2137,13 @@ class TestRateLimiterBucket(unittest.TestCase): b1 = object_updater.RateLimiterBucket(10) b2 = object_updater.RateLimiterBucket(10) - b2.last_time = next(time_iter) + b2.running_time = next(time_iter) buckets = PriorityQueue() buckets.put(b1) buckets.put(b2) self.assertEqual([b1, b2], [buckets.get_nowait() for _ in range(2)]) - b1.last_time = next(time_iter) + b1.running_time = next(time_iter) buckets.put(b1) buckets.put(b2) self.assertEqual([b2, b1], [buckets.get_nowait() for _ in range(2)]) diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index 27e888e1f..ac73465da 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -2148,7 +2148,7 @@ class TestContainerController(TestRingBase): 'X-Backend-Sharding-State': sharding_state}) self.assertEqual( [mock.call.get('container/a/c'), - mock.call.get('shard-listing/a/c'), + mock.call.get('shard-listing/a/c', raise_on_error=True), mock.call.set('shard-listing/a/c', self.sr_dicts, time=exp_recheck_listing), # Since there was a backend request, we go ahead and cache @@ -2177,7 +2177,7 @@ class TestContainerController(TestRingBase): 'X-Backend-Sharding-State': sharding_state}) self.assertEqual( [mock.call.get('container/a/c'), - mock.call.get('shard-listing/a/c')], + mock.call.get('shard-listing/a/c', raise_on_error=True)], self.memcache.calls) self.assertIn('swift.infocache', req.environ) self.assertIn('shard-listing/a/c', req.environ['swift.infocache']) @@ -2237,7 +2237,7 @@ class TestContainerController(TestRingBase): 'X-Backend-Sharding-State': sharding_state}) self.assertEqual( [mock.call.get('container/a/c'), - mock.call.get('shard-listing/a/c')], + mock.call.get('shard-listing/a/c', raise_on_error=True)], self.memcache.calls) self.assertIn('swift.infocache', req.environ) self.assertIn('shard-listing/a/c', req.environ['swift.infocache']) @@ -2390,7 +2390,7 @@ class TestContainerController(TestRingBase): # deleted from cache self.assertEqual( [mock.call.get('container/a/c'), - mock.call.get('shard-listing/a/c'), + mock.call.get('shard-listing/a/c', raise_on_error=True), mock.call.set('container/a/c', mock.ANY, time=6.0)], self.memcache.calls) self.assertEqual(404, self.memcache.calls[2][1][1]['status']) @@ -2400,6 +2400,39 @@ class TestContainerController(TestRingBase): 'container.shard_listing.backend.404': 1}, self.logger.get_increment_counts()) + def test_GET_shard_ranges_read_from_cache_error(self): + self._setup_shard_range_stubs() + self.memcache = FakeMemcache() + self.memcache.delete_all() + self.logger.clear() + info = headers_to_container_info(self.root_resp_hdrs) + info['status'] = 200 + info['sharding_state'] = 'sharded' + self.memcache.set('container/a/c', info) + self.memcache.clear_calls() + self.memcache.error_on_get = [False, True] + + req = self._build_request({'X-Backend-Record-Type': 'shard'}, + {'states': 'listing'}, {}) + backend_req, resp = self._capture_backend_request( + req, 404, b'', {}, num_resp=2 * self.CONTAINER_REPLICAS) + self._check_backend_req( + req, backend_req, + extra_hdrs={'X-Backend-Record-Type': 'shard', + 'X-Backend-Override-Shard-Name-Filter': 'sharded'}) + self.assertNotIn('X-Backend-Cached-Results', resp.headers) + self.assertEqual( + [mock.call.get('container/a/c'), + mock.call.get('shard-listing/a/c', raise_on_error=True), + mock.call.set('container/a/c', mock.ANY, time=6.0)], + self.memcache.calls) + self.assertEqual(404, self.memcache.calls[2][1][1]['status']) + self.assertEqual(b'', resp.body) + self.assertEqual(404, resp.status_int) + self.assertEqual({'container.shard_listing.cache.error': 1, + 'container.shard_listing.backend.404': 1}, + self.logger.get_increment_counts()) + def _do_test_GET_shard_ranges_read_from_cache(self, params, record_type): # pre-warm cache with container metadata and shard ranges and verify # that shard range listing are read from cache when appropriate @@ -2417,7 +2450,7 @@ class TestContainerController(TestRingBase): resp = req.get_response(self.app) self.assertEqual( [mock.call.get('container/a/c'), - mock.call.get('shard-listing/a/c')], + mock.call.get('shard-listing/a/c', raise_on_error=True)], self.memcache.calls) self.assertEqual({'container.shard_listing.cache.hit': 1}, self.logger.get_increment_counts()) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 01440906d..93c0ada20 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -1627,6 +1627,31 @@ class TestReplicatedObjController(CommonObjectControllerMixin, policy_opts.rebalance_missing_suppression_count = 2 do_test([Timeout(), 404, 404], 503) + # overloaded primary after double rebalance + # ... opts should increase rebalance_missing_suppression_count + policy_opts.rebalance_missing_suppression_count = 2 + do_test([Timeout(), 404, 404], 503) + + # two primaries out, but no rebalance + # ... default is fine for tombstones + policy_opts.rebalance_missing_suppression_count = 1 + do_test([Timeout(), Exception('kaboom!'), 404], 404, + include_timestamp=True) + # ... but maybe not ideal for missing names + # (N.B. 503 isn't really a BAD response here) + do_test([Timeout(), Exception('kaboom!'), 404], 503) + # still ... ops might think they should tune it down + policy_opts.rebalance_missing_suppression_count = 0 + do_test([Timeout(), Exception('kaboom!'), 404], 404) + # and we could maybe leave it like this for the next rebalance + do_test([Timeout(), 404, 404], 404) + # ... but it gets bad when faced with timeouts, b/c we can't trust a + # single primary 404 response during rebalance + do_test([Timeout(), Timeout(), 404], 404) + # ops needs to fix configs to get the 503 + policy_opts.rebalance_missing_suppression_count = 1 + do_test([Timeout(), Timeout(), 404], 503) + def test_GET_primaries_mixed_explode_and_timeout(self): req = swift.common.swob.Request.blank('/v1/a/c/o') primaries = [] diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 5158a09cc..a03e8b43e 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -4413,6 +4413,23 @@ class TestReplicatedObjectController( expected[device] = '10.0.0.%d:100%d' % (i, i) self.assertEqual(container_headers, expected) + # shard lookup in memcache may error... + req = Request.blank( + '/v1/a/c/o', {'swift.cache': cache}, + method=method, body='', headers={'Content-Type': 'text/plain'}) + cache.error_on_get = [False, True] + with mock.patch('random.random', return_value=1.0), \ + mocked_http_conn(*status_codes, headers=resp_headers, + body=body): + resp = req.get_response(self.app) + + self.assertEqual(resp.status_int, 202) + stats = self.app.logger.get_increment_counts() + self.assertEqual({'object.shard_updating.cache.skip': 1, + 'object.shard_updating.cache.hit': 1, + 'object.shard_updating.cache.error': 1, + 'object.shard_updating.backend.200': 2}, stats) + do_test('POST', 'sharding') do_test('POST', 'sharded') do_test('DELETE', 'sharding') |