diff options
| -rw-r--r-- | doc/s3api/rnc/list_bucket_result.rnc | 2 | ||||
| -rw-r--r-- | swift/common/middleware/s3api/controllers/bucket.py | 6 | ||||
| -rw-r--r-- | swift/common/middleware/s3api/schema/list_bucket_result.rng | 4 | ||||
| -rw-r--r-- | swift/common/middleware/slo.py | 43 | ||||
| -rw-r--r-- | swift/common/middleware/symlink.py | 2 | ||||
| -rw-r--r-- | swift/common/middleware/versioned_writes.py | 7 | ||||
| -rw-r--r-- | swift/container/replicator.py | 2 | ||||
| -rw-r--r-- | test/functional/test_versioned_writes.py | 41 | ||||
| -rw-r--r-- | test/unit/common/middleware/s3api/test_bucket.py | 44 | ||||
| -rw-r--r-- | test/unit/common/middleware/test_slo.py | 158 | ||||
| -rw-r--r-- | test/unit/container/test_sync.py | 12 | ||||
| -rw-r--r-- | test/unit/obj/test_replicator.py | 2 |
12 files changed, 301 insertions, 22 deletions
diff --git a/doc/s3api/rnc/list_bucket_result.rnc b/doc/s3api/rnc/list_bucket_result.rnc index e7f572b7b..eb86c0804 100644 --- a/doc/s3api/rnc/list_bucket_result.rnc +++ b/doc/s3api/rnc/list_bucket_result.rnc @@ -16,8 +16,8 @@ start = ) ), element MaxKeys { xsd:int }, - element EncodingType { xsd:string }?, element Delimiter { xsd:string }?, + element EncodingType { xsd:string }?, element IsTruncated { xsd:boolean }, element Contents { element Key { xsd:string }, diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index b4f662acb..d3ad14d19 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -177,16 +177,16 @@ class BucketController(Controller): else: name = objects[-1]['subdir'] if encoding_type == 'url': - name = quote(name) + name = quote(name.encode('utf-8')) SubElement(elem, 'NextMarker').text = name elif listing_type == 'version-2': if is_truncated: if 'name' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ - b64encode(objects[-1]['name'].encode('utf8')) + b64encode(objects[-1]['name'].encode('utf-8')) if 'subdir' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ - b64encode(objects[-1]['subdir'].encode('utf8')) + b64encode(objects[-1]['subdir'].encode('utf-8')) if 'continuation-token' in req.params: SubElement(elem, 'ContinuationToken').text = \ req.params['continuation-token'] diff --git a/swift/common/middleware/s3api/schema/list_bucket_result.rng b/swift/common/middleware/s3api/schema/list_bucket_result.rng index 9c6640c69..b3181238e 100644 --- a/swift/common/middleware/s3api/schema/list_bucket_result.rng +++ b/swift/common/middleware/s3api/schema/list_bucket_result.rng @@ -45,12 +45,12 @@ <data type="int"/> </element> <optional> - <element name="EncodingType"> + <element name="Delimiter"> <data type="string"/> </element> </optional> <optional> - <element name="Delimiter"> + <element name="EncodingType"> <data type="string"/> </element> </optional> diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index c347ca6cb..9cf2f24ad 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -328,12 +328,14 @@ from swift.common.middleware.listing_formats import \ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \ HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \ - HTTPUnauthorized, HTTPConflict, HTTPUnprocessableEntity, Response, Range, \ + HTTPUnauthorized, HTTPConflict, HTTPUnprocessableEntity, \ + HTTPServiceUnavailable, Response, Range, \ RESPONSE_REASONS from swift.common.utils import get_logger, config_true_value, \ get_valid_utf8_str, override_bytes_from_content_type, split_path, \ register_swift_info, RateLimitedIterator, quote, close_if_possible, \ - closing_if_possible, LRUCache, StreamingPile, strict_b64decode + closing_if_possible, LRUCache, StreamingPile, strict_b64decode, \ + Timestamp from swift.common.request_helpers import SegmentedIterable, \ get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS @@ -717,7 +719,7 @@ class SloGetContext(WSGIContext): content_range = value break # e.g. Content-Range: bytes 0-14289/14290 - match = re.match('bytes (\d+)-(\d+)/(\d+)$', content_range) + match = re.match(r'bytes (\d+)-(\d+)/(\d+)$', content_range) if not match: # Malformed or missing, so we don't know what we got. return True @@ -750,7 +752,7 @@ class SloGetContext(WSGIContext): resp_iter = self._app_call(req.environ) # make sure this response is for a static large object manifest - slo_marker = slo_etag = slo_size = None + slo_marker = slo_etag = slo_size = slo_timestamp = None for header, value in self._response_headers: header = header.lower() if header == SYSMETA_SLO_ETAG: @@ -760,8 +762,10 @@ class SloGetContext(WSGIContext): elif (header == 'x-static-large-object' and config_true_value(value)): slo_marker = value + elif header == 'x-backend-timestamp': + slo_timestamp = value - if slo_marker and slo_etag and slo_size: + if slo_marker and slo_etag and slo_size and slo_timestamp: break if not slo_marker: @@ -819,6 +823,35 @@ class SloGetContext(WSGIContext): headers={'x-auth-token': req.headers.get('x-auth-token')}, agent='%(orig)s SLO MultipartGET', swift_source='SLO') resp_iter = self._app_call(get_req.environ) + slo_marker = config_true_value(self._response_header_value( + 'x-static-large-object')) + if not slo_marker: # will also catch non-2xx responses + got_timestamp = self._response_header_value( + 'x-backend-timestamp') or '0' + if Timestamp(got_timestamp) >= Timestamp(slo_timestamp): + # We've got a newer response available, so serve that. + # Note that if there's data, it's going to be a 200 now, + # not a 206, and we're not going to drop bytes in the + # proxy on the client's behalf. Fortunately, the RFC is + # pretty forgiving for a server; there's no guarantee that + # a Range header will be respected. + resp = Response( + status=self._response_status, + headers=self._response_headers, + app_iter=resp_iter, + request=req, + conditional_etag=resolve_etag_is_at_header( + req, self._response_headers), + conditional_response=is_success( + int(self._response_status[:3]))) + return resp(req.environ, start_response) + else: + # We saw newer data that indicated it's an SLO, but + # couldn't fetch the whole thing; 503 seems reasonable? + close_if_possible(resp_iter) + raise HTTPServiceUnavailable(request=req) + # NB: we might have gotten an out-of-date manifest -- that's OK; + # we'll just try to serve the old data # Any Content-Range from a manifest is almost certainly wrong for the # full large object. diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py index 94e6c8edf..e8cbd15c8 100644 --- a/swift/common/middleware/symlink.py +++ b/swift/common/middleware/symlink.py @@ -447,7 +447,7 @@ class SymlinkObjectContext(WSGIContext): if TGT_ACCT_SYSMETA_SYMLINK_HDR in req.headers: etag_override.append( 'symlink_target_account=%s' % - req.headers[TGT_OBJ_SYSMETA_SYMLINK_HDR]) + req.headers[TGT_ACCT_SYSMETA_SYMLINK_HDR]) req.headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \ '; '.join(etag_override) diff --git a/swift/common/middleware/versioned_writes.py b/swift/common/middleware/versioned_writes.py index 516f3f26e..d4f9b5e34 100644 --- a/swift/common/middleware/versioned_writes.py +++ b/swift/common/middleware/versioned_writes.py @@ -385,13 +385,18 @@ class VersionedWritesContext(WSGIContext): return source_resp def _put_versioned_obj(self, req, put_path_info, source_resp): - # Create a new Request object to PUT to the versions container, copying + # Create a new Request object to PUT to the container, copying # all headers from the source object apart from x-timestamp. put_req = make_pre_authed_request( req.environ, path=quote(put_path_info), method='PUT', swift_source='VW') copy_header_subset(source_resp, put_req, lambda k: k.lower() != 'x-timestamp') + slo_size = put_req.headers.get('X-Object-Sysmeta-Slo-Size') + if slo_size: + put_req.headers['Content-Type'] += '; swift_bytes=' + slo_size + put_req.environ['swift.content_type_overridden'] = True + put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) put_resp = put_req.get_response(self.app) close_if_possible(source_resp.app_iter) diff --git a/swift/container/replicator.py b/swift/container/replicator.py index 1077e7f4e..67a988432 100644 --- a/swift/container/replicator.py +++ b/swift/container/replicator.py @@ -138,7 +138,7 @@ class ContainerReplicator(db_replicator.Replicator): def _fetch_and_merge_shard_ranges(self, http, broker): with Timeout(self.node_timeout): response = http.replicate('get_shard_ranges') - if is_success(response.status): + if response and is_success(response.status): broker.merge_shard_ranges(json.loads( response.data.decode('ascii'))) diff --git a/test/functional/test_versioned_writes.py b/test/functional/test_versioned_writes.py index 205ba0663..0be71c315 100644 --- a/test/functional/test_versioned_writes.py +++ b/test/functional/test_versioned_writes.py @@ -1047,3 +1047,44 @@ class TestSloWithVersioning(unittest2.TestCase): # expect the original manifest file to be restored self._assert_is_manifest(file_item, 'a') self._assert_is_object(file_item, 'a') + + def test_slo_manifest_version_size(self): + file_item = self._create_manifest('a') + # sanity check: read the manifest, then the large object + self._assert_is_manifest(file_item, 'a') + self._assert_is_object(file_item, b'a') + + # original manifest size + primary_list = self.container.files(parms={'format': 'json'}) + self.assertEqual(1, len(primary_list)) + org_size = primary_list[0]['bytes'] + + # upload new manifest + file_item = self._create_manifest('b') + # sanity check: read the manifest, then the large object + self._assert_is_manifest(file_item, 'b') + self._assert_is_object(file_item, b'b') + + versions_list = self.versions_container.files(parms={'format': 'json'}) + self.assertEqual(1, len(versions_list)) + version_file = self.versions_container.file(versions_list[0]['name']) + version_file_size = versions_list[0]['bytes'] + # check the version is still a manifest + self._assert_is_manifest(version_file, 'a') + self._assert_is_object(version_file, b'a') + + # check the version size is correct + self.assertEqual(version_file_size, org_size) + + # delete the newest manifest + file_item.delete() + + # expect the original manifest file to be restored + self._assert_is_manifest(file_item, 'a') + self._assert_is_object(file_item, b'a') + + primary_list = self.container.files(parms={'format': 'json'}) + self.assertEqual(1, len(primary_list)) + primary_file_size = primary_list[0]['bytes'] + # expect the original manifest file size to be the same + self.assertEqual(primary_file_size, org_size) diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index 85dd7437c..a530433ae 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -17,6 +17,7 @@ import unittest import cgi import mock +import six from six.moves.urllib.parse import quote from swift.common import swob @@ -95,7 +96,7 @@ class TestS3ApiBucket(S3ApiTestCase): '/v1/AUTH_test/subdirs?delimiter=/&format=json&limit=3', swob.HTTPOk, {}, json.dumps([ {'subdir': 'nothing/'}, - {'subdir': 'but/'}, + {'subdir': u'but-\u062a/'}, {'subdir': 'subdirs/'}, ])) @@ -242,7 +243,46 @@ class TestS3ApiBucket(S3ApiTestCase): status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./IsTruncated').text, 'true') - self.assertEqual(elem.find('./NextMarker').text, 'but/') + if six.PY2: + self.assertEqual(elem.find('./NextMarker').text, + u'but-\u062a/'.encode('utf-8')) + else: + self.assertEqual(elem.find('./NextMarker').text, + u'but-\u062a/') + + def test_bucket_GET_is_truncated_url_encoded(self): + bucket_name = 'junk' + + req = Request.blank( + '/%s?encoding-type=url&max-keys=%d' % ( + bucket_name, len(self.objects)), + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + elem = fromstring(body, 'ListBucketResult') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + + req = Request.blank( + '/%s?encoding-type=url&max-keys=%d' % ( + bucket_name, len(self.objects) - 1), + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + elem = fromstring(body, 'ListBucketResult') + self.assertEqual(elem.find('./IsTruncated').text, 'true') + + req = Request.blank('/subdirs?encoding-type=url&delimiter=/&' + 'max-keys=2', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + elem = fromstring(body, 'ListBucketResult') + self.assertEqual(elem.find('./IsTruncated').text, 'true') + self.assertEqual(elem.find('./NextMarker').text, + quote(u'but-\u062a/'.encode('utf-8'))) def test_bucket_GET_v2_is_truncated(self): bucket_name = 'junk' diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 7da774f96..95bc89327 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -2387,6 +2387,164 @@ class TestSloGetManifest(SloTestCase): ('GET', '/v1/AUTH_test/gettest/big_seg?multipart-manifest=get')]) + def test_range_get_beyond_manifest_refetch_fails(self): + big = 'e' * 1024 * 1024 + big_etag = md5hex(big) + big_manifest = json.dumps( + [{'name': '/gettest/big_seg', 'hash': big_etag, + 'bytes': 1024 * 1024, 'content_type': 'application/foo'}]) + self.app.register_responses( + 'GET', '/v1/AUTH_test/gettest/big_manifest', + [(swob.HTTPOk, {'Content-Type': 'application/octet-stream', + 'X-Static-Large-Object': 'true', + 'X-Backend-Timestamp': '1234', + 'Etag': md5hex(big_manifest)}, + big_manifest), + (swob.HTTPNotFound, {}, None)]) + + req = Request.blank( + '/v1/AUTH_test/gettest/big_manifest', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Range': 'bytes=100000-199999'}) + status, headers, body = self.call_slo(req) + headers = HeaderKeyDict(headers) + + self.assertEqual(status, '503 Service Unavailable') + self.assertNotIn('X-Static-Large-Object', headers) + self.assertEqual(self.app.calls, [ + # has Range header, gets 416 + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + # retry the first one + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + ]) + + def test_range_get_beyond_manifest_refetch_finds_old(self): + big = 'e' * 1024 * 1024 + big_etag = md5hex(big) + big_manifest = json.dumps( + [{'name': '/gettest/big_seg', 'hash': big_etag, + 'bytes': 1024 * 1024, 'content_type': 'application/foo'}]) + self.app.register_responses( + 'GET', '/v1/AUTH_test/gettest/big_manifest', + [(swob.HTTPOk, {'Content-Type': 'application/octet-stream', + 'X-Static-Large-Object': 'true', + 'X-Backend-Timestamp': '1234', + 'Etag': md5hex(big_manifest)}, + big_manifest), + (swob.HTTPOk, {'X-Backend-Timestamp': '1233'}, [b'small body'])]) + + req = Request.blank( + '/v1/AUTH_test/gettest/big_manifest', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Range': 'bytes=100000-199999'}) + status, headers, body = self.call_slo(req) + headers = HeaderKeyDict(headers) + + self.assertEqual(status, '503 Service Unavailable') + self.assertNotIn('X-Static-Large-Object', headers) + self.assertEqual(self.app.calls, [ + # has Range header, gets 416 + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + # retry the first one + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + ]) + + def test_range_get_beyond_manifest_refetch_small_non_slo(self): + big = 'e' * 1024 * 1024 + big_etag = md5hex(big) + big_manifest = json.dumps( + [{'name': '/gettest/big_seg', 'hash': big_etag, + 'bytes': 1024 * 1024, 'content_type': 'application/foo'}]) + self.app.register_responses( + 'GET', '/v1/AUTH_test/gettest/big_manifest', + [(swob.HTTPOk, {'Content-Type': 'application/octet-stream', + 'X-Static-Large-Object': 'true', + 'X-Backend-Timestamp': '1234', + 'Etag': md5hex(big_manifest)}, + big_manifest), + (swob.HTTPOk, {'X-Backend-Timestamp': '1235'}, [b'small body'])]) + + req = Request.blank( + '/v1/AUTH_test/gettest/big_manifest', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Range': 'bytes=100000-199999'}) + status, headers, body = self.call_slo(req) + headers = HeaderKeyDict(headers) + + self.assertEqual(status, '416 Requested Range Not Satisfiable') + self.assertNotIn('X-Static-Large-Object', headers) + self.assertEqual(self.app.calls, [ + # has Range header, gets 416 + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + # retry the first one + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + ]) + + def test_range_get_beyond_manifest_refetch_big_non_slo(self): + big = 'e' * 1024 * 1024 + big_etag = md5hex(big) + big_manifest = json.dumps( + [{'name': '/gettest/big_seg', 'hash': big_etag, + 'bytes': 1024 * 1024, 'content_type': 'application/foo'}]) + self.app.register_responses( + 'GET', '/v1/AUTH_test/gettest/big_manifest', + [(swob.HTTPOk, {'Content-Type': 'application/octet-stream', + 'X-Static-Large-Object': 'true', + 'X-Backend-Timestamp': '1234', + 'Etag': md5hex(big_manifest)}, + big_manifest), + (swob.HTTPOk, {'X-Backend-Timestamp': '1235'}, + [b'x' * 1024 * 1024])]) + + req = Request.blank( + '/v1/AUTH_test/gettest/big_manifest', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Range': 'bytes=100000-199999'}) + status, headers, body = self.call_slo(req) + headers = HeaderKeyDict(headers) + + self.assertEqual(status, '200 OK') # NOT 416 or 206! + self.assertNotIn('X-Static-Large-Object', headers) + self.assertEqual(len(body), 1024 * 1024) + self.assertEqual(body, b'x' * 1024 * 1024) + self.assertEqual(self.app.calls, [ + # has Range header, gets 416 + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + # retry the first one + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + ]) + + def test_range_get_beyond_manifest_refetch_tombstone(self): + big = 'e' * 1024 * 1024 + big_etag = md5hex(big) + big_manifest = json.dumps( + [{'name': '/gettest/big_seg', 'hash': big_etag, + 'bytes': 1024 * 1024, 'content_type': 'application/foo'}]) + self.app.register_responses( + 'GET', '/v1/AUTH_test/gettest/big_manifest', + [(swob.HTTPOk, {'Content-Type': 'application/octet-stream', + 'X-Static-Large-Object': 'true', + 'X-Backend-Timestamp': '1234', + 'Etag': md5hex(big_manifest)}, + big_manifest), + (swob.HTTPNotFound, {'X-Backend-Timestamp': '1345'}, None)]) + + req = Request.blank( + '/v1/AUTH_test/gettest/big_manifest', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Range': 'bytes=100000-199999'}) + status, headers, body = self.call_slo(req) + headers = HeaderKeyDict(headers) + + self.assertEqual(status, '404 Not Found') + self.assertNotIn('X-Static-Large-Object', headers) + self.assertEqual(self.app.calls, [ + # has Range header, gets 416 + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + # retry the first one + ('GET', '/v1/AUTH_test/gettest/big_manifest'), + ]) + def test_range_get_bogus_content_range(self): # Just a little paranoia; Swift currently sends back valid # Content-Range headers, but if somehow someone sneaks an invalid one diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index d3c7e468d..833304bfa 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -793,11 +793,13 @@ class TestContainerSync(unittest.TestCase): # Succeeds because no rows match log_line = cs.logger.get_lines_for_level('info')[0] lines = log_line.split(',') - self.assertTrue('sync_point2: 5', lines.pop().strip()) - self.assertTrue('sync_point1: 5', lines.pop().strip()) - self.assertTrue('bytes: 1100', lines.pop().strip()) - self.assertTrue('deletes: 2', lines.pop().strip()) - self.assertTrue('puts: 3', lines.pop().strip()) + self.assertEqual('total_rows: 1', lines.pop().strip()) + self.assertEqual('sync_point2: None', lines.pop().strip()) + self.assertEqual('sync_point1: 5', lines.pop().strip()) + self.assertEqual('bytes: 0', lines.pop().strip()) + self.assertEqual('deletes: 0', lines.pop().strip()) + self.assertEqual('posts: 0', lines.pop().strip()) + self.assertEqual('puts: 0', lines.pop().strip()) def test_container_sync_row_delete(self): self._test_container_sync_row_delete(None, None) diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index 781a3de17..20dd320ec 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -469,7 +469,7 @@ class TestObjectReplicator(unittest.TestCase): for job in jobs: jobs_by_pol_part[str(int(job['policy'])) + job['partition']] = job self.assertEqual(len(jobs_to_delete), 2) - self.assertTrue('1', jobs_to_delete[0]['partition']) + self.assertEqual('1', jobs_to_delete[0]['partition']) self.assertEqual( [node['id'] for node in jobs_by_pol_part['00']['nodes']], [1, 2]) self.assertEqual( |