diff options
author | Tim Burke <tim.burke@gmail.com> | 2018-09-20 20:39:09 +0000 |
---|---|---|
committer | Tim Burke <tim.burke@gmail.com> | 2019-10-21 15:08:55 -0700 |
commit | 6b8c333ef31d2fe9983603c411a3648c612b3928 (patch) | |
tree | 76ddba9383677500446b7846ba3b7c229cfaf325 | |
parent | c6e2fbb41775e065be7cdab2366d846785b57804 (diff) | |
download | swift-6b8c333ef31d2fe9983603c411a3648c612b3928.tar.gz |
s3api: Increase max body size for Delete Multiple Objects requests
Sub-64k was *way* too low, particularly when clients expect to be able to
delete as many as 1000 objects at a time.
I have *no idea* where the previous limit came from.
Change-Id: Ifead1f9ca6509d50dbbef294b7cb3d7f11a09229
(cherry picked from commit ecebea18203d1ca53da8709026acf3e83c3f5357)
-rw-r--r-- | swift/common/middleware/s3api/controllers/multi_delete.py | 13 | ||||
-rw-r--r-- | test/functional/s3api/test_multi_delete.py | 16 | ||||
-rw-r--r-- | test/unit/common/middleware/s3api/test_multi_delete.py | 27 |
3 files changed, 43 insertions, 13 deletions
diff --git a/swift/common/middleware/s3api/controllers/multi_delete.py b/swift/common/middleware/s3api/controllers/multi_delete.py index a7e95a7bd..e12dc8267 100644 --- a/swift/common/middleware/s3api/controllers/multi_delete.py +++ b/swift/common/middleware/s3api/controllers/multi_delete.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from swift.common.constraints import MAX_OBJECT_NAME_LENGTH from swift.common.utils import public from swift.common.middleware.s3api.controllers.base import Controller, \ @@ -23,8 +24,6 @@ from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \ NoSuchKey, ErrorResponse, MalformedXML, UserKeyMustBeSpecified, \ AccessDenied, MissingRequestBodyError -MAX_MULTI_DELETE_BODY_SIZE = 61365 - class MultiObjectDeleteController(Controller): """ @@ -61,8 +60,16 @@ class MultiObjectDeleteController(Controller): yield key, version + max_body_size = min( + # FWIW, AWS limits multideletes to 1000 keys, and swift limits + # object names to 1024 bytes (by default). Add a factor of two to + # allow some slop. + 2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH, + # But, don't let operators shoot themselves in the foot + 10 * 1024 * 1024) + try: - xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE) + xml = req.xml(max_body_size) if not xml: raise MissingRequestBodyError() diff --git a/test/functional/s3api/test_multi_delete.py b/test/functional/s3api/test_multi_delete.py index bd90166ce..e0d9fd2b8 100644 --- a/test/functional/s3api/test_multi_delete.py +++ b/test/functional/s3api/test_multi_delete.py @@ -18,8 +18,6 @@ import os import test.functional as tf from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \ SubElement -from swift.common.middleware.s3api.controllers.multi_delete import \ - MAX_MULTI_DELETE_BODY_SIZE from test.functional.s3api import S3ApiBase from test.functional.s3api.s3_test_client import Connection @@ -172,11 +170,12 @@ class TestS3ApiMultiDelete(S3ApiBase): query=query) self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified') + max_deletes = tf.cluster_info.get('s3api', {}).get( + 'max_multi_delete_objects', 1000) # specified number of objects are over max_multi_delete_objects - # (Default 1000), but xml size is smaller than 61365 bytes. - req_objects = ['obj%s' for var in xrange(1001)] + # (Default 1000), but xml size is relatively small + req_objects = ['obj%s' for var in xrange(max_deletes + 1)] xml = self._gen_multi_delete_xml(req_objects) - self.assertTrue(len(xml.encode('utf-8')) <= MAX_MULTI_DELETE_BODY_SIZE) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, @@ -184,12 +183,11 @@ class TestS3ApiMultiDelete(S3ApiBase): query=query) self.assertEqual(get_error_code(body), 'MalformedXML') - # specified xml size is over 61365 bytes, but number of objects are + # specified xml size is large, but number of objects are # smaller than max_multi_delete_objects. - obj = 'a' * 1024 - req_objects = [obj + str(var) for var in xrange(999)] + obj = 'a' * 102400 + req_objects = [obj + str(var) for var in xrange(max_deletes - 1)] xml = self._gen_multi_delete_xml(req_objects) - self.assertTrue(len(xml.encode('utf-8')) > MAX_MULTI_DELETE_BODY_SIZE) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, diff --git a/test/unit/common/middleware/s3api/test_multi_delete.py b/test/unit/common/middleware/s3api/test_multi_delete.py index 91f8e8d22..a1ba1483a 100644 --- a/test/unit/common/middleware/s3api/test_multi_delete.py +++ b/test/unit/common/middleware/s3api/test_multi_delete.py @@ -180,11 +180,36 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertEqual(self._get_error_code(body), 'InvalidRequest') @s3acl + def test_object_multi_DELETE_lots_of_keys(self): + elem = Element('Delete') + for i in range(self.conf.max_multi_delete_objects): + name = 'x' * 1000 + str(i) + self.swift.register('HEAD', '/v1/AUTH_test/bucket/%s' % name, + swob.HTTPNotFound, {}, None) + obj = SubElement(elem, 'Object') + SubElement(obj, 'Key').text = name + body = tostring(elem, use_s3ns=False) + content_md5 = md5(body).digest().encode('base64').strip() + + req = Request.blank('/bucket?delete', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-MD5': content_md5}, + body=body) + status, headers, body = self.call_s3api(req) + self.assertEqual('200 OK', status) + + elem = fromstring(body) + self.assertEqual(len(elem.findall('Deleted')), + self.conf.max_multi_delete_objects) + + @s3acl def test_object_multi_DELETE_too_many_keys(self): elem = Element('Delete') for i in range(self.conf.max_multi_delete_objects + 1): obj = SubElement(elem, 'Object') - SubElement(obj, 'Key').text = str(i) + SubElement(obj, 'Key').text = 'x' * 1000 + str(i) body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() |