1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
# Copyright (c) 2010-2014 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import json
from swift.common.constraints import MAX_OBJECT_NAME_LENGTH
from swift.common.http import HTTP_NO_CONTENT
from swift.common.swob import str_to_wsgi
from swift.common.utils import public, StreamingPile
from swift.common.registry import get_swift_info
from swift.common.middleware.s3api.controllers.base import Controller, \
bucket_operation
from swift.common.middleware.s3api.etree import Element, SubElement, \
fromstring, tostring, XMLSyntaxError, DocumentInvalid
from swift.common.middleware.s3api.s3response import HTTPOk, \
S3NotImplemented, NoSuchKey, ErrorResponse, MalformedXML, \
UserKeyMustBeSpecified, AccessDenied, MissingRequestBodyError
class MultiObjectDeleteController(Controller):
"""
Handles Delete Multiple Objects, which is logged as a MULTI_OBJECT_DELETE
operation in the S3 server log.
"""
def _gen_error_body(self, error, elem, delete_list):
for key, version in delete_list:
error_elem = SubElement(elem, 'Error')
SubElement(error_elem, 'Key').text = key
if version is not None:
SubElement(error_elem, 'VersionId').text = version
SubElement(error_elem, 'Code').text = error.__class__.__name__
SubElement(error_elem, 'Message').text = error._msg
return tostring(elem)
@public
@bucket_operation
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
max_body_size = min(
# FWIW, AWS limits multideletes to 1000 keys, and swift limits
# object names to 1024 bytes (by default). Add a factor of two to
# allow some slop.
2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
# But, don't let operators shoot themselves in the foot
10 * 1024 * 1024)
try:
xml = req.xml(max_body_size)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
self.quiet = quiet is not None and quiet.text.lower() == 'true'
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
if 'object_versioning' not in get_swift_info() and any(
version not in ('null', None)
for _key, version in delete_list):
raise S3NotImplemented()
def do_delete(base_req, key, version):
req = copy.copy(base_req)
req.environ = copy.copy(base_req.environ)
req.object_name = str_to_wsgi(key)
if version:
req.params = {'version-id': version, 'symlink': 'get'}
try:
try:
query = req.gen_multipart_manifest_delete_query(
self.app, version=version)
except NoSuchKey:
query = {}
if version:
query['version-id'] = version
query['symlink'] = 'get'
resp = req.get_response(self.app, method='DELETE', query=query,
headers={'Accept': 'application/json'})
# If async segment cleanup is available, we expect to get
# back a 204; otherwise, the delete is synchronous and we
# have to read the response to actually do the SLO delete
if query.get('multipart-manifest') and \
resp.status_int != HTTP_NO_CONTENT:
try:
delete_result = json.loads(resp.body)
if delete_result['Errors']:
# NB: bulk includes 404s in "Number Not Found",
# not "Errors"
msg_parts = [delete_result['Response Status']]
msg_parts.extend(
'%s: %s' % (obj, status)
for obj, status in delete_result['Errors'])
return key, {'code': 'SLODeleteError',
'message': '\n'.join(msg_parts)}
# else, all good
except (ValueError, TypeError, KeyError):
# Logs get all the gory details
self.logger.exception(
'Could not parse SLO delete response (%s): %s',
resp.status, resp.body)
# Client gets something more generic
return key, {'code': 'SLODeleteError',
'message': 'Unexpected swift response'}
except NoSuchKey:
pass
except ErrorResponse as e:
return key, {'code': e.__class__.__name__, 'message': e._msg}
except Exception:
self.logger.exception(
'Unexpected Error handling DELETE of %r %r' % (
req.container_name, key))
return key, {'code': 'Server Error', 'message': 'Server Error'}
return key, None
with StreamingPile(self.conf.multi_delete_concurrency) as pile:
for key, err in pile.asyncstarmap(do_delete, (
(req, key, version) for key, version in delete_list)):
if err:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = err['code']
SubElement(error, 'Message').text = err['message']
elif not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
|
