diff options
Diffstat (limited to 'swift/common/middleware/s3api/s3request.py')
| -rw-r--r-- | swift/common/middleware/s3api/s3request.py | 210 |
1 files changed, 160 insertions, 50 deletions
diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index 5833921f8..86dd6f75f 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -14,7 +14,7 @@ # limitations under the License. import base64 -from collections import defaultdict +from collections import defaultdict, OrderedDict from email.header import Header from hashlib import sha1, sha256, md5 import hmac @@ -24,7 +24,7 @@ import six from six.moves.urllib.parse import quote, unquote, parse_qsl import string -from swift.common.utils import split_path +from swift.common.utils import split_path, json, get_swift_info from swift.common import swob from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \ HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \ @@ -44,7 +44,7 @@ from swift.common.middleware.s3api.controllers import ServiceController, \ UploadController, UploadsController, VersioningController, \ UnsupportedController, S3AclController, BucketController from swift.common.middleware.s3api.s3response import AccessDenied, \ - InvalidArgument, InvalidDigest, \ + InvalidArgument, InvalidDigest, BucketAlreadyOwnedByYou, \ RequestTimeTooSkewed, S3Response, SignatureDoesNotMatch, \ BucketAlreadyExists, BucketNotEmpty, EntityTooLarge, \ InternalError, NoSuchBucket, NoSuchKey, PreconditionFailed, InvalidRange, \ @@ -116,9 +116,10 @@ class SigV4Mixin(object): """ def check_signature(self, secret): + secret = utf8encode(secret) user_signature = self.signature derived_secret = 'AWS4' + secret - for scope_piece in self.scope: + for scope_piece in self.scope.values(): derived_secret = hmac.new( derived_secret, scope_piece, sha256).digest() valid_signature = hmac.new( @@ -176,6 +177,8 @@ class SigV4Mixin(object): err = None try: expires = int(self.params['X-Amz-Expires']) + except KeyError: + raise AccessDenied() except ValueError: err = 'X-Amz-Expires should be a number' else: @@ -193,6 +196,15 @@ class SigV4Mixin(object): if int(self.timestamp) + expires < S3Timestamp.now(): raise AccessDenied('Request has expired') + def _parse_credential(self, credential_string): + parts = credential_string.split("/") + # credential must be in following format: + # <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request + if not parts[0] or len(parts) != 5: + raise AccessDenied() + return dict(zip(['access', 'date', 'region', 'service', 'terminal'], + parts)) + def _parse_query_authentication(self): """ Parse v4 query authentication @@ -205,10 +217,11 @@ class SigV4Mixin(object): raise InvalidArgument('X-Amz-Algorithm', self.params.get('X-Amz-Algorithm')) try: - cred_param = self.params['X-Amz-Credential'].split("/") - access = cred_param[0] + cred_param = self._parse_credential( + self.params['X-Amz-Credential']) sig = self.params['X-Amz-Signature'] - expires = self.params['X-Amz-Expires'] + if not sig: + raise AccessDenied() except KeyError: raise AccessDenied() @@ -220,12 +233,26 @@ class SigV4Mixin(object): self._signed_headers = set(signed_headers.split(';')) - # credential must be in following format: - # <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request - if not all([access, sig, len(cred_param) == 5, expires]): - raise AccessDenied() + invalid_messages = { + 'date': 'Invalid credential date "%s". This date is not the same ' + 'as X-Amz-Date: "%s".', + 'region': "Error parsing the X-Amz-Credential parameter; " + "the region '%s' is wrong; expecting '%s'", + 'service': 'Error parsing the X-Amz-Credential parameter; ' + 'incorrect service "%s". This endpoint belongs to "%s".', + 'terminal': 'Error parsing the X-Amz-Credential parameter; ' + 'incorrect terminal "%s". This endpoint uses "%s".', + } + for key in ('date', 'region', 'service', 'terminal'): + if cred_param[key] != self.scope[key]: + kwargs = {} + if key == 'region': + kwargs = {'region': self.scope['region']} + raise AuthorizationQueryParametersError( + invalid_messages[key] % (cred_param[key], self.scope[key]), + **kwargs) - return access, sig + return cred_param['access'], sig def _parse_header_authentication(self): """ @@ -237,23 +264,39 @@ class SigV4Mixin(object): """ auth_str = self.headers['Authorization'] - cred_param = auth_str.partition( - "Credential=")[2].split(',')[0].split("/") - access = cred_param[0] + cred_param = self._parse_credential(auth_str.partition( + "Credential=")[2].split(',')[0]) sig = auth_str.partition("Signature=")[2].split(',')[0] + if not sig: + raise AccessDenied() signed_headers = auth_str.partition( "SignedHeaders=")[2].split(',', 1)[0] - # credential must be in following format: - # <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request - if not all([access, sig, len(cred_param) == 5]): - raise AccessDenied() if not signed_headers: # TODO: make sure if is it Malformed? raise AuthorizationHeaderMalformed() + invalid_messages = { + 'date': 'Invalid credential date "%s". This date is not the same ' + 'as X-Amz-Date: "%s".', + 'region': "The authorization header is malformed; the region '%s' " + "is wrong; expecting '%s'", + 'service': 'The authorization header is malformed; incorrect ' + 'service "%s". This endpoint belongs to "%s".', + 'terminal': 'The authorization header is malformed; incorrect ' + 'terminal "%s". This endpoint uses "%s".', + } + for key in ('date', 'region', 'service', 'terminal'): + if cred_param[key] != self.scope[key]: + kwargs = {} + if key == 'region': + kwargs = {'region': self.scope['region']} + raise AuthorizationHeaderMalformed( + invalid_messages[key] % (cred_param[key], self.scope[key]), + **kwargs) + self._signed_headers = set(signed_headers.split(';')) - return access, sig + return cred_param['access'], sig def _canonical_query_string(self): return '&'.join( @@ -363,8 +406,12 @@ class SigV4Mixin(object): @property def scope(self): - return [self.timestamp.amz_date_format.split('T')[0], - self.location, SERVICE, 'aws4_request'] + return OrderedDict([ + ('date', self.timestamp.amz_date_format.split('T')[0]), + ('region', self.location), + ('service', SERVICE), + ('terminal', 'aws4_request'), + ]) def _string_to_sign(self): """ @@ -372,9 +419,19 @@ class SigV4Mixin(object): """ return '\n'.join(['AWS4-HMAC-SHA256', self.timestamp.amz_date_format, - '/'.join(self.scope), + '/'.join(self.scope.values()), sha256(self._canonical_request()).hexdigest()]) + def signature_does_not_match_kwargs(self): + kwargs = super(SigV4Mixin, self).signature_does_not_match_kwargs() + cr = self._canonical_request() + kwargs.update({ + 'canonical_request': cr, + 'canonical_request_bytes': ' '.join( + format(ord(c), '02x') for c in cr), + }) + return kwargs + def get_request_class(env, s3_acl): """ @@ -404,8 +461,8 @@ class S3Request(swob.Request): bucket_acl = _header_acl_property('container') object_acl = _header_acl_property('object') - def __init__(self, env, app=None, slo_enabled=True, - storage_domain='', location='US', force_request_log=False, + def __init__(self, env, app=None, slo_enabled=True, storage_domain='', + location='us-east-1', force_request_log=False, dns_compliant_bucket_names=True, allow_multipart_uploads=True, allow_no_owner=False): # NOTE: app and allow_no_owner are not used by this class, need for @@ -434,20 +491,12 @@ class S3Request(swob.Request): self.user_id = None self.slo_enabled = slo_enabled - # NOTE(andrey-mp): substitute authorization header for next modules - # in pipeline (s3token). it uses this and X-Auth-Token in specific - # format. - # (kota_): yeah, the reason we need this is s3token only supports - # v2 like header consists of AWS access:signature. Since the commit - # b626a3ca86e467fc7564eac236b9ee2efd49bdcc, the s3token is in swift3 - # repo so probably we need to change s3token to support v4 format. - self.headers['Authorization'] = 'AWS %s:%s' % ( - self.access_key, self.signature) # Avoids that swift.swob.Response replaces Location header value # by full URL when absolute path given. See swift.swob for more detail. self.environ['swift.leave_relative_location'] = True def check_signature(self, secret): + secret = utf8encode(secret) user_signature = self.signature valid_signature = base64.b64encode(hmac.new( secret, self.string_to_sign, sha1).digest()).strip() @@ -577,8 +626,10 @@ class S3Request(swob.Request): :raises: NotS3Request """ if self._is_query_auth: + self._validate_expire_param() return self._parse_query_authentication() elif self._is_header_auth: + self._validate_dates() return self._parse_header_authentication() else: # if this request is neither query auth nor header auth @@ -593,7 +644,7 @@ class S3Request(swob.Request): # Expires header is a float since epoch try: ex = S3Timestamp(float(self.params['Expires'])) - except ValueError: + except (KeyError, ValueError): raise AccessDenied() if S3Timestamp.now() > ex: @@ -610,11 +661,6 @@ class S3Request(swob.Request): :raises: AccessDenied :raises: RequestTimeTooSkewed """ - if self._is_query_auth: - self._validate_expire_param() - # TODO: make sure the case if timestamp param in query - return - date_header = self.headers.get('Date') amz_date_header = self.headers.get('X-Amz-Date') if not date_header and not amz_date_header: @@ -642,8 +688,6 @@ class S3Request(swob.Request): raise InvalidArgument('Content-Length', self.environ['CONTENT_LENGTH']) - self._validate_dates() - value = _header_strip(self.headers.get('Content-MD5')) if value is not None: if not re.match('^[A-Za-z0-9+/]+={0,2}$', value): @@ -688,12 +732,33 @@ class S3Request(swob.Request): if 'x-amz-mfa' in self.headers: raise S3NotImplemented('MFA Delete is not supported.') - if 'x-amz-server-side-encryption' in self.headers: - raise S3NotImplemented('Server-side encryption is not supported.') + sse_value = self.headers.get('x-amz-server-side-encryption') + if sse_value is not None: + if sse_value not in ('aws:kms', 'AES256'): + raise InvalidArgument( + 'x-amz-server-side-encryption', sse_value, + 'The encryption method specified is not supported') + encryption_enabled = get_swift_info(admin=True)['admin'].get( + 'encryption', {}).get('enabled') + if not encryption_enabled or sse_value != 'AES256': + raise S3NotImplemented( + 'Server-side encryption is not supported.') if 'x-amz-website-redirect-location' in self.headers: raise S3NotImplemented('Website redirection is not supported.') + # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html + # describes some of what would be required to support this + if any(['aws-chunked' in self.headers.get('content-encoding', ''), + 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' == self.headers.get( + 'x-amz-content-sha256', ''), + 'x-amz-decoded-content-length' in self.headers]): + raise S3NotImplemented('Transfering payloads in multiple chunks ' + 'using aws-chunked is not supported.') + + if 'x-amz-tagging' in self.headers: + raise S3NotImplemented('Object tagging is not supported.') + @property def body(self): """ @@ -866,6 +931,15 @@ class S3Request(swob.Request): buf.append(path) return '\n'.join(buf) + def signature_does_not_match_kwargs(self): + return { + 'a_w_s_access_key_id': self.access_key, + 'string_to_sign': self.string_to_sign, + 'signature_provided': self.signature, + 'string_to_sign_bytes': ' '.join( + format(ord(c), '02x') for c in self.string_to_sign), + } + @property def controller_name(self): return self.controller.__name__[:-len('Controller')] @@ -982,6 +1056,22 @@ class S3Request(swob.Request): env['HTTP_X_COPY_FROM'] = env['HTTP_X_AMZ_COPY_SOURCE'] del env['HTTP_X_AMZ_COPY_SOURCE'] env['CONTENT_LENGTH'] = '0' + # Content type cannot be modified on COPY + env.pop('CONTENT_TYPE', None) + if env.pop('HTTP_X_AMZ_METADATA_DIRECTIVE', None) == 'REPLACE': + env['HTTP_X_FRESH_METADATA'] = 'True' + else: + copy_exclude_headers = ('HTTP_CONTENT_DISPOSITION', + 'HTTP_CONTENT_ENCODING', + 'HTTP_CONTENT_LANGUAGE', + 'HTTP_EXPIRES', + 'HTTP_CACHE_CONTROL', + 'HTTP_X_ROBOTS_TAG') + for key in copy_exclude_headers: + env.pop(key, None) + for key in list(env.keys()): + if key.startswith('HTTP_X_OBJECT_META_'): + del env[key] if self.force_request_log: env['swift.proxy_access_log_made'] = False @@ -1059,6 +1149,7 @@ class S3Request(swob.Request): ], 'PUT': [ HTTP_CREATED, + HTTP_ACCEPTED, # For SLO with heartbeating ], 'POST': [ HTTP_ACCEPTED, @@ -1071,6 +1162,20 @@ class S3Request(swob.Request): return code_map[method] + def _bucket_put_accepted_error(self, container, app): + sw_req = self.to_swift_req('HEAD', container, None) + info = get_container_info(sw_req.environ, app) + sysmeta = info.get('sysmeta', {}) + try: + acl = json.loads(sysmeta.get('s3api-acl', + sysmeta.get('swift3-acl', '{}'))) + owner = acl.get('Owner') + except (ValueError, TypeError, KeyError): + owner = None + if owner is None or owner == self.user_id: + raise BucketAlreadyOwnedByYou(container) + raise BucketAlreadyExists(container) + def _swift_error_codes(self, method, container, obj, env, app): """ Returns a dict from expected Swift error codes to the corresponding S3 @@ -1092,7 +1197,8 @@ class S3Request(swob.Request): HTTP_NOT_FOUND: (NoSuchBucket, container), }, 'PUT': { - HTTP_ACCEPTED: (BucketAlreadyExists, container), + HTTP_ACCEPTED: (self._bucket_put_accepted_error, container, + app), }, 'POST': { HTTP_NOT_FOUND: (NoSuchBucket, container), @@ -1202,7 +1308,8 @@ class S3Request(swob.Request): if status == HTTP_BAD_REQUEST: raise BadSwiftRequest(err_msg) if status == HTTP_UNAUTHORIZED: - raise SignatureDoesNotMatch() + raise SignatureDoesNotMatch( + **self.signature_does_not_match_kwargs()) if status == HTTP_FORBIDDEN: raise AccessDenied() @@ -1294,8 +1401,8 @@ class S3AclRequest(S3Request): """ S3Acl request object. """ - def __init__(self, env, app, slo_enabled=True, - storage_domain='', location='US', force_request_log=False, + def __init__(self, env, app, slo_enabled=True, storage_domain='', + location='us-east-1', force_request_log=False, dns_compliant_bucket_names=True, allow_multipart_uploads=True, allow_no_owner=False): super(S3AclRequest, self).__init__( @@ -1325,7 +1432,8 @@ class S3AclRequest(S3Request): sw_resp = sw_req.get_response(app) if not sw_req.remote_user: - raise SignatureDoesNotMatch() + raise SignatureDoesNotMatch( + **self.signature_does_not_match_kwargs()) _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'], 2, 3, True) @@ -1341,9 +1449,11 @@ class S3AclRequest(S3Request): # tempauth self.user_id = self.access_key + sw_req.environ.get('swift.authorize', lambda req: None)(sw_req) + self.environ['swift_owner'] = sw_req.environ.get('swift_owner', False) + # Need to skip S3 authorization on subsequent requests to prevent # overwriting the account in PATH_INFO - del self.headers['Authorization'] del self.environ['s3api.auth_details'] def to_swift_req(self, method, container, obj, query=None, |