From 0a4e41701dbb5795ff4cab7a2c68a41c90bd51e7 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 25 Feb 2022 10:47:31 -0800 Subject: Add cap_length helper Change-Id: Ib864c7dc6c8c7bb849f4f97a1239eb5cc04c424c --- swift/common/middleware/proxy_logging.py | 6 ++---- swift/common/utils/__init__.py | 9 +++++++++ swift/obj/ssync_receiver.py | 6 ++++-- swift/obj/ssync_sender.py | 8 ++++---- swift/proxy/server.py | 4 ++-- test/unit/common/test_utils.py | 9 +++++++++ 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/swift/common/middleware/proxy_logging.py b/swift/common/middleware/proxy_logging.py index d89343479..3cf2aee62 100644 --- a/swift/common/middleware/proxy_logging.py +++ b/swift/common/middleware/proxy_logging.py @@ -78,7 +78,7 @@ from swift.common.middleware.catch_errors import enforce_byte_count from swift.common.swob import Request from swift.common.utils import (get_logger, get_remote_client, config_true_value, reiterate, - close_if_possible, + close_if_possible, cap_length, InputProxy, list_from_csv, get_policy_index, split_path, StrAnonymizer, StrFormatTime, LogStringFormatter) @@ -200,9 +200,7 @@ class ProxyLoggingMiddleware(object): env['swift.proxy_access_log_made'] = True def obscure_sensitive(self, value): - if value and len(value) > self.reveal_sensitive_prefix: - return value[:self.reveal_sensitive_prefix] + '...' - return value + return cap_length(value, self.reveal_sensitive_prefix) def obscure_req(self, req): for header in get_sensitive_headers(): diff --git a/swift/common/utils/__init__.py b/swift/common/utils/__init__.py index 9c560aa71..3b4db177e 100644 --- a/swift/common/utils/__init__.py +++ b/swift/common/utils/__init__.py @@ -6483,6 +6483,15 @@ def strict_b64decode(value, allow_line_breaks=False): raise ValueError +def cap_length(value, max_length): + if value and len(value) > max_length: + if isinstance(value, bytes): + return value[:max_length] + b'...' + else: + return value[:max_length] + '...' + return value + + MD5_BLOCK_READ_BYTES = 4096 diff --git a/swift/obj/ssync_receiver.py b/swift/obj/ssync_receiver.py index b1e2ab072..345728a83 100644 --- a/swift/obj/ssync_receiver.py +++ b/swift/obj/ssync_receiver.py @@ -380,7 +380,8 @@ class Receiver(object): raise SsyncClientDisconnected if line.strip() != b':MISSING_CHECK: START': raise Exception( - 'Looking for :MISSING_CHECK: START got %r' % line[:1024]) + 'Looking for :MISSING_CHECK: START got %r' + % utils.cap_length(line, 1024)) object_hashes = [] nlines = 0 while True: @@ -442,7 +443,8 @@ class Receiver(object): # Guess they hung up waiting for us to process the missing check raise SsyncClientDisconnected if line.strip() != b':UPDATES: START': - raise Exception('Looking for :UPDATES: START got %r' % line[:1024]) + raise Exception('Looking for :UPDATES: START got %r' + % utils.cap_length(line, 1024)) successes = 0 failures = 0 updates = 0 diff --git a/swift/obj/ssync_sender.py b/swift/obj/ssync_sender.py index 3f3bd9477..57f02e0e2 100644 --- a/swift/obj/ssync_sender.py +++ b/swift/obj/ssync_sender.py @@ -262,7 +262,7 @@ class Sender(object): self.daemon.node_timeout, 'connect receive'): response = connection.getresponse() if response.status != http.HTTP_OK: - err_msg = response.read()[:1024] + err_msg = utils.cap_length(response.read(), 1024) raise exceptions.ReplicationException( 'Expected status %s; got %s (%s)' % (http.HTTP_OK, response.status, err_msg)) @@ -354,7 +354,7 @@ class Sender(object): except UnicodeDecodeError: pass raise exceptions.ReplicationException( - 'Unexpected response: %r' % line[:1024]) + 'Unexpected response: %r' % utils.cap_length(line, 1024)) while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'missing_check line wait'): @@ -443,7 +443,7 @@ class Sender(object): except UnicodeDecodeError: pass raise exceptions.ReplicationException( - 'Unexpected response: %r' % line[:1024]) + 'Unexpected response: %r' % utils.cap_length(line, 1024)) while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'updates line wait'): @@ -460,7 +460,7 @@ class Sender(object): except UnicodeDecodeError: pass raise exceptions.ReplicationException( - 'Unexpected response: %r' % line[:1024]) + 'Unexpected response: %r' % utils.cap_length(line, 1024)) def send_subrequest(self, connection, method, url_path, headers, df): msg = [b'%s %s' % (method.encode('ascii'), url_path.encode('utf8'))] diff --git a/swift/proxy/server.py b/swift/proxy/server.py index a5ba1d858..64a14ff1d 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -36,7 +36,7 @@ from swift.common.utils import Watchdog, get_logger, \ get_remote_client, split_path, config_true_value, generate_trans_id, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \ parse_prefixed_conf, config_auto_int_value, node_to_string, \ - config_request_node_count_value, config_percent_value + config_request_node_count_value, config_percent_value, cap_length from swift.common.registry import register_swift_info from swift.common.constraints import check_utf8, valid_api_version from swift.proxy.controllers import AccountController, ContainerController, \ @@ -736,7 +736,7 @@ class Application(object): else: fmt = 'ERROR %(status)d %(body)s Trying to %(method)s ' \ '%(path)s From %(type)s Server' - values['body'] = body[:1024] + values['body'] = cap_length(body, 1024) self.error_occurred(node, fmt % values) else: ok = True diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 772019c5a..3ad0432c3 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -4617,6 +4617,15 @@ cluster_dfw1 = http://dfw1.host/v1/ self.fail('Invalid results from pure function:\n%s' % '\n'.join(failures)) + def test_cap_length(self): + self.assertEqual(utils.cap_length(None, 3), None) + self.assertEqual(utils.cap_length('', 3), '') + self.assertEqual(utils.cap_length('asdf', 3), 'asd...') + self.assertEqual(utils.cap_length('asdf', 5), 'asdf') + + self.assertEqual(utils.cap_length(b'asdf', 3), b'asd...') + self.assertEqual(utils.cap_length(b'asdf', 5), b'asdf') + def test_get_partition_for_hash(self): hex_hash = 'af088baea4806dcaba30bf07d9e64c77' self.assertEqual(43, utils.get_partition_for_hash(hex_hash, 6)) -- cgit v1.2.1