Shard expiring object container

All the expiring objects for a given X-Delete-At are funnelled into the
same expiring object container- this can act as a bottleneck.

Change-Id: I288a177a7ae3e213c727a2a81fa76d4ef9cf7eb3

5 files changed, 38 insertions, 21 deletions

diff --git a/swift/common/utils.py b/swift/common/utils.py
index 37fbeaea0..f7d70f1c0 100644
--- a/swift/common/utils.py
+++ b/swift/common/utils.py
@@ -2960,3 +2960,13 @@ def quote(value, safe='/'):
     Patched version of urllib.quote that encodes utf-8 strings before quoting
     """
     return _quote(get_valid_utf8_str(value), safe)
+
+
+def get_expirer_container(x_delete_at, expirer_divisor, acc, cont, obj):
+    """
+    Returns a expiring object container name for given X-Delete-At and
+    a/c/o.
+    """
+    shard_int = int(hash_path(acc, cont, obj), 16) % 100
+    return normalize_delete_at_timestamp(
+        int(x_delete_at) / expirer_divisor * expirer_divisor - shard_int)
diff --git a/swift/obj/server.py b/swift/obj/server.py
index dc2975856..0fa1d7622 100644
--- a/swift/obj/server.py
+++ b/swift/obj/server.py
@@ -29,7 +29,8 @@ from eventlet import sleep, Timeout
 
 from swift.common.utils import public, get_logger, \
     config_true_value, timing_stats, replication, \
-    normalize_delete_at_timestamp, get_log_line, Timestamp
+    normalize_delete_at_timestamp, get_log_line, Timestamp, \
+    get_expirer_container
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_object_creation, \
     valid_timestamp, check_utf8
@@ -284,9 +285,9 @@ class ObjectController(object):
                     'best guess as to the container name for now.' % op)
                 # TODO(gholt): In a future release, change the above warning to
                 # a raised exception and remove the guess code below.
-                delete_at_container = (
-                    int(delete_at) / self.expiring_objects_container_divisor *
-                    self.expiring_objects_container_divisor)
+                delete_at_container = get_expirer_container(
+                    delete_at, self.expiring_objects_container_divisor,
+                    account, container, obj)
             partition = headers_in.get('X-Delete-At-Partition', None)
             hosts = headers_in.get('X-Delete-At-Host', '')
             contdevices = headers_in.get('X-Delete-At-Device', '')
@@ -307,9 +308,9 @@ class ObjectController(object):
             # exist there and the original data is left where it is, where
             # it will be ignored when the expirer eventually tries to issue the
             # object DELETE later since the X-Delete-At value won't match up.
-            delete_at_container = str(
-                int(delete_at) / self.expiring_objects_container_divisor *
-                self.expiring_objects_container_divisor)
+            delete_at_container = get_expirer_container(
+                delete_at, self.expiring_objects_container_divisor,
+                account, container, obj)
         delete_at_container = normalize_delete_at_timestamp(
             delete_at_container)
 
diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py
index 7b3eba440..b10e2dd18 100644
--- a/swift/proxy/controllers/obj.py
+++ b/swift/proxy/controllers/obj.py
@@ -38,7 +38,7 @@ from eventlet.timeout import Timeout
 from swift.common.utils import (
     clean_content_type, config_true_value, ContextPool, csv_append,
     GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp,
-    normalize_delete_at_timestamp, public, quorum_size)
+    normalize_delete_at_timestamp, public, quorum_size, get_expirer_container)
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_metadata, check_object_creation, \
     check_copy_from_header
@@ -285,6 +285,7 @@ class ObjectController(Controller):
             req.headers['X-Backend-Storage-Policy-Index'] = policy_index
             partition, nodes = obj_ring.get_nodes(
                 self.account_name, self.container_name, self.object_name)
+
             req.headers['X-Timestamp'] = Timestamp(time.time()).internal
 
             headers = self._backend_requests(
@@ -449,10 +450,11 @@ class ObjectController(Controller):
 
             req.environ.setdefault('swift.log_info', []).append(
                 'x-delete-at:%s' % x_delete_at)
-            delete_at_container = normalize_delete_at_timestamp(
-                x_delete_at /
-                self.app.expiring_objects_container_divisor *
-                self.app.expiring_objects_container_divisor)
+
+            delete_at_container = get_expirer_container(
+                x_delete_at, self.app.expiring_objects_container_divisor,
+                self.account_name, self.container_name, self.object_name)
+
             delete_at_part, delete_at_nodes = \
                 self.app.container_ring.get_nodes(
                     self.app.expiring_objects_account, delete_at_container)
diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py
index 4637e88fa..e2137484e 100755
--- a/test/unit/obj/test_server.py
+++ b/test/unit/obj/test_server.py
@@ -3144,8 +3144,14 @@ class TestObjectController(unittest.TestCase):
                      'X-Trans-Id': '1234'})
         self.object_controller.delete_at_update(
             'DELETE', 12345678901, 'a', 'c', 'o', req, 'sda1', 0)
+        expiring_obj_container = given_args.pop(2)
+        expected_exp_cont = utils.get_expirer_container(
+            utils.normalize_delete_at_timestamp(12345678901),
+            86400, 'a', 'c', 'o')
+        self.assertEqual(expiring_obj_container, expected_exp_cont)
+
         self.assertEquals(given_args, [
-            'DELETE', '.expiring_objects', '9999936000', '9999999999-a/c/o',
+            'DELETE', '.expiring_objects', '9999999999-a/c/o',
             None, None, None,
             HeaderKeyDict({
                 'X-Backend-Storage-Policy-Index': 0,
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index 65a476ff1..7a09516c8 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -4810,10 +4810,9 @@ class TestObjectController(unittest.TestCase):
         self.app.container_ring.set_replicas(2)
 
         delete_at_timestamp = int(time.time()) + 100000
-        delete_at_container = str(
-            delete_at_timestamp /
-            self.app.expiring_objects_container_divisor *
-            self.app.expiring_objects_container_divisor)
+        delete_at_container = utils.get_expirer_container(
+            delete_at_timestamp, self.app.expiring_objects_container_divisor,
+            'a', 'c', 'o')
         req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={'Content-Type': 'application/stuff',
                                      'Content-Length': '0',
@@ -4847,10 +4846,9 @@ class TestObjectController(unittest.TestCase):
         self.app.expiring_objects_container_divisor = 60
 
         delete_at_timestamp = int(time.time()) + 100000
-        delete_at_container = str(
-            delete_at_timestamp /
-            self.app.expiring_objects_container_divisor *
-            self.app.expiring_objects_container_divisor)
+        delete_at_container = utils.get_expirer_container(
+            delete_at_timestamp, self.app.expiring_objects_container_divisor,
+            'a', 'c', 'o')
         req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={'Content-Type': 'application/stuff',
                                      'Content-Length': 0,