Identify deleted shards as shards

When a DB gets deleted, we clear out its metadata. This included sysmeta
such as that used to tell shards the name of their root DB.

Previously, this would cause deleted shards to pop back to life as roots
that claimed to have objects still sitting in whatever container they
sharnk into.

Now, use the metadata if it's available, but when it's not, go by the
state of the DB's "own shard range" -- deleted shards should be marked
deleted, while roots never are.

This allows us to actually clean up the database files; you can test
this by doing something like

 * Run `nosetests test/probe/test_sharder.py:TestContainerSharding.test_shrinking`
 * Run `find /srv/*/*/containers -name '*.db'` to see how many databases
   are left on disk. There should be 15: 3 for the root container, 6 for
   the two shards on the first pass, and another 6 for the two shards on
   the second pass.
 * Edit container configs to decrease reclaim_age -- even 1 should be
   fine.
 * Run `swift-init main start` to restart the servers.
 * Run `swift-init container-sharder once` to have the shards get marked
   deleted.
 * Run `swift-init container-updater once` to ensure all containers have
   reported.
 * Run `swift-init container-replicator once` to clean up the
   containers.
 * Run `find /srv/*/*/containers -name '*.db'` again to verify no
   containers remain on disk.

Cherry-Picked-From: Icba98f1c9e17e8ade3f0e1b9a23360cf5ab8c86b
Change-Id: I7b7be3b62e033ab668316e3c698ba667ceef31f8

2 files changed, 50 insertions, 10 deletions

diff --git a/swift/container/backend.py b/swift/container/backend.py
index bdf34f7d8..6886ccd69 100644
--- a/swift/container/backend.py
+++ b/swift/container/backend.py
@@ -2061,6 +2061,21 @@ class ContainerBroker(DatabaseBroker):
         else:
             return {k: v[0] for k, v in info.items()}
 
+    def _get_root_meta(self):
+        """
+        Get the (unquoted) root path, plus the header the info came from.
+        If no info available, returns ``(None, None)``
+        """
+        path = self.get_sharding_sysmeta('Quoted-Root')
+        if path:
+            return 'X-Container-Sysmeta-Shard-Quoted-Root', unquote(path)
+
+        path = self.get_sharding_sysmeta('Root')
+        if path:
+            return 'X-Container-Sysmeta-Shard-Root', path
+
+        return None, None
+
     def _load_root_info(self):
         """
         Load the root container name and account for the container represented
@@ -2073,13 +2088,7 @@ class ContainerBroker(DatabaseBroker):
         ``container`` attributes respectively.
 
         """
-        path = self.get_sharding_sysmeta('Quoted-Root')
-        hdr = 'X-Container-Sysmeta-Shard-Quoted-Root'
-        if path:
-            path = unquote(path)
-        else:
-            path = self.get_sharding_sysmeta('Root')
-            hdr = 'X-Container-Sysmeta-Shard-Root'
+        hdr, path = self._get_root_meta()
 
         if not path:
             # Ensure account/container get populated
@@ -2118,9 +2127,25 @@ class ContainerBroker(DatabaseBroker):
         A root container is a container that is not a shard of another
         container.
         """
-        self._populate_instance_cache()
-        return (self.root_account == self.account and
-                self.root_container == self.container)
+        _, path = self._get_root_meta()
+        if path is not None:
+            # We have metadata telling us where the root is; it's authoritative
+            return self.path == path
+
+        # Else, we're either a root or a deleted shard.
+
+        # Use internal method so we don't try to update stats.
+        own_shard_range = self._own_shard_range(no_default=True)
+        if not own_shard_range:
+            return True  # Never been sharded
+
+        if own_shard_range.deleted:
+            # When shard ranges shrink, they get marked deleted
+            return False
+        else:
+            # But even when a root collapses, empties, and gets deleted, its
+            # own_shard_range is left alive
+            return True
 
     def _get_next_shard_range_upper(self, shard_size, last_upper=None):
         """
diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py
index 37308c154..eb976c01f 100644
--- a/test/unit/container/test_backend.py
+++ b/test/unit/container/test_backend.py
@@ -442,6 +442,7 @@ class TestContainerBroker(unittest.TestCase):
             self.assertTrue(broker_to_test.empty())
 
         self.assertTrue(broker.empty())
+        self.assertFalse(broker.is_root_container())
         check_object_counted(broker, broker)
 
         # own shard range is not considered for object count
@@ -498,6 +499,20 @@ class TestContainerBroker(unittest.TestCase):
         own_sr.update_meta(3, 4, meta_timestamp=next(self.ts))
         broker.merge_shard_ranges([own_sr])
         self.assertTrue(broker.empty())
+        self.assertFalse(broker.is_deleted())
+        self.assertFalse(broker.is_root_container())
+
+        # sharder won't call delete_db() unless own_shard_range is deleted
+        own_sr.deleted = True
+        own_sr.timestamp = next(self.ts)
+        broker.merge_shard_ranges([own_sr])
+        broker.delete_db(next(self.ts).internal)
+
+        # Get a fresh broker, with instance cache unset
+        broker = ContainerBroker(db_path, account='.shards_a', container='cc')
+        self.assertTrue(broker.empty())
+        self.assertTrue(broker.is_deleted())
+        self.assertFalse(broker.is_root_container())
 
     def test_reclaim(self):
         broker = ContainerBroker(':memory:', account='test_account',