RBD: Fix total_capacitystable/xena

Ceph has changed the meaning of the ``bytes_used`` column in the pools
reported by the ``df`` command, which means that in some deployments the
rbd driver is not reporting the expected information ot the schedulers.

The information we should be used for the calculations is returned in
the ``stored`` field in those systems.

This patch uses ``stored`` when present and fallbacks to ``bytes_used``
if not.

Closes-Bug: #1960206
Change-Id: I0ca25789a0b279d82f766091235f24f429405da6
(cherry picked from commit 86d9ec5d5932557ade18e7893cc2b8f564b5b2d8)
(cherry picked from commit 97926fb888a6cdb96640ebe4e06785e6ce198226)
(cherry picked from commit 2f798cb06a34a669459f2de6ab55d06aa985d221)

3 files changed, 57 insertions, 2 deletions

diff --git a/cinder/tests/unit/volume/drivers/test_rbd.py b/cinder/tests/unit/volume/drivers/test_rbd.py
index 30a33cfde..a2dad955a 100644
--- a/cinder/tests/unit/volume/drivers/test_rbd.py
+++ b/cinder/tests/unit/volume/drivers/test_rbd.py
@@ -1833,6 +1833,51 @@ class RBDTestCase(test.TestCase):
         ])
         self.assertEqual((free_capacity, total_capacity), result)
 
+    @ddt.data(
+        # Normal case, no quota and dynamic total
+        {'free_capacity': 27.0, 'total_capacity': 28.44},
+        # No quota and static total
+        {'dynamic_total': False,
+         'free_capacity': 27.0, 'total_capacity': 59.96},
+        # Quota and dynamic total
+        {'quota_max_bytes': 3221225472, 'max_avail': 1073741824,
+         'free_capacity': 1, 'total_capacity': 2.44},
+        # Quota and static total
+        {'quota_max_bytes': 3221225472, 'max_avail': 1073741824,
+         'dynamic_total': False,
+         'free_capacity': 1, 'total_capacity': 3.00},
+        # Quota and dynamic total when free would be negative
+        {'quota_max_bytes': 1073741824,
+         'free_capacity': 0, 'total_capacity': 1.44},
+    )
+    @ddt.unpack
+    @common_mocks
+    def test_get_pool_nautilus(self, free_capacity, total_capacity,
+                               max_avail=28987613184, quota_max_bytes=0,
+                               dynamic_total=True):
+        client = self.mock_client.return_value
+        client.__enter__.return_value = client
+        client.cluster.mon_command.side_effect = [
+            (0, '{"stats":{"total_bytes":64385286144,'
+             '"total_used_bytes":3289628672,"total_avail_bytes":61095657472},'
+             '"pools":[{"name":"rbd","id":2,"stats":{"kb_used":1510197,'
+             '"stored":1546440971,"bytes_used":4639322913,"max_avail":%s,'
+             '"objects":412}},{"name":"volumes","id":3,"stats":{"kb_used":0,'
+             '"bytes_used":0,"max_avail":28987613184,"objects":0}}]}\n' %
+             max_avail, ''),
+            (0, '{"pool_name":"volumes","pool_id":4,"quota_max_objects":0,'
+             '"quota_max_bytes":%s}\n' % quota_max_bytes, ''),
+        ]
+        with mock.patch.object(self.driver.configuration, 'safe_get',
+                               return_value=dynamic_total):
+            result = self.driver._get_pool_stats()
+        client.cluster.mon_command.assert_has_calls([
+            mock.call('{"prefix":"df", "format":"json"}', b''),
+            mock.call('{"prefix":"osd pool get-quota", "pool": "rbd",'
+                      ' "format":"json"}', b''),
+        ])
+        self.assertEqual((free_capacity, total_capacity), result)
+
     @common_mocks
     def test_get_pool_bytes(self):
         """Test for mon_commands returning bytes instead of strings."""
diff --git a/cinder/volume/drivers/rbd.py b/cinder/volume/drivers/rbd.py
index eacddd18c..1fdd4c5f3 100644
--- a/cinder/volume/drivers/rbd.py
+++ b/cinder/volume/drivers/rbd.py
@@ -596,12 +596,14 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD,
         pool_stats = [pool for pool in df_data['pools']
                       if pool['name'] == pool_name][0]['stats']
 
+        # In Nautilus bytes_used was renamed to stored
+        bytes_used = pool_stats.get('stored', pool_stats['bytes_used'])
         quota_outbuf = encodeutils.safe_decode(quota_outbuf)
         bytes_quota = json.loads(quota_outbuf)['quota_max_bytes']
         # With quota the total is the quota limit and free is quota - used
         if bytes_quota:
             total_capacity = bytes_quota
-            free_capacity = max(min(total_capacity - pool_stats['bytes_used'],
+            free_capacity = max(min(total_capacity - bytes_used,
                                     pool_stats['max_avail']),
                                 0)
         # Without quota free is pools max available and total is global size
@@ -611,7 +613,7 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD,
 
         # If we want dynamic total capacity (default behavior)
         if self.configuration.safe_get('report_dynamic_total_capacity'):
-            total_capacity = free_capacity + pool_stats['bytes_used']
+            total_capacity = free_capacity + bytes_used
 
         free_capacity = round((float(free_capacity) / units.Gi), 2)
         total_capacity = round((float(total_capacity) / units.Gi), 2)
diff --git a/releasenotes/notes/rbd-total_capacity-60f10b45e3a8c8ea.yaml b/releasenotes/notes/rbd-total_capacity-60f10b45e3a8c8ea.yaml
new file mode 100644
index 000000000..1efa65097
--- /dev/null
+++ b/releasenotes/notes/rbd-total_capacity-60f10b45e3a8c8ea.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    RBD driver `bug #1960206
+    <https://bugs.launchpad.net/cinder/+bug/1960206>`_: Fixed
+    ``total_capacity`` reported by the driver to the scheduler on Ceph clusters
+    that have renamed the ``bytes_used`` field to ``stored``. (e.g., `Nautilus
+    <https://docs.ceph.com/en/nautilus/releases/nautilus/#upgrade-compatibility-notes>`_).