Merge "Fix and unify capacity calculations" into stable/xena

7 files changed, 260 insertions, 38 deletions

diff --git a/cinder/scheduler/filters/capacity_filter.py b/cinder/scheduler/filters/capacity_filter.py
index d1eb7fb7b..bd4cce839 100644
--- a/cinder/scheduler/filters/capacity_filter.py
+++ b/cinder/scheduler/filters/capacity_filter.py
@@ -17,11 +17,10 @@
 #    under the License.
 
 
-import math
-
 from oslo_log import log as logging
 
 from cinder.scheduler import filters
+from cinder import utils
 
 
 LOG = logging.getLogger(__name__)
@@ -103,10 +102,6 @@ class CapacityFilter(filters.BaseBackendFilter):
                          "grouping_name": backend_state.backend_id})
             return False
 
-        # Calculate how much free space is left after taking into account
-        # the reserved space.
-        free = free_space - math.floor(total * reserved)
-
         # NOTE(xyang): If 'provisioning:type' is 'thick' in extra_specs,
         # we will not use max_over_subscription_ratio and
         # provisioned_capacity_gb to determine whether a volume can be
@@ -118,18 +113,45 @@ class CapacityFilter(filters.BaseBackendFilter):
         if provision_type == 'thick':
             thin = False
 
+        thin_support = backend_state.thin_provisioning_support
+        if thin_support:
+            max_over_subscription_ratio = (
+                backend_state.max_over_subscription_ratio
+            )
+        else:
+            max_over_subscription_ratio = 1
+
+        # NOTE(hemna): this takes into consideration all major factors
+        # including reserved space, free_space (reported by driver),
+        # and over subscription ratio.
+        factors = utils.calculate_capacity_factors(
+            total_space,
+            free_space,
+            backend_state.provisioned_capacity_gb,
+            thin_support,
+            max_over_subscription_ratio,
+            backend_state.reserved_percentage,
+            thin
+        )
+        virtual_free_space = factors["virtual_free_capacity"]
+        LOG.debug("Storage Capacity factors %s", factors)
+
         msg_args = {"grouping_name": backend_state.backend_id,
                     "grouping": grouping,
                     "requested": requested_size,
-                    "available": free}
+                    "available": virtual_free_space}
+
         # Only evaluate using max_over_subscription_ratio if
         # thin_provisioning_support is True. Check if the ratio of
         # provisioned capacity over total capacity has exceeded over
         # subscription ratio.
         if (thin and backend_state.thin_provisioning_support and
                 backend_state.max_over_subscription_ratio >= 1):
-            provisioned_ratio = ((backend_state.provisioned_capacity_gb +
-                                  requested_size) / total)
+            provisioned_ratio = (
+                (backend_state.provisioned_capacity_gb + requested_size) / (
+                    factors["total_available_capacity"]
+                )
+            )
             LOG.debug("Checking provisioning for request of %s GB. "
                       "Backend: %s", requested_size, backend_state)
             if provisioned_ratio > backend_state.max_over_subscription_ratio:
@@ -149,14 +171,12 @@ class CapacityFilter(filters.BaseBackendFilter):
             else:
                 # Thin provisioning is enabled and projected over-subscription
                 # ratio does not exceed max_over_subscription_ratio. The host
-                # passes if "adjusted" free virtual capacity is enough to
+                # passes if virtual free capacity is enough to
                 # accommodate the volume. Adjusted free virtual capacity is
                 # the currently available free capacity (taking into account
                 # of reserved space) which we can over-subscribe.
-                adjusted_free_virtual = (
-                    free * backend_state.max_over_subscription_ratio)
-                msg_args["available"] = adjusted_free_virtual
-                res = adjusted_free_virtual >= requested_size
+                msg_args["available"] = virtual_free_space
+                res = virtual_free_space >= requested_size
                 if not res:
                     LOG.warning("Insufficient free virtual space "
                                 "(%(available)sGB) to accommodate thin "
@@ -179,7 +199,7 @@ class CapacityFilter(filters.BaseBackendFilter):
                          "grouping_name": backend_state.backend_id})
             return False
 
-        if free < requested_size:
+        if virtual_free_space < requested_size:
             LOG.warning("Insufficient free space for volume creation "
                         "on %(grouping)s %(grouping_name)s (requested / "
                         "avail): %(requested)s/%(available)s",
diff --git a/cinder/tests/unit/scheduler/test_capacity_weigher.py b/cinder/tests/unit/scheduler/test_capacity_weigher.py
index 8a90e94ee..81259f373 100644
--- a/cinder/tests/unit/scheduler/test_capacity_weigher.py
+++ b/cinder/tests/unit/scheduler/test_capacity_weigher.py
@@ -114,7 +114,7 @@ class CapacityWeigherTestCase(test.TestCase):
         {'volume_type': {'extra_specs': {'provisioning:type': 'thin'}},
          'winner': 'host4'},
         {'volume_type': {'extra_specs': {'provisioning:type': 'thick'}},
-         'winner': 'host2'},
+         'winner': 'host4'},
         {'volume_type': {'extra_specs': {}},
          'winner': 'host4'},
         {'volume_type': {},
diff --git a/cinder/tests/unit/scheduler/test_host_filters.py b/cinder/tests/unit/scheduler/test_host_filters.py
index 461982997..4606e2e9e 100644
--- a/cinder/tests/unit/scheduler/test_host_filters.py
+++ b/cinder/tests/unit/scheduler/test_host_filters.py
@@ -99,7 +99,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
     def test_filter_fails(self, _mock_serv_is_up):
         _mock_serv_is_up.return_value = True
         filt_cls = self.class_map['CapacityFilter']()
-        filter_properties = {'size': 100,
+        filter_properties = {'size': 121,
                              'request_spec': {'volume_id': fake.VOLUME_ID}}
         service = {'disabled': False}
         host = fakes.FakeBackendState('host1',
@@ -282,7 +282,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
     def test_filter_thin_true_passes2(self, _mock_serv_is_up):
         _mock_serv_is_up.return_value = True
         filt_cls = self.class_map['CapacityFilter']()
-        filter_properties = {'size': 3000,
+        filter_properties = {'size': 2400,
                              'capabilities:thin_provisioning_support':
                                  '<is> True',
                              'capabilities:thick_provisioning_support':
@@ -462,7 +462,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
     def test_filter_reserved_thin_thick_true_fails(self, _mock_serv_is_up):
         _mock_serv_is_up.return_value = True
         filt_cls = self.class_map['CapacityFilter']()
-        filter_properties = {'size': 100,
+        filter_properties = {'size': 151,
                              'capabilities:thin_provisioning_support':
                                  '<is> True',
                              'capabilities:thick_provisioning_support':
diff --git a/cinder/tests/unit/scheduler/test_host_manager.py b/cinder/tests/unit/scheduler/test_host_manager.py
index 5f4980be3..42b7f2060 100644
--- a/cinder/tests/unit/scheduler/test_host_manager.py
+++ b/cinder/tests/unit/scheduler/test_host_manager.py
@@ -1029,7 +1029,7 @@ class HostManagerTestCase(test.TestCase):
              "free": 18.01,
              "allocated": 2.0,
              "provisioned": 2.0,
-             "virtual_free": 37.02,
+             "virtual_free": 36.02,
              "reported_at": 40000},
             {"name_to_id": 'host1@backend1',
              "type": "backend",
@@ -1037,7 +1037,7 @@ class HostManagerTestCase(test.TestCase):
              "free": 46.02,
              "allocated": 4.0,
              "provisioned": 4.0,
-             "virtual_free": 64.03,
+             "virtual_free": 63.03,
              "reported_at": 40000}]
 
         expected2 = [
@@ -1055,7 +1055,7 @@ class HostManagerTestCase(test.TestCase):
              "free": 46.02,
              "allocated": 4.0,
              "provisioned": 4.0,
-             "virtual_free": 95.04,
+             "virtual_free": 94.04,
              "reported_at": 40000}]
 
         def sort_func(data):
diff --git a/cinder/tests/unit/test_utils.py b/cinder/tests/unit/test_utils.py
index e41841792..6676f2181 100644
--- a/cinder/tests/unit/test_utils.py
+++ b/cinder/tests/unit/test_utils.py
@@ -1083,10 +1083,10 @@ class TestCalculateVirtualFree(test.TestCase):
          'is_thin_lun': False, 'expected': 27.01},
         {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
          'thin_support': True, 'thick_support': False,
-         'is_thin_lun': True, 'expected': 37.02},
+         'is_thin_lun': True, 'expected': 36.02},
         {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
          'thin_support': True, 'thick_support': True,
-         'is_thin_lun': True, 'expected': 37.02},
+         'is_thin_lun': True, 'expected': 36.02},
         {'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 2.0,
          'thin_support': True, 'thick_support': True,
          'is_thin_lun': False, 'expected': 27.01},
@@ -1114,6 +1114,98 @@ class TestCalculateVirtualFree(test.TestCase):
 
         self.assertEqual(expected, free_capacity)
 
+    @ddt.data(
+        {'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 1.0,
+         'thin_support': False, 'thick_support': True,
+         'is_thin_lun': False, 'reserved_percentage': 5,
+         'expected_total_capacity': 30.01,
+         'expected_reserved_capacity': 1,
+         'expected_free_capacity': 28.01,
+         'expected_total_available_capacity': 29.01,
+         'expected_virtual_free': 27.01,
+         'expected_free_percent': 93.11,
+         'expected_provisioned_type': 'thick',
+         'expected_provisioned_ratio': 0.07},
+        {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
+         'thin_support': True, 'thick_support': False,
+         'is_thin_lun': True, 'reserved_percentage': 10,
+         'expected_total_capacity': 20.01,
+         'expected_reserved_capacity': 2,
+         'expected_free_capacity': 18.01,
+         'expected_total_available_capacity': 36.02,
+         'expected_virtual_free': 34.02,
+         'expected_free_percent': 94.45,
+         'expected_provisioned_type': 'thin',
+         'expected_provisioned_ratio': 0.06},
+        {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
+         'thin_support': True, 'thick_support': True,
+         'is_thin_lun': True, 'reserved_percentage': 20,
+         'expected_total_capacity': 20.01,
+         'expected_reserved_capacity': 4,
+         'expected_free_capacity': 18.01,
+         'expected_total_available_capacity': 32.02,
+         'expected_virtual_free': 30.02,
+         'expected_free_percent': 93.75,
+         'expected_provisioned_type': 'thin',
+         'expected_provisioned_ratio': 0.06},
+        {'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 2.0,
+         'thin_support': True, 'thick_support': True,
+         'is_thin_lun': False, 'reserved_percentage': 10,
+         'expected_total_capacity': 30.01,
+         'expected_reserved_capacity': 3,
+         'expected_free_capacity': 28.01,
+         'expected_total_available_capacity': 27.01,
+         'expected_virtual_free': 25.01,
+         'expected_free_percent': 92.6,
+         'expected_provisioned_type': 'thick',
+         'expected_provisioned_ratio': 0.07},
+    )
+    @ddt.unpack
+    def test_utils_calculate_capacity_factors(
+            self, total, free, provisioned, max_ratio, thin_support,
+            thick_support, is_thin_lun, reserved_percentage,
+            expected_total_capacity,
+            expected_reserved_capacity,
+            expected_free_capacity,
+            expected_total_available_capacity,
+            expected_virtual_free,
+            expected_free_percent,
+            expected_provisioned_type,
+            expected_provisioned_ratio):
+        host_stat = {'total_capacity_gb': total,
+                     'free_capacity_gb': free,
+                     'provisioned_capacity_gb': provisioned,
+                     'max_over_subscription_ratio': max_ratio,
+                     'thin_provisioning_support': thin_support,
+                     'thick_provisioning_support': thick_support,
+                     'reserved_percentage': reserved_percentage}
+
+        factors = utils.calculate_capacity_factors(
+            host_stat['total_capacity_gb'],
+            host_stat['free_capacity_gb'],
+            host_stat['provisioned_capacity_gb'],
+            host_stat['thin_provisioning_support'],
+            host_stat['max_over_subscription_ratio'],
+            host_stat['reserved_percentage'],
+            is_thin_lun)
+
+        self.assertEqual(expected_total_capacity,
+                         factors['total_capacity'])
+        self.assertEqual(expected_reserved_capacity,
+                         factors['reserved_capacity'])
+        self.assertEqual(expected_free_capacity,
+                         factors['free_capacity'])
+        self.assertEqual(expected_total_available_capacity,
+                         factors['total_available_capacity'])
+        self.assertEqual(expected_virtual_free,
+                         factors['virtual_free_capacity'])
+        self.assertEqual(expected_free_percent,
+                         factors['free_percent'])
+        self.assertEqual(expected_provisioned_type,
+                         factors['provisioned_type'])
+        self.assertEqual(expected_provisioned_ratio,
+                         factors['provisioned_ratio'])
+
 
 class Comparable(utils.ComparableMixin):
     def __init__(self, value):
diff --git a/cinder/utils.py b/cinder/utils.py
index 5aa4395c9..28fc77fb5 100644
--- a/cinder/utils.py
+++ b/cinder/utils.py
@@ -706,14 +706,118 @@ def build_or_str(elements: Union[None, str, Iterable[str]],
     return elements
 
 
+def calculate_capacity_factors(total_capacity: float,
+                               free_capacity: float,
+                               provisioned_capacity: float,
+                               thin_provisioning_support: bool,
+                               max_over_subscription_ratio: float,
+                               reserved_percentage: int,
+                               thin: bool) -> dict:
+    """Create the various capacity factors of the a particular backend.
+
+    Based off of definition of terms
+    cinder-specs/specs/queens/provisioning-improvements.html
+    Description of factors calculated where units of gb are Gibibytes.
+    reserved_capacity - The amount of space reserved from the total_capacity
+    as reported by the backend.
+    total_reserved_available_capacity - The total capacity minus reserved
+    capacity
+    total_available_capacity - The total capacity available to cinder
+    calculated from total_reserved_available_capacity (for thick) OR
+    for thin total_reserved_available_capacity max_over_subscription_ratio
+    calculated_free_capacity - total_available_capacity - provisioned_capacity
+    virtual_free_capacity - The calculated free capacity available to cinder
+    to allocate new storage.
+    For thin: calculated_free_capacity
+    For thick: the reported free_capacity can be less than the calculated
+    capacity, so we use free_capacity - reserved_capacity.
+
+    free_percent - the percentage of the virtual_free_capacity and
+    total_available_capacity is left over
+    provisioned_ratio - The ratio of provisioned storage to
+    total_available_capacity
+
+    :param total_capacity: The reported total capacity in the backend.
+    :type total_capacity: float
+    :param free_capacity: The free space/capacity as reported by the backend.
+    :type free_capacity: float
+    :param provisioned_capacity: as reported by backend or volume manager from
+        allocated_capacity_gb
+    :type provisioned_capacity: float
+    :param thin_provisioning_support: Is thin provisioning supported?
+    :type thin_provisioning_support: bool
+    :param max_over_subscription_ratio: as reported by the backend
+    :type max_over_subscription_ratio: float
+    :param reserved_percentage: the % amount to reserve as unavailable. 0-100
+    :type reserved_percentage: int, 0-100
+    :param thin: calculate based on thin provisioning if enabled by
+        thin_provisioning_support
+    :type thin: bool
+    :return: A dictionary of all of the capacity factors.
+    :rtype: dict
+
+    """
+
+    total = float(total_capacity)
+    reserved = float(reserved_percentage) / 100
+    reserved_capacity = math.floor(total * reserved)
+    total_reserved_available = total - reserved_capacity
+
+    if thin and thin_provisioning_support:
+        total_available_capacity = (
+            total_reserved_available * max_over_subscription_ratio
+        )
+        calculated_free = total_available_capacity - provisioned_capacity
+        virtual_free = calculated_free
+        provisioned_type = 'thin'
+    else:
+        # Calculate how much free space is left after taking into
+        # account the reserved space.
+        total_available_capacity = total_reserved_available
+        calculated_free = total_available_capacity - provisioned_capacity
+        virtual_free = calculated_free
+        if free_capacity < calculated_free:
+            virtual_free = free_capacity
+
+        provisioned_type = 'thick'
+
+    if total_available_capacity:
+        provisioned_ratio = provisioned_capacity / total_available_capacity
+        free_percent = (virtual_free / total_available_capacity) * 100
+    else:
+        provisioned_ratio = 0
+        free_percent = 0
+
+    def _limit(x):
+        """Limit our floating points to 2 decimal places."""
+        return round(x, 2)
+
+    return {
+        "total_capacity": total,
+        "free_capacity": free_capacity,
+        "reserved_capacity": reserved_capacity,
+        "total_reserved_available_capacity": _limit(total_reserved_available),
+        "max_over_subscription_ratio": (
+            max_over_subscription_ratio if provisioned_type == 'thin' else None
+        ),
+        "total_available_capacity": _limit(total_available_capacity),
+        "provisioned_capacity": provisioned_capacity,
+        "calculated_free_capacity": _limit(calculated_free),
+        "virtual_free_capacity": _limit(virtual_free),
+        "free_percent": _limit(free_percent),
+        "provisioned_ratio": _limit(provisioned_ratio),
+        "provisioned_type": provisioned_type
+    }
+
+
 def calculate_virtual_free_capacity(total_capacity: float,
                                     free_capacity: float,
                                     provisioned_capacity: float,
                                     thin_provisioning_support: bool,
                                     max_over_subscription_ratio: float,
-                                    reserved_percentage: float,
+                                    reserved_percentage: int,
                                     thin: bool) -> float:
-    """Calculate the virtual free capacity based on thin provisioning support.
+    """Calculate the virtual free capacity based on multiple factors.
 
     :param total_capacity:  total_capacity_gb of a host_state or pool.
     :param free_capacity:   free_capacity_gb of a host_state or pool.
@@ -729,18 +833,16 @@ def calculate_virtual_free_capacity(total_capacity: float,
     :returns: the calculated virtual free capacity.
     """
 
-    total = float(total_capacity)
-    reserved = float(reserved_percentage) / 100
-
-    if thin and thin_provisioning_support:
-        free = (total * max_over_subscription_ratio
-                - provisioned_capacity
-                - math.floor(total * reserved))
-    else:
-        # Calculate how much free space is left after taking into
-        # account the reserved space.
-        free = free_capacity - math.floor(total * reserved)
-    return free
+    factors = calculate_capacity_factors(
+        total_capacity,
+        free_capacity,
+        provisioned_capacity,
+        thin_provisioning_support,
+        max_over_subscription_ratio,
+        reserved_percentage,
+        thin
+    )
+    return factors["virtual_free_capacity"]
 
 
 def calculate_max_over_subscription_ratio(
diff --git a/releasenotes/notes/slug-b6a0fc3db0a2dd45.yaml b/releasenotes/notes/slug-b6a0fc3db0a2dd45.yaml
new file mode 100644
index 000000000..a6392c35b
--- /dev/null
+++ b/releasenotes/notes/slug-b6a0fc3db0a2dd45.yaml
@@ -0,0 +1,8 @@
+---
+other:
+  - |
+    Unified how cinder calculates the virtual free storage space for a pool.
+    Previously Cinder had 2 different mechanisms for calculating the
+    virtual free storage.  Now both the Capacity Filter and the Capacity
+    Weigher use the same mechanism, which is based upon the defined terms in
+    https://specs.openstack.org/openstack/cinder-specs/specs/queens/provisioning-improvements.html