Fix and unify capacity calculations

This patch updates how cinder calculates it's free capacity.
The new calculations are based off of the queens cinder specs
that describes each of the capacity factors here:
https://specs.openstack.org/openstack/cinder-specs/specs/queens/provisioning-improvements.html

This patch updates the capacity filter to use the new capacity factors
calculations, which is also used by the capacity weigher.

The new calculate_capacity_factors describes each of the factors and
returns a dictionary of each of the factors as calculated.

Change-Id: Ic1b5737281e542d2782089a369e4b7941fc3d921
(cherry picked from commit 856d3e108035ac1ed9c6a3cec4de47b1eb6ad18b)

7 files changed, 260 insertions, 38 deletions

diff --git a/cinder/scheduler/filters/capacity_filter.py b/cinder/scheduler/filters/capacity_filter.py
index d1eb7fb7b..bd4cce839 100644
--- a/cinder/scheduler/filters/capacity_filter.py
+++ b/cinder/scheduler/filters/capacity_filter.py
@@ -17,11 +17,10 @@
 #    under the License.
 
 
-import math
-
 from oslo_log import log as logging
 
 from cinder.scheduler import filters
+from cinder import utils
 
 
 LOG = logging.getLogger(__name__)
@@ -103,10 +102,6 @@ class CapacityFilter(filters.BaseBackendFilter):
                          "grouping_name": backend_state.backend_id})
             return False
 
-        # Calculate how much free space is left after taking into account
-        # the reserved space.
-        free = free_space - math.floor(total * reserved)
-
         # NOTE(xyang): If 'provisioning:type' is 'thick' in extra_specs,
         # we will not use max_over_subscription_ratio and
         # provisioned_capacity_gb to determine whether a volume can be
@@ -118,18 +113,45 @@ class CapacityFilter(filters.BaseBackendFilter):
         if provision_type == 'thick':
             thin = False
 
+        thin_support = backend_state.thin_provisioning_support
+        if thin_support:
+            max_over_subscription_ratio = (
+                backend_state.max_over_subscription_ratio
+            )
+        else:
+            max_over_subscription_ratio = 1
+
+        # NOTE(hemna): this takes into consideration all major factors
+        # including reserved space, free_space (reported by driver),
+        # and over subscription ratio.
+        factors = utils.calculate_capacity_factors(
+            total_space,
+            free_space,
+            backend_state.provisioned_capacity_gb,
+            thin_support,
+            max_over_subscription_ratio,
+            backend_state.reserved_percentage,
+            thin
+        )
+        virtual_free_space = factors["virtual_free_capacity"]
+        LOG.debug("Storage Capacity factors %s", factors)
+
         msg_args = {"grouping_name": backend_state.backend_id,
                     "grouping": grouping,
                     "requested": requested_size,
-                    "available": free}
+                    "available": virtual_free_space}
+
         # Only evaluate using max_over_subscription_ratio if
         # thin_provisioning_support is True. Check if the ratio of
         # provisioned capacity over total capacity has exceeded over
         # subscription ratio.
         if (thin and backend_state.thin_provisioning_support and
                 backend_state.max_over_subscription_ratio >= 1):
-            provisioned_ratio = ((backend_state.provisioned_capacity_gb +
-                                  requested_size) / total)
+            provisioned_ratio = (
+                (backend_state.provisioned_capacity_gb + requested_size) / (
+                    factors["total_available_capacity"]
+                )
+            )
             LOG.debug("Checking provisioning for request of %s GB. "
                       "Backend: %s", requested_size, backend_state)
             if provisioned_ratio > backend_state.max_over_subscription_ratio:
@@ -149,14 +171,12 @@ class CapacityFilter(filters.BaseBackendFilter):
             else:
                 # Thin provisioning is enabled and projected over-subscription
                 # ratio does not exceed max_over_subscription_ratio. The host
-                # passes if "adjusted" free virtual capacity is enough to
+                # passes if virtual free capacity is enough to
                 # accommodate the volume. Adjusted free virtual capacity is
                 # the currently available free capacity (taking into account
                 # of reserved space) which we can over-subscribe.
-                adjusted_free_virtual = (
-                    free * backend_state.max_over_subscription_ratio)
-                msg_args["available"] = adjusted_free_virtual
-                res = adjusted_free_virtual >= requested_size
+                msg_args["available"] = virtual_free_space
+                res = virtual_free_space >= requested_size
                 if not res:
                     LOG.warning("Insufficient free virtual space "
                                 "(%(available)sGB) to accommodate thin "
@@ -179,7 +199,7 @@ class CapacityFilter(filters.BaseBackendFilter):
                          "grouping_name": backend_state.backend_id})
             return False
 
-        if free < requested_size:
+        if virtual_free_space < requested_size:
             LOG.warning("Insufficient free space for volume creation "
                         "on %(grouping)s %(grouping_name)s (requested / "
                         "avail): %(requested)s/%(available)s",
diff --git a/cinder/tests/unit/scheduler/test_capacity_weigher.py b/cinder/tests/unit/scheduler/test_capacity_weigher.py
index 8a90e94ee..81259f373 100644
--- a/cinder/tests/unit/scheduler/test_capacity_weigher.py
+++ b/cinder/tests/unit/scheduler/test_capacity_weigher.py
@@ -114,7 +114,7 @@ class CapacityWeigherTestCase(test.TestCase):
         {'volume_type': {'extra_specs': {'provisioning:type': 'thin'}},
          'winner': 'host4'},
         {'volume_type': {'extra_specs': {'provisioning:type': 'thick'}},
-         'winner': 'host2'},
+         'winner': 'host4'},
         {'volume_type': {'extra_specs': {}},
          'winner': 'host4'},
         {'volume_type': {},
diff --git a/cinder/tests/unit/scheduler/test_host_filters.py b/cinder/tests/unit/scheduler/test_host_filters.py
index 815a1c611..767b649e9 100644
--- a/cinder/tests/unit/scheduler/test_host_filters.py
+++ b/cinder/tests/unit/scheduler/test_host_filters.py
@@ -99,7 +99,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
     def test_filter_fails(self, _mock_serv_is_up):
         _mock_serv_is_up.return_value = True
         filt_cls = self.class_map['CapacityFilter']()
-        filter_properties = {'size': 100,
+        filter_properties = {'size': 121,
                              'request_spec': {'volume_id': fake.VOLUME_ID}}
         service = {'disabled': False}
         host = fakes.FakeBackendState('host1',
@@ -282,7 +282,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
     def test_filter_thin_true_passes2(self, _mock_serv_is_up):
         _mock_serv_is_up.return_value = True
         filt_cls = self.class_map['CapacityFilter']()
-        filter_properties = {'size': 3000,
+        filter_properties = {'size': 2400,
                              'capabilities:thin_provisioning_support':
                                  '<is> True',
                              'capabilities:thick_provisioning_support':
@@ -462,7 +462,7 @@ class CapacityFilterTestCase(BackendFiltersTestCase):
     def test_filter_reserved_thin_thick_true_fails(self, _mock_serv_is_up):
         _mock_serv_is_up.return_value = True
         filt_cls = self.class_map['CapacityFilter']()
-        filter_properties = {'size': 100,
+        filter_properties = {'size': 151,
                              'capabilities:thin_provisioning_support':
                                  '<is> True',
                              'capabilities:thick_provisioning_support':
diff --git a/cinder/tests/unit/scheduler/test_host_manager.py b/cinder/tests/unit/scheduler/test_host_manager.py
index 5f4980be3..42b7f2060 100644
--- a/cinder/tests/unit/scheduler/test_host_manager.py
+++ b/cinder/tests/unit/scheduler/test_host_manager.py
@@ -1029,7 +1029,7 @@ class HostManagerTestCase(test.TestCase):
              "free": 18.01,
              "allocated": 2.0,
              "provisioned": 2.0,
-             "virtual_free": 37.02,
+             "virtual_free": 36.02,
              "reported_at": 40000},
             {"name_to_id": 'host1@backend1',
              "type": "backend",
@@ -1037,7 +1037,7 @@ class HostManagerTestCase(test.TestCase):
              "free": 46.02,
              "allocated": 4.0,
              "provisioned": 4.0,
-             "virtual_free": 64.03,
+             "virtual_free": 63.03,
              "reported_at": 40000}]
 
         expected2 = [
@@ -1055,7 +1055,7 @@ class HostManagerTestCase(test.TestCase):
              "free": 46.02,
              "allocated": 4.0,
              "provisioned": 4.0,
-             "virtual_free": 95.04,
+             "virtual_free": 94.04,
              "reported_at": 40000}]
 
         def sort_func(data):
diff --git a/cinder/tests/unit/test_utils.py b/cinder/tests/unit/test_utils.py
index e41841792..6676f2181 100644
--- a/cinder/tests/unit/test_utils.py
+++ b/cinder/tests/unit/test_utils.py
@@ -1083,10 +1083,10 @@ class TestCalculateVirtualFree(test.TestCase):
          'is_thin_lun': False, 'expected': 27.01},
         {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
          'thin_support': True, 'thick_support': False,
-         'is_thin_lun': True, 'expected': 37.02},
+         'is_thin_lun': True, 'expected': 36.02},
         {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
          'thin_support': True, 'thick_support': True,
-         'is_thin_lun': True, 'expected': 37.02},
+         'is_thin_lun': True, 'expected': 36.02},
         {'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 2.0,
          'thin_support': True, 'thick_support': True,
          'is_thin_lun': False, 'expected': 27.01},
@@ -1114,6 +1114,98 @@ class TestCalculateVirtualFree(test.TestCase):
 
         self.assertEqual(expected, free_capacity)
 
+    @ddt.data(
+        {'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 1.0,
+         'thin_support': False, 'thick_support': True,
+         'is_thin_lun': False, 'reserved_percentage': 5,
+         'expected_total_capacity': 30.01,
+         'expected_reserved_capacity': 1,
+         'expected_free_capacity': 28.01,
+         'expected_total_available_capacity': 29.01,
+         'expected_virtual_free': 27.01,
+         'expected_free_percent': 93.11,
+         'expected_provisioned_type': 'thick',
+         'expected_provisioned_ratio': 0.07},
+        {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
+         'thin_support': True, 'thick_support': False,
+         'is_thin_lun': True, 'reserved_percentage': 10,
+         'expected_total_capacity': 20.01,
+         'expected_reserved_capacity': 2,
+         'expected_free_capacity': 18.01,
+         'expected_total_available_capacity': 36.02,
+         'expected_virtual_free': 34.02,
+         'expected_free_percent': 94.45,
+         'expected_provisioned_type': 'thin',
+         'expected_provisioned_ratio': 0.06},
+        {'total': 20.01, 'free': 18.01, 'provisioned': 2.0, 'max_ratio': 2.0,
+         'thin_support': True, 'thick_support': True,
+         'is_thin_lun': True, 'reserved_percentage': 20,
+         'expected_total_capacity': 20.01,
+         'expected_reserved_capacity': 4,
+         'expected_free_capacity': 18.01,
+         'expected_total_available_capacity': 32.02,
+         'expected_virtual_free': 30.02,
+         'expected_free_percent': 93.75,
+         'expected_provisioned_type': 'thin',
+         'expected_provisioned_ratio': 0.06},
+        {'total': 30.01, 'free': 28.01, 'provisioned': 2.0, 'max_ratio': 2.0,
+         'thin_support': True, 'thick_support': True,
+         'is_thin_lun': False, 'reserved_percentage': 10,
+         'expected_total_capacity': 30.01,
+         'expected_reserved_capacity': 3,
+         'expected_free_capacity': 28.01,
+         'expected_total_available_capacity': 27.01,
+         'expected_virtual_free': 25.01,
+         'expected_free_percent': 92.6,
+         'expected_provisioned_type': 'thick',
+         'expected_provisioned_ratio': 0.07},
+    )
+    @ddt.unpack
+    def test_utils_calculate_capacity_factors(
+            self, total, free, provisioned, max_ratio, thin_support,
+            thick_support, is_thin_lun, reserved_percentage,
+            expected_total_capacity,
+            expected_reserved_capacity,
+            expected_free_capacity,
+            expected_total_available_capacity,
+            expected_virtual_free,
+            expected_free_percent,
+            expected_provisioned_type,
+            expected_provisioned_ratio):
+        host_stat = {'total_capacity_gb': total,
+                     'free_capacity_gb': free,
+                     'provisioned_capacity_gb': provisioned,
+                     'max_over_subscription_ratio': max_ratio,
+                     'thin_provisioning_support': thin_support,
+                     'thick_provisioning_support': thick_support,
+                     'reserved_percentage': reserved_percentage}
+
+        factors = utils.calculate_capacity_factors(
+            host_stat['total_capacity_gb'],
+            host_stat['free_capacity_gb'],
+            host_stat['provisioned_capacity_gb'],
+            host_stat['thin_provisioning_support'],
+            host_stat['max_over_subscription_ratio'],
+            host_stat['reserved_percentage'],
+            is_thin_lun)
+
+        self.assertEqual(expected_total_capacity,
+                         factors['total_capacity'])
+        self.assertEqual(expected_reserved_capacity,
+                         factors['reserved_capacity'])
+        self.assertEqual(expected_free_capacity,
+                         factors['free_capacity'])
+        self.assertEqual(expected_total_available_capacity,
+                         factors['total_available_capacity'])
+        self.assertEqual(expected_virtual_free,
+                         factors['virtual_free_capacity'])
+        self.assertEqual(expected_free_percent,
+                         factors['free_percent'])
+        self.assertEqual(expected_provisioned_type,
+                         factors['provisioned_type'])
+        self.assertEqual(expected_provisioned_ratio,
+                         factors['provisioned_ratio'])
+
 
 class Comparable(utils.ComparableMixin):
     def __init__(self, value):
diff --git a/cinder/utils.py b/cinder/utils.py
index 1ff4e783e..d58733d5f 100644
--- a/cinder/utils.py
+++ b/cinder/utils.py
@@ -710,14 +710,118 @@ def build_or_str(elements: Union[None, str, Iterable[str]],
     return elements
 
 
+def calculate_capacity_factors(total_capacity: float,
+                               free_capacity: float,
+                               provisioned_capacity: float,
+                               thin_provisioning_support: bool,
+                               max_over_subscription_ratio: float,
+                               reserved_percentage: int,
+                               thin: bool) -> dict:
+    """Create the various capacity factors of the a particular backend.
+
+    Based off of definition of terms
+    cinder-specs/specs/queens/provisioning-improvements.html
+    Description of factors calculated where units of gb are Gibibytes.
+    reserved_capacity - The amount of space reserved from the total_capacity
+    as reported by the backend.
+    total_reserved_available_capacity - The total capacity minus reserved
+    capacity
+    total_available_capacity - The total capacity available to cinder
+    calculated from total_reserved_available_capacity (for thick) OR
+    for thin total_reserved_available_capacity max_over_subscription_ratio
+    calculated_free_capacity - total_available_capacity - provisioned_capacity
+    virtual_free_capacity - The calculated free capacity available to cinder
+    to allocate new storage.
+    For thin: calculated_free_capacity
+    For thick: the reported free_capacity can be less than the calculated
+    capacity, so we use free_capacity - reserved_capacity.
+
+    free_percent - the percentage of the virtual_free_capacity and
+    total_available_capacity is left over
+    provisioned_ratio - The ratio of provisioned storage to
+    total_available_capacity
+
+    :param total_capacity: The reported total capacity in the backend.
+    :type total_capacity: float
+    :param free_capacity: The free space/capacity as reported by the backend.
+    :type free_capacity: float
+    :param provisioned_capacity: as reported by backend or volume manager from
+        allocated_capacity_gb
+    :type provisioned_capacity: float
+    :param thin_provisioning_support: Is thin provisioning supported?
+    :type thin_provisioning_support: bool
+    :param max_over_subscription_ratio: as reported by the backend
+    :type max_over_subscription_ratio: float
+    :param reserved_percentage: the % amount to reserve as unavailable. 0-100
+    :type reserved_percentage: int, 0-100
+    :param thin: calculate based on thin provisioning if enabled by
+        thin_provisioning_support
+    :type thin: bool
+    :return: A dictionary of all of the capacity factors.
+    :rtype: dict
+
+    """
+
+    total = float(total_capacity)
+    reserved = float(reserved_percentage) / 100
+    reserved_capacity = math.floor(total * reserved)
+    total_reserved_available = total - reserved_capacity
+
+    if thin and thin_provisioning_support:
+        total_available_capacity = (
+            total_reserved_available * max_over_subscription_ratio
+        )
+        calculated_free = total_available_capacity - provisioned_capacity
+        virtual_free = calculated_free
+        provisioned_type = 'thin'
+    else:
+        # Calculate how much free space is left after taking into
+        # account the reserved space.
+        total_available_capacity = total_reserved_available
+        calculated_free = total_available_capacity - provisioned_capacity
+        virtual_free = calculated_free
+        if free_capacity < calculated_free:
+            virtual_free = free_capacity
+
+        provisioned_type = 'thick'
+
+    if total_available_capacity:
+        provisioned_ratio = provisioned_capacity / total_available_capacity
+        free_percent = (virtual_free / total_available_capacity) * 100
+    else:
+        provisioned_ratio = 0
+        free_percent = 0
+
+    def _limit(x):
+        """Limit our floating points to 2 decimal places."""
+        return round(x, 2)
+
+    return {
+        "total_capacity": total,
+        "free_capacity": free_capacity,
+        "reserved_capacity": reserved_capacity,
+        "total_reserved_available_capacity": _limit(total_reserved_available),
+        "max_over_subscription_ratio": (
+            max_over_subscription_ratio if provisioned_type == 'thin' else None
+        ),
+        "total_available_capacity": _limit(total_available_capacity),
+        "provisioned_capacity": provisioned_capacity,
+        "calculated_free_capacity": _limit(calculated_free),
+        "virtual_free_capacity": _limit(virtual_free),
+        "free_percent": _limit(free_percent),
+        "provisioned_ratio": _limit(provisioned_ratio),
+        "provisioned_type": provisioned_type
+    }
+
+
 def calculate_virtual_free_capacity(total_capacity: float,
                                     free_capacity: float,
                                     provisioned_capacity: float,
                                     thin_provisioning_support: bool,
                                     max_over_subscription_ratio: float,
-                                    reserved_percentage: float,
+                                    reserved_percentage: int,
                                     thin: bool) -> float:
-    """Calculate the virtual free capacity based on thin provisioning support.
+    """Calculate the virtual free capacity based on multiple factors.
 
     :param total_capacity:  total_capacity_gb of a host_state or pool.
     :param free_capacity:   free_capacity_gb of a host_state or pool.
@@ -733,18 +837,16 @@ def calculate_virtual_free_capacity(total_capacity: float,
     :returns: the calculated virtual free capacity.
     """
 
-    total = float(total_capacity)
-    reserved = float(reserved_percentage) / 100
-
-    if thin and thin_provisioning_support:
-        free = (total * max_over_subscription_ratio
-                - provisioned_capacity
-                - math.floor(total * reserved))
-    else:
-        # Calculate how much free space is left after taking into
-        # account the reserved space.
-        free = free_capacity - math.floor(total * reserved)
-    return free
+    factors = calculate_capacity_factors(
+        total_capacity,
+        free_capacity,
+        provisioned_capacity,
+        thin_provisioning_support,
+        max_over_subscription_ratio,
+        reserved_percentage,
+        thin
+    )
+    return factors["virtual_free_capacity"]
 
 
 def calculate_max_over_subscription_ratio(
diff --git a/releasenotes/notes/slug-b6a0fc3db0a2dd45.yaml b/releasenotes/notes/slug-b6a0fc3db0a2dd45.yaml
new file mode 100644
index 000000000..a6392c35b
--- /dev/null
+++ b/releasenotes/notes/slug-b6a0fc3db0a2dd45.yaml
@@ -0,0 +1,8 @@
+---
+other:
+  - |
+    Unified how cinder calculates the virtual free storage space for a pool.
+    Previously Cinder had 2 different mechanisms for calculating the
+    virtual free storage.  Now both the Capacity Filter and the Capacity
+    Weigher use the same mechanism, which is based upon the defined terms in
+    https://specs.openstack.org/openstack/cinder-specs/specs/queens/provisioning-improvements.html