Use physical utilisation for cached images

Since glance images are downloaded and snapshotted before they are used.
Only a small proportion of its VDI will be in use and it will never grow.
So the real disk usage for VDIs varies with their capability to expand.
Disks connected to VMs continue to use the virtual utilisation as they
are able to expand.

Change-Id: Ie7aae58a47e6651af5b609fb9abc13ab5d61e4df
Closes-bug: 1612057

2 files changed, 109 insertions, 3 deletions

diff --git a/nova/tests/unit/virt/xenapi/test_xenapi.py b/nova/tests/unit/virt/xenapi/test_xenapi.py
index 2978c1b53d..6354b5fa0a 100644
--- a/nova/tests/unit/virt/xenapi/test_xenapi.py
+++ b/nova/tests/unit/virt/xenapi/test_xenapi.py
@@ -2089,7 +2089,7 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
         stats = self.conn.host_state.get_host_stats(False)
         # Values from fake.create_local_srs (ext SR)
         self.assertEqual(stats['disk_total'], 40000)
-        self.assertEqual(stats['disk_used'], 20000)
+        self.assertEqual(stats['disk_used'], 0)
         # Values from fake._plugin_xenhost_host_data
         self.assertEqual(stats['host_memory_total'], 10)
         self.assertEqual(stats['host_memory_overhead'], 20)
@@ -2213,6 +2213,7 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
         self.mox.StubOutWithMock(vm_utils, 'scan_default_sr')
         self.mox.StubOutWithMock(vm_utils, 'list_vms')
         self.mox.StubOutWithMock(self.conn._session, 'call_xenapi')
+        self.mox.StubOutWithMock(host.HostState, 'get_disk_used')
         data = {'disk_total': 0,
                 'disk_used': 0,
                 'disk_available': 0,
@@ -2233,6 +2234,7 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
             vm_utils.list_vms(self.conn._session).AndReturn([])
             self.conn._session.call_xenapi('SR.get_record', "ref").AndReturn(
                 sr_rec)
+            host.HostState.get_disk_used("ref").AndReturn((0, 0))
             if i == 2:
                 # On the third call (the second below) change the hostname
                 data = dict(data, host_hostname='bar')
@@ -2244,6 +2246,72 @@ class XenAPIHostTestCase(stubs.XenAPITestBase):
         self.assertEqual('foo', stats['hypervisor_hostname'])
 
 
+@mock.patch.object(host.HostState, 'update_status')
+class XenAPIHostStateTestCase(stubs.XenAPITestBaseNoDB):
+
+    def _test_get_disk_used(self, vdis, attached_vbds):
+        session = mock.MagicMock()
+        host_state = host.HostState(session)
+
+        sr_ref = 'sr_ref'
+
+        session.SR.get_VDIs.return_value = vdis.keys()
+        session.VDI.get_virtual_size.side_effect = \
+            lambda vdi_ref: vdis[vdi_ref]['virtual_size']
+        session.VDI.get_physical_utilisation.side_effect = \
+            lambda vdi_ref: vdis[vdi_ref]['physical_utilisation']
+        session.VDI.get_VBDs.side_effect = \
+            lambda vdi_ref: vdis[vdi_ref]['VBDs']
+        session.VBD.get_currently_attached.side_effect = \
+            lambda vbd_ref: vbd_ref in attached_vbds
+
+        disk_used = host_state.get_disk_used(sr_ref)
+        session.SR.get_VDIs.assert_called_once_with(sr_ref)
+        return disk_used
+
+    def test_get_disk_used_virtual(self, mock_update_status):
+        # Both VDIs are attached
+        attached_vbds = ['vbd_1', 'vbd_2']
+        vdis = {
+            'vdi_1': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_1']},
+            'vdi_2': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_2']}
+        }
+        disk_used = self._test_get_disk_used(vdis, attached_vbds)
+        self.assertEqual((200, 2), disk_used)
+
+    def test_get_disk_used_physical(self, mock_update_status):
+        # Neither VDIs are attached
+        attached_vbds = []
+        vdis = {
+            'vdi_1': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_1']},
+            'vdi_2': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_2']}
+        }
+        disk_used = self._test_get_disk_used(vdis, attached_vbds)
+        self.assertEqual((2, 2), disk_used)
+
+    def test_get_disk_used_both(self, mock_update_status):
+        # One VDI is attached
+        attached_vbds = ['vbd_1']
+        vdis = {
+            'vdi_1': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_1']},
+            'vdi_2': {'physical_utilisation': 1,
+                      'virtual_size': 100,
+                      'VBDs': ['vbd_2']}
+        }
+        disk_used = self._test_get_disk_used(vdis, attached_vbds)
+        self.assertEqual((101, 2), disk_used)
+
+
 class ToSupportedInstancesTestCase(test.NoDBTestCase):
     def test_default_return_value(self):
         self.assertEqual([],
diff --git a/nova/virt/xenapi/host.py b/nova/virt/xenapi/host.py
index 7ff5e9b511..85d6f56a53 100644
--- a/nova/virt/xenapi/host.py
+++ b/nova/virt/xenapi/host.py
@@ -225,6 +225,44 @@ class HostState(object):
             self.update_status()
         return self._stats
 
+    def get_disk_used(self, sr_ref):
+        """Since glance images are downloaded and snapshotted before they are
+        used, only a small proportion of its VDI will be in use and it will
+        never grow.  We only need to count the virtual size for disks that
+        are attached to a VM - every other disk can count physical.
+        """
+
+        def _vdi_attached(vdi_ref):
+            try:
+                vbds = self._session.VDI.get_VBDs(vdi_ref)
+                for vbd in vbds:
+                    if self._session.VBD.get_currently_attached(vbd):
+                        return True
+            except self._session.XenAPI.Failure:
+                # VDI or VBD may no longer exist - in which case, it's
+                # not attached
+                pass
+            return False
+
+        allocated = 0
+        physical_used = 0
+
+        all_vdis = self._session.SR.get_VDIs(sr_ref)
+        for vdi_ref in all_vdis:
+            try:
+                vdi_physical = \
+                    int(self._session.VDI.get_physical_utilisation(vdi_ref))
+                if _vdi_attached(vdi_ref):
+                    allocated += \
+                        int(self._session.VDI.get_virtual_size(vdi_ref))
+                else:
+                    allocated += vdi_physical
+                physical_used += vdi_physical
+            except (ValueError, self._session.XenAPI.Failure):
+                LOG.exception(_LE('Unable to get size for vdi %s'), vdi_ref)
+
+        return (allocated, physical_used)
+
     def update_status(self):
         """Since under Xenserver, a compute node runs on a given host,
         we can get host status information using xenapi.
@@ -235,10 +273,10 @@ class HostState(object):
             sr_ref = vm_utils.scan_default_sr(self._session)
             sr_rec = self._session.SR.get_record(sr_ref)
             total = int(sr_rec["physical_size"])
-            used = int(sr_rec["physical_utilisation"])
+            (allocated, used) = self.get_disk_used(sr_ref)
             data["disk_total"] = total
             data["disk_used"] = used
-            data["disk_allocated"] = int(sr_rec["virtual_allocation"])
+            data["disk_allocated"] = allocated
             data["disk_available"] = total - used
             data["supported_instances"] = to_supported_instances(
                 data.get("host_capabilities")