Merge "SR-IOV: Add Agent QoS driver to support bandwidth limit" into feature/qosfeature/qos

11 files changed, 287 insertions, 8 deletions

diff --git a/doc/source/devref/quality_of_service.rst b/doc/source/devref/quality_of_service.rst
index 3a4d6f3c94..bd4a8c716d 100644
--- a/doc/source/devref/quality_of_service.rst
+++ b/doc/source/devref/quality_of_service.rst
@@ -253,8 +253,13 @@ with them.
 Agent backends
 --------------
 
-At the moment, QoS is supported by Open vSwitch backend only, so
-QosOVSAgentDriver is the only driver that implements QosAgentDriver interface.
+At the moment, QoS is supported by Open vSwitch and SR-IOV ml2 drivers.
+
+Each agent backend defines a QoS driver that implements the QosAgentDriver
+interface:
+
+* Open vSwitch (QosOVSAgentDriver);
+* SR-IOV (QosSRIOVAgentDriver).
 
 
 Open vSwitch
@@ -274,6 +279,24 @@ That approach is less flexible than linux-htb, Queues and OvS QoS profiles,
 which we may explore in the future, but which will need to be used in
 combination with openflow rules.
 
+SR-IOV
+~~~~~~
+
+SR-IOV bandwidth limit implementation relies on the new pci_lib function:
+
+* set_vf_max_rate
+
+As the name of the function suggests, the limit is applied on a Virtual
+Function (VF).
+
+ip link interface has the following limitation for bandwidth limit: it uses
+Mbps as units of bandwidth measurement, not kbps, and does not support float
+numbers. So in case the limit is set to something less than 1000 kbps, it's set
+to 1 Mbps only. If the limit is set to something that does not divide to 1000
+kbps chunks, then the effective limit is rounded to the nearest integer Mbps
+value.
+
+
 Configuration
 =============
 
diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py
index 938db45900..12168883e8 100644
--- a/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py
+++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py
@@ -354,3 +354,17 @@ class ESwitchManager(object):
                             {"device_mac": device_mac, "pci_slot": pci_slot})
                 embedded_switch = None
         return embedded_switch
+
+    def get_pci_slot_by_mac(self, device_mac):
+        """Get pci slot by mac.
+
+        Get pci slot by device mac
+        @param device_mac: device mac
+        """
+        result = None
+        for pci_slot, embedded_switch in self.pci_slot_map.items():
+            used_device_mac = embedded_switch.get_pci_device(pci_slot)
+            if used_device_mac == device_mac:
+                result = pci_slot
+                break
+        return result
diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py
new file mode 100755
index 0000000000..e69de29bb2
--- /dev/null
+++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py
diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/qos_driver.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/qos_driver.py
new file mode 100755
index 0000000000..8c30817a1a
--- /dev/null
+++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/qos_driver.py
@@ -0,0 +1,84 @@
+# Copyright 2015 Mellanox Technologies, Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log as logging
+
+from neutron.agent.l2.extensions import qos
+from neutron.i18n import _LE, _LI, _LW
+from neutron.plugins.ml2.drivers.mech_sriov.agent.common import (
+    exceptions as exc)
+from neutron.plugins.ml2.drivers.mech_sriov.agent import eswitch_manager as esm
+from neutron.plugins.ml2.drivers.mech_sriov.mech_driver import (
+    mech_driver)
+
+LOG = logging.getLogger(__name__)
+
+
+class QosSRIOVAgentDriver(qos.QosAgentDriver):
+
+    _SUPPORTED_RULES = (
+        mech_driver.SriovNicSwitchMechanismDriver.supported_qos_rule_types)
+
+    def __init__(self):
+        super(QosSRIOVAgentDriver, self).__init__()
+        self.eswitch_mgr = None
+
+    def initialize(self):
+        self.eswitch_mgr = esm.ESwitchManager()
+
+    def create(self, port, qos_policy):
+        self._handle_rules('create', port, qos_policy)
+
+    def update(self, port, qos_policy):
+        self._handle_rules('update', port, qos_policy)
+
+    def delete(self, port, qos_policy):
+        # TODO(QoS): consider optimizing flushing of all QoS rules from the
+        # port by inspecting qos_policy.rules contents
+        self._delete_bandwidth_limit(port)
+
+    def _handle_rules(self, action, port, qos_policy):
+        for rule in qos_policy.rules:
+            if rule.rule_type in self._SUPPORTED_RULES:
+                handler_name = ("".join(("_", action, "_", rule.rule_type)))
+                handler = getattr(self, handler_name)
+                handler(port, rule)
+            else:
+                LOG.warning(_LW('Unsupported QoS rule type for %(rule_id)s: '
+                            '%(rule_type)s; skipping'),
+                            {'rule_id': rule.id, 'rule_type': rule.rule_type})
+
+    def _create_bandwidth_limit(self, port, rule):
+        self._update_bandwidth_limit(port, rule)
+
+    def _update_bandwidth_limit(self, port, rule):
+        pci_slot = port['profile'].get('pci_slot')
+        device = port['device']
+        self._set_vf_max_rate(device, pci_slot, rule.max_kbps)
+
+    def _delete_bandwidth_limit(self, port):
+        pci_slot = port['profile'].get('pci_slot')
+        device = port['device']
+        self._set_vf_max_rate(device, pci_slot)
+
+    def _set_vf_max_rate(self, device, pci_slot, max_kbps=0):
+        if self.eswitch_mgr.device_exists(device, pci_slot):
+            try:
+                self.eswitch_mgr.set_device_max_rate(
+                    device, pci_slot, max_kbps)
+            except exc.SriovNicError:
+                LOG.exception(
+                    _LE("Failed to set device %s max rate"), device)
+        else:
+            LOG.info(_LI("No device with MAC %s defined on agent."), device)
diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py
index 7bf2979555..13210aa515 100644
--- a/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py
+++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py
@@ -26,6 +26,7 @@ from oslo_log import log as logging
 import oslo_messaging
 from oslo_service import loopingcall
 
+from neutron.agent.l2.extensions import manager as ext_manager
 from neutron.agent import rpc as agent_rpc
 from neutron.agent import securitygroups_rpc as sg_rpc
 from neutron.common import config as common_config
@@ -34,7 +35,7 @@ from neutron.common import topics
 from neutron.common import utils as n_utils
 from neutron import context
 from neutron.i18n import _LE, _LI, _LW
-from neutron.plugins.ml2.drivers.mech_sriov.agent.common import config  # noqa
+from neutron.plugins.ml2.drivers.mech_sriov.agent.common import config
 from neutron.plugins.ml2.drivers.mech_sriov.agent.common \
     import exceptions as exc
 from neutron.plugins.ml2.drivers.mech_sriov.agent import eswitch_manager as esm
@@ -72,12 +73,13 @@ class SriovNicSwitchAgent(object):
                  polling_interval):
 
         self.polling_interval = polling_interval
+        self.conf = cfg.CONF
         self.setup_eswitch_mgr(physical_devices_mappings,
                                exclude_devices)
         configurations = {'device_mappings': physical_devices_mappings}
         self.agent_state = {
             'binary': 'neutron-sriov-nic-agent',
-            'host': cfg.CONF.host,
+            'host': self.conf.host,
             'topic': n_constants.L2_AGENT_TOPIC,
             'configurations': configurations,
             'agent_type': n_constants.AGENT_TYPE_NIC_SWITCH,
@@ -92,6 +94,10 @@ class SriovNicSwitchAgent(object):
         self.sg_agent = sg_rpc.SecurityGroupAgentRpc(self.context,
                 self.sg_plugin_rpc)
         self._setup_rpc()
+        self.ext_manager = self._create_agent_extension_manager(
+            self.connection)
+        # The initialization is complete; we can start receiving messages
+        self.connection.consume_in_threads()
         # Initialize iteration counter
         self.iter_num = 0
 
@@ -111,7 +117,8 @@ class SriovNicSwitchAgent(object):
                      [topics.SECURITY_GROUP, topics.UPDATE]]
         self.connection = agent_rpc.create_consumers(self.endpoints,
                                                      self.topic,
-                                                     consumers)
+                                                     consumers,
+                                                     start_listening=False)
 
         report_interval = cfg.CONF.AGENT.report_interval
         if report_interval:
@@ -129,6 +136,12 @@ class SriovNicSwitchAgent(object):
         except Exception:
             LOG.exception(_LE("Failed reporting state!"))
 
+    def _create_agent_extension_manager(self, connection):
+        ext_manager.register_opts(self.conf)
+        mgr = ext_manager.AgentExtensionsManager(self.conf)
+        mgr.initialize(connection, 'sriov')
+        return mgr
+
     def setup_eswitch_mgr(self, device_mappings, exclude_devices={}):
         self.eswitch_mgr = esm.ESwitchManager()
         self.eswitch_mgr.discover_devices(device_mappings, exclude_devices)
@@ -225,6 +238,7 @@ class SriovNicSwitchAgent(object):
                                   profile.get('pci_slot'),
                                   device_details['admin_state_up'],
                                   spoofcheck)
+                self.ext_manager.handle_port(self.context, device_details)
             else:
                 LOG.info(_LI("Device with MAC %s not defined on plugin"),
                          device)
@@ -235,6 +249,16 @@ class SriovNicSwitchAgent(object):
         for device in devices:
             LOG.info(_LI("Removing device with mac_address %s"), device)
             try:
+                pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(device)
+                if pci_slot:
+                    profile = {'pci_slot': pci_slot}
+                    port = {'device': device, 'profile': profile}
+                    self.ext_manager.delete_port(self.context, port)
+                else:
+                    LOG.warning(_LW("Failed to find pci slot for device "
+                                    "%(device)s; skipping extension port "
+                                    "cleanup"), device)
+
                 dev_details = self.plugin_rpc.update_device_down(self.context,
                                                                  device,
                                                                  self.agent_id,
diff --git a/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py b/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py
index 50a95e2268..dcb7e52d38 100644
--- a/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py
+++ b/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py
@@ -24,6 +24,7 @@ from neutron.plugins.common import constants as p_const
 from neutron.plugins.ml2 import driver_api as api
 from neutron.plugins.ml2.drivers.mech_sriov.mech_driver \
     import exceptions as exc
+from neutron.services.qos import qos_consts
 
 
 LOG = log.getLogger(__name__)
@@ -61,6 +62,8 @@ class SriovNicSwitchMechanismDriver(api.MechanismDriver):
 
     """
 
+    supported_qos_rule_types = [qos_consts.RULE_TYPE_BANDWIDTH_LIMIT]
+
     def __init__(self,
                  agent_type=constants.AGENT_TYPE_NIC_SWITCH,
                  vif_type=portbindings.VIF_TYPE_HW_VEB,
diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py
new file mode 100755
index 0000000000..e69de29bb2
--- /dev/null
+++ b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py
diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/test_qos_driver.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/test_qos_driver.py
new file mode 100755
index 0000000000..7ccb74507c
--- /dev/null
+++ b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/test_qos_driver.py
@@ -0,0 +1,92 @@
+# Copyright 2015 Mellanox Technologies, Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+from oslo_utils import uuidutils
+
+from neutron import context
+from neutron.objects.qos import policy
+from neutron.objects.qos import rule
+from neutron.plugins.ml2.drivers.mech_sriov.agent.common import exceptions
+from neutron.plugins.ml2.drivers.mech_sriov.agent.extension_drivers import (
+    qos_driver)
+from neutron.tests import base
+
+
+class QosSRIOVAgentDriverTestCase(base.BaseTestCase):
+
+    ASSIGNED_MAC = '00:00:00:00:00:66'
+    PCI_SLOT = '0000:06:00.1'
+
+    def setUp(self):
+        super(QosSRIOVAgentDriverTestCase, self).setUp()
+        self.context = context.get_admin_context()
+        self.qos_driver = qos_driver.QosSRIOVAgentDriver()
+        self.qos_driver.initialize()
+        self.qos_driver.eswitch_mgr = mock.Mock()
+        self.qos_driver.eswitch_mgr.set_device_max_rate = mock.Mock()
+        self.max_rate_mock = self.qos_driver.eswitch_mgr.set_device_max_rate
+        self.rule = self._create_bw_limit_rule_obj()
+        self.qos_policy = self._create_qos_policy_obj([self.rule])
+        self.port = self._create_fake_port()
+
+    def _create_bw_limit_rule_obj(self):
+        rule_obj = rule.QosBandwidthLimitRule()
+        rule_obj.id = uuidutils.generate_uuid()
+        rule_obj.max_kbps = 2
+        rule_obj.max_burst_kbps = 200
+        rule_obj.obj_reset_changes()
+        return rule_obj
+
+    def _create_qos_policy_obj(self, rules):
+        policy_dict = {'id': uuidutils.generate_uuid(),
+                'tenant_id': uuidutils.generate_uuid(),
+                'name': 'test',
+                'description': 'test',
+                'shared': False,
+                'rules': rules}
+        policy_obj = policy.QosPolicy(self.context, **policy_dict)
+        policy_obj.obj_reset_changes()
+        return policy_obj
+
+    def _create_fake_port(self):
+        return {'port_id': uuidutils.generate_uuid(),
+                'profile': {'pci_slot': self.PCI_SLOT},
+                'device': self.ASSIGNED_MAC}
+
+    def test_create_rule(self):
+        self.qos_driver.create(self.port, self.qos_policy)
+        self.max_rate_mock.assert_called_once_with(
+            self.ASSIGNED_MAC, self.PCI_SLOT, self.rule.max_kbps)
+
+    def test_update_rule(self):
+        self.qos_driver.update(self.port, self.qos_policy)
+        self.max_rate_mock.assert_called_once_with(
+            self.ASSIGNED_MAC, self.PCI_SLOT, self.rule.max_kbps)
+
+    def test_delete_rules(self):
+        self.qos_driver.delete(self.port, self.qos_policy)
+        self.max_rate_mock.assert_called_once_with(
+            self.ASSIGNED_MAC, self.PCI_SLOT, 0)
+
+    def test__set_vf_max_rate_captures_sriov_failure(self):
+        self.max_rate_mock.side_effect = exceptions.SriovNicError()
+        self.qos_driver._set_vf_max_rate(self.ASSIGNED_MAC, self.PCI_SLOT)
+
+    def test__set_vf_max_rate_unknown_device(self):
+        with mock.patch.object(self.qos_driver.eswitch_mgr, 'device_exists',
+                               return_value=False):
+            self.qos_driver._set_vf_max_rate(self.ASSIGNED_MAC, self.PCI_SLOT)
+            self.assertFalse(self.max_rate_mock.called)
diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py
index 2d30a05070..b3a7d958a8 100644
--- a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py
+++ b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py
@@ -194,6 +194,26 @@ class TestESwitchManagerApi(base.BaseTestCase):
                                              'device_mac': self.WRONG_MAC})
                 self.assertFalse(result)
 
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
+                "PciDeviceIPWrapper.get_assigned_macs",
+                return_value=[ASSIGNED_MAC])
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                "eswitch_manager.PciOsWrapper.is_assigned_vf",
+                return_value=True)
+    def test_get_pci_slot_by_existing_mac(self, *args):
+        pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(self.ASSIGNED_MAC)
+        self.assertIsNotNone(pci_slot)
+
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
+                "PciDeviceIPWrapper.get_assigned_macs",
+                return_value=[ASSIGNED_MAC])
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                "eswitch_manager.PciOsWrapper.is_assigned_vf",
+                return_value=True)
+    def test_get_pci_slot_by_not_existing_mac(self, *args):
+        pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(self.WRONG_MAC)
+        self.assertIsNone(pci_slot)
+
 
 class TestEmbSwitch(base.BaseTestCase):
     DEV_NAME = "eth2"
diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py
index ccbb04435a..8ebc73ce5f 100644
--- a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py
+++ b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py
@@ -49,7 +49,13 @@ class TestSriovAgent(base.BaseTestCase):
 
         self.agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
 
-    def test_treat_devices_removed_with_existed_device(self):
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
+                "PciDeviceIPWrapper.get_assigned_macs",
+                return_value=[DEVICE_MAC])
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                "eswitch_manager.PciOsWrapper.is_assigned_vf",
+                return_value=True)
+    def test_treat_devices_removed_with_existed_device(self, *args):
         agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
         devices = [DEVICE_MAC]
         with mock.patch.object(agent.plugin_rpc,
@@ -63,7 +69,13 @@ class TestSriovAgent(base.BaseTestCase):
                 self.assertFalse(resync)
                 self.assertTrue(fn_udd.called)
 
-    def test_treat_devices_removed_with_not_existed_device(self):
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
+                "PciDeviceIPWrapper.get_assigned_macs",
+                return_value=[DEVICE_MAC])
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                "eswitch_manager.PciOsWrapper.is_assigned_vf",
+                return_value=True)
+    def test_treat_devices_removed_with_not_existed_device(self, *args):
         agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
         devices = [DEVICE_MAC]
         with mock.patch.object(agent.plugin_rpc,
@@ -77,7 +89,13 @@ class TestSriovAgent(base.BaseTestCase):
                 self.assertFalse(resync)
                 self.assertTrue(fn_udd.called)
 
-    def test_treat_devices_removed_failed(self):
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
+                "PciDeviceIPWrapper.get_assigned_macs",
+                return_value=[DEVICE_MAC])
+    @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                "eswitch_manager.PciOsWrapper.is_assigned_vf",
+                return_value=True)
+    def test_treat_devices_removed_failed(self, *args):
         agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
         devices = [DEVICE_MAC]
         with mock.patch.object(agent.plugin_rpc,
diff --git a/setup.cfg b/setup.cfg
index c9ff7b7c0d..3ae7a72303 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -200,6 +200,7 @@ neutron.agent.l2.extensions =
     qos = neutron.agent.l2.extensions.qos:QosAgentExtension
 neutron.qos.agent_drivers =
     ovs = neutron.plugins.ml2.drivers.openvswitch.agent.extension_drivers.qos_driver:QosOVSAgentDriver
+    sriov = neutron.plugins.ml2.drivers.mech_sriov.agent.extension_drivers.qos_driver:QosSRIOVAgentDriver
 # These are for backwards compat with Icehouse notification_driver configuration values
 oslo.messaging.notify.drivers =
     neutron.openstack.common.notifier.log_notifier = oslo_messaging.notify._impl_log:LogDriver