Merge "Provide a path to set explicit ipxe bootloaders" into stable/train

11 files changed, 224 insertions, 42 deletions

diff --git a/devstack/lib/ironic b/devstack/lib/ironic
index cc3f49bad..fea0ff61c 100644
--- a/devstack/lib/ironic
+++ b/devstack/lib/ironic
@@ -1548,10 +1548,8 @@ function configure_ironic_conductor {
         pxebin=`basename $IRONIC_PXE_BOOT_IMAGE`
         uefipxebin=`basename $(get_uefi_ipxe_boot_file)`
         iniset $IRONIC_CONF_FILE pxe ipxe_enabled True
-        iniset $IRONIC_CONF_FILE pxe pxe_config_template '$pybasedir/drivers/modules/ipxe_config.template'
-        iniset $IRONIC_CONF_FILE pxe pxe_bootfile_name $pxebin
-        iniset $IRONIC_CONF_FILE pxe uefi_pxe_config_template '$pybasedir/drivers/modules/ipxe_config.template'
-        iniset $IRONIC_CONF_FILE pxe uefi_pxe_bootfile_name $uefipxebin
+        iniset $IRONIC_CONF_FILE pxe ipxe_bootfile_name $pxebin
+        iniset $IRONIC_CONF_FILE pxe uefi_ipxe_bootfile_name $uefipxebin
         iniset $IRONIC_CONF_FILE deploy http_root $IRONIC_HTTP_DIR
         iniset $IRONIC_CONF_FILE deploy http_url "http://$IRONIC_HTTP_SERVER:$IRONIC_HTTP_PORT"
         if [[ "$IRONIC_IPXE_USE_SWIFT" == "True" ]]; then
diff --git a/doc/source/install/configure-pxe.rst b/doc/source/install/configure-pxe.rst
index 88b8c1b7d..f4f44abf0 100644
--- a/doc/source/install/configure-pxe.rst
+++ b/doc/source/install/configure-pxe.rst
@@ -357,14 +357,14 @@ on the Bare Metal service node(s) where ``ironic-conductor`` is running.
 
    Ubuntu::
 
-       cp /usr/lib/ipxe/{undionly.kpxe,ipxe.efi} /tftpboot
+       cp /usr/lib/ipxe/{undionly.kpxe,ipxe.efi,snponly.efi} /tftpboot
 
    Fedora/RHEL7/CentOS7::
 
-       cp /usr/share/ipxe/{undionly.kpxe,ipxe.efi} /tftpboot
+       cp /usr/share/ipxe/{undionly.kpxe,ipxe.efi,snponly.efi} /tftpboot
 
-#. Enable/Configure iPXE in the Bare Metal Service's configuration file
-   (/etc/ironic/ironic.conf):
+#. Enable/Configure iPXE overrides in the Bare Metal Service's configuration
+   file **if required** (/etc/ironic/ironic.conf):
 
    .. code-block:: ini
 
@@ -374,24 +374,45 @@ on the Bare Metal service node(s) where ``ironic-conductor`` is running.
       ipxe_enabled=True
 
       # Neutron bootfile DHCP parameter. (string value)
-      pxe_bootfile_name=undionly.kpxe
+      ipxe_bootfile_name=undionly.kpxe
 
       # Bootfile DHCP parameter for UEFI boot mode. (string value)
-      uefi_pxe_bootfile_name=ipxe.efi
+      uefi_ipxe_bootfile_name=ipxe.efi
 
       # Template file for PXE configuration. (string value)
-      pxe_config_template=$pybasedir/drivers/modules/ipxe_config.template
+      ipxe_config_template=$pybasedir/drivers/modules/ipxe_config.template
 
-      # Template file for PXE configuration for UEFI boot loader.
-      # (string value)
-      uefi_pxe_config_template=$pybasedir/drivers/modules/ipxe_config.template
+   .. note::
+      Most UEFI systems have integrated networking which means the
+      ``[pxe]uefi_ipxe_bootfile_name`` setting should be set to
+      ``snponly.efi``.
 
    .. note::
-      The ``[pxe]ipxe_enabled`` option has been deprecated and will be removed
-      in the T* development cycle. Users should instead consider use of the
-      ``ipxe`` boot interface. The same default use of iPXE functionality can
-      be achieved by setting the ``[DEFAULT]default_boot_interface`` option
-      to ``ipxe``.
+      Setting the iPXE parameters noted in the code block above to no value,
+      in other words setting a line to something like ``ipxe_bootfile_name=``
+      will result in ironic falling back to the default values of the non-iPXE
+      PXE settings. This is for backwards compatability.
+
+#. Ensure iPXE is the default PXE, if applicable.
+
+   In earlier versions of ironic, a ``[pxe]ipxe_enabled`` setting allowing
+   operators to declare the behavior of the conductor to exclusively operate
+   as if only iPXE was to be used. As time moved on, iPXE functionality was
+   moved to it's own ``ipxe`` boot interface.
+
+   If you want to emulate that same hehavior, set the following in the
+   configuration file (/etc/ironic/ironic.conf):
+
+   .. code-block:: ini
+
+      [DEFAULT]
+      default_boot_interface=ipxe
+      enabled_boot_interfaces=ipxe,pxe
+
+   .. note::
+      The ``[DEFAULT]enabled_boot_interfaces`` setting may be exclusively set
+      to ``ipxe``, however ironic has multiple interfaces available depending
+      on the hardware types available for use.
 
 #. It is possible to configure the Bare Metal service in such a way
    that nodes will boot into the deploy image directly from Object Storage.
@@ -442,7 +463,6 @@ on the Bare Metal service node(s) where ``ironic-conductor`` is running.
 
      sudo service ironic-conductor restart
 
-
 PXE multi-architecture setup
 ----------------------------
 
@@ -491,6 +511,18 @@ nodes will be deployed by 'grubaa64.efi', and ppc64 nodes by 'bootppc64'::
     # aarch64:/opt/share/grubaa64_pxe_config.template (dict value)
     pxe_config_template_by_arch=aarch64:pxe_grubaa64_config.template,ppc64:pxe_ppc64_config.template
 
+.. note::
+   The grub implementation may vary on different architecture, you may need to
+   tweak the pxe config template for a specific arch. For example, grubaa64.efi
+   shipped with CentoOS7 does not support ``linuxefi`` and ``initrdefi``
+   commands, you'll need to switch to use ``linux`` and ``initrd`` command
+   instead.
+
+.. note::
+   A ``[pxe]ipxe_bootfile_name_by_arch`` setting is available for multi-arch
+   iPXE based deployment, and defaults to the same behavior as the comperable
+   ``[pxe]pxe_bootfile_by_arch`` setting for standard PXE.
+
 PXE timeouts tuning
 -------------------
 
diff --git a/ironic/common/pxe_utils.py b/ironic/common/pxe_utils.py
index bb24e14a4..b308abc71 100644
--- a/ironic/common/pxe_utils.py
+++ b/ironic/common/pxe_utils.py
@@ -273,7 +273,10 @@ def create_pxe_config(task, pxe_options, template=None, ipxe_enabled=False):
     """
     LOG.debug("Building PXE config for node %s", task.node.uuid)
     if template is None:
-        template = deploy_utils.get_pxe_config_template(task.node)
+        if ipxe_enabled:
+            template = deploy_utils.get_ipxe_config_template(task.node)
+        else:
+            template = deploy_utils.get_pxe_config_template(task.node)
 
     _ensure_config_dirs_exist(task, ipxe_enabled)
 
@@ -388,7 +391,16 @@ def _dhcp_option_file_or_url(task, urlboot=False):
                      URL should be returned to the file as opposed
                      to a file.
     """
-    boot_file = deploy_utils.get_pxe_boot_file(task.node)
+    try:
+        if task.driver.boot.ipxe_enabled:
+            boot_file = deploy_utils.get_ipxe_boot_file(task.node)
+        else:
+            boot_file = deploy_utils.get_pxe_boot_file(task.node)
+    except AttributeError:
+        # Support boot interfaces that lack an explicit ipxe_enabled
+        # attribute flag.
+        boot_file = deploy_utils.get_pxe_boot_file(task.node)
+
     # NOTE(TheJulia): There are additional cases as we add new
     # features, so the logic below is in the form of if/elif/elif
     if not urlboot:
@@ -800,7 +812,10 @@ def build_service_pxe_config(task, instance_image_info,
         pxe_options = build_pxe_config_options(task, instance_image_info,
                                                service=True,
                                                ipxe_enabled=ipxe_enabled)
-        pxe_config_template = deploy_utils.get_pxe_config_template(node)
+        if ipxe_enabled:
+            pxe_config_template = deploy_utils.get_ipxe_config_template(node)
+        else:
+            pxe_config_template = deploy_utils.get_pxe_config_template(node)
         create_pxe_config(task, pxe_options, pxe_config_template,
                           ipxe_enabled=ipxe_enabled)
     iwdi = node.driver_internal_info.get('is_whole_disk_image')
@@ -929,8 +944,12 @@ def prepare_instance_pxe_config(task, image_info,
         pxe_options = build_pxe_config_options(
             task, image_info, service=ramdisk_boot,
             ipxe_enabled=ipxe_enabled)
-        pxe_config_template = (
-            deploy_utils.get_pxe_config_template(node))
+        if ipxe_enabled:
+            pxe_config_template = (
+                deploy_utils.get_ipxe_config_template(node))
+        else:
+            pxe_config_template = (
+                deploy_utils.get_pxe_config_template(node))
         create_pxe_config(
             task, pxe_options, pxe_config_template,
             ipxe_enabled=ipxe_enabled)
diff --git a/ironic/conf/pxe.py b/ironic/conf/pxe.py
index 05b955ab8..7bb81e9f6 100644
--- a/ironic/conf/pxe.py
+++ b/ironic/conf/pxe.py
@@ -51,13 +51,20 @@ opts = [
                default=os.path.join(
                    '$pybasedir', 'drivers/modules/pxe_config.template'),
                help=_('On ironic-conductor node, template file for PXE '
-                      'configuration.')),
+                      'loader configuration.')),
+    cfg.StrOpt('ipxe_config_template',
+               default=os.path.join(
+                   '$pybasedir', 'drivers/modules/ipxe_config.template'),
+               mutable=True,
+               help=_('On ironic-conductor node, template file for iPXE '
+                      'operations.')),
     cfg.StrOpt('uefi_pxe_config_template',
                default=os.path.join(
                    '$pybasedir',
                    'drivers/modules/pxe_grub_config.template'),
                help=_('On ironic-conductor node, template file for PXE '
-                      'configuration for UEFI boot loader.')),
+                      'configuration for UEFI boot loader. Generally this '
+                      'is used for GRUB specific templates.')),
     cfg.DictOpt('pxe_config_template_by_arch',
                 default={},
                 help=_('On ironic-conductor node, template file for PXE '
@@ -102,10 +109,22 @@ opts = [
     cfg.StrOpt('uefi_pxe_bootfile_name',
                default='bootx64.efi',
                help=_('Bootfile DHCP parameter for UEFI boot mode.')),
+    cfg.StrOpt('ipxe_bootfile_name',
+               default='undionly.kpxe',
+               help=_('Bootfile DHCP parameter.')),
+    cfg.StrOpt('uefi_ipxe_bootfile_name',
+               default='ipxe.efi',
+               help=_('Bootfile DHCP parameter for UEFI boot mode. If you '
+                      'experience problems with booting using it, try '
+                      'snponly.efi.')),
     cfg.DictOpt('pxe_bootfile_name_by_arch',
                 default={},
                 help=_('Bootfile DHCP parameter per node architecture. '
                        'For example: aarch64:grubaa64.efi')),
+    cfg.DictOpt('ipxe_bootfile_name_by_arch',
+                default={},
+                help=_('Bootfile DHCP parameter per node architecture. '
+                       'For example: aarch64:ipxe_aa64.efi')),
     cfg.BoolOpt('ipxe_enabled',
                 default=False,
                 help=_('Defaults the PXE interface to only use iPXE.'),
diff --git a/ironic/drivers/modules/deploy_utils.py b/ironic/drivers/modules/deploy_utils.py
index 339fe92eb..1a257354b 100644
--- a/ironic/drivers/modules/deploy_utils.py
+++ b/ironic/drivers/modules/deploy_utils.py
@@ -741,6 +741,54 @@ def get_pxe_boot_file(node):
     return boot_file
 
 
+def get_ipxe_boot_file(node):
+    """Return the iPXE boot file name requested for deploy.
+
+    This method returns iPXE boot file name to be used for deploy.
+    Architecture specific boot file is searched first. BIOS/UEFI
+    boot file is used if no valid architecture specific file found.
+
+    If no valid value is found, the default reverts to the
+    ``get_pxe_boot_file`` method and thus the
+    ``[pxe]pxe_bootfile_name`` and ``[pxe]uefi_ipxe_bootfile_name``
+    settings.
+
+    :param node: A single Node.
+    :returns: The iPXE boot file name.
+    """
+    cpu_arch = node.properties.get('cpu_arch')
+    boot_file = CONF.pxe.ipxe_bootfile_name_by_arch.get(cpu_arch)
+    if boot_file is None:
+        if boot_mode_utils.get_boot_mode(node) == 'uefi':
+            boot_file = CONF.pxe.uefi_ipxe_bootfile_name
+        else:
+            boot_file = CONF.pxe.ipxe_bootfile_name
+
+    if boot_file is None:
+        boot_file = get_pxe_boot_file(node)
+
+    return boot_file
+
+
+def get_ipxe_config_template(node):
+    """Return the iPXE config template file name requested of deploy.
+
+    This method returns the iPXE configuration template file.
+
+    :param node: A single Node.
+    :returns: The iPXE config template file name.
+    """
+    # NOTE(TheJulia): iPXE configuration files don't change based upon the
+    # architecture and we're not trying to support multiple different boot
+    # loaders by architecture as they are all consistent. Where as PXE
+    # could need to be grub for one arch, PXELINUX for another.
+    configured_template = CONF.pxe.ipxe_config_template
+    override_template = node.driver_info.get('pxe_template')
+    if override_template:
+        configured_template = override_template
+    return configured_template or get_pxe_config_template(node)
+
+
 def get_pxe_config_template(node):
     """Return the PXE config template file name requested for deploy.
 
diff --git a/ironic/drivers/modules/ipxe.py b/ironic/drivers/modules/ipxe.py
index bfe1d1cd4..bbf06e3be 100644
--- a/ironic/drivers/modules/ipxe.py
+++ b/ironic/drivers/modules/ipxe.py
@@ -164,7 +164,7 @@ class iPXEBoot(pxe_base.PXEBaseMixin, base.BootInterface):
                                                          ipxe_enabled=True)
         pxe_options.update(ramdisk_params)
 
-        pxe_config_template = deploy_utils.get_pxe_config_template(node)
+        pxe_config_template = deploy_utils.get_ipxe_config_template(node)
 
         pxe_utils.create_pxe_config(task, pxe_options,
                                     pxe_config_template,
diff --git a/ironic/tests/unit/common/test_pxe_utils.py b/ironic/tests/unit/common/test_pxe_utils.py
index e934e2051..7b391ea1a 100644
--- a/ironic/tests/unit/common/test_pxe_utils.py
+++ b/ironic/tests/unit/common/test_pxe_utils.py
@@ -667,7 +667,7 @@ class TestPXEUtils(db_base.DbTestCase):
                                       'config'),
                          pxe_utils.get_pxe_config_file_path(self.node.uuid))
 
-    def _dhcp_options_for_instance(self, ip_version=4):
+    def _dhcp_options_for_instance(self, ip_version=4, ipxe=False):
         self.config(ip_version=ip_version, group='pxe')
         if ip_version == 4:
             self.config(tftp_server='192.0.2.1', group='pxe')
@@ -675,6 +675,10 @@ class TestPXEUtils(db_base.DbTestCase):
             self.config(tftp_server='ff80::1', group='pxe')
         self.config(pxe_bootfile_name='fake-bootfile', group='pxe')
         self.config(tftp_root='/tftp-path/', group='pxe')
+        if ipxe:
+            bootfile = 'fake-bootfile-ipxe'
+        else:
+            bootfile = 'fake-bootfile'
 
         if ip_version == 6:
             # NOTE(TheJulia): DHCPv6 RFCs seem to indicate that the prior
@@ -682,11 +686,11 @@ class TestPXEUtils(db_base.DbTestCase):
             # by vendors. The apparent proper option is to return a
             # URL in the field https://tools.ietf.org/html/rfc5970#section-3
             expected_info = [{'opt_name': '59',
-                              'opt_value': 'tftp://[ff80::1]/fake-bootfile',
+                              'opt_value': 'tftp://[ff80::1]/%s' % bootfile,
                               'ip_version': ip_version}]
         elif ip_version == 4:
             expected_info = [{'opt_name': '67',
-                              'opt_value': 'fake-bootfile',
+                              'opt_value': bootfile,
                               'ip_version': ip_version},
                              {'opt_name': '210',
                               'opt_value': '/tftp-path/',
diff --git a/ironic/tests/unit/drivers/modules/test_deploy_utils.py b/ironic/tests/unit/drivers/modules/test_deploy_utils.py
index 4b6ffeabf..1404c5e17 100644
--- a/ironic/tests/unit/drivers/modules/test_deploy_utils.py
+++ b/ironic/tests/unit/drivers/modules/test_deploy_utils.py
@@ -1116,6 +1116,34 @@ class GetPxeBootConfigTestCase(db_base.DbTestCase):
         result = utils.get_pxe_boot_file(self.node)
         self.assertEqual('bios-bootfile', result)
 
+    def test_get_ipxe_boot_file(self):
+        self.config(ipxe_bootfile_name='meow', group='pxe')
+        result = utils.get_ipxe_boot_file(self.node)
+        self.assertEqual('meow', result)
+
+    def test_get_ipxe_boot_file_uefi(self):
+        self.config(uefi_ipxe_bootfile_name='ipxe-uefi-bootfile', group='pxe')
+        properties = {'capabilities': 'boot_mode:uefi'}
+        self.node.properties = properties
+        result = utils.get_ipxe_boot_file(self.node)
+        self.assertEqual('ipxe-uefi-bootfile', result)
+
+    def test_get_ipxe_boot_file_other_arch(self):
+        arch_names = {'aarch64': 'ipxe-aa64.efi',
+                      'x86_64': 'ipxe.kpxe'}
+        self.config(ipxe_bootfile_name_by_arch=arch_names, group='pxe')
+        properties = {'cpu_arch': 'aarch64', 'capabilities': 'boot_mode:uefi'}
+        self.node.properties = properties
+        result = utils.get_ipxe_boot_file(self.node)
+        self.assertEqual('ipxe-aa64.efi', result)
+
+    def test_get_ipxe_boot_file_fallback(self):
+        self.config(ipxe_bootfile_name=None, group='pxe')
+        self.config(uefi_ipxe_bootfile_name=None, group='pxe')
+        self.config(pxe_bootfile_name='lolcat', group='pxe')
+        result = utils.get_ipxe_boot_file(self.node)
+        self.assertEqual('lolcat', result)
+
     def test_get_pxe_config_template_emtpy_property(self):
         self.node.properties = {}
         self.config(pxe_config_template_by_arch=self.template_by_arch,
@@ -1131,6 +1159,28 @@ class GetPxeBootConfigTestCase(db_base.DbTestCase):
         result = utils.get_pxe_config_template(node)
         self.assertEqual('fake-template', result)
 
+    def test_get_ipxe_config_template(self):
+        node = obj_utils.create_test_node(
+            self.context, driver='fake-hardware')
+        self.assertIn('ipxe_config.template',
+                      utils.get_ipxe_config_template(node))
+
+    def test_get_ipxe_config_template_none(self):
+        self.config(ipxe_config_template=None, group='pxe')
+        self.config(pxe_config_template='magical_bootloader',
+                    group='pxe')
+        node = obj_utils.create_test_node(
+            self.context, driver='fake-hardware')
+        self.assertEqual('magical_bootloader',
+                         utils.get_ipxe_config_template(node))
+
+    def test_get_ipxe_config_template_override_pxe_fallback(self):
+        node = obj_utils.create_test_node(
+            self.context, driver='fake-hardware',
+            driver_info={'pxe_template': 'magical'})
+        self.assertEqual('magical',
+                         utils.get_ipxe_config_template(node))
+
 
 @mock.patch('time.sleep', lambda sec: None)
 class OtherFunctionTestCase(db_base.DbTestCase):
diff --git a/ironic/tests/unit/drivers/modules/test_ipxe.py b/ironic/tests/unit/drivers/modules/test_ipxe.py
index e86e71ea5..9a7d41d04 100644
--- a/ironic/tests/unit/drivers/modules/test_ipxe.py
+++ b/ironic/tests/unit/drivers/modules/test_ipxe.py
@@ -291,14 +291,9 @@ class iPXEBootTestCase(db_base.DbTestCase):
                     mock_cache_r_k.assert_called_once_with(
                         task, {'rescue_kernel': 'a', 'rescue_ramdisk': 'r'},
                         ipxe_enabled=True)
-            if uefi:
-                mock_pxe_config.assert_called_once_with(
-                    task, {'foo': 'bar'}, CONF.pxe.uefi_pxe_config_template,
-                    ipxe_enabled=True)
-            else:
-                mock_pxe_config.assert_called_once_with(
-                    task, {'foo': 'bar'}, CONF.pxe.pxe_config_template,
-                    ipxe_enabled=True)
+            mock_pxe_config.assert_called_once_with(
+                task, {'foo': 'bar'}, CONF.pxe.ipxe_config_template,
+                ipxe_enabled=True)
 
     def test_prepare_ramdisk(self):
         self.node.provision_state = states.DEPLOYING
@@ -678,7 +673,7 @@ class iPXEBootTestCase(db_base.DbTestCase):
                                                ipxe_enabled=True)
             provider_mock.update_dhcp.assert_called_once_with(task, dhcp_opts)
             create_pxe_config_mock.assert_called_once_with(
-                task, mock.ANY, CONF.pxe.pxe_config_template,
+                task, mock.ANY, CONF.pxe.ipxe_config_template,
                 ipxe_enabled=True)
             switch_pxe_config_mock.assert_called_once_with(
                 pxe_config_path, "30212642-09d3-467f-8e09-21685826ab50",
@@ -786,7 +781,7 @@ class iPXEBootTestCase(db_base.DbTestCase):
             self.assertFalse(cache_mock.called)
             provider_mock.update_dhcp.assert_called_once_with(task, dhcp_opts)
             create_pxe_config_mock.assert_called_once_with(
-                task, mock.ANY, CONF.pxe.pxe_config_template,
+                task, mock.ANY, CONF.pxe.ipxe_config_template,
                 ipxe_enabled=True)
             switch_pxe_config_mock.assert_called_once_with(
                 pxe_config_path, None, boot_modes.LEGACY_BIOS, False,
diff --git a/ironic/tests/unit/drivers/modules/test_pxe.py b/ironic/tests/unit/drivers/modules/test_pxe.py
index f3df5dae3..b301418a1 100644
--- a/ironic/tests/unit/drivers/modules/test_pxe.py
+++ b/ironic/tests/unit/drivers/modules/test_pxe.py
@@ -786,7 +786,7 @@ class PXEBootTestCase(db_base.DbTestCase):
             self.assertFalse(cache_mock.called)
             provider_mock.update_dhcp.assert_called_once_with(task, dhcp_opts)
             create_pxe_config_mock.assert_called_once_with(
-                task, mock.ANY, CONF.pxe.pxe_config_template,
+                task, mock.ANY, CONF.pxe.ipxe_config_template,
                 ipxe_enabled=True)
             switch_pxe_config_mock.assert_called_once_with(
                 pxe_config_path, None, boot_modes.LEGACY_BIOS, False,
diff --git a/releasenotes/notes/explicit_ipxe_config_options-d7bf9a743a13f523.yaml b/releasenotes/notes/explicit_ipxe_config_options-d7bf9a743a13f523.yaml
new file mode 100644
index 000000000..acf5daccf
--- /dev/null
+++ b/releasenotes/notes/explicit_ipxe_config_options-d7bf9a743a13f523.yaml
@@ -0,0 +1,17 @@
+---
+upgrade:
+  - |
+    Operators upgrading from earlier versions using PXE should explicitly set
+    ``[pxe]ipxe_bootfile_name``, ``[pxe]uefi_ipxe_bootfile_name``, and
+    possibly ``[pxe]ipxe_bootfile_name_by_arch`` settings, as well as a
+    iPXE specific ``[pxe]ipxe_config_template`` override, if required.
+
+    Setting the ``[pxe]ipxe_config_template`` to no value will result in the
+    ``[pxe]pxe_config_template`` being used. The default value points to the
+    supplied standard iPXE template, so only highly customized operators may
+    have to tune this setting.
+fixes:
+  - |
+    Addresses the lack of an ability to explicitly set different bootloaders
+    for ``iPXE`` and ``PXE`` based boot operations via their respective
+    ``ipxe`` and ``pxe`` boot interfaces.