LVM nvmet: Add support for multiple ip addresses

The nvmet target driver only supports single portals, which was all that
was available back on the original implementation, but now that it
supports the new connection information format it can provide
multiple portals.

This patch adds support to provide multiple portals when attaching a new
volume, that way os-brick can try the different portals when connecting a
volume until it finds one that works, making it more robust.

Thanks to this features it will also enable multipathing automatically
(without additional changes) once the NVMe-oF os-brick connector
supports it.

Since the new connection information format is necessary to pass
multiple portals it requires that the configuration option
``nvmeof_conn_info_version`` is set to ``2``.

The patch also deprecates the ``iscsi_secondary_ip_addresses``
configuration option in favor of the new
``target_secondary_ip_addresses``.  This is something we already did a
while back for ``iscsi_ip_address`` which was renamed in the same way to
``target_ip_address``.

Change-Id: Iccfbe62406b6202446e974487e0f91465a5d0fa3

17 files changed, 215 insertions, 85 deletions

diff --git a/cinder/tests/unit/targets/test_nvmeof_driver.py b/cinder/tests/unit/targets/test_nvmeof_driver.py
index 289e492fd..21d30b739 100644
--- a/cinder/tests/unit/targets/test_nvmeof_driver.py
+++ b/cinder/tests/unit/targets/test_nvmeof_driver.py
@@ -62,7 +62,7 @@ class TestNVMeOFDriver(tf.TargetDriverFixture):
                      "ngn.%s-%s" % (
                          self.nvmet_subsystem_name,
                          self.fake_volume_id),
-                     self.target_ip,
+                     [self.target_ip],
                      self.target_port,
                      self.nvme_transport_type,
                      self.nvmet_ns_id
@@ -92,7 +92,7 @@ class TestNVMeOFDriver(tf.TargetDriverFixture):
         mock_create_nvme_target.assert_called_once_with(
             self.fake_volume_id,
             self.configuration.target_prefix,
-            self.target_ip,
+            [self.target_ip],
             self.target_port,
             self.nvme_transport_type,
             self.nvmet_port_id,
@@ -117,7 +117,31 @@ class TestNVMeOFDriver(tf.TargetDriverFixture):
         mock_uuid.assert_called_once_with(self.testvol)
         mock_get_conn_props.assert_called_once_with(
             f'ngn.{self.nvmet_subsystem_name}-{self.fake_volume_id}',
-            self.target_ip,
+            [self.target_ip],
+            str(self.target_port),
+            self.nvme_transport_type,
+            str(self.nvmet_ns_id),
+            mock_uuid.return_value)
+
+    @mock.patch.object(nvmeof.NVMeOF, '_get_nvme_uuid')
+    @mock.patch.object(nvmeof.NVMeOF, '_get_connection_properties')
+    def test__get_connection_properties_multiple_addresses(
+            self, mock_get_conn_props, mock_uuid):
+        """Test connection properties from a volume with multiple ips."""
+        self.testvol['provider_location'] = self.target.get_nvmeof_location(
+            f"ngn.{self.nvmet_subsystem_name}-{self.fake_volume_id}",
+            [self.target_ip, '127.0.0.1'],
+            self.target_port,
+            self.nvme_transport_type,
+            self.nvmet_ns_id
+        )
+
+        res = self.target._get_connection_properties_from_vol(self.testvol)
+        self.assertEqual(mock_get_conn_props.return_value, res)
+        mock_uuid.assert_called_once_with(self.testvol)
+        mock_get_conn_props.assert_called_once_with(
+            f'ngn.{self.nvmet_subsystem_name}-{self.fake_volume_id}',
+            [self.target_ip, '127.0.0.1'],
             str(self.target_port),
             self.nvme_transport_type,
             str(self.nvmet_ns_id),
@@ -134,7 +158,7 @@ class TestNVMeOFDriver(tf.TargetDriverFixture):
             'ns_id': str(self.nvmet_ns_id)
         }
         res = self.target._get_connection_properties(nqn,
-                                                     self.target_ip,
+                                                     [self.target_ip],
                                                      str(self.target_port),
                                                      self.nvme_transport_type,
                                                      str(self.nvmet_ns_id),
@@ -158,7 +182,7 @@ class TestNVMeOFDriver(tf.TargetDriverFixture):
                          expected_transport)],
         }
         res = self.target._get_connection_properties(nqn,
-                                                     self.target_ip,
+                                                     [self.target_ip],
                                                      str(self.target_port),
                                                      transport,
                                                      str(self.nvmet_ns_id),
@@ -182,6 +206,22 @@ class TestNVMeOFDriver(tf.TargetDriverFixture):
                           root_helper=utils.get_root_helper(),
                           configuration=self.configuration)
 
+    def test_invalid_secondary_ips_old_conn_info_combination(self):
+        """Secondary IPS are only supported with new connection information."""
+        self.configuration.target_secondary_ip_addresses = ['127.0.0.1']
+        self.configuration.nvmeof_conn_info_version = 1
+        self.assertRaises(exception.InvalidConfigurationValue,
+                          FakeNVMeOFDriver,
+                          root_helper=utils.get_root_helper(),
+                          configuration=self.configuration)
+
+    def test_valid_secondary_ips_old_conn_info_combination(self):
+        """Secondary IPS are supported with new connection information."""
+        self.configuration.target_secondary_ip_addresses = ['127.0.0.1']
+        self.configuration.nvmeof_conn_info_version = 2
+        FakeNVMeOFDriver(root_helper=utils.get_root_helper(),
+                         configuration=self.configuration)
+
     def test_are_same_connector(self):
         res = self.target.are_same_connector({'nqn': 'nvme'}, {'nqn': 'nvme'})
         self.assertTrue(res)
@@ -192,3 +232,20 @@ class TestNVMeOFDriver(tf.TargetDriverFixture):
     def test_are_same_connector_different(self, a_conn_props, b_conn_props):
         res = self.target.are_same_connector(a_conn_props, b_conn_props)
         self.assertFalse(bool(res))
+
+    def test_get_nvmeof_location(self):
+        """Serialize connection information into location."""
+        result = self.target.get_nvmeof_location(
+            'ngn.subsys_name-vol_id', ['127.0.0.1'], 4420, 'tcp', 10)
+
+        expected = '127.0.0.1:4420 tcp ngn.subsys_name-vol_id 10'
+        self.assertEqual(expected, result)
+
+    def test_get_nvmeof_location_multiple_ips(self):
+        """Serialize connection information with multiple ips into location."""
+        result = self.target.get_nvmeof_location(
+            'ngn.subsys_name-vol_id', ['127.0.0.1', '192.168.1.1'], 4420,
+            'tcp', 10)
+
+        expected = '127.0.0.1,192.168.1.1:4420 tcp ngn.subsys_name-vol_id 10'
+        self.assertEqual(expected, result)
diff --git a/cinder/tests/unit/targets/test_nvmet_driver.py b/cinder/tests/unit/targets/test_nvmet_driver.py
index ad1a3307c..abfd138b7 100644
--- a/cinder/tests/unit/targets/test_nvmet_driver.py
+++ b/cinder/tests/unit/targets/test_nvmet_driver.py
@@ -76,7 +76,7 @@ class TestNVMETDriver(tf.TargetDriverFixture):
         mock_uuid.assert_called_once_with(vol)
         mock_get_conn_props.assert_called_once_with(
             mock.sentinel.nqn,
-            self.target.target_ip,
+            self.target.target_ips,
             self.target.target_port,
             self.target.nvme_transport_type,
             mock.sentinel.nsid,
@@ -119,7 +119,7 @@ class TestNVMETDriver(tf.TargetDriverFixture):
         mock_map.assert_called_once_with(mock.sentinel.vol,
                                          mock.sentinel.volume_path)
         mock_location.assert_called_once_with(mock.sentinel.nqn,
-                                              self.target.target_ip,
+                                              self.target.target_ips,
                                               self.target.target_port,
                                               self.target.nvme_transport_type,
                                               mock.sentinel.nsid)
@@ -160,7 +160,7 @@ class TestNVMETDriver(tf.TargetDriverFixture):
                                             mock.sentinel.volume_path,
                                             mock_uuid.return_value)
         mock_port.assert_called_once_with(mock_nqn.return_value,
-                                          self.target.target_ip,
+                                          self.target.target_ips,
                                           self.target.target_port,
                                           self.target.nvme_transport_type,
                                           self.target.nvmet_port_id)
@@ -197,7 +197,7 @@ class TestNVMETDriver(tf.TargetDriverFixture):
             mock_port.assert_not_called()
         else:
             mock_port.assert_called_once_with(mock.sentinel.nqn,
-                                              self.target.target_ip,
+                                              self.target.target_ips,
                                               self.target.target_port,
                                               self.target.nvme_transport_type,
                                               self.target.nvmet_port_id)
@@ -345,13 +345,14 @@ class TestNVMETDriver(tf.TargetDriverFixture):
     def test__ensure_port_exports_already_does(self, mock_port):
         """Skips port creation and subsystem export since they both exist."""
         nqn = 'nqn.nvme-subsystem-1-uuid'
+        port_id = 1
         mock_port.return_value.subsystems = [nqn]
         self.target._ensure_port_exports(nqn,
-                                         mock.sentinel.addr,
+                                         [mock.sentinel.addr],
                                          mock.sentinel.port,
                                          mock.sentinel.transport,
-                                         mock.sentinel.port_id)
-        mock_port.assert_called_once_with(mock.sentinel.port_id)
+                                         port_id)
+        mock_port.assert_called_once_with(port_id)
         mock_port.setup.assert_not_called()
         mock_port.return_value.add_subsystem.assert_not_called()
 
@@ -359,13 +360,14 @@ class TestNVMETDriver(tf.TargetDriverFixture):
     def test__ensure_port_exports_port_exists_not_exported(self, mock_port):
         """Skips port creation if exists but exports subsystem."""
         nqn = 'nqn.nvme-subsystem-1-vol-2-uuid'
+        port_id = 1
         mock_port.return_value.subsystems = ['nqn.nvme-subsystem-1-vol-1-uuid']
         self.target._ensure_port_exports(nqn,
-                                         mock.sentinel.addr,
+                                         [mock.sentinel.addr],
                                          mock.sentinel.port,
                                          mock.sentinel.transport,
-                                         mock.sentinel.port_id)
-        mock_port.assert_called_once_with(mock.sentinel.port_id)
+                                         port_id)
+        mock_port.assert_called_once_with(port_id)
         mock_port.setup.assert_not_called()
         mock_port.return_value.add_subsystem.assert_called_once_with(nqn)
 
@@ -373,23 +375,35 @@ class TestNVMETDriver(tf.TargetDriverFixture):
     def test__ensure_port_exports_port(self, mock_port):
         """Creates the port and export the subsystem when they don't exist."""
         nqn = 'nqn.nvme-subsystem-1-vol-2-uuid'
+        port_id = 1
         mock_port.side_effect = priv_nvmet.NotFound
         self.target._ensure_port_exports(nqn,
-                                         mock.sentinel.addr,
+                                         [mock.sentinel.addr,
+                                          mock.sentinel.addr2],
                                          mock.sentinel.port,
                                          mock.sentinel.transport,
-                                         mock.sentinel.port_id)
-        mock_port.assert_called_once_with(mock.sentinel.port_id)
-        new_port = {'addr': {'adrfam': 'ipv4',
-                             'traddr': mock.sentinel.addr,
-                             'treq': 'not specified',
-                             'trsvcid': mock.sentinel.port,
-                             'trtype': mock.sentinel.transport},
-                    'portid': mock.sentinel.port_id,
-                    'referrals': [],
-                    'subsystems': [nqn]}
-        mock_port.setup.assert_called_once_with(self.target._nvmet_root,
-                                                new_port)
+                                         port_id)
+        new_port1 = {'addr': {'adrfam': 'ipv4',
+                              'traddr': mock.sentinel.addr,
+                              'treq': 'not specified',
+                              'trsvcid': mock.sentinel.port,
+                              'trtype': mock.sentinel.transport},
+                     'portid': port_id,
+                     'referrals': [],
+                     'subsystems': [nqn]}
+        new_port2 = new_port1.copy()
+        new_port2['portid'] = port_id + 1
+        new_port2['addr'] = new_port1['addr'].copy()
+        new_port2['addr']['traddr'] = mock.sentinel.addr2
+
+        self.assertEqual(2, mock_port.call_count)
+        self.assertEqual(2, mock_port.setup.call_count)
+        mock_port.assert_has_calls([
+            mock.call(port_id),
+            mock.call.setup(self.target._nvmet_root, new_port1),
+            mock.call(port_id + 1),
+            mock.call.setup(self.target._nvmet_root, new_port2)
+        ])
         mock_port.return_value.assert_not_called()
 
     @mock.patch.object(nvmet.NVMET, '_locked_unmap_volume')
diff --git a/cinder/tests/unit/targets/test_spdknvmf.py b/cinder/tests/unit/targets/test_spdknvmf.py
index e99923991..d64d52759 100644
--- a/cinder/tests/unit/targets/test_spdknvmf.py
+++ b/cinder/tests/unit/targets/test_spdknvmf.py
@@ -349,6 +349,7 @@ class SpdkNvmfDriverTestCase(test.TestCase):
         super(SpdkNvmfDriverTestCase, self).setUp()
         self.configuration = mock.Mock(conf.Configuration)
         self.configuration.target_ip_address = '192.168.0.1'
+        self.configuration.target_secondary_ip_addresses = []
         self.configuration.target_port = '4420'
         self.configuration.target_prefix = ""
         self.configuration.nvmet_port_id = "1"
diff --git a/cinder/tests/unit/volume/drivers/test_lvm_driver.py b/cinder/tests/unit/volume/drivers/test_lvm_driver.py
index 74004e677..f899a4829 100644
--- a/cinder/tests/unit/volume/drivers/test_lvm_driver.py
+++ b/cinder/tests/unit/volume/drivers/test_lvm_driver.py
@@ -65,6 +65,23 @@ class LVMVolumeDriverTestCase(test_driver.BaseDriverTestCase):
                           lvm.LVMVolumeDriver,
                           configuration=self.configuration)
 
+    def test___init___secondary_ips_not_supported(self):
+        """Fail to use secondary ips if target driver doesn't support it."""
+        original_import = importutils.import_object
+
+        def wrap_target_as_no_secondary_ips_support(*args, **kwargs):
+            res = original_import(*args, **kwargs)
+            self.mock_object(res, 'SECONDARY_IP_SUPPORT', False)
+            return res
+
+        self.patch('oslo_utils.importutils.import_object',
+                   side_effect=wrap_target_as_no_secondary_ips_support)
+
+        self.configuration.target_secondary_ip_addresses = True
+        self.assertRaises(exception.InvalidConfigurationValue,
+                          lvm.LVMVolumeDriver,
+                          configuration=self.configuration)
+
     def test___init___share_target_supported(self):
         """OK to use shared targets if target driver supports it."""
         original_import = importutils.import_object
diff --git a/cinder/tests/unit/volume/drivers/test_spdk.py b/cinder/tests/unit/volume/drivers/test_spdk.py
index e0bf823a7..cf725ffba 100644
--- a/cinder/tests/unit/volume/drivers/test_spdk.py
+++ b/cinder/tests/unit/volume/drivers/test_spdk.py
@@ -502,6 +502,7 @@ class SpdkDriverTestCase(test.TestCase):
         self.configuration = mock.Mock(conf.Configuration)
         self.configuration.target_helper = ""
         self.configuration.target_ip_address = "192.168.0.1"
+        self.configuration.target_secondary_ip_addresses = []
         self.configuration.target_port = 4420
         self.configuration.target_prefix = "nqn.2014-08.io.spdk"
         self.configuration.nvmeof_conn_info_version = 1
@@ -796,7 +797,7 @@ class SpdkDriverTestCase(test.TestCase):
                             self.configuration.nvmet_subsystem_name,
                             self.driver.target_driver._get_first_free_node()
                         ),
-                        self.configuration.target_ip_address,
+                        [self.configuration.target_ip_address],
                         self.configuration.target_port, "rdma",
                         self.configuration.nvmet_ns_id
                     ),
diff --git a/cinder/tests/unit/windows/test_iscsi.py b/cinder/tests/unit/windows/test_iscsi.py
index 282557fca..166def263 100644
--- a/cinder/tests/unit/windows/test_iscsi.py
+++ b/cinder/tests/unit/windows/test_iscsi.py
@@ -82,7 +82,7 @@ class TestWindowsISCSIDriver(test.TestCase):
         self._driver.configuration = mock.Mock()
         self._driver.configuration.target_port = iscsi_port
         self._driver.configuration.target_ip_address = requested_ips[0]
-        self._driver.configuration.iscsi_secondary_ip_addresses = (
+        self._driver.configuration.target_secondary_ip_addresses = (
             requested_ips[1:])
 
         self._driver._tgt_utils.get_portal_locations.return_value = (
diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py
index 2f70bf06e..f623a39a0 100644
--- a/cinder/volume/driver.py
+++ b/cinder/volume/driver.py
@@ -57,7 +57,8 @@ volume_opts = [
                default='$my_ip',
                help='The IP address that the iSCSI/NVMEoF daemon is '
                     'listening on'),
-    cfg.ListOpt('iscsi_secondary_ip_addresses',
+    cfg.ListOpt('target_secondary_ip_addresses',
+                deprecated_name='iscsi_secondary_ip_addresses',
                 default=[],
                 help='The list of secondary IP addresses of the '
                      'iSCSI/NVMEoF daemon'),
@@ -276,7 +277,9 @@ nvmeof_opts = [
 nvmet_opts = [
     cfg.PortOpt('nvmet_port_id',
                 default=1,
-                help='The port that the NVMe target is listening on.'),
+                help='The id of the NVMe target port definition when not '
+                     'sharing targets.  The starting port id value when '
+                     'sharing, incremented for each secondary ip address.'),
     cfg.IntOpt('nvmet_ns_id',
                default=10,
                help='Namespace id for the subsystem for the LVM volume when '
diff --git a/cinder/volume/drivers/lvm.py b/cinder/volume/drivers/lvm.py
index 440b00e2c..ffa55a1d9 100644
--- a/cinder/volume/drivers/lvm.py
+++ b/cinder/volume/drivers/lvm.py
@@ -118,6 +118,12 @@ class LVMVolumeDriver(driver.VolumeDriver):
                 and not self.target_driver.SHARED_TARGET_SUPPORT):
             raise exception.InvalidConfigurationValue(
                 f"{target_driver} doesn't support shared targets")
+
+        if (self.configuration.target_secondary_ip_addresses
+                and not self.target_driver.SECONDARY_IP_SUPPORT):
+            raise exception.InvalidConfigurationValue(
+                f"{target_driver} doesn't support secondary addresses")
+
         self._sparse_copy_volume = False
 
     @classmethod
@@ -129,7 +135,7 @@ class LVMVolumeDriver(driver.VolumeDriver):
             'target_ip_address', 'target_helper', 'target_protocol',
             'volume_clear', 'volume_clear_size', 'reserved_percentage',
             'max_over_subscription_ratio', 'volume_dd_blocksize',
-            'target_prefix', 'volumes_dir', 'iscsi_secondary_ip_addresses',
+            'target_prefix', 'volumes_dir', 'target_secondary_ip_addresses',
             'target_port',
             'iscsi_write_cache', 'iscsi_target_flags',  # TGT
             'iet_conf', 'iscsi_iotype',  # IET
diff --git a/cinder/volume/drivers/synology/synology_common.py b/cinder/volume/drivers/synology/synology_common.py
index dc2d1b87b..a571be7dc 100644
--- a/cinder/volume/drivers/synology/synology_common.py
+++ b/cinder/volume/drivers/synology/synology_common.py
@@ -993,7 +993,7 @@ class SynoCommon(object):
     def get_provider_location(self, iqn, trg_id):
         portals = ['%(ip)s:%(port)d' % {'ip': self.get_ip(),
                                         'port': self.target_port}]
-        sec_ips = self.config.safe_get('iscsi_secondary_ip_addresses')
+        sec_ips = self.config.safe_get('target_secondary_ip_addresses')
         for ip in sec_ips:
             portals.append('%(ip)s:%(port)d' %
                            {'ip': ip,
@@ -1288,7 +1288,7 @@ class SynoCommon(object):
             'access_mode': 'rw',
             'discard': False
         }
-        ips = self.config.safe_get('iscsi_secondary_ip_addresses')
+        ips = self.config.safe_get('target_secondary_ip_addresses')
         if ips:
             target_portals = [iscsi_properties['target_portal']]
             for ip in ips:
diff --git a/cinder/volume/drivers/synology/synology_iscsi.py b/cinder/volume/drivers/synology/synology_iscsi.py
index 6d9ee86c5..8b2044e6b 100644
--- a/cinder/volume/drivers/synology/synology_iscsi.py
+++ b/cinder/volume/drivers/synology/synology_iscsi.py
@@ -49,7 +49,7 @@ class SynoISCSIDriver(driver.ISCSIDriver):
         additional_opts = cls._get_oslo_driver_opts(
             'target_ip_address', 'target_protocol', 'target_port',
             'driver_use_ssl', 'use_chap_auth', 'chap_username',
-            'chap_password', 'iscsi_secondary_ip_addresses', 'target_prefix',
+            'chap_password', 'target_secondary_ip_addresses', 'target_prefix',
             'reserved_percentage', 'max_over_subscription_ratio')
         return common.cinder_opts + additional_opts
 
diff --git a/cinder/volume/drivers/windows/iscsi.py b/cinder/volume/drivers/windows/iscsi.py
index 616356127..9ca2b3608 100644
--- a/cinder/volume/drivers/windows/iscsi.py
+++ b/cinder/volume/drivers/windows/iscsi.py
@@ -93,7 +93,7 @@ class WindowsISCSIDriver(driver.ISCSIDriver):
 
         iscsi_port = self.configuration.target_port
         iscsi_ips = ([self.configuration.target_ip_address] +
-                     self.configuration.iscsi_secondary_ip_addresses)
+                     self.configuration.target_secondary_ip_addresses)
         requested_portals = {':'.join([iscsi_ip, str(iscsi_port)])
                              for iscsi_ip in iscsi_ips}
 
diff --git a/cinder/volume/targets/driver.py b/cinder/volume/targets/driver.py
index b4c0d1e28..9e186629b 100644
--- a/cinder/volume/targets/driver.py
+++ b/cinder/volume/targets/driver.py
@@ -33,6 +33,7 @@ class Target(object, metaclass=abc.ABCMeta):
     """
     storage_protocol = None
     SHARED_TARGET_SUPPORT = False
+    SECONDARY_IP_SUPPORT = True
 
     def __init__(self, *args, **kwargs):
         # TODO(stephenfin): Drop this in favour of using 'db' directly
diff --git a/cinder/volume/targets/iscsi.py b/cinder/volume/targets/iscsi.py
index 46bea8ef3..1f4db6c7c 100644
--- a/cinder/volume/targets/iscsi.py
+++ b/cinder/volume/targets/iscsi.py
@@ -167,8 +167,8 @@ class ISCSITarget(driver.Target):
 
     def _get_portals_config(self):
         # Prepare portals configuration
-        portals_ips = ([self.configuration.target_ip_address]
-                       + self.configuration.iscsi_secondary_ip_addresses or [])
+        portals_ips = ([self.configuration.target_ip_address] +
+                       self.configuration.target_secondary_ip_addresses or [])
 
         return {'portals_ips': portals_ips,
                 'portals_port': self.configuration.target_port}
@@ -201,7 +201,7 @@ class ISCSITarget(driver.Target):
         data = {}
         data['location'] = self._iscsi_location(
             self.configuration.target_ip_address, tid, iscsi_name, lun,
-            self.configuration.iscsi_secondary_ip_addresses)
+            self.configuration.target_secondary_ip_addresses)
         LOG.debug('Set provider_location to: %s', data['location'])
         data['auth'] = self._iscsi_authentication(
             'CHAP', *chap_auth)
diff --git a/cinder/volume/targets/nvmeof.py b/cinder/volume/targets/nvmeof.py
index a858c8654..238edd00c 100644
--- a/cinder/volume/targets/nvmeof.py
+++ b/cinder/volume/targets/nvmeof.py
@@ -40,7 +40,8 @@ class NVMeOF(driver.Target):
         """Reads NVMeOF configurations."""
 
         super(NVMeOF, self).__init__(*args, **kwargs)
-        self.target_ip = self.configuration.target_ip_address
+        self.target_ips = ([self.configuration.target_ip_address] +
+                           self.configuration.target_secondary_ip_addresses)
         self.target_port = self.configuration.target_port
         self.nvmet_port_id = self.configuration.nvmet_port_id
         self.nvmet_ns_id = self.configuration.nvmet_ns_id
@@ -57,6 +58,13 @@ class NVMeOF(driver.Target):
                 protocol=target_protocol
             )
 
+        # Secondary ip addresses only work with new connection info
+        if (self.configuration.target_secondary_ip_addresses
+                and self.configuration.nvmeof_conn_info_version == 1):
+            raise exception.InvalidConfigurationValue(
+                'Secondary addresses need to use NVMe-oF connection properties'
+                ' format version 2 or greater (nvmeof_conn_info_version).')
+
     def initialize_connection(self, volume, connector):
         """Returns the connection info.
 
@@ -95,20 +103,22 @@ class NVMeOF(driver.Target):
         method.
 
         :return: dictionary with the connection properties using one of the 2
-                 existing formats depending on the nvmeof_new_conn_info
+                 existing formats depending on the nvmeof_conn_info_version
                  configuration option.
         """
         location = volume['provider_location']
         target_connection, nvme_transport_type, nqn, nvmet_ns_id = (
             location.split(' '))
-        target_portal, target_port = target_connection.split(':')
+        target_portals, target_port = target_connection.split(':')
+        target_portals = target_portals.split(',')
 
         uuid = self._get_nvme_uuid(volume)
-        return self._get_connection_properties(nqn, target_portal, target_port,
+        return self._get_connection_properties(nqn,
+                                               target_portals, target_port,
                                                nvme_transport_type,
                                                nvmet_ns_id, uuid)
 
-    def _get_connection_properties(self, nqn, portal, port, transport, ns_id,
+    def _get_connection_properties(self, nqn, portals, port, transport, ns_id,
                                    uuid):
         """Get connection properties dictionary.
 
@@ -150,13 +160,13 @@ class NVMeOF(driver.Target):
             return {
                 'target_nqn': nqn,
                 'vol_uuid': uuid,
-                'portals': [(portal, port, transport)],
+                'portals': [(portal, port, transport) for portal in portals],
                 'ns_id': ns_id,
             }
 
         # NVMe-oF Connection Information Version 1
         result = {
-            'target_portal': portal,
+            'target_portal': portals[0],
             'target_port': port,
             'nqn': nqn,
             'transport_type': transport,
@@ -173,12 +183,12 @@ class NVMeOF(driver.Target):
         """
         return None
 
-    def get_nvmeof_location(self, nqn, target_ip, target_port,
+    def get_nvmeof_location(self, nqn, target_ips, target_port,
                             nvme_transport_type, nvmet_ns_id):
         """Serializes driver data into single line string."""
 
         return "%(ip)s:%(port)s %(transport)s %(nqn)s %(ns_id)s" % (
-            {'ip': target_ip,
+            {'ip': ','.join(target_ips),
              'port': target_port,
              'transport': nvme_transport_type,
              'nqn': nqn,
@@ -198,7 +208,7 @@ class NVMeOF(driver.Target):
         return self.create_nvmeof_target(
             volume['id'],
             self.configuration.target_prefix,
-            self.target_ip,
+            self.target_ips,
             self.target_port,
             self.nvme_transport_type,
             self.nvmet_port_id,
@@ -222,7 +232,7 @@ class NVMeOF(driver.Target):
     def create_nvmeof_target(self,
                              volume_id,
                              subsystem_name,
-                             target_ip,
+                             target_ips,
                              target_port,
                              transport_type,
                              nvmet_port_id,
diff --git a/cinder/volume/targets/nvmet.py b/cinder/volume/targets/nvmet.py
index 8de15336f..8ffcf99cd 100644
--- a/cinder/volume/targets/nvmet.py
+++ b/cinder/volume/targets/nvmet.py
@@ -61,7 +61,7 @@ class NVMET(nvmeof.NVMeOF):
         return {
             'driver_volume_type': self.protocol,
             'data': self._get_connection_properties(nqn,
-                                                    self.target_ip,
+                                                    self.target_ips,
                                                     self.target_port,
                                                     self.nvme_transport_type,
                                                     ns_id, uuid),
@@ -75,7 +75,7 @@ class NVMET(nvmeof.NVMeOF):
         else:
             nqn, ns_id = self._map_volume(volume, volume_path)
             location = self.get_nvmeof_location(nqn,
-                                                self.target_ip,
+                                                self.target_ips,
                                                 self.target_port,
                                                 self.nvme_transport_type,
                                                 ns_id)
@@ -92,7 +92,7 @@ class NVMET(nvmeof.NVMeOF):
 
             ns_id = self._ensure_subsystem_exists(nqn, volume_path, uuid)
 
-            self._ensure_port_exports(nqn, self.target_ip, self.target_port,
+            self._ensure_port_exports(nqn, self.target_ips, self.target_port,
                                       self.nvme_transport_type,
                                       self.nvmet_port_id)
         except Exception:
@@ -195,36 +195,39 @@ class NVMET(nvmeof.NVMeOF):
     def _get_nvme_uuid(self, volume):
         return volume.name_id
 
-    def _ensure_port_exports(self, nqn, addr, port, transport_type, port_id):
-        # Assume if port exists, it has the right configuration
-        try:
-            port = nvmet.Port(port_id)
-            LOG.debug('Skip creating port %s as it already exists.', port_id)
-        except nvmet.NotFound:
-            LOG.debug('Creating port %s.', port_id)
-
-            # Port section
-            port_section = {
-                "addr": {
-                    "adrfam": "ipv4",
-                    "traddr": addr,
-                    "treq": "not specified",
-                    "trsvcid": port,
-                    "trtype": transport_type,
-                },
-                "portid": port_id,
-                "referrals": [],
-                "subsystems": [nqn]
-            }
-            nvmet.Port.setup(self._nvmet_root, port_section)  # privsep
-            LOG.debug('Added port: %s', port_id)
+    def _ensure_port_exports(self, nqn, addrs, port, transport_type, port_id):
+        for addr in addrs:
+            # Assume if port exists, it has the right configuration
+            try:
+                nvme_port = nvmet.Port(port_id)
+                LOG.debug('Skip creating port %s as it already exists.',
+                          port_id)
+            except nvmet.NotFound:
+                LOG.debug('Creating port %s.', port_id)
+
+                # Port section
+                port_section = {
+                    "addr": {
+                        "adrfam": "ipv4",
+                        "traddr": addr,
+                        "treq": "not specified",
+                        "trsvcid": port,
+                        "trtype": transport_type,
+                    },
+                    "portid": port_id,
+                    "referrals": [],
+                    "subsystems": [nqn]
+                }
+                nvmet.Port.setup(self._nvmet_root, port_section)  # privsep
+                LOG.debug('Added port: %s', port_id)
 
-        else:
-            if nqn in port.subsystems:
-                LOG.debug('%s already exported on port %s', nqn, port_id)
             else:
-                port.add_subsystem(nqn)  # privsep
-                LOG.debug('Exported %s on port %s', nqn, port_id)
+                if nqn in nvme_port.subsystems:
+                    LOG.debug('%s already exported on port %s', nqn, port_id)
+                else:
+                    nvme_port.add_subsystem(nqn)  # privsep
+                    LOG.debug('Exported %s on port %s', nqn, port_id)
+            port_id += 1
 
     # #######  Connection termination methods ########
 
diff --git a/cinder/volume/targets/spdknvmf.py b/cinder/volume/targets/spdknvmf.py
index a90b0b77d..e7b39fa89 100644
--- a/cinder/volume/targets/spdknvmf.py
+++ b/cinder/volume/targets/spdknvmf.py
@@ -51,6 +51,7 @@ LOG = logging.getLogger(__name__)
 
 
 class SpdkNvmf(nvmeof.NVMeOF):
+    SECONDARY_IP_SUPPORT = False
 
     def __init__(self, *args, **kwargs):
         super(SpdkNvmf, self).__init__(*args, **kwargs)
@@ -131,7 +132,7 @@ class SpdkNvmf(nvmeof.NVMeOF):
     def create_nvmeof_target(self,
                              volume_id,
                              subsystem_name,
-                             target_ip,
+                             target_ips,
                              target_port,
                              transport_type,
                              nvmet_port_id,
@@ -158,7 +159,7 @@ class SpdkNvmf(nvmeof.NVMeOF):
 
             listen_address = {
                 'trtype': transport_type,
-                'traddr': target_ip,
+                'traddr': target_ips[0],
                 'trsvcid': str(target_port),
             }
             params = {
@@ -179,7 +180,7 @@ class SpdkNvmf(nvmeof.NVMeOF):
 
         location = self.get_nvmeof_location(
             nqn,
-            target_ip,
+            target_ips,
             target_port,
             transport_type,
             ns_id)
diff --git a/releasenotes/notes/nvmet-multipath-d35f55286f263e72.yaml b/releasenotes/notes/nvmet-multipath-d35f55286f263e72.yaml
new file mode 100644
index 000000000..7e4f0db5a
--- /dev/null
+++ b/releasenotes/notes/nvmet-multipath-d35f55286f263e72.yaml
@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    nvmet target driver: Added support to serve volumes on multiple addresses
+    using the ``target_secondary_ip_addresses`` configuration option. This
+    allows os-brick to iterate through them in search of one connection that
+    works, and once os-brick supports NVMe-oF multipathing it will be
+    automatically supported.
+
+    This requires that ``nvmeof_conn_info_version`` configuration option is set
+    to ``2`` as well.
+deprecations:
+  - |
+    Configuration option ``iscsi_secondary_ip_addresses`` is deprecated in
+    favor of ``target_secondary_ip_addresses`` to follow the same naming
+    convention of ``target_ip_address``.