Fix prepare_for_replace/restore_prev_rsrc handing for server

Now, we set 'fixed_ips' to [] for server ports when prepare
for server replacement, but the ports are still in-use if only
set 'fixed_ips' to []. So this patch will to detach the ports from
nova server to make sure same ports can be attached to new one in
prepare_for_replace(). Also, when restoring server, we need to detach
ports from existing server, and then to attach them to previous
server.

We check the interface attach/detach complete by list the
server.interfaces, this change will use 'retry' wrapper to re-poll
the server interfaces ten times, then will raise
exception if the attach/detach still not complete.

Closes-Bug: #1533076
(cherry picked from commit 163d46bdc8bbfa2e7da2989f5a5d608826de2dcc)

Change-Id: I7b322f9cf16c100dcd0365bc3091c289f00f0548

3 files changed, 91 insertions, 29 deletions

diff --git a/heat/engine/clients/client_plugin.py b/heat/engine/clients/client_plugin.py
index 4b0306ceb..e8c15a707 100644
--- a/heat/engine/clients/client_plugin.py
+++ b/heat/engine/clients/client_plugin.py
@@ -228,3 +228,7 @@ class ClientPlugin(object):
             return True
         except exceptions.EndpointNotFound:
             return False
+
+
+def retry_if_result_is_false(result):
+    return result is False
diff --git a/heat/engine/clients/os/nova.py b/heat/engine/clients/os/nova.py
index a83fa4347..a778c152b 100644
--- a/heat/engine/clients/os/nova.py
+++ b/heat/engine/clients/os/nova.py
@@ -25,6 +25,7 @@ from novaclient import exceptions
 from oslo_config import cfg
 from oslo_serialization import jsonutils
 from oslo_utils import uuidutils
+from retrying import retry
 import six
 from six.moves.urllib import parse as urlparse
 
@@ -656,6 +657,30 @@ echo -e '%s\tALL=(ALL)\tNOPASSWD: ALL' >> /etc/sudoers
         else:
             return False
 
+    @retry(stop_max_attempt_number=10,
+           wait_fixed=500,
+           retry_on_result=client_plugin.retry_if_result_is_false)
+    def check_interface_detach(self, server_id, port_id):
+        server = self.fetch_server(server_id)
+        if server:
+            interfaces = server.interface_list()
+            for iface in interfaces:
+                if iface.port_id == port_id:
+                    return False
+        return True
+
+    @retry(stop_max_attempt_number=10,
+           wait_fixed=500,
+           retry_on_result=client_plugin.retry_if_result_is_false)
+    def check_interface_attach(self, server_id, port_id):
+        server = self.fetch_server(server_id)
+        if server:
+            interfaces = server.interface_list()
+            for iface in interfaces:
+                if iface.port_id == port_id:
+                    return True
+        return False
+
     def has_extension(self, alias):
         """Check if extension is present."""
         extensions = self.client().list_extensions.show_all()
diff --git a/heat/engine/resources/openstack/nova/server_network_mixin.py b/heat/engine/resources/openstack/nova/server_network_mixin.py
index 49af5abc2..fe1693d62 100644
--- a/heat/engine/resources/openstack/nova/server_network_mixin.py
+++ b/heat/engine/resources/openstack/nova/server_network_mixin.py
@@ -16,6 +16,7 @@ import itertools
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
 from oslo_utils import netutils
+import retrying
 
 from heat.common import exception
 from heat.common.i18n import _
@@ -190,7 +191,6 @@ class ServerNetworkMixin(object):
     def _build_nics(self, networks, security_groups=None):
         if not networks:
             return None
-
         nics = []
 
         for idx, net in enumerate(networks):
@@ -355,39 +355,72 @@ class ServerNetworkMixin(object):
             [('external_ports', port)
              for port in self._data_get_ports('external_ports')]))
         for port_type, port in port_data:
-            # store port fixed_ips for restoring after failed update
-            port_details = self.client('neutron').show_port(port['id'])['port']
-            fixed_ips = port_details.get('fixed_ips', [])
-            data[port_type].append({'id': port['id'], 'fixed_ips': fixed_ips})
-
-        if data.get('internal_ports'):
-            self.data_set('internal_ports',
-                          jsonutils.dumps(data['internal_ports']))
-        if data.get('external_ports'):
-            self.data_set('external_ports',
-                          jsonutils.dumps(data['external_ports']))
-        # reset fixed_ips for these ports by setting for each of them
-        # fixed_ips to []
+            data[port_type].append({'id': port['id']})
+
+        # detach the ports from the server
+        server_id = self.resource_id
         for port_type, port in port_data:
-            self.client('neutron').update_port(
-                port['id'], {'port': {'fixed_ips': []}})
+            self.client_plugin().interface_detach(server_id, port['id'])
+            try:
+                if self.client_plugin().check_interface_detach(
+                        server_id, port['id']):
+                    LOG.info(_LI('Detach interface %(port)s successful '
+                                 'from server %(server)s when prepare '
+                                 'for replace.')
+                             % {'port': port['id'],
+                                'server': server_id})
+            except retrying.RetryError:
+                raise exception.InterfaceDetachFailed(
+                    port=port['id'], server=server_id)
 
     def restore_ports_after_rollback(self):
         if not self.is_using_neutron():
             return
 
-        old_server = self.stack._backup_stack().resources.get(self.name)
+        backup_res = self.stack._backup_stack().resources.get(self.name)
+        prev_server = backup_res
+        existing_server = self
+
+        port_data = itertools.chain(
+            existing_server._data_get_ports(),
+            existing_server._data_get_ports('external_ports')
+        )
 
-        port_data = itertools.chain(self._data_get_ports(),
-                                    self._data_get_ports('external_ports'))
+        existing_server_id = existing_server.resource_id
         for port in port_data:
-            self.client('neutron').update_port(port['id'],
-                                               {'port': {'fixed_ips': []}})
-
-        old_port_data = itertools.chain(
-            old_server._data_get_ports(),
-            old_server._data_get_ports('external_ports'))
-        for port in old_port_data:
-            fixed_ips = port['fixed_ips']
-            self.client('neutron').update_port(
-                port['id'], {'port': {'fixed_ips': fixed_ips}})
+            # detach the ports from current resource
+            self.client_plugin().interface_detach(
+                existing_server_id, port['id'])
+            try:
+                if self.client_plugin().check_interface_detach(
+                        existing_server_id, port['id']):
+                    LOG.info(_LI('Detach interface %(port)s successful from '
+                                 'server %(server)s when restore after '
+                                 'rollback.')
+                             % {'port': port['id'],
+                                'server': existing_server_id})
+            except retrying.RetryError:
+                raise exception.InterfaceDetachFailed(
+                    port=port['id'], server=existing_server_id)
+
+        # attach the ports for old resource
+        prev_port_data = itertools.chain(
+            prev_server._data_get_ports(),
+            prev_server._data_get_ports('external_ports'))
+
+        prev_server_id = prev_server.resource_id
+
+        for port in prev_port_data:
+            self.client_plugin().interface_attach(prev_server_id,
+                                                  port['id'])
+            try:
+                if self.client_plugin().check_interface_attach(
+                        prev_server_id, port['id']):
+                    LOG.info(_LI('Attach interface %(port)s successful to '
+                                 'server %(server)s when restore after '
+                                 'rollback.')
+                             % {'port': port['id'],
+                                'server': prev_server_id})
+            except retrying.RetryError:
+                raise exception.InterfaceAttachFailed(
+                    port=port['id'], server=prev_server_id)