Fix the race condition for creating and deleting instance

Story: 2008353
Task: 41255

Change-Id: I02bb0434862819a8dbf3ecf3802c3420731b9ca7

2 files changed, 21 insertions, 5 deletions

diff --git a/releasenotes/notes/wallaby-fix-race-condition-create-delete.yaml b/releasenotes/notes/wallaby-fix-race-condition-create-delete.yaml
new file mode 100644
index 00000000..9bd41f0e
--- /dev/null
+++ b/releasenotes/notes/wallaby-fix-race-condition-create-delete.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - Fixed a race condition that instance becomes ERROR when Trove is handling
+    creating and deleting at the same time.
diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py
index 786ebbdb..c33b1762 100755
--- a/trove/taskmanager/models.py
+++ b/trove/taskmanager/models.py
@@ -440,6 +440,14 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                 self.reset_task_status()
             TroveInstanceCreate(instance=self,
                                 instance_size=flavor['ram']).notify()
+        except exception.ComputeInstanceNotFound:
+            # Check if the instance has been deleted by another request.
+            instance = DBInstance.find_by(id=self.id)
+            if (instance.deleted or
+                    instance.task_status == InstanceTasks.DELETING):
+                LOG.warning(f"Instance {self.id} has been deleted during "
+                            f"waiting for creation")
+                return
         except (TroveError, PollTimeOut) as ex:
             LOG.error("Failed to create instance %s, error: %s.",
                       self.id, str(ex))
@@ -786,11 +794,15 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
         try:
             server = self.nova_client.servers.get(c_id)
         except Exception as e:
-            raise TroveError(
-                _("Failed to get server %(server)s for instance %(instance)s, "
-                  "error: %(error)s"),
-                server=c_id, instance=self.id, error=str(e)
-            )
+            if getattr(e, 'message', '') == 'Not found':
+                raise exception.ComputeInstanceNotFound(instance_id=self.id,
+                                                        server_id=c_id)
+            else:
+                raise TroveError(
+                    _("Failed to get server %(server)s for instance "
+                      "%(instance)s, error: %(error)s"),
+                    server=c_id, instance=self.id, error=str(e)
+                )
 
         server_status = server.status
         if server_status in [InstanceStatus.ERROR,