Add logging to find test cases leaking libvirt threads

We see functional test failures due to leaked libvirt event handling
thread weaking up after its original test finished and importing
libvirt. If it happens when the libvirt package import is poisoned then
the currently executing test will fail. This patch logs the name of the
test case that leaked the libvirt event handling thread.

We will revert his before RC1.

Change-Id: I3146e9afb411056d004fc118ccfa31126a3c6b15
Related-Bug: #1946339

2 files changed, 44 insertions, 0 deletions

diff --git a/nova/tests/fixtures/nova.py b/nova/tests/fixtures/nova.py
index 5fd893e7dc..9a652c02cb 100644
--- a/nova/tests/fixtures/nova.py
+++ b/nova/tests/fixtures/nova.py
@@ -1822,6 +1822,24 @@ class ImportModulePoisonFixture(fixtures.Fixture):
 
         def find_spec(self, fullname, path, target=None):
             if fullname in self.modules:
+                current = eventlet.getcurrent()
+                # NOTE(gibi) not all eventlet spawn is under our control, so
+                # there can be senders without test_case_id set, find the first
+                # ancestor that was spawned from nova.utils.spawn[_n] and
+                # therefore has the id set.
+                while (
+                    current is not None and
+                    not getattr(current, 'test_case_id', None)
+                ):
+                    current = current.parent
+
+                if current is not None:
+                    self.test.tc_id = current.test_case_id
+                    LOG.warning(
+                        "!!!---!!! TestCase ID %s hit the import poison while "
+                        "importing %s. If you see this in a failed functional "
+                        "test then please let #openstack-nova on IRC know "
+                        "about it. !!!---!!!", current.test_case_id, fullname)
                 self.test.fail_message = (
                     f"This test imports the '{fullname}' module, which it "
                     f'should not in the test environment. Please add '
@@ -1832,6 +1850,7 @@ class ImportModulePoisonFixture(fixtures.Fixture):
     def __init__(self, module_names):
         self.module_names = module_names
         self.fail_message = ''
+        self.tc_id = None
         if isinstance(module_names, str):
             self.module_names = {module_names}
         self.meta_path_finder = self.ForbiddenModules(self, self.module_names)
@@ -1849,6 +1868,13 @@ class ImportModulePoisonFixture(fixtures.Fixture):
         # there (which is also what self.assert* and self.fail() do underneath)
         # will not work to cause a failure in the test.
         if self.fail_message:
+            if self.tc_id is not None:
+                LOG.warning(
+                    "!!!---!!! TestCase ID %s hit the import poison. If you "
+                    "see this in a failed functional test then please let "
+                    "#openstack-nova on IRC know about it. !!!---!!!",
+                    self.tc_id
+                )
             raise ImportError(self.fail_message)
 
 
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 542383cbad..5b0fa799c3 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -10061,6 +10061,24 @@ class LibvirtDriver(driver.ComputeDriver):
         :param instance: instance object that is in migration
 
         """
+        current = eventlet.getcurrent()
+        # NOTE(gibi) not all eventlet spawn is under our control, so
+        # there can be senders without test_case_id set, find the first
+        # ancestor that was spawned from nova.utils.spawn[_n] and
+        # therefore has the id set.
+        while (
+                current is not None and
+                not getattr(current, 'test_case_id', None)
+        ):
+            current = current.parent
+
+        if current is not None:
+            LOG.warning(
+                "!!!---!!! live_migration_abort thread was spawned by "
+                "TestCase ID: %s. If you see this in a failed functional test "
+                "then please let #openstack-nova on IRC know about it. "
+                "!!!---!!!", current.test_case_id
+            )
 
         guest = self._host.get_guest(instance)
         dom = guest._domain