diff options
Diffstat (limited to 'nova/tests/fixtures/nova.py')
| -rw-r--r-- | nova/tests/fixtures/nova.py | 375 |
1 files changed, 290 insertions, 85 deletions
diff --git a/nova/tests/fixtures/nova.py b/nova/tests/fixtures/nova.py index ef873f6654..abfc3ecc6c 100644 --- a/nova/tests/fixtures/nova.py +++ b/nova/tests/fixtures/nova.py @@ -20,14 +20,17 @@ import collections import contextlib from contextlib import contextmanager import functools +from importlib.abc import MetaPathFinder import logging as std_logging import os +import sys +import time +from unittest import mock import warnings import eventlet import fixtures import futurist -import mock from openstack import service_description from oslo_concurrency import lockutils from oslo_config import cfg @@ -62,6 +65,7 @@ from nova.scheduler import weights from nova import service from nova.tests.functional.api import client from nova import utils +from nova.virt import node CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -451,6 +455,13 @@ class CellDatabases(fixtures.Fixture): # yield to do the actual work. We can do schedulable things # here and not exclude other threads from making progress. # If an exception is raised, we capture that and save it. + # Note that it is possible that another thread has changed the + # global state (step #2) after we released the writer lock but + # before we acquired the reader lock. If this happens, we will + # detect the global state change and retry step #2 a limited number + # of times. If we happen to race repeatedly with another thread and + # exceed our retry limit, we will give up and raise a RuntimeError, + # which will fail the test. # 4. If we changed state in #2, we need to change it back. So we grab # a writer lock again and do that. # 5. Finally, if an exception was raised in #3 while state was @@ -469,29 +480,47 @@ class CellDatabases(fixtures.Fixture): raised_exc = None - with self._cell_lock.write_lock(): - if cell_mapping is not None: - # This assumes the next local DB access is the same cell that - # was targeted last time. - self._last_ctxt_mgr = desired + def set_last_ctxt_mgr(): + with self._cell_lock.write_lock(): + if cell_mapping is not None: + # This assumes the next local DB access is the same cell + # that was targeted last time. + self._last_ctxt_mgr = desired - with self._cell_lock.read_lock(): - if self._last_ctxt_mgr != desired: - # NOTE(danms): This is unlikely to happen, but it's possible - # another waiting writer changed the state between us letting - # it go and re-acquiring as a reader. If lockutils supported - # upgrading and downgrading locks, this wouldn't be a problem. - # Regardless, assert that it is still as we left it here - # so we don't hit the wrong cell. If this becomes a problem, - # we just need to retry the write section above until we land - # here with the cell we want. - raise RuntimeError('Global DB state changed underneath us') + # Set last context manager to the desired cell's context manager. + set_last_ctxt_mgr() + # Retry setting the last context manager if we detect that a writer + # changed global DB state before we take the read lock. + for retry_time in range(0, 3): try: - with self._real_target_cell(context, cell_mapping) as ccontext: - yield ccontext - except Exception as exc: - raised_exc = exc + with self._cell_lock.read_lock(): + if self._last_ctxt_mgr != desired: + # NOTE(danms): This is unlikely to happen, but it's + # possible another waiting writer changed the state + # between us letting it go and re-acquiring as a + # reader. If lockutils supported upgrading and + # downgrading locks, this wouldn't be a problem. + # Regardless, assert that it is still as we left it + # here so we don't hit the wrong cell. If this becomes + # a problem, we just need to retry the write section + # above until we land here with the cell we want. + raise RuntimeError( + 'Global DB state changed underneath us') + try: + with self._real_target_cell( + context, cell_mapping + ) as ccontext: + yield ccontext + except Exception as exc: + raised_exc = exc + # Leave the retry loop after calling target_cell + break + except RuntimeError: + # Give other threads a chance to make progress, increasing the + # wait time between attempts. + time.sleep(retry_time) + set_last_ctxt_mgr() with self._cell_lock.write_lock(): # Once we have returned from the context, we need @@ -537,11 +566,10 @@ class CellDatabases(fixtures.Fixture): call_monitor_timeout=None): """Mirror rpc.get_client() but with our special sauce.""" serializer = CheatingSerializer(serializer) - return messaging.RPCClient(rpc.TRANSPORT, - target, - version_cap=version_cap, - serializer=serializer, - call_monitor_timeout=call_monitor_timeout) + return messaging.get_rpc_client(rpc.TRANSPORT, target, + version_cap=version_cap, + serializer=serializer, + call_monitor_timeout=call_monitor_timeout) def add_cell_database(self, connection_str, default=False): """Add a cell database to the fixture. @@ -780,7 +808,7 @@ class WarningsFixture(fixtures.Fixture): """Filters out warnings during test runs.""" def setUp(self): - super(WarningsFixture, self).setUp() + super().setUp() self._original_warning_filters = warnings.filters[:] @@ -793,15 +821,19 @@ class WarningsFixture(fixtures.Fixture): # forward on is_admin, the deprecation is definitely really premature. warnings.filterwarnings( 'ignore', - message='Policy enforcement is depending on the value of is_admin.' - ' This key is deprecated. Please update your policy ' - 'file to use the standard policy values.') + message=( + 'Policy enforcement is depending on the value of is_admin. ' + 'This key is deprecated. Please update your policy ' + 'file to use the standard policy values.' + ), + ) # NOTE(mriedem): Ignore scope check UserWarnings from oslo.policy. warnings.filterwarnings( 'ignore', message="Policy .* failed scope check", - category=UserWarning) + category=UserWarning, + ) # NOTE(gibi): The UUIDFields emits a warning if the value is not a # valid UUID. Let's escalate that to an exception in the test to @@ -813,70 +845,36 @@ class WarningsFixture(fixtures.Fixture): # how to handle (or isn't given a fallback callback). warnings.filterwarnings( 'error', - message="Cannot convert <oslo_db.sqlalchemy.enginefacade" - "._Default object at ", - category=UserWarning) - - warnings.filterwarnings( - 'error', message='Evaluating non-mapped column expression', - category=sqla_exc.SAWarning) + message=( + 'Cannot convert <oslo_db.sqlalchemy.enginefacade._Default ' + 'object at ' + ), + category=UserWarning, + ) # Enable deprecation warnings for nova itself to capture upcoming # SQLAlchemy changes warnings.filterwarnings( 'ignore', - category=sqla_exc.SADeprecationWarning) + category=sqla_exc.SADeprecationWarning, + ) warnings.filterwarnings( 'error', module='nova', - category=sqla_exc.SADeprecationWarning) - - # ...but filter everything out until we get around to fixing them - # TODO(stephenfin): Fix all of these - - warnings.filterwarnings( - 'ignore', - module='nova', - message=r'The current statement is being autocommitted .*', - category=sqla_exc.SADeprecationWarning) - - warnings.filterwarnings( - 'ignore', - module='nova', - message=r'The Column.copy\(\) method is deprecated .*', - category=sqla_exc.SADeprecationWarning) - - warnings.filterwarnings( - 'ignore', - module='nova', - message=r'The Connection.connect\(\) method is considered .*', - category=sqla_exc.SADeprecationWarning) - - warnings.filterwarnings( - 'ignore', - module='nova', - message=r'Using strings to indicate column or relationship .*', - category=sqla_exc.SADeprecationWarning) + category=sqla_exc.SADeprecationWarning, + ) - warnings.filterwarnings( - 'ignore', - module='nova', - message=r'Using strings to indicate relationship names .*', - category=sqla_exc.SADeprecationWarning) + # Enable general SQLAlchemy warnings also to ensure we're not doing + # silly stuff. It's possible that we'll need to filter things out here + # with future SQLAlchemy versions, but that's a good thing warnings.filterwarnings( - 'ignore', + 'error', module='nova', - message=r'Invoking and_\(\) without arguments is deprecated, .*', - category=sqla_exc.SADeprecationWarning) - - # TODO(stephenfin): Remove once we fix this in placement 5.0.2 or 6.0.0 - warnings.filterwarnings( - 'ignore', - message='Implicit coercion of SELECT and textual SELECT .*', - category=sqla_exc.SADeprecationWarning) + category=sqla_exc.SAWarning, + ) self.addCleanup(self._reset_warning_filters) @@ -1006,9 +1004,15 @@ class OSAPIFixture(fixtures.Fixture): self.api = client.TestOpenStackClient( 'fake', base_url, project_id=self.project_id, roles=['reader', 'member']) + self.alternative_api = client.TestOpenStackClient( + 'fake', base_url, project_id=self.project_id, + roles=['reader', 'member']) self.admin_api = client.TestOpenStackClient( 'admin', base_url, project_id=self.project_id, roles=['reader', 'member', 'admin']) + self.alternative_admin_api = client.TestOpenStackClient( + 'admin', base_url, project_id=self.project_id, + roles=['reader', 'member', 'admin']) self.reader_api = client.TestOpenStackClient( 'reader', base_url, project_id=self.project_id, roles=['reader']) @@ -1104,9 +1108,9 @@ class PoisonFunctions(fixtures.Fixture): # Don't poison the function if it's already mocked import nova.virt.libvirt.host if not isinstance(nova.virt.libvirt.host.Host._init_events, mock.Mock): - self.useFixture(fixtures.MockPatch( + self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.host.Host._init_events', - side_effect=evloop)) + evloop)) class IndirectionAPIFixture(fixtures.Fixture): @@ -1314,6 +1318,77 @@ class PrivsepFixture(fixtures.Fixture): nova.privsep.sys_admin_pctxt, 'client_mode', False)) +class CGroupsFixture(fixtures.Fixture): + """Mocks checks made for available subsystems on the host's control group. + + The fixture mocks all calls made on the host to verify the capabilities + provided by its kernel. Through this, one can simulate the underlying + system hosts work on top of and have tests react to expected outcomes from + such. + + Use sample: + >>> cgroups = self.useFixture(CGroupsFixture()) + >>> cgroups = self.useFixture(CGroupsFixture(version=2)) + >>> cgroups = self.useFixture(CGroupsFixture()) + ... cgroups.version = 2 + + :attr version: Arranges mocks to simulate the host interact with nova + following the given version of cgroups. + Available values are: + - 0: All checks related to cgroups will return False. + - 1: Checks related to cgroups v1 will return True. + - 2: Checks related to cgroups v2 will return True. + Defaults to 1. + """ + + def __init__(self, version=1): + self._cpuv1 = None + self._cpuv2 = None + + self._version = version + + @property + def version(self): + return self._version + + @version.setter + def version(self, value): + self._version = value + self._update_mocks() + + def setUp(self): + super().setUp() + self._cpuv1 = self.useFixture(fixtures.MockPatch( + 'nova.virt.libvirt.host.Host._has_cgroupsv1_cpu_controller')).mock + self._cpuv2 = self.useFixture(fixtures.MockPatch( + 'nova.virt.libvirt.host.Host._has_cgroupsv2_cpu_controller')).mock + self._update_mocks() + + def _update_mocks(self): + if not self._cpuv1: + return + + if not self._cpuv2: + return + + if self.version == 0: + self._cpuv1.return_value = False + self._cpuv2.return_value = False + return + + if self.version == 1: + self._cpuv1.return_value = True + self._cpuv2.return_value = False + return + + if self.version == 2: + self._cpuv1.return_value = False + self._cpuv2.return_value = True + return + + raise ValueError(f"Unknown cgroups version: '{self.version}'.") + + class NoopQuotaDriverFixture(fixtures.Fixture): """A fixture to run tests using the NoopQuotaDriver. @@ -1459,7 +1534,7 @@ class AvailabilityZoneFixture(fixtures.Fixture): ``get_availability_zones``. ``get_instance_availability_zone`` will return the availability_zone - requested when creating a server otherwise the instance.availabilty_zone + requested when creating a server otherwise the instance.availability_zone or default_availability_zone is returned. """ @@ -1611,7 +1686,11 @@ class GenericPoisonFixture(fixtures.Fixture): current = __import__(components[0], {}, {}) for component in components[1:]: current = getattr(current, component) - if not isinstance(getattr(current, attribute), mock.Mock): + + # NOTE(stephenfin): There are a couple of mock libraries in use + # (including mocked versions of mock from oslotest) so we can't + # use isinstance checks here + if 'mock' not in str(type(getattr(current, attribute))): self.useFixture(fixtures.MonkeyPatch( meth, poison_configure(meth, why))) except ImportError: @@ -1733,3 +1812,129 @@ class ReaderWriterLock(lockutils.ReaderWriterLock): 'threading.current_thread', eventlet.getcurrent) with mpatch if eventlet_patched else contextlib.ExitStack(): super().__init__(*a, **kw) + + +class SysFsPoisonFixture(fixtures.Fixture): + + def inject_poison(self, module_name, function_name): + import importlib + mod = importlib.import_module(module_name) + orig_f = getattr(mod, function_name) + if ( + isinstance(orig_f, mock.Mock) or + # FIXME(gibi): Is this a bug in unittest.mock? If I remove this + # then LibvirtReportSevTraitsTests fails as builtins.open is mocked + # there at import time via @test.patch_open. That injects a + # MagicMock instance to builtins.open which we check here against + # Mock (or even MagicMock) via isinstance and that check says it is + # not a mock. More interestingly I cannot reproduce the same + # issue with @test.patch_open and isinstance in a simple python + # interpreter. So to make progress I'm checking the class name + # here instead as that works. + orig_f.__class__.__name__ == "MagicMock" + ): + # the target is already mocked, probably via a decorator run at + # import time, so we don't need to inject our poison + return + + full_name = module_name + "." + function_name + + def toxic_wrapper(*args, **kwargs): + path = args[0] + if isinstance(path, bytes): + pattern = b'/sys' + elif isinstance(path, str): + pattern = '/sys' + else: + # we ignore the rest of the potential pathlike types for now + pattern = None + + if pattern and path.startswith(pattern): + raise Exception( + 'This test invokes %s on %s. It is bad, you ' + 'should mock it.' + % (full_name, path) + ) + else: + return orig_f(*args, **kwargs) + + self.useFixture(fixtures.MonkeyPatch(full_name, toxic_wrapper)) + + def setUp(self): + super().setUp() + self.inject_poison("os.path", "isdir") + self.inject_poison("builtins", "open") + self.inject_poison("glob", "iglob") + self.inject_poison("os", "listdir") + self.inject_poison("glob", "glob") + # TODO(gibi): Would be good to poison these too but that makes + # a bunch of test to fail + # self.inject_poison("os.path", "exists") + # self.inject_poison("os", "stat") + + +class ImportModulePoisonFixture(fixtures.Fixture): + """Poison imports of modules unsuitable for the test environment. + + Examples are guestfs and libvirt. Ordinarily, these would not be installed + in the test environment but if they _are_ present, it can result in + actual calls to libvirt, for example, which could cause tests to fail. + + This fixture will inspect module imports and if they are in the disallowed + list, it will fail the test with a helpful message about mocking needed in + the test. + """ + + class ForbiddenModules(MetaPathFinder): + def __init__(self, test, modules): + super().__init__() + self.test = test + self.modules = modules + + def find_spec(self, fullname, path, target=None): + if fullname in self.modules: + self.test.fail_message = ( + f"This test imports the '{fullname}' module, which it " + f'should not in the test environment. Please add ' + f'appropriate mocking to this test.' + ) + raise ImportError(fullname) + + def __init__(self, module_names): + self.module_names = module_names + self.fail_message = '' + if isinstance(module_names, str): + self.module_names = {module_names} + self.meta_path_finder = self.ForbiddenModules(self, self.module_names) + + def setUp(self): + super().setUp() + self.addCleanup(self.cleanup) + sys.meta_path.insert(0, self.meta_path_finder) + + def cleanup(self): + sys.meta_path.remove(self.meta_path_finder) + # We use a flag and check it during the cleanup phase to fail the test + # if needed. This is done because some module imports occur inside of a + # try-except block that ignores all exceptions, so raising an exception + # there (which is also what self.assert* and self.fail() do underneath) + # will not work to cause a failure in the test. + if self.fail_message: + raise ImportError(self.fail_message) + + +class ComputeNodeIdFixture(fixtures.Fixture): + def setUp(self): + super().setUp() + + node.LOCAL_NODE_UUID = None + self.useFixture(fixtures.MockPatch( + 'nova.virt.node.read_local_node_uuid', + lambda: None)) + self.useFixture(fixtures.MockPatch( + 'nova.virt.node.write_local_node_uuid', + lambda uuid: None)) + self.useFixture(fixtures.MockPatch( + 'nova.compute.manager.ComputeManager.' + '_ensure_existing_node_identity', + mock.DEFAULT)) |