diff options
author | Ed Cranford <ed.cranford@rackspace.com> | 2013-01-02 16:51:40 -0600 |
---|---|---|
committer | Ed Cranford <ed.cranford@rackspace.com> | 2013-01-17 16:59:16 -0600 |
commit | ce6b98e2d13cad0ebe7a2d99f1046b5303c1ea21 (patch) | |
tree | 22da51d48d66a5f8ecce6864c35f814e96cbd971 | |
parent | bdb54cc49402bd36cec5fdcffda2136b6c708915 (diff) | |
download | trove-ce6b98e2d13cad0ebe7a2d99f1046b5303c1ea21.tar.gz |
Checks guest status during migration.
During a migration, waits for the guest and service to start up
following the status change to VERIFY_RESIZE. Confirms if everything
is all right, or reverts if not.
Implements blueprint migration-check-guest-status
Change-Id: Ia7c7ed1fd0070429fed93323ca559d1c0742bd8f
-rw-r--r-- | etc/reddwarf/reddwarf.conf.test | 2 | ||||
-rw-r--r-- | reddwarf/common/cfg.py | 2 | ||||
-rw-r--r-- | reddwarf/common/wsgi.py | 1 | ||||
-rw-r--r-- | reddwarf/guestagent/api.py | 5 | ||||
-rw-r--r-- | reddwarf/guestagent/dbaas.py | 28 | ||||
-rw-r--r-- | reddwarf/guestagent/manager.py | 4 | ||||
-rw-r--r-- | reddwarf/instance/tasks.py | 3 | ||||
-rw-r--r-- | reddwarf/openstack/common/processutils.py | 4 | ||||
-rw-r--r-- | reddwarf/taskmanager/models.py | 270 | ||||
-rw-r--r-- | reddwarf/tests/api/instances.py | 26 | ||||
-rw-r--r-- | reddwarf/tests/api/instances_actions.py | 41 | ||||
-rw-r--r-- | reddwarf/tests/api/instances_mysql_down.py | 35 | ||||
-rw-r--r-- | reddwarf/tests/api/instances_resize.py | 198 | ||||
-rw-r--r-- | reddwarf/tests/fakes/common.py | 8 | ||||
-rw-r--r-- | reddwarf/tests/fakes/guestagent.py | 38 | ||||
-rw-r--r-- | reddwarf/tests/fakes/nova.py | 70 | ||||
-rw-r--r-- | run_tests.py | 1 | ||||
-rw-r--r-- | tools/test-requires | 1 |
18 files changed, 543 insertions, 194 deletions
diff --git a/etc/reddwarf/reddwarf.conf.test b/etc/reddwarf/reddwarf.conf.test index 706c0831..c922fda1 100644 --- a/etc/reddwarf/reddwarf.conf.test +++ b/etc/reddwarf/reddwarf.conf.test @@ -81,6 +81,8 @@ agent_call_high_timeout = 100 server_delete_time_out=10 use_nova_server_volume = False +dns_time_out = 120 +resize_time_out = 120 # ============ notifer queue kombu connection options ======================== diff --git a/reddwarf/common/cfg.py b/reddwarf/common/cfg.py index 6322fcc4..f2ccc000 100644 --- a/reddwarf/common/cfg.py +++ b/reddwarf/common/cfg.py @@ -88,6 +88,8 @@ common_opts = [ cfg.IntOpt('volume_time_out', default=2), cfg.IntOpt('reboot_time_out', default=60 * 2), cfg.StrOpt('service_options', default=['mysql']), + cfg.IntOpt('dns_time_out', default=60 * 2), + cfg.IntOpt('resize_time_out', default=60 * 10), ] diff --git a/reddwarf/common/wsgi.py b/reddwarf/common/wsgi.py index 6258837f..16b7b8ab 100644 --- a/reddwarf/common/wsgi.py +++ b/reddwarf/common/wsgi.py @@ -17,7 +17,6 @@ """Wsgi helper utilities for reddwarf""" import eventlet.wsgi -import os import paste.urlmap import re import traceback diff --git a/reddwarf/guestagent/api.py b/reddwarf/guestagent/api.py index 38f7eb52..081af5bd 100644 --- a/reddwarf/guestagent/api.py +++ b/reddwarf/guestagent/api.py @@ -188,10 +188,11 @@ class API(proxy.RpcProxy): self._call("start_mysql_with_conf_changes", AGENT_HIGH_TIMEOUT, updated_memory_size=updated_memory_size) - def stop_mysql(self): + def stop_mysql(self, do_not_start_on_reboot=False): """Stop the MySQL server.""" LOG.debug(_("Sending the call to stop MySQL on the Guest.")) - self._call("stop_mysql", AGENT_HIGH_TIMEOUT) + self._call("stop_mysql", AGENT_HIGH_TIMEOUT, + do_not_start_on_reboot=do_not_start_on_reboot) def upgrade(self): """Make an asynchronous call to self upgrade the guest agent""" diff --git a/reddwarf/guestagent/dbaas.py b/reddwarf/guestagent/dbaas.py index e5bae01e..9cc37437 100644 --- a/reddwarf/guestagent/dbaas.py +++ b/reddwarf/guestagent/dbaas.py @@ -587,8 +587,32 @@ class MySqlApp(object): LOG.debug(_("Finished installing mysql server")) #TODO(rnirmal): Add checks to make sure the package got installed - def stop_mysql(self, update_db=False): + def _enable_mysql_on_boot(self): + ''' + # This works in Debian Squeeze, but Ubuntu Precise has other plans. + # Use update-rc.d to enable or disable mysql at boot. + # update-rc.d is idempotent; any substitute method should be, too. + flag = "enable" if enabled else "disable" + LOG.info("Setting mysql to '%s' in rc.d" % flag) + utils.execute_with_timeout("sudo", "update-rc.d", "mysql", flag) + ''' + LOG.info("Enabling mysql on boot.") + conf = "/etc/init/mysql.conf" + command = "sudo sed -i '/^manual$/d' %(conf)s" + command = command % locals() + utils.execute_with_timeout(command, with_shell=True) + + def _disable_mysql_on_boot(self): + LOG.info("Disabling mysql on boot.") + conf = "/etc/init/mysql.conf" + command = '''sudo sh -c "echo manual >> %(conf)s"''' + command = command % locals() + utils.execute_with_timeout(command, with_shell=True) + + def stop_mysql(self, update_db=False, do_not_start_on_reboot=False): LOG.info(_("Stopping mysql...")) + if do_not_start_on_reboot: + self._disable_mysql_on_boot() utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "stop") if not self.status.wait_for_real_status_to_change_to( rd_models.ServiceStatuses.SHUTDOWN, @@ -712,6 +736,8 @@ class MySqlApp(object): # Essentially what happens is thaty mysql start fails, but does not # die. It is then impossible to kill the original, so + self._enable_mysql_on_boot() + try: utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "start") except ProcessExecutionError: diff --git a/reddwarf/guestagent/manager.py b/reddwarf/guestagent/manager.py index e649373e..57c2e369 100644 --- a/reddwarf/guestagent/manager.py +++ b/reddwarf/guestagent/manager.py @@ -80,9 +80,9 @@ class Manager(periodic_task.PeriodicTasks): app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get()) app.start_mysql_with_conf_changes(updated_memory_size) - def stop_mysql(self, context): + def stop_mysql(self, context, do_not_start_on_reboot=False): app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get()) - app.stop_mysql() + app.stop_mysql(do_not_start_on_reboot=do_not_start_on_reboot) def get_filesystem_stats(self, context, fs_path): """ Gets the filesystem stats for the path given """ diff --git a/reddwarf/instance/tasks.py b/reddwarf/instance/tasks.py index 26a25ab6..815e15a8 100644 --- a/reddwarf/instance/tasks.py +++ b/reddwarf/instance/tasks.py @@ -59,6 +59,9 @@ class InstanceTask(object): return None return cls._lookup[code] + def __str__(self): + return "(%d %s %s)" % (self._code, self._action, self._db_text) + class InstanceTasks(object): NONE = InstanceTask(0x01, 'NONE', 'No tasks for the instance.') diff --git a/reddwarf/openstack/common/processutils.py b/reddwarf/openstack/common/processutils.py index 2c3f8f62..f9f4696f 100644 --- a/reddwarf/openstack/common/processutils.py +++ b/reddwarf/openstack/common/processutils.py @@ -86,6 +86,7 @@ def execute(*cmd, **kwargs): attempts = kwargs.pop('attempts', 1) run_as_root = kwargs.pop('run_as_root', False) root_helper = kwargs.pop('root_helper', '') + with_shell = kwargs.pop('with_shell', False) if len(kwargs): raise UnknownArgumentError(_('Got unknown keyword args ' 'to utils.execute: %r') % kwargs) @@ -102,7 +103,8 @@ def execute(*cmd, **kwargs): stdin=_PIPE, stdout=_PIPE, stderr=_PIPE, - close_fds=True) + close_fds=True, + shell=with_shell) result = None if process_input is not None: result = obj.communicate(process_input) diff --git a/reddwarf/taskmanager/models.py b/reddwarf/taskmanager/models.py index 0b905415..5fe260c6 100644 --- a/reddwarf/taskmanager/models.py +++ b/reddwarf/taskmanager/models.py @@ -47,6 +47,10 @@ from reddwarf.openstack.common.gettextutils import _ LOG = logging.getLogger(__name__) CONF = cfg.CONF +VOLUME_TIME_OUT = CONF.volume_time_out # seconds. +DNS_TIME_OUT = CONF.dns_time_out # seconds. +RESIZE_TIME_OUT = CONF.resize_time_out # seconds. + use_nova_server_volume = CONF.use_nova_server_volume @@ -179,7 +183,7 @@ class FreshInstanceTasks(FreshInstance): lambda: volume_client.volumes.get(volume_ref.id), lambda v_ref: v_ref.status in ['available', 'error'], sleep_time=2, - time_out=2 * 60) + time_out=VOLUME_TIME_OUT) v_ref = volume_client.volumes.get(volume_ref.id) if v_ref.status in ['error']: @@ -257,7 +261,7 @@ class FreshInstanceTasks(FreshInstance): LOG.error(msg % (self.id, server.status)) raise ReddwarfError(status=server.status) poll_until(get_server, ip_is_available, - sleep_time=1, time_out=60 * 2) + sleep_time=1, time_out=DNS_TIME_OUT) server = nova_client.servers.get(self.db_info.compute_instance_id) LOG.info("Creating dns entry...") dns_client.create_instance_entry(self.id, @@ -328,7 +332,6 @@ class BuiltInstanceTasks(BuiltInstance): self.update_db(volume_size=volume.size) self.nova_client.volumes.rescan_server_volume(self.server, self.volume_id) - self.guest.resize_fs(self.get_volume_mountpoint()) except PollTimeOut as pto: LOG.error("Timeout trying to rescan or resize the attached volume " "filesystem for volume: %s" % self.volume_id) @@ -342,104 +345,12 @@ class BuiltInstanceTasks(BuiltInstance): def resize_flavor(self, new_flavor_id, old_memory_size, new_memory_size): - self._resize_flavor(new_flavor_id, old_memory_size, - new_memory_size) + action = ResizeAction(self, new_flavor_id, new_memory_size) + action.execute() def migrate(self): - self._resize_flavor() - - def _resize_flavor(self, new_flavor_id=None, old_memory_size=None, - new_memory_size=None): - def resize_status_msg(): - return "instance_id=%s, status=%s, flavor_id=%s, "\ - "dest. flavor id=%s)" % (self.db_info.id, - self.server.status, - str(self.server.flavor['id']), - str(new_flavor_id)) - - try: - LOG.debug("Instance %s calling stop_mysql..." % self.db_info.id) - self.guest.stop_mysql() - try: - LOG.debug("Instance %s calling Compute resize..." - % self.db_info.id) - if new_flavor_id: - LOG.debug("Instance with new flavor id") - self.server.resize(new_flavor_id) - else: - LOG.debug("Migrating instance %s without flavor change ..." - % self.db_info.id) - self.server.migrate() - - LOG.debug("refreshing the compute status of the instance") - # Do initial check and confirm the status is appropriate. - self._refresh_compute_server_info() - if (self.server.status != "RESIZE" and - self.server.status != "VERIFY_RESIZE"): - msg = "Unexpected status after call to resize! : %s" - raise ReddwarfError(msg % resize_status_msg()) - - LOG.debug("the compute status of the instance : (%s)" - % self.server.status) - - # Wait for the flavor to change. - def update_server_info(): - self._refresh_compute_server_info() - LOG.debug("refreshed... compute status (%s)" - % self.server.status) - return self.server.status != 'RESIZE' - - LOG.debug("polling the server until its not RESIZE") - utils.poll_until( - update_server_info, - sleep_time=2, - time_out=60 * 10) - - LOG.debug("compute status should not be RESIZE now") - LOG.debug("instance_id=%s, status=%s, " - "dest. flavor id=%s)" % (self.db_info.id, - self.server.status, - str(new_flavor_id))) - - # Do check to make sure the status and flavor id are correct. - if new_flavor_id: - if str(self.server.flavor['id']) != str(new_flavor_id): - msg = ("Assertion failed! flavor_id=%s and not %s" - % (self.server.flavor['id'], new_flavor_id)) - raise ReddwarfError(msg) - if (self.server.status != "VERIFY_RESIZE"): - msg = ("Assertion failed! status=%s and not %s" - % (self.server.status, 'VERIFY_RESIZE')) - raise ReddwarfError(msg) - - LOG.debug("wait a sec man!!!") - time.sleep(5) - # Confirm the resize with Nova. - LOG.debug("Instance %s calling Compute confirm resize..." - % self.db_info.id) - self.server.confirm_resize() - LOG.debug("Compute confirm resize DONE ...") - if new_flavor_id: - # Record the new flavor_id in our database. - LOG.debug("Updating instance %s to flavor_id %s." - % (self.id, new_flavor_id)) - self.update_db(flavor_id=new_flavor_id) - except Exception as ex: - new_memory_size = old_memory_size - new_flavor_id = None - LOG.exception("Error resizing instance %s." % self.db_info.id) - finally: - # Tell the guest to restart MySQL with the new RAM size. - # This is in the finally because we have to call this, or - # else MySQL could stay turned off on an otherwise usable - # instance. - LOG.debug("Instance %s starting mysql..." % self.db_info.id) - if new_flavor_id: - self.guest.start_mysql_with_conf_changes(new_memory_size) - else: - self.guest.restart() - finally: - self.update_db(task_status=inst_models.InstanceTasks.NONE) + action = MigrateAction(self) + action.execute() def reboot(self): try: @@ -461,9 +372,7 @@ class BuiltInstanceTasks(BuiltInstance): # Set the status to PAUSED. The guest agent will reset the status # when the reboot completes and MySQL is running. - status = InstanceServiceStatus.find_by(instance_id=self.id) - status.set_status(inst_models.ServiceStatuses.PAUSED) - status.save() + self._set_service_status_to_paused() LOG.debug("Successfully rebooted instance %s" % self.id) except Exception, e: LOG.error("Failed to reboot instance %s: %s" % (self.id, str(e))) @@ -486,3 +395,160 @@ class BuiltInstanceTasks(BuiltInstance): """Refreshes the compute server field.""" server = self.nova_client.servers.get(self.server.id) self.server = server + + def _refresh_compute_service_status(self): + """Refreshes the service status info for an instance.""" + service = InstanceServiceStatus.find_by(instance_id=self.id) + self.service_status = service.get_status() + + def _set_service_status_to_paused(self): + status = InstanceServiceStatus.find_by(instance_id=self.id) + status.set_status(inst_models.ServiceStatuses.PAUSED) + status.save() + + +class ResizeActionBase(object): + """Base class for executing a resize action.""" + + def __init__(self, instance): + self.instance = instance + + def _assert_guest_is_ok(self): + # The guest will never set the status to PAUSED. + self.instance._set_service_status_to_paused() + # Now we wait until it sets it to anything at all, + # so we know it's alive. + utils.poll_until( + self._guest_is_awake, + sleep_time=2, + time_out=RESIZE_TIME_OUT) + + def _assert_nova_was_successful(self): + # Make sure Nova thinks things went well. + if self.instance.server.status != "VERIFY_RESIZE": + msg = "Migration failed! status=%s and not %s" \ + % (self.instance.server.status, 'VERIFY_RESIZE') + raise ReddwarfError(msg) + + def _assert_mysql_is_ok(self): + # Tell the guest to turn on MySQL, and ensure the status becomes + # ACTIVE. + self._start_mysql() + # The guest should do this for us... but sometimes it walks funny. + self.instance._refresh_compute_service_status() + if self.instance.service_status != ServiceStatuses.RUNNING: + raise Exception("Migration failed! Service status was %s." + % self.instance.service_status) + + def _assert_processes_are_ok(self): + """Checks the procs; if anything is wrong, reverts the operation.""" + # Tell the guest to turn back on, and make sure it can start. + self._assert_guest_is_ok() + LOG.debug("Nova guest is fine.") + self._assert_mysql_is_ok() + LOG.debug("Mysql is good, too.") + + def _confirm_nova_action(self): + LOG.debug("Instance %s calling Compute confirm resize..." + % self.instance.id) + self.instance.server.confirm_resize() + + def _revert_nova_action(self): + LOG.debug("Instance %s calling Compute revert resize..." + % self.instance.id) + self.instance.server.revert_resize() + + def execute(self): + """Initiates the action.""" + try: + LOG.debug("Instance %s calling stop_mysql..." + % self.instance.id) + self.instance.guest.stop_mysql(do_not_start_on_reboot=True) + self._perform_nova_action() + finally: + self.instance.update_db(task_status=inst_models.InstanceTasks.NONE) + + def _guest_is_awake(self): + self.instance._refresh_compute_service_status() + return self.instance.service_status != ServiceStatuses.PAUSED + + def _perform_nova_action(self): + """Calls Nova to resize or migrate an instance, and confirms.""" + need_to_revert = False + try: + LOG.debug("Initiating nova action") + self._initiate_nova_action() + LOG.debug("Waiting for nova action") + self._wait_for_nova_action() + LOG.debug("Asserting success") + self._assert_nova_was_successful() + LOG.debug("Asserting processes are OK") + need_to_revert = True + LOG.debug("* * * REVERT BARRIER PASSED * * *") + self._assert_processes_are_ok() + LOG.debug("Confirming nova action") + self._confirm_nova_action() + except Exception as ex: + LOG.exception("Exception during nova action.") + if need_to_revert: + LOG.error("Reverting action for instance %s" % + self.instance.id) + self._revert_nova_action() + self.instance.guest.restart() + LOG.error("Error resizing instance %s." % self.instance.id) + raise ex + + LOG.debug("Recording success") + self._record_action_success() + + def _wait_for_nova_action(self): + # Wait for the flavor to change. + def update_server_info(): + self.instance._refresh_compute_server_info() + return self.instance.server.status != 'RESIZE' + utils.poll_until( + update_server_info, + sleep_time=2, + time_out=RESIZE_TIME_OUT) + + +class ResizeAction(ResizeActionBase): + + def __init__(self, instance, new_flavor_id=None, new_memory_size=None): + self.instance = instance + self.new_flavor_id = new_flavor_id + self.new_memory_size = new_memory_size + + def _assert_nova_was_successful(self): + # Do check to make sure the status and flavor id are correct. + if str(self.instance.server.flavor['id']) != str(self.new_flavor_id): + msg = "Assertion failed! flavor_id=%s and not %s" \ + % (self.instance.server.flavor['id'], self.new_flavor_id) + raise ReddwarfError(msg) + super(ResizeAction, self)._assert_nova_was_successful() + + def _initiate_nova_action(self): + self.instance.server.resize(self.new_flavor_id) + + def _record_action_success(self): + LOG.debug("Updating instance %s to flavor_id %s." + % (self.instance.id, self.new_flavor_id)) + self.instance.update_db(flavor_id=self.new_flavor_id) + + def _start_mysql(self): + self.instance.guest.start_mysql_with_conf_changes(self.new_memory_size) + + +class MigrateAction(ResizeActionBase): + + def _initiate_nova_action(self): + LOG.debug("Migrating instance %s without flavor change ..." + % self.instance.id) + self.instance.server.migrate() + + def _record_action_success(self): + LOG.debug("Successfully finished Migration to %s: %s" % + (self.hostname, self.instance.id)) + + def _start_mysql(self): + self.instance.guest.restart() diff --git a/reddwarf/tests/api/instances.py b/reddwarf/tests/api/instances.py index 8c772554..7025d96d 100644 --- a/reddwarf/tests/api/instances.py +++ b/reddwarf/tests/api/instances.py @@ -241,7 +241,7 @@ class CreateInstance(unittest.TestCase): def test_instance_size_too_big(self): vol_ok = CONFIG.get('reddwarf_volume_support', False) if 'reddwarf_max_accepted_volume_size' in CONFIG.values and vol_ok: - too_big = CONFIG.values['reddwarf_max_accepted_volume_size'] + too_big = CONFIG.reddwarf_max_accepted_volume_size assert_raises(exceptions.OverLimit, dbaas.instances.create, "way_too_large", instance_info.dbaas_flavor_href, {'size': too_big + 1}, []) @@ -296,7 +296,7 @@ class CreateInstance(unittest.TestCase): 'name', 'status', 'updated'] if CONFIG.values['reddwarf_volume_support']: expected_attrs.append('volume') - if CONFIG.values['reddwarf_dns_support']: + if CONFIG.reddwarf_dns_support: expected_attrs.append('hostname') with CheckInstance(result._info) as check: @@ -510,7 +510,7 @@ class TestGuestProcess(object): Test that the guest process is started with all the right parameters """ - @test(enabled=CONFIG.values['use_local_ovz']) + @test(enabled=CONFIG.use_local_ovz) @time_out(60 * 10) def check_process_alive_via_local_ovz(self): init_re = ("[\w\W\|\-\s\d,]*nova-guest " @@ -559,8 +559,8 @@ class TestGuestProcess(object): @test(depends_on_classes=[CreateInstance], - groups=[GROUP, GROUP_START, - GROUP_START_SIMPLE, GROUP_TEST, "nova.volumes.instance"], + groups=[GROUP, GROUP_START, GROUP_START_SIMPLE, GROUP_TEST, + "nova.volumes.instance"], enabled=CONFIG.white_box) class TestVolume(unittest.TestCase): """Make sure the volume is attached to instance correctly.""" @@ -638,6 +638,14 @@ class TestInstanceListing(object): check.links(instance_dict['links']) check.used_volume() + @test(enabled=CONFIG.reddwarf_dns_support) + def test_instance_hostname(self): + instance = dbaas.instances.get(instance_info.id) + assert_equal(200, dbaas.last_http_code) + hostname_prefix = ("%s" % (hashlib.sha1(instance.id).hexdigest())) + instance_hostname_prefix = instance.hostname.split('.')[0] + assert_equal(hostname_prefix, instance_hostname_prefix) + @test def test_get_instance_status(self): result = dbaas.instances.get(instance_info.id) @@ -685,7 +693,7 @@ class TestInstanceListing(object): assert_raises(exceptions.NotFound, self.other_client.instances.delete, instance_info.id) - @test(enabled=CONFIG.values['test_mgmt']) + @test(enabled=CONFIG.test_mgmt) def test_mgmt_get_instance_after_started(self): result = dbaas_admin.management.show(instance_info.id) expected_attrs = ['account_id', 'addresses', 'created', 'databases', @@ -756,8 +764,8 @@ class DeleteInstance(object): except exceptions.NotFound: pass except Exception as ex: - fail("A failure occured when trying to GET instance %s for the %d " - "time: %s" % (str(instance_info.id), attempts, str(ex))) + fail("A failure occured when trying to GET instance %s for the %d" + " time: %s" % (str(instance_info.id), attempts, str(ex))) @time_out(30) @test(enabled=CONFIG.values["reddwarf_volume_support"], @@ -779,7 +787,7 @@ class DeleteInstance(object): @test(depends_on_classes=[CreateInstance, VerifyGuestStarted, WaitForGuestInstallationToFinish], groups=[GROUP, GROUP_START, GROUP_START_SIMPLE], - enabled=CONFIG.values['test_mgmt']) + enabled=CONFIG.test_mgmt) class VerifyInstanceMgmtInfo(object): @before_class diff --git a/reddwarf/tests/api/instances_actions.py b/reddwarf/tests/api/instances_actions.py index 9109569f..5314fe8e 100644 --- a/reddwarf/tests/api/instances_actions.py +++ b/reddwarf/tests/api/instances_actions.py @@ -18,12 +18,7 @@ import time from proboscis import after_class from proboscis import before_class from proboscis import test -from proboscis.asserts import assert_equal -from proboscis.asserts import assert_false -from proboscis.asserts import assert_not_equal -from proboscis.asserts import assert_raises -from proboscis.asserts import assert_true -from proboscis.asserts import fail +from proboscis.asserts import * from proboscis.decorators import time_out from proboscis import SkipTest @@ -50,11 +45,11 @@ GROUP_RESTART = "dbaas.api.instances.actions.restart" GROUP_STOP_MYSQL = "dbaas.api.instances.actions.stop" MYSQL_USERNAME = "test_user" MYSQL_PASSWORD = "abcde" -FAKE_MODE = CONFIG.values['fake_mode'] +FAKE_MODE = CONFIG.fake_mode # If true, then we will actually log into the database. -USE_IP = not CONFIG.values['fake_mode'] +USE_IP = not FAKE_MODE # If true, then we will actually search for the process -USE_LOCAL_OVZ = CONFIG.values['use_local_ovz'] +USE_LOCAL_OVZ = CONFIG.use_local_ovz class MySqlConnection(object): @@ -99,10 +94,10 @@ class ActionTestBase(object): def set_up(self): """If you're using this as a base class, call this method first.""" + self.dbaas = instance_info.dbaas if USE_IP: address = instance_info.get_address() self.connection = MySqlConnection(address) - self.dbaas = instance_info.dbaas @property def instance(self): @@ -181,6 +176,8 @@ class RebootTestBase(ActionTestBase): """Wait until our connection breaks.""" if not USE_IP: return + if not hasattr(self, "connection"): + return poll_until(self.connection.is_connected, lambda connected: not connected, time_out=TIME_OUT_TIME) @@ -189,8 +186,6 @@ class RebootTestBase(ActionTestBase): """Wait until status becomes running.""" def is_finished_rebooting(): instance = self.instance - instance.get() - print(instance.status) if instance.status == "REBOOT": return False assert_equal("ACTIVE", instance.status) @@ -277,6 +272,8 @@ class RestartTests(RebootTestBase): @test(depends_on=[test_ensure_mysql_is_running], enabled=not FAKE_MODE) def test_unsuccessful_restart(self): """Restart MySQL via the REST when it should fail, assert it does.""" + if FAKE_MODE: + raise SkipTest("Cannot run this in fake mode.") self.unsuccessful_restart() @test(depends_on=[test_set_up], @@ -344,18 +341,22 @@ class RebootTests(RebootTestBase): @before_class def test_set_up(self): self.set_up() + assert_true(hasattr(self, 'dbaas')) + assert_true(self.dbaas is not None) @test def test_ensure_mysql_is_running(self): """Make sure MySQL is accessible before restarting.""" self.ensure_mysql_is_running() - @test(depends_on=[test_ensure_mysql_is_running], enabled=not FAKE_MODE) + @test(depends_on=[test_ensure_mysql_is_running]) def test_unsuccessful_restart(self): """Restart MySQL via the REST when it should fail, assert it does.""" + if FAKE_MODE: + raise SkipTest("Cannot run this in fake mode.") self.unsuccessful_restart() - @after_class(always_run=True) + @after_class(depends_on=[test_set_up]) def test_successful_restart(self): """Restart MySQL via the REST API successfully.""" self.successful_restart() @@ -414,6 +415,8 @@ class ResizeInstanceTest(ActionTestBase): assert_equal(len(flavors), 1, "Number of flavors with name '%s' " "found was '%d'." % (flavor_name, len(flavors))) flavor = flavors[0] + self.old_dbaas_flavor = instance_info.dbaas_flavor + instance_info.dbaas_flavor = flavor assert_true(flavor is not None, "Flavor '%s' not found!" % flavor_name) flavor_href = self.dbaas.find_flavor_self_href(flavor) assert_true(flavor_href is not None, @@ -424,6 +427,8 @@ class ResizeInstanceTest(ActionTestBase): def test_status_changed_to_resize(self): self.log_current_users() self.obtain_flavor_ids() + if CONFIG.simulate_events: + raise SkipTest("Cannot simulate this test.") self.dbaas.instances.resize_instance( self.instance_id, self.get_flavor_href(flavor_id=self.expected_new_flavor_id)) @@ -460,6 +465,8 @@ class ResizeInstanceTest(ActionTestBase): @test(depends_on=[test_instance_returns_to_active_after_resize], runs_after=[resize_should_not_delete_users]) def test_make_sure_mysql_is_running_after_resize(self): + if CONFIG.simulate_events: + raise SkipTest("Cannot simulate this test.") self.ensure_mysql_is_running() actual = self.get_flavor_href(self.instance.flavor['id']) expected = self.get_flavor_href(flavor_id=self.expected_new_flavor_id) @@ -468,6 +475,8 @@ class ResizeInstanceTest(ActionTestBase): @test(depends_on=[test_make_sure_mysql_is_running_after_resize]) @time_out(TIME_OUT_TIME) def test_resize_down(self): + if CONFIG.simulate_events: + raise SkipTest("Cannot simulate this test.") expected_dbaas_flavor = self.expected_dbaas_flavor self.dbaas.instances.resize_instance( self.instance_id, @@ -489,7 +498,7 @@ def resize_should_not_delete_users(): fail("Somehow, the resize made the test user disappear.") -@test(depends_on_classes=[ResizeInstanceTest], depends_on=[create_user], +@test(runs_after=[ResizeInstanceTest], depends_on=[create_user], groups=[GROUP, tests.INSTANCES], enabled=CONFIG.reddwarf_volume_support) class ResizeInstanceVolume(object): @@ -513,7 +522,7 @@ class ResizeInstanceVolume(object): instance_info.dbaas.instances.resize_volume(instance_info.id, self.new_volume_size) - @test + @test(depends_on=[test_volume_resize]) @time_out(300) def test_volume_resize_success(self): diff --git a/reddwarf/tests/api/instances_mysql_down.py b/reddwarf/tests/api/instances_mysql_down.py index bbca1307..e94e13bf 100644 --- a/reddwarf/tests/api/instances_mysql_down.py +++ b/reddwarf/tests/api/instances_mysql_down.py @@ -20,16 +20,8 @@ from proboscis.decorators import time_out from proboscis import after_class from proboscis import before_class from proboscis import test -from proboscis.asserts import assert_equal -from proboscis.asserts import assert_false -from proboscis.asserts import assert_is -from proboscis.asserts import assert_is_not -from proboscis.asserts import assert_is_none -from proboscis.asserts import assert_not_equal -from proboscis.asserts import assert_raises -from proboscis.asserts import assert_true -from proboscis.asserts import Check -from proboscis.asserts import fail +from proboscis import SkipTest +from proboscis.asserts import * import time from datetime import datetime @@ -42,6 +34,7 @@ from reddwarf.tests.util import test_config @test(groups=["dbaas.api.instances.down"]) class TestBase(object): + """Base class for instance-down tests.""" @before_class def set_up(self): @@ -63,11 +56,6 @@ class TestBase(object): lambda instance: instance.status == "ACTIVE", time_out=(60 * 8)) - def _wait_for_new_volume_size(self, new_size): - poll_until(lambda: self.client.instances.get(self.id), - lambda instance: instance.volume['size'] == new_size, - time_out=(60 * 8)) - @test def create_instance(self): initial = self.client.instances.create(self.name, self.flavor_id, @@ -75,11 +63,14 @@ class TestBase(object): self.id = initial.id self._wait_for_active() - @test(depends_on=[create_instance]) - def put_into_shutdown_state(self): + def _shutdown_instance(self): instance = self.client.instances.get(self.id) self.mgmt_client.management.stop(self.id) + @test(depends_on=[create_instance]) + def put_into_shutdown_state(self): + self._shutdown_instance() + @test(depends_on=[put_into_shutdown_state]) @time_out(60 * 5) def resize_instance_in_shutdown_state(self): @@ -89,20 +80,20 @@ class TestBase(object): @test(depends_on=[create_instance], runs_after=[resize_instance_in_shutdown_state]) def put_into_shutdown_state_2(self): - instance = self.client.instances.get(self.id) - self.mgmt_client.management.stop(self.id) + self._shutdown_instance() @test(depends_on=[put_into_shutdown_state_2]) @time_out(60 * 5) def resize_volume_in_shutdown_state(self): self.client.instances.resize_volume(self.id, 2) - self._wait_for_new_volume_size(2) + poll_until(lambda: self.client.instances.get(self.id), + lambda instance: instance.volume['size'] == 2, + time_out=(60 * 8)) @test(depends_on=[create_instance], runs_after=[resize_volume_in_shutdown_state]) def put_into_shutdown_state_3(self): - instance = self.client.instances.get(self.id) - self.mgmt_client.management.stop(self.id) + self._shutdown_instance() @test(depends_on=[create_instance], runs_after=[put_into_shutdown_state_3]) diff --git a/reddwarf/tests/api/instances_resize.py b/reddwarf/tests/api/instances_resize.py new file mode 100644 index 00000000..b720c40f --- /dev/null +++ b/reddwarf/tests/api/instances_resize.py @@ -0,0 +1,198 @@ +# Copyright 2013 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mox +from testtools import TestCase +from proboscis import test + +from novaclient.exceptions import BadRequest +from novaclient.v1_1.servers import Server + +from reddwarf.common.exception import PollTimeOut +from reddwarf.common import utils +from reddwarf.common.context import ReddwarfContext +from reddwarf.guestagent import api as guest +from reddwarf.instance.models import DBInstance +from reddwarf.instance.models import ServiceStatuses +from reddwarf.instance.tasks import InstanceTasks +from reddwarf.openstack.common.rpc.common import RPCException +from reddwarf.taskmanager import models as models + +GROUP = 'dbaas.api.instances.resize' + +OLD_FLAVOR_ID = 1 +NEW_FLAVOR_ID = 2 + + +class ResizeTestBase(TestCase): + + def _init(self): + self.mock = mox.Mox() + self.instance_id = 500 + context = ReddwarfContext() + self.db_info = DBInstance.create( + name="instance", + flavor_id=OLD_FLAVOR_ID, + tenant_id=999, + volume_size=None, + task_status=InstanceTasks.RESIZING) + self.server = self.mock.CreateMock(Server) + self.instance = models.BuiltInstanceTasks(context, + self.db_info, + self.server, + service_status="ACTIVE") + self.instance.server.flavor = {'id': OLD_FLAVOR_ID} + self.guest = self.mock.CreateMock(guest.API) + self.instance._guest = self.guest + self.instance._refresh_compute_server_info = lambda: None + self.instance._refresh_compute_service_status = lambda: None + self.mock.StubOutWithMock(self.instance, 'update_db') + self.mock.StubOutWithMock(self.instance, + '_set_service_status_to_paused') + self.action = None + + def _teardown(self): + try: + self.instance.update_db(task_status=InstanceTasks.NONE) + self.mock.ReplayAll() + self.assertRaises(Exception, self.action.execute) + self.mock.VerifyAll() + finally: + self.mock.UnsetStubs() + self.db_info.delete() + + def _stop_mysql(self, reboot=True): + self.guest.stop_mysql(do_not_start_on_reboot=reboot) + + def _server_changes_to(self, new_status, new_flavor_id): + def change(): + self.server.status = new_status + self.instance.server.flavor['id'] = new_flavor_id + + self.mock.StubOutWithMock(utils, "poll_until") + utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)\ + .WithSideEffects(lambda ignore, sleep_time, time_out: change()) + + def _nova_resizes_successfully(self): + self.server.resize(NEW_FLAVOR_ID) + self._server_changes_to("VERIFY_RESIZE", NEW_FLAVOR_ID) + + +@test(groups=[GROUP, GROUP + '.resize']) +class ResizeTests(ResizeTestBase): + + def setUp(self): + super(ResizeTests, self).setUp() + self._init() + self.action = models.ResizeAction(self.instance, + new_flavor_id=NEW_FLAVOR_ID) + + def tearDown(self): + super(ResizeTests, self).tearDown() + self._teardown() + + def _start_mysql(self): + self.instance.guest.start_mysql_with_conf_changes(None) + + def test_guest_wont_stop_mysql(self): + self.guest.stop_mysql(do_not_start_on_reboot=True)\ + .AndRaise(RPCException("Could not stop MySQL!")) + + def test_nova_wont_resize(self): + self._stop_mysql() + self.server.resize(NEW_FLAVOR_ID).AndRaise(BadRequest) + self.guest.restart() + + def test_nova_resize_timeout(self): + self._stop_mysql() + self.server.resize(NEW_FLAVOR_ID) + + self.mock.StubOutWithMock(utils, 'poll_until') + utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)\ + .AndRaise(PollTimeOut) + self.guest.restart() + + def test_nova_doesnt_change_flavor(self): + self._stop_mysql() + self.server.resize(NEW_FLAVOR_ID) + self._server_changes_to("VERIFY_RESIZE", OLD_FLAVOR_ID) + self.guest.restart() + + def test_nova_resize_fails(self): + self._stop_mysql() + self.server.resize(NEW_FLAVOR_ID) + self._server_changes_to("ACTIVE", OLD_FLAVOR_ID) + self.guest.restart() + + def test_nova_resizes_in_weird_state(self): + self._stop_mysql() + self.server.resize(NEW_FLAVOR_ID) + self._server_changes_to("ACTIVE", NEW_FLAVOR_ID) + self.guest.restart() + + def test_guest_is_not_okay(self): + self._stop_mysql() + self._nova_resizes_successfully() + self.instance._set_service_status_to_paused() + self.instance.service_status = ServiceStatuses.PAUSED + utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)\ + .AndRaise(PollTimeOut) + self.instance.server.revert_resize() + self.guest.restart() + + def test_mysql_is_not_okay(self): + self._stop_mysql() + self._nova_resizes_successfully() + self.instance._set_service_status_to_paused() + self.instance.service_status = ServiceStatuses.SHUTDOWN + utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120) + self._start_mysql() + self.instance.server.revert_resize() + + def test_confirm_resize_fails(self): + self._stop_mysql() + self._nova_resizes_successfully() + self.instance._set_service_status_to_paused() + self.instance.service_status = ServiceStatuses.RUNNING + utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120) + self._start_mysql() + self.server.status = "SHUTDOWN" + self.instance.server.confirm_resize() + + +@test(groups=[GROUP, GROUP + '.migrate']) +class MigrateTests(ResizeTestBase): + + def setUp(self): + super(MigrateTests, self).setUp() + self._init() + self.action = models.MigrateAction(self.instance) + + def tearDown(self): + super(MigrateTests, self).tearDown() + self._teardown() + + def _start_mysql(self): + self.guest.restart() + + def test_successful_migrate(self): + self._stop_mysql() + self.server.migrate() + self._server_changes_to("VERIFY_RESIZE", NEW_FLAVOR_ID) + self.instance._set_service_status_to_paused() + self.instance.service_status = ServiceStatuses.RUNNING + utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120) + self._start_mysql() + self.instance.server.confirm_resize() diff --git a/reddwarf/tests/fakes/common.py b/reddwarf/tests/fakes/common.py index ec955606..d70a7088 100644 --- a/reddwarf/tests/fakes/common.py +++ b/reddwarf/tests/fakes/common.py @@ -18,8 +18,6 @@ """Common code to help in faking the models.""" import time -import traceback -import sys from novaclient import exceptions as nova_exceptions from reddwarf.common import cfg @@ -65,7 +63,6 @@ def event_simulator_sleep(time_to_sleep): global pending_events while time_to_sleep > 0: itr_sleep = 0.5 - print pending_events for i in range(len(pending_events)): event = pending_events[i] event["time"] = event["time"] - itr_sleep @@ -77,10 +74,7 @@ def event_simulator_sleep(time_to_sleep): try: func() except Exception as e: - type_, value, tb = sys.exc_info() - LOG.info("Simulated event error.") - LOG.info((traceback.format_exception(type_, value, tb))) - pass # Ignore exceptions, which can potentially occur. + LOG.exception("Simulated event error.") time_to_sleep -= itr_sleep sleep_entrance_count -= 1 diff --git a/reddwarf/tests/fakes/guestagent.py b/reddwarf/tests/fakes/guestagent.py index f2e7080f..b66b0723 100644 --- a/reddwarf/tests/fakes/guestagent.py +++ b/reddwarf/tests/fakes/guestagent.py @@ -110,45 +110,51 @@ class FakeGuest(object): def prepare(self, memory_mb, databases, users, device_path=None, mount_point=None): + from reddwarf.instance.models import DBInstance from reddwarf.instance.models import InstanceServiceStatus from reddwarf.instance.models import ServiceStatuses from reddwarf.guestagent.models import AgentHeartBeat LOG.debug("users... %s" % users) LOG.debug("databases... %s" % databases) + instance_name = DBInstance.find_by(id=self.id).name self.create_user(users) self.create_database(databases) def update_db(): status = InstanceServiceStatus.find_by(instance_id=self.id) - status.status = ServiceStatuses.RUNNING + if instance_name.endswith('GUEST_ERROR'): + status.status = ServiceStatuses.FAILED + else: + status.status = ServiceStatuses.RUNNING status.save() AgentHeartBeat.create(instance_id=self.id) self.event_spawn(1.0, update_db) - def restart(self): + def _set_status(self, new_status='RUNNING'): from reddwarf.instance.models import InstanceServiceStatus from reddwarf.instance.models import ServiceStatuses + print("Setting status to %s" % new_status) + states = {'RUNNING': ServiceStatuses.RUNNING, + 'SHUTDOWN': ServiceStatuses.SHUTDOWN, + } + status = InstanceServiceStatus.find_by(instance_id=self.id) + status.status = states[new_status] + status.save() + + def restart(self): # All this does is restart, and shut off the status updates while it # does so. So there's actually nothing to do to fake this out except # take a nap. + print("Sleeping for a second.") time.sleep(1) - status = InstanceServiceStatus.find_by(instance_id=self.id) - status.status = ServiceStatuses.RUNNING - status.save() + self._set_status('RUNNING') def start_mysql_with_conf_changes(self, updated_memory_size): - from reddwarf.instance.models import InstanceServiceStatus - from reddwarf.instance.models import ServiceStatuses - status = InstanceServiceStatus.find_by(instance_id=self.id) - status.status = ServiceStatuses.RUNNING - status.save() + time.sleep(2) + self._set_status('RUNNING') - def stop_mysql(self): - from reddwarf.instance.models import InstanceServiceStatus - from reddwarf.instance.models import ServiceStatuses - status = InstanceServiceStatus.find_by(instance_id=self.id) - status.status = ServiceStatuses.SHUTDOWN - status.save() + def stop_mysql(self, do_not_start_on_reboot=False): + self._set_status('SHUTDOWN') def get_volume_info(self): """Return used volume information in bytes.""" diff --git a/reddwarf/tests/fakes/nova.py b/reddwarf/tests/fakes/nova.py index 20c95ba7..441f122d 100644 --- a/reddwarf/tests/fakes/nova.py +++ b/reddwarf/tests/fakes/nova.py @@ -15,17 +15,19 @@ # License for the specific language governing permissions and limitations # under the License. -import eventlet -from reddwarf.openstack.common import log as logging -from novaclient.v1_1.client import Client from novaclient import exceptions as nova_exceptions -import uuid +from novaclient.v1_1.client import Client +from reddwarf.common.exception import PollTimeOut +from reddwarf.common.utils import poll_until +from reddwarf.openstack.common import log as logging from reddwarf.tests.fakes.common import authorize from reddwarf.tests.fakes.common import get_event_spawer -from reddwarf.common.utils import poll_until -from reddwarf.common.exception import PollTimeOut + +import eventlet +import uuid LOG = logging.getLogger(__name__) +FAKE_HOSTS = ["fake_host_1", "fake_host_2"] class FakeFlavor(object): @@ -100,6 +102,7 @@ class FakeServer(object): self.name = name self.image_id = image_id self.flavor_ref = flavor_ref + self.old_flavor_ref = None self.event_spawn = get_event_spawer() self._current_status = "BUILD" self.volumes = volumes @@ -111,7 +114,8 @@ class FakeServer(object): for volume in self.volumes: info_vols.append({'id': volume.id}) volume.set_attachment(id) - self.host = "fake_host" + self.host = FAKE_HOSTS[0] + self.old_host = None self._info = {'os:volumes': info_vols} @@ -124,13 +128,23 @@ class FakeServer(object): raise RuntimeError("Not in resize confirm mode.") self._current_status = "ACTIVE" + def revert_resize(self): + if self.status != "VERIFY_RESIZE": + raise RuntimeError("Not in resize confirm mode.") + self.host = self.old_host + self.old_host = None + self.flavor_ref = self.old_flavor_ref + self.old_flavor_ref = None + self._current_status = "ACTIVE" + def reboot(self): LOG.debug("Rebooting server %s" % (self.id)) - self._current_status = "REBOOT" def set_to_active(): self._current_status = "ACTIVE" self.parent.schedule_simulate_running_server(self.id, 1.5) + + self._current_status = "REBOOT" self.event_spawn(1, set_to_active) def delete(self): @@ -157,7 +171,10 @@ class FakeServer(object): return [{"href": url, "rel": link_type} for link_type in ['self', 'bookmark']] - def resize(self, new_flavor_id): + def migrate(self): + self.resize(None) + + def resize(self, new_flavor_id=None): self._current_status = "RESIZE" if self.name.endswith("_RESIZE_TIMEOUT"): raise PollTimeOut() @@ -165,15 +182,32 @@ class FakeServer(object): def set_to_confirm_mode(): self._current_status = "VERIFY_RESIZE" + def set_to_active(): + self.parent.schedule_simulate_running_server(self.id, 1.5) + self.event_spawn(1, set_to_active) + + def change_host(): + self.old_host = self.host + self.host = [host for host in FAKE_HOSTS if host != self.host][0] + def set_flavor(): if self.name.endswith("_RESIZE_ERROR"): self._current_status = "ACTIVE" + return + if new_flavor_id is None: + # Migrations are flavorless flavor resizes. + # A resize MIGHT change the host, but a migrate + # deliberately does. + LOG.debug("Migrating fake instance.") + self.event_spawn(0.75, change_host) else: + LOG.debug("Resizing fake instance.") + self.old_flavor_ref = self.flavor_ref flavor = self.parent.flavors.get(new_flavor_id) self.flavor_ref = flavor.links[0]['href'] - self.event_spawn(1, set_to_confirm_mode) + self.event_spawn(1, set_to_confirm_mode) - self.event_spawn(1, set_flavor) + self.event_spawn(0.8, set_flavor) def schedule_status(self, new_status, time_from_now): """Makes a new status take effect at the given time.""" @@ -291,10 +325,11 @@ class FakeServers(object): self.event_spawn(time_from_now, delete_server) def schedule_simulate_running_server(self, id, time_from_now): + from reddwarf.instance.models import DBInstance + from reddwarf.instance.models import InstanceServiceStatus + from reddwarf.instance.models import ServiceStatuses + def set_server_running(): - from reddwarf.instance.models import DBInstance - from reddwarf.instance.models import InstanceServiceStatus - from reddwarf.instance.models import ServiceStatuses instance = DBInstance.find_by(compute_instance_id=id) LOG.debug("Setting server %s to running" % instance.id) status = InstanceServiceStatus.find_by(instance_id=instance.id) @@ -530,6 +565,10 @@ class FakeHost(object): self.totalRAM = 2004 # 16384 self.usedRAM = 0 for server in self.servers.list(): + print server + if server.host != self.name: + print "\t...not on this host." + continue self.instances.append({ 'uuid': server.id, 'name': server.name, @@ -550,7 +589,8 @@ class FakeHosts(object): def __init__(self, servers): self.hosts = {} - self.add_host(FakeHost('fake_host', servers)) + for host in FAKE_HOSTS: + self.add_host(FakeHost(host, servers)) def add_host(self, host): self.hosts[host.name] = host diff --git a/run_tests.py b/run_tests.py index 3d121c47..b4da1311 100644 --- a/run_tests.py +++ b/run_tests.py @@ -116,6 +116,7 @@ if __name__=="__main__": from reddwarf.tests.api import instances_actions from reddwarf.tests.api import instances_delete from reddwarf.tests.api import instances_mysql_down + from reddwarf.tests.api import instances_resize from reddwarf.tests.api import databases from reddwarf.tests.api import root from reddwarf.tests.api import users diff --git a/tools/test-requires b/tools/test-requires index 2dc96eef..f81a3e53 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -12,6 +12,7 @@ wsgi_intercept proboscis python-reddwarfclient mock +mox testtools>=0.9.22 pexpect discover |