diff options
48 files changed, 1428 insertions, 503 deletions
diff --git a/bin/trove-guestagent b/bin/trove-guestagent index f10a2812..3f15c8f2 100755 --- a/bin/trove-guestagent +++ b/bin/trove-guestagent @@ -53,9 +53,10 @@ if __name__ == '__main__': try: get_db_api().configure_db(CONF) - manager = dbaas.service_registry().get(CONF.service_type) + manager = dbaas.datastore_registry().get(CONF.datastore_manager) if not manager: - msg = "Manager not found for service type " + CONF.service_type + msg = ("Manager class not registered for datastore manager " + + CONF.datastore_manager) raise RuntimeError(msg) server = rpc_service.RpcService(manager=manager, host=CONF.guest_id) launcher = openstack_service.launch(server) diff --git a/bin/trove-manage b/bin/trove-manage index 84e641c0..4dfe081b 100755 --- a/bin/trove-manage +++ b/bin/trove-manage @@ -36,11 +36,13 @@ if os.path.exists(os.path.join(possible_topdir, 'trove', '__init__.py')): from trove import version from trove.common import cfg +from trove.common import exception from trove.common import utils from trove.db import get_db_api from trove.openstack.common import log as logging from trove.openstack.common import uuidutils from trove.instance import models as instance_models +from trove.datastore import models as datastore_models CONF = cfg.CONF @@ -69,28 +71,30 @@ class Commands(object): kwargs[arg] = getattr(CONF.action, arg) exec_method(**kwargs) - def image_update(self, service_name, image_id): - self.db_api.configure_db(CONF) - image = self.db_api.find_by(instance_models.ServiceImage, - service_name=service_name) - if image is None: - # Create a new one - image = instance_models.ServiceImage() - image.id = uuidutils.generate_uuid() - image.service_name = service_name - image.image_id = image_id - self.db_api.save(image) - - def db_wipe(self, repo_path, service_name, image_id): + def datastore_update(self, datastore_name, manager, default_version): + try: + datastore_models.update_datastore(datastore_name, manager, + default_version) + print("Datastore '%s' updated." % datastore_name) + except exception.DatastoreVersionNotFound as e: + print(e) + + def datastore_version_update(self, datastore, version_name, image_id, + packages, active): + try: + datastore_models.update_datastore_version(datastore, + version_name, image_id, + packages, active) + print("Datastore version '%s' updated." % version_name) + except exception.DatastoreNotFound as e: + print(e) + + def db_wipe(self, repo_path): """Drops the database and recreates it.""" from trove.instance import models from trove.db.sqlalchemy import session self.db_api.drop_db(CONF) self.db_sync() - # Sets up database engine, so the next line will work... - session.configure_db(CONF) - models.ServiceImage.create(service_name=service_name, - image_id=image_id) def params_of(self, command_name): if Commands.has(command_name): @@ -108,13 +112,18 @@ if __name__ == '__main__': parser = subparser.add_parser('db_downgrade') parser.add_argument('version') parser.add_argument('--repo_path') - parser = subparser.add_parser('image_update') - parser.add_argument('service_name') + parser = subparser.add_parser('datastore_update') + parser.add_argument('datastore_name') + parser.add_argument('manager') + parser.add_argument('default_version') + parser = subparser.add_parser('datastore_version_update') + parser.add_argument('datastore') + parser.add_argument('version_name') parser.add_argument('image_id') + parser.add_argument('packages') + parser.add_argument('active') parser = subparser.add_parser('db_wipe') parser.add_argument('repo_path') - parser.add_argument('service_name') - parser.add_argument('image_id') cfg.custom_parser('action', actions) cfg.parse_args(sys.argv) diff --git a/etc/trove/trove.conf.test b/etc/trove/trove.conf.test index 59d28886..2c8b9195 100644 --- a/etc/trove/trove.conf.test +++ b/etc/trove/trove.conf.test @@ -92,6 +92,9 @@ http_post_rate = 200 http_put_rate = 200 http_delete_rate = 200 +# default datastore +default_datastore = a00000a0-00a0-0a00-00a0-000a000000aa + # Auth admin_roles = admin diff --git a/run_tests.py b/run_tests.py index 8a9bbe87..3e517ca5 100644 --- a/run_tests.py +++ b/run_tests.py @@ -52,16 +52,41 @@ def initialize_trove(config_file): return pastedeploy.paste_deploy_app(config_file, 'trove', {}) +def datastore_init(): + # Adds the datastore for mysql (needed to make most calls work). + from trove.datastore import models + models.DBDatastore.create(id="a00000a0-00a0-0a00-00a0-000a000000aa", + name=CONFIG.dbaas_datastore, manager='mysql', + default_version_id= + "b00000b0-00b0-0b00-00b0-000b000000bb") + models.DBDatastore.create(id="e00000e0-00e0-0e00-00e0-000e000000ee", + name='Test_Datastore_1', manager='manager1', + default_version_id=None) + models.DBDatastoreVersion.create(id="b00000b0-00b0-0b00-00b0-000b000000bb", + datastore_id= + "a00000a0-00a0-0a00-00a0-000a000000aa", + name=CONFIG.dbaas_datastore_version, + image_id= + 'c00000c0-00c0-0c00-00c0-000c000000cc', + packages='test packages', + active=1) + models.DBDatastoreVersion.create(id="d00000d0-00d0-0d00-00d0-000d000000dd", + datastore_id= + "a00000a0-00a0-0a00-00a0-000a000000aa", + name='mysql_inactive_version', + image_id= + 'c00000c0-00c0-0c00-00c0-000c000000cc', + packages=None, active=0) + + def initialize_database(): from trove.db import get_db_api - from trove.instance import models from trove.db.sqlalchemy import session db_api = get_db_api() db_api.drop_db(CONF) # Destroys the database, if it exists. db_api.db_sync(CONF) session.configure_db(CONF) - # Adds the image for mysql (needed to make most calls work). - models.ServiceImage.create(service_name="mysql", image_id="fake") + datastore_init() db_api.configure_db(CONF) @@ -125,6 +150,7 @@ if __name__ == "__main__": from trove.tests.api import instances_mysql_down from trove.tests.api import instances_resize from trove.tests.api import databases + from trove.tests.api import datastores from trove.tests.api import root from trove.tests.api import root_on_create from trove.tests.api import users diff --git a/trove/common/api.py b/trove/common/api.py index df7203b5..311bddd1 100644 --- a/trove/common/api.py +++ b/trove/common/api.py @@ -20,6 +20,7 @@ from trove.instance.service import InstanceController from trove.limits.service import LimitsController from trove.backup.service import BackupController from trove.versions import VersionsController +from trove.datastore.service import DatastoreController class API(wsgi.Router): @@ -28,6 +29,7 @@ class API(wsgi.Router): mapper = routes.Mapper() super(API, self).__init__(mapper) self._instance_router(mapper) + self._datastore_router(mapper) self._flavor_router(mapper) self._versions_router(mapper) self._limits_router(mapper) @@ -37,6 +39,17 @@ class API(wsgi.Router): versions_resource = VersionsController().create_resource() mapper.connect("/", controller=versions_resource, action="show") + def _datastore_router(self, mapper): + datastore_resource = DatastoreController().create_resource() + mapper.resource("datastore", "/{tenant_id}/datastores", + controller=datastore_resource) + mapper.connect("/{tenant_id}/datastores/{datastore}/versions", + controller=datastore_resource, + action="version_index") + mapper.connect("/{tenant_id}/datastores/{datastore}/versions/{id}", + controller=datastore_resource, + action="version_show") + def _instance_router(self, mapper): instance_resource = InstanceController().create_resource() path = "/{tenant_id}/instances" diff --git a/trove/common/apischema.py b/trove/common/apischema.py index b415b324..dbc7a19a 100644 --- a/trove/common/apischema.py +++ b/trove/common/apischema.py @@ -183,7 +183,6 @@ instance = { "volume": volume, "databases": databases_def, "users": users_list, - "service_type": non_empty_string, "restorePoint": { "type": "object", "required": ["backupRef"], @@ -192,7 +191,15 @@ instance = { "backupRef": uuid } }, - "availability_zone": non_empty_string + "availability_zone": non_empty_string, + "datastore": { + "type": "object", + "additionalProperties": True, + "properties": { + "type": non_empty_string, + "version": non_empty_string + } + } } } } diff --git a/trove/common/cfg.py b/trove/common/cfg.py index f884e3e9..04d0f92c 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -64,8 +64,6 @@ common_opts = [ cfg.IntOpt('periodic_interval', default=60), cfg.BoolOpt('trove_dns_support', default=False), cfg.StrOpt('db_api_implementation', default='trove.db.sqlalchemy.api'), - cfg.StrOpt('mysql_pkg', default='mysql-server-5.5'), - cfg.StrOpt('percona_pkg', default='percona-server-server-5.5'), cfg.StrOpt('dns_driver', default='trove.dns.driver.DnsDriver'), cfg.StrOpt('dns_instance_entry_factory', default='trove.dns.driver.DnsInstanceEntryFactory'), @@ -111,13 +109,18 @@ common_opts = [ cfg.BoolOpt('use_heat', default=False), cfg.StrOpt('device_path', default='/dev/vdb'), cfg.StrOpt('mount_point', default='/var/lib/mysql'), - cfg.StrOpt('service_type', default='mysql'), + cfg.StrOpt('default_datastore', default=None, + help="The default datastore id or name to use if one is not " + "provided by the user. If the default value is None, the field" + " becomes required in the instance-create request."), + cfg.StrOpt('datastore_manager', default=None, + help='manager class in guestagent, setup by taskmanager on ' + 'instance provision'), cfg.StrOpt('block_device_mapping', default='vdb'), cfg.IntOpt('server_delete_time_out', default=60), cfg.IntOpt('volume_time_out', default=60), cfg.IntOpt('heat_time_out', default=60), cfg.IntOpt('reboot_time_out', default=60 * 2), - cfg.StrOpt('service_options', default=['mysql']), cfg.IntOpt('dns_time_out', default=60 * 2), cfg.IntOpt('resize_time_out', default=60 * 10), cfg.IntOpt('revert_time_out', default=60 * 10), @@ -212,10 +215,10 @@ common_opts = [ cfg.StrOpt('guest_config', default='$pybasedir/etc/trove/trove-guestagent.conf.sample', help="Path to guestagent config file"), - cfg.DictOpt('service_registry_ext', default=dict(), - help='Extention for default service managers.' + cfg.DictOpt('datastore_registry_ext', default=dict(), + help='Extention for default datastore managers.' ' Allows to use custom managers for each of' - ' service type supported in trove'), + ' datastore supported in trove'), cfg.StrOpt('template_path', default='/etc/trove/templates/', help='Path which leads to datastore templates'), diff --git a/trove/common/exception.py b/trove/common/exception.py index 59ab2524..83e2d6d5 100644 --- a/trove/common/exception.py +++ b/trove/common/exception.py @@ -96,6 +96,41 @@ class DnsRecordNotFound(NotFound): message = _("DnsRecord with name= %(name)s not found.") +class DatastoreNotFound(NotFound): + + message = _("Datastore '%(datastore)s' cannot be found.") + + +class DatastoreVersionNotFound(NotFound): + + message = _("Datastore version '%(version)s' cannot be found.") + + +class DatastoresNotFound(NotFound): + + message = _("Datastores cannot be found.") + + +class DatastoreNoVersion(TroveError): + + message = _("Datastore '%(datastore)s' has no version '%(version)s'.") + + +class DatastoreVersionInactive(TroveError): + + message = _("Datastore version '%(version)s' is not active.") + + +class DatastoreDefaultDatastoreNotFound(TroveError): + + message = _("Please specify datastore.") + + +class DatastoreDefaultVersionNotFound(TroveError): + + message = _("Default version for datastore '%(datastore)s' not found.") + + class OverLimit(TroveError): internal_message = _("The server rejected the request due to its size or " diff --git a/trove/common/template.py b/trove/common/template.py index 4b9ace30..6c4c2afa 100644 --- a/trove/common/template.py +++ b/trove/common/template.py @@ -30,10 +30,10 @@ class SingleInstanceConfigTemplate(object): """ This class selects a single configuration file by database type for rendering on the guest """ - def __init__(self, datastore_type, flavor_dict, instance_id): + def __init__(self, datastore_manager, flavor_dict, instance_id): """ Constructor - :param datastore_type: The database type. + :param datastore_manager: The datastore manager. :type name: str. :param flavor_dict: dict containing flavor details for use in jinja. :type flavor_dict: dict. @@ -42,7 +42,7 @@ class SingleInstanceConfigTemplate(object): """ self.flavor_dict = flavor_dict - template_filename = "%s/config.template" % datastore_type + template_filename = "%s/config.template" % datastore_manager self.template = ENV.get_template(template_filename) self.instance_id = instance_id @@ -66,12 +66,12 @@ class SingleInstanceConfigTemplate(object): return abs(hash(self.instance_id) % (2 ** 31)) -def load_heat_template(datastore_type): - template_filename = "%s/heat.template" % datastore_type +def load_heat_template(datastore_manager): + template_filename = "%s/heat.template" % datastore_manager try: template_obj = ENV.get_template(template_filename) return template_obj except jinja2.TemplateNotFound: - msg = "Missing heat template for %s" % datastore_type + msg = "Missing heat template for %s" % datastore_manager LOG.error(msg) raise exception.TroveError(msg) diff --git a/trove/datastore/__init__.py b/trove/datastore/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/trove/datastore/__init__.py diff --git a/trove/datastore/models.py b/trove/datastore/models.py new file mode 100644 index 00000000..8c962134 --- /dev/null +++ b/trove/datastore/models.py @@ -0,0 +1,185 @@ +from trove.common import cfg +from trove.common import exception +from trove.db import models as dbmodels +from trove.db import get_db_api +from trove.openstack.common import uuidutils + + +CONF = cfg.CONF +db_api = get_db_api() + + +def persisted_models(): + return { + 'datastore': DBDatastore, + 'datastore_version': DBDatastoreVersion, + } + + +class DBDatastore(dbmodels.DatabaseModelBase): + + _data_fields = ['id', 'name', 'manager', 'default_version_id'] + + +class DBDatastoreVersion(dbmodels.DatabaseModelBase): + + _data_fields = ['id', 'datastore_id', 'name', 'image_id', 'packages', + 'active'] + + +class Datastore(object): + + def __init__(self, db_info): + self.db_info = db_info + + @classmethod + def load(cls, id_or_name): + try: + return cls(DBDatastore.find_by(id=id_or_name)) + except exception.ModelNotFoundError: + try: + return cls(DBDatastore.find_by(name=id_or_name)) + except exception.ModelNotFoundError: + raise exception.DatastoreNotFound(datastore=id_or_name) + + @property + def id(self): + return self.db_info.id + + @property + def name(self): + return self.db_info.name + + @property + def manager(self): + return self.db_info.manager + + @property + def default_version_id(self): + return self.db_info.default_version_id + + +class Datastores(object): + + def __init__(self, db_info): + self.db_info = db_info + + @classmethod + def load(cls): + return cls(DBDatastore.find_all()) + + def __iter__(self): + for item in self.db_info: + yield item + + +class DatastoreVersion(object): + + def __init__(self, db_info): + self.db_info = db_info + + @classmethod + def load(cls, id_or_name): + try: + return cls(DBDatastoreVersion.find_by(id=id_or_name)) + except exception.ModelNotFoundError: + try: + return cls(DBDatastoreVersion.find_by(name=id_or_name)) + except exception.ModelNotFoundError: + raise exception.DatastoreVersionNotFound(version=id_or_name) + + @property + def id(self): + return self.db_info.id + + @property + def datastore_id(self): + return self.db_info.datastore_id + + @property + def name(self): + return self.db_info.name + + @property + def image_id(self): + return self.db_info.image_id + + @property + def packages(self): + return self.db_info.packages + + @property + def active(self): + return self.db_info.active + + +class DatastoreVersions(object): + + def __init__(self, db_info): + self.db_info = db_info + + @classmethod + def load(cls, id_or_name, active=True): + datastore = Datastore.load(id_or_name) + return cls(DBDatastoreVersion.find_all(datastore_id=datastore.id, + active=active)) + + def __iter__(self): + for item in self.db_info: + yield item + + +def get_datastore_version(type=None, version=None): + datastore = type or CONF.default_datastore + if not datastore: + raise exception.DatastoreDefaultDatastoreNotFound() + datastore = Datastore.load(datastore) + version = version or datastore.default_version_id + if not version: + raise exception.DatastoreDefaultVersionNotFound(datastore= + datastore.name) + datastore_version = DatastoreVersion.load(version) + if datastore_version.datastore_id != datastore.id: + raise exception.DatastoreNoVersion(datastore=datastore.name, + version=datastore_version.name) + if not datastore_version.active: + raise exception.DatastoreVersionInactive(version= + datastore_version.name) + return (datastore, datastore_version) + + +def update_datastore(name, manager, default_version): + db_api.configure_db(CONF) + if default_version: + version = DatastoreVersion.load(default_version) + if not version.active: + raise exception.DatastoreVersionInactive(version= + version.name) + try: + datastore = DBDatastore.find_by(name=name) + except exception.ModelNotFoundError: + # Create a new one + datastore = DBDatastore() + datastore.id = uuidutils.generate_uuid() + datastore.name = name + datastore.manager = manager + if default_version: + datastore.default_version_id = version.id + db_api.save(datastore) + + +def update_datastore_version(datastore, name, image_id, packages, active): + db_api.configure_db(CONF) + datastore = Datastore.load(datastore) + try: + version = DBDatastoreVersion.find_by(name=name) + except exception.ModelNotFoundError: + # Create a new one + version = DBDatastoreVersion() + version.id = uuidutils.generate_uuid() + version.name = name + version.datastore_id = datastore.id + version.image_id = image_id + version.packages = packages + version.active = active + db_api.save(version) diff --git a/trove/datastore/service.py b/trove/datastore/service.py new file mode 100644 index 00000000..8cdf2e3b --- /dev/null +++ b/trove/datastore/service.py @@ -0,0 +1,31 @@ +from trove.common import cfg +from trove.common import exception +from trove.common import utils +from trove.common import wsgi +from trove.datastore import models, views + + +class DatastoreController(wsgi.Controller): + + def show(self, req, tenant_id, id): + datastore = models.Datastore.load(id) + return wsgi.Result(views. + DatastoreView(datastore, req).data(), 200) + + def index(self, req, tenant_id): + datastores = models.Datastores.load() + return wsgi.Result(views. + DatastoresView(datastores, req).data(), + 200) + + def version_show(self, req, tenant_id, datastore, id): + datastore, datastore_version = models.get_datastore_version(datastore, + id) + return wsgi.Result(views.DatastoreVersionView(datastore_version, + req).data(), 200) + + def version_index(self, req, tenant_id, datastore): + datastore_versions = models.DatastoreVersions.load(datastore) + return wsgi.Result(views. + DatastoreVersionsView(datastore_versions, + req).data(), 200) diff --git a/trove/datastore/views.py b/trove/datastore/views.py new file mode 100644 index 00000000..a2b29085 --- /dev/null +++ b/trove/datastore/views.py @@ -0,0 +1,76 @@ +from trove.common.views import create_links + + +class DatastoreView(object): + + def __init__(self, datastore, req=None): + self.datastore = datastore + self.req = req + + def data(self): + datastore_dict = { + "id": self.datastore.id, + "name": self.datastore.name, + "links": self._build_links(), + } + + return {"datastore": datastore_dict} + + def _build_links(self): + return create_links("datastores", self.req, + self.datastore.id) + + +class DatastoresView(object): + + def __init__(self, datastores, req=None): + self.datastores = datastores + self.req = req + + def data(self): + data = [] + for datastore in self.datastores: + data.append(self.data_for_datastore(datastore)) + return {'datastores': data} + + def data_for_datastore(self, datastore): + view = DatastoreView(datastore, req=self.req) + return view.data()['datastore'] + + +class DatastoreVersionView(object): + + def __init__(self, datastore_version, req=None): + self.datastore_version = datastore_version + self.req = req + + def data(self): + datastore_version_dict = { + "id": self.datastore_version.id, + "name": self.datastore_version.name, + "links": self._build_links(), + } + + return {"version": datastore_version_dict} + + def _build_links(self): + return create_links("datastores/versions", + self.req, self.datastore_version.id) + + +class DatastoreVersionsView(object): + + def __init__(self, datastore_versions, req=None): + self.datastore_versions = datastore_versions + self.req = req + + def data(self): + data = [] + for datastore_version in self.datastore_versions: + data.append(self. + data_for_datastore_version(datastore_version)) + return {'versions': data} + + def data_for_datastore_version(self, datastore_version): + view = DatastoreVersionView(datastore_version, req=self.req) + return view.data()['version'] diff --git a/trove/db/__init__.py b/trove/db/__init__.py index f9cfdebd..0b8943cb 100644 --- a/trove/db/__init__.py +++ b/trove/db/__init__.py @@ -51,6 +51,10 @@ class Query(object): return self.db_api.count(self._query_func, self._model, **self._conditions) + def first(self): + return self.db_api.first(self._query_func, self._model, + **self._conditions) + def __iter__(self): return iter(self.all()) diff --git a/trove/db/sqlalchemy/api.py b/trove/db/sqlalchemy/api.py index 99217641..71905b50 100644 --- a/trove/db/sqlalchemy/api.py +++ b/trove/db/sqlalchemy/api.py @@ -35,6 +35,10 @@ def count(query, *args, **kwargs): return query(*args, **kwargs).count() +def first(query, *args, **kwargs): + return query(*args, **kwargs).first() + + def find_all(model, **conditions): return _query_by(model, **conditions) diff --git a/trove/db/sqlalchemy/mappers.py b/trove/db/sqlalchemy/mappers.py index 3b4a8853..835eec8d 100644 --- a/trove/db/sqlalchemy/mappers.py +++ b/trove/db/sqlalchemy/mappers.py @@ -30,8 +30,10 @@ def map(engine, models): orm.mapper(models['instance'], Table('instances', meta, autoload=True)) orm.mapper(models['root_enabled_history'], Table('root_enabled_history', meta, autoload=True)) - orm.mapper(models['service_image'], - Table('service_images', meta, autoload=True)) + orm.mapper(models['datastore'], + Table('datastores', meta, autoload=True)) + orm.mapper(models['datastore_version'], + Table('datastore_versions', meta, autoload=True)) orm.mapper(models['service_statuses'], Table('service_statuses', meta, autoload=True)) orm.mapper(models['dns_records'], diff --git a/trove/db/sqlalchemy/migrate_repo/versions/016_add_datastore_type.py b/trove/db/sqlalchemy/migrate_repo/versions/016_add_datastore_type.py new file mode 100644 index 00000000..831345eb --- /dev/null +++ b/trove/db/sqlalchemy/migrate_repo/versions/016_add_datastore_type.py @@ -0,0 +1,76 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import ForeignKey +from sqlalchemy.schema import Column +from sqlalchemy.schema import MetaData +from sqlalchemy.schema import UniqueConstraint + +from trove.db.sqlalchemy.migrate_repo.schema import Boolean +from trove.db.sqlalchemy.migrate_repo.schema import create_tables +from trove.db.sqlalchemy.migrate_repo.schema import DateTime +from trove.db.sqlalchemy.migrate_repo.schema import drop_tables +from trove.db.sqlalchemy.migrate_repo.schema import Integer +from trove.db.sqlalchemy.migrate_repo.schema import BigInteger +from trove.db.sqlalchemy.migrate_repo.schema import String +from trove.db.sqlalchemy.migrate_repo.schema import Table + + +meta = MetaData() + + +datastores = Table( + 'datastores', + meta, + Column('id', String(36), primary_key=True, nullable=False), + Column('name', String(255), unique=True), + Column('manager', String(255), nullable=False), + Column('default_version_id', String(36)), +) + + +datastore_versions = Table( + 'datastore_versions', + meta, + Column('id', String(36), primary_key=True, nullable=False), + Column('datastore_id', String(36), ForeignKey('datastores.id')), + Column('name', String(255), unique=True), + Column('image_id', String(36), nullable=False), + Column('packages', String(511)), + Column('active', Boolean(), nullable=False), + UniqueConstraint('datastore_id', 'name', name='ds_versions') +) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + create_tables([datastores, datastore_versions]) + instances = Table('instances', meta, autoload=True) + datastore_version_id = Column('datastore_version_id', String(36), + ForeignKey('datastore_versions.id')) + instances.create_column(datastore_version_id) + instances.drop_column('service_type') + # Table 'service_images' is deprecated since this version. + # Leave it for few releases. + #drop_tables([service_images]) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + drop_tables([datastores, datastore_versions]) + instances = Table('instances', meta, autoload=True) + instances.drop_column('datastore_version_id') + service_type = Column('service_type', String(36)) + instances.create_column(service_type) + instances.update().values({'service_type': 'mysql'}).execute() diff --git a/trove/db/sqlalchemy/session.py b/trove/db/sqlalchemy/session.py index 12108a25..4cd94b07 100644 --- a/trove/db/sqlalchemy/session.py +++ b/trove/db/sqlalchemy/session.py @@ -42,6 +42,7 @@ def configure_db(options, models_mapper=None): models_mapper.map(_ENGINE) else: from trove.instance import models as base_models + from trove.datastore import models as datastores_models from trove.dns import models as dns_models from trove.extensions.mysql import models as mysql_models from trove.guestagent import models as agent_models @@ -51,6 +52,7 @@ def configure_db(options, models_mapper=None): model_modules = [ base_models, + datastores_models, dns_models, mysql_models, agent_models, diff --git a/trove/extensions/mgmt/instances/models.py b/trove/extensions/mgmt/instances/models.py index e9532bc3..a819e4b2 100644 --- a/trove/extensions/mgmt/instances/models.py +++ b/trove/extensions/mgmt/instances/models.py @@ -180,14 +180,14 @@ class NotificationTransformer(object): subsecond=True) return audit_start, audit_end - def _get_service_id(self, service_type, id_map): - if service_type in id_map: - service_type_id = id_map[service_type] + def _get_service_id(self, datastore_manager, id_map): + if datastore_manager in id_map: + datastore_manager_id = id_map[datastore_manager] else: - service_type_id = cfg.UNKNOWN_SERVICE_ID - LOG.error("Service ID for Type (%s) is not configured" - % service_type) - return service_type_id + datastore_manager_id = cfg.UNKNOWN_SERVICE_ID + LOG.error("Datastore ID for Manager (%s) is not configured" + % datastore_manager) + return datastore_manager_id def transform_instance(self, instance, audit_start, audit_end): payload = { @@ -206,7 +206,7 @@ class NotificationTransformer(object): 'tenant_id': instance.tenant_id } payload['service_id'] = self._get_service_id( - instance.service_type, CONF.notification_service_id) + instance.datastore.manager, CONF.notification_service_id) return payload def __call__(self): diff --git a/trove/guestagent/api.py b/trove/guestagent/api.py index 5d064ddb..8be215b4 100644 --- a/trove/guestagent/api.py +++ b/trove/guestagent/api.py @@ -212,7 +212,7 @@ class API(proxy.RpcProxy): LOG.debug(_("Check diagnostics on Instance %s"), self.id) return self._call("get_diagnostics", AGENT_LOW_TIMEOUT) - def prepare(self, memory_mb, databases, users, + def prepare(self, memory_mb, packages, databases, users, device_path='/dev/vdb', mount_point='/mnt/volume', backup_id=None, config_contents=None, root_password=None): """Make an asynchronous call to prepare the guest @@ -220,10 +220,10 @@ class API(proxy.RpcProxy): """ LOG.debug(_("Sending the call to prepare the Guest")) self._cast_with_consumer( - "prepare", databases=databases, memory_mb=memory_mb, - users=users, device_path=device_path, mount_point=mount_point, - backup_id=backup_id, config_contents=config_contents, - root_password=root_password) + "prepare", packages=packages, databases=databases, + memory_mb=memory_mb, users=users, device_path=device_path, + mount_point=mount_point, backup_id=backup_id, + config_contents=config_contents, root_password=root_password) def restart(self): """Restart the MySQL server.""" diff --git a/trove/guestagent/common/operating_system.py b/trove/guestagent/common/operating_system.py index b0fdd3de..e591db48 100644 --- a/trove/guestagent/common/operating_system.py +++ b/trove/guestagent/common/operating_system.py @@ -25,3 +25,46 @@ def get_os(): return REDHAT else: return DEBIAN + + +def service_discovery(service_candidates): + """ + This function discovering how to start, stop, enable, disable service + in current environment. "service_candidates" is array with possible + system service names. Works for upstart, systemd, sysvinit. + """ + result = {} + for service in service_candidates: + # check upstart + if os.path.isfile("/etc/init/%s.conf" % service): + # upstart returns error code when service already started/stopped + result['cmd_start'] = "sudo start %s || true" % service + result['cmd_stop'] = "sudo stop %s || true" % service + result['cmd_enable'] = ("sudo sed -i '/^manual$/d' " + "/etc/init/%s.conf" % service) + result['cmd_disable'] = ("sudo sh -c 'echo manual >> " + "/etc/init/%s.conf'" % service) + break + # check sysvinit + if os.path.isfile("/etc/init.d/%s" % service): + result['cmd_start'] = "sudo service %s start" % service + result['cmd_stop'] = "sudo service %s stop" % service + if os.path.isfile("/usr/sbin/update-rc.d"): + result['cmd_enable'] = "sudo update-rc.d %s defaults; sudo " \ + "update-rc.d %s enable" % (service, + service) + result['cmd_disable'] = "sudo update-rc.d %s defaults; sudo " \ + "update-rc.d %s disable" % (service, + service) + elif os.path.isfile("/sbin/chkconfig"): + result['cmd_enable'] = "sudo chkconfig %s on" % service + result['cmd_disable'] = "sudo chkconfig %s off" % service + break + # check systemd + if os.path.isfile("/lib/systemd/system/%s.service" % service): + result['cmd_start'] = "sudo systemctl start %s" % service + result['cmd_stop'] = "sudo systemctl stop %s" % service + result['cmd_enable'] = "sudo systemctl enable %s" % service + result['cmd_disable'] = "sudo systemctl disable %s" % service + break + return result diff --git a/trove/guestagent/datastore/mysql/manager.py b/trove/guestagent/datastore/mysql/manager.py index a21a0170..2163411b 100644 --- a/trove/guestagent/datastore/mysql/manager.py +++ b/trove/guestagent/datastore/mysql/manager.py @@ -84,32 +84,26 @@ class Manager(periodic_task.PeriodicTasks): raise LOG.info(_("Restored database successfully")) - def prepare(self, context, databases, memory_mb, users, device_path=None, - mount_point=None, backup_id=None, config_contents=None, - root_password=None): + def prepare(self, context, packages, databases, memory_mb, users, + device_path=None, mount_point=None, backup_id=None, + config_contents=None, root_password=None): """Makes ready DBAAS on a Guest container.""" MySqlAppStatus.get().begin_install() # status end_mysql_install set with secure() app = MySqlApp(MySqlAppStatus.get()) - restart_mysql = False + app.install_if_needed(packages) if device_path: + #stop and do not update database + app.stop_db() device = volume.VolumeDevice(device_path) device.format() - #if a /var/lib/mysql folder exists, back it up. if os.path.exists(CONF.mount_point): - #stop and do not update database - app.stop_db() #rsync exiting data - if not backup_id: - restart_mysql = True - device.migrate_data(CONF.mount_point) + device.migrate_data(CONF.mount_point) #mount the volume device.mount(mount_point) LOG.debug(_("Mounted the volume.")) - #check mysql was installed and stopped - if restart_mysql: - app.start_mysql() - app.install_if_needed() + app.start_mysql() if backup_id: self._perform_restore(backup_id, context, CONF.mount_point, app) LOG.info(_("Securing mysql now.")) diff --git a/trove/guestagent/datastore/mysql/service.py b/trove/guestagent/datastore/mysql/service.py index b82cb94b..c1457c5e 100644 --- a/trove/guestagent/datastore/mysql/service.py +++ b/trove/guestagent/datastore/mysql/service.py @@ -13,11 +13,11 @@ from trove.common import cfg from trove.common import utils as utils from trove.common import exception from trove.common import instance as rd_instance +from trove.guestagent.common import operating_system from trove.guestagent.common import sql_query from trove.guestagent.db import models from trove.guestagent import pkg from trove.guestagent.datastore import service -from trove.guestagent.datastore.mysql import system from trove.openstack.common import log as logging from trove.openstack.common.gettextutils import _ from trove.extensions.mysql.models import RootHistory @@ -26,7 +26,6 @@ ADMIN_USER_NAME = "os_admin" LOG = logging.getLogger(__name__) FLUSH = text(sql_query.FLUSH) ENGINE = None -MYSQLD_ARGS = None PREPARING = False UUID = False @@ -39,6 +38,11 @@ INCLUDE_MARKER_OPERATORS = { False: ">" } +MYSQL_CONFIG = "/etc/mysql/my.cnf" +MYSQL_SERVICE_CANDIDATES = ["mysql", "mysqld", "mysql-server"] +MYSQL_BIN_CANDIDATES = ["/usr/sbin/mysqld", "/usr/libexec/mysqld"] + + # Create a package impl packager = pkg.Package() @@ -47,12 +51,41 @@ def generate_random_password(): return passlib.utils.generate_password(size=CONF.default_password_length) +def clear_expired_password(): + """ + Some mysql installations generating random root password + and save it in /root/.mysql_secret, this password is + expired and should be changed by client that supports expired passwords. + """ + LOG.debug("Removing expired password.") + secret_file = "/root/.mysql_secret" + try: + out, err = utils.execute("cat", secret_file, + run_as_root=True, root_helper="sudo") + except exception.ProcessExecutionError: + LOG.debug("/root/.mysql_secret is not exists.") + return + m = re.match('# The random password set for the root user at .*: (.*)', + out) + if m: + try: + out, err = utils.execute("mysqladmin", "-p%s" % m.group(1), + "password", "", run_as_root=True, + root_helper="sudo") + except exception.ProcessExecutionError: + LOG.error("Cannot change mysql password.") + return + utils.execute("rm", "-f", secret_file, run_as_root=True, + root_helper="sudo") + LOG.debug("Expired password removed.") + + def get_auth_password(): pwd, err = utils.execute_with_timeout( "sudo", "awk", "/password\\t=/{print $3; exit}", - system.MYSQL_CONFIG) + MYSQL_CONFIG) if err: LOG.error(err) raise RuntimeError("Problem reading my.cnf! : %s" % err) @@ -76,8 +109,13 @@ def get_engine(): def load_mysqld_options(): + #find mysqld bin + for bin in MYSQL_BIN_CANDIDATES: + if os.path.isfile(bin): + mysqld_bin = bin + break try: - out, err = utils.execute(system.MYSQL_BIN, "--print-defaults", + out, err = utils.execute(mysqld_bin, "--print-defaults", run_as_root=True, root_helper="sudo") arglist = re.split("\n", out)[1].split() args = {} @@ -89,7 +127,7 @@ def load_mysqld_options(): args[item.lstrip("--")] = None return args except exception.ProcessExecutionError: - return None + return {} class MySqlAppStatus(service.BaseDbStatus): @@ -100,7 +138,6 @@ class MySqlAppStatus(service.BaseDbStatus): return cls._instance def _get_actual_db_status(self): - global MYSQLD_ARGS try: out, err = utils.execute_with_timeout( "/usr/bin/mysqladmin", @@ -119,10 +156,9 @@ class MySqlAppStatus(service.BaseDbStatus): LOG.info("Service Status is BLOCKED.") return rd_instance.ServiceStatuses.BLOCKED except exception.ProcessExecutionError: - if not MYSQLD_ARGS: - MYSQLD_ARGS = load_mysqld_options() - pid_file = MYSQLD_ARGS.get('pid_file', - '/var/run/mysqld/mysqld.pid') + mysql_args = load_mysqld_options() + pid_file = mysql_args.get('pid_file', + '/var/run/mysqld/mysqld.pid') if os.path.exists(pid_file): LOG.info("Service Status is CRASHED.") return rd_instance.ServiceStatuses.CRASHED @@ -492,10 +528,6 @@ class MySqlApp(object): """Prepares DBaaS on a Guest container.""" TIME_OUT = 1000 - if CONF.service_type == "mysql": - MYSQL_PACKAGE_VERSION = CONF.mysql_pkg - elif CONF.service_type == "percona": - MYSQL_PACKAGE_VERSION = CONF.percona_pkg def __init__(self, status): """ By default login with root no password for initial setup. """ @@ -522,11 +554,19 @@ class MySqlApp(object): t = text(str(uu)) client.execute(t) - def install_if_needed(self): + def install_if_needed(self, packages): """Prepare the guest machine with a secure mysql server installation""" LOG.info(_("Preparing Guest as MySQL Server")) - if not self.is_installed(): - self._install_mysql() + if not packager.pkg_is_installed(packages): + LOG.debug(_("Installing mysql server")) + self._clear_mysql_config() + # set blank password on pkg configuration stage + pkg_opts = {'root_password': '', + 'root_password_again': ''} + packager.pkg_install(packages, pkg_opts, self.TIME_OUT) + self._create_mysql_confd_dir() + LOG.debug(_("Finished installing mysql server")) + self.start_mysql() LOG.info(_("Dbaas install_if_needed complete")) def complete_install_or_restart(self): @@ -535,7 +575,7 @@ class MySqlApp(object): def secure(self, config_contents): LOG.info(_("Generating admin password...")) admin_password = generate_random_password() - + clear_expired_password() engine = sqlalchemy.create_engine("mysql://root:@localhost:3306", echo=True) with LocalSqlClient(engine) as client: @@ -549,22 +589,25 @@ class MySqlApp(object): LOG.info(_("Dbaas secure complete.")) def secure_root(self, secure_remote_root=True): - engine = sqlalchemy.create_engine("mysql://root:@localhost:3306", - echo=True) - with LocalSqlClient(engine) as client: + with LocalSqlClient(get_engine()) as client: LOG.info(_("Preserving root access from restore")) self._generate_root_password(client) if secure_remote_root: self._remove_remote_root_access(client) - def _install_mysql(self): - """Install mysql server. The current version is 5.5""" - LOG.debug(_("Installing mysql server")) - self._create_mysql_confd_dir() - packager.pkg_install(self.MYSQL_PACKAGE_VERSION, self.TIME_OUT) - self.start_mysql() - LOG.debug(_("Finished installing mysql server")) - #TODO(rnirmal): Add checks to make sure the package got installed + def _clear_mysql_config(self): + """Clear old configs, which can be incompatible with new version """ + LOG.debug("Clearing old mysql config") + random_uuid = str(uuid.uuid4()) + configs = ["/etc/my.cnf", "/etc/mysql/conf.d", "/etc/mysql/my.cnf"] + for config in configs: + command = "mv %s %s_%s" % (config, config, random_uuid) + try: + utils.execute_with_timeout(command, shell=True, + root_helper="sudo") + LOG.debug("%s saved to %s_%s" % (config, config, random_uuid)) + except exception.ProcessExecutionError: + pass def _create_mysql_confd_dir(self): conf_dir = "/etc/mysql/conf.d" @@ -573,44 +616,33 @@ class MySqlApp(object): utils.execute_with_timeout(command, shell=True) def _enable_mysql_on_boot(self): - """ - There is a difference between the init.d mechanism and the upstart - The stock mysql uses the upstart mechanism, therefore, there is a - mysql.conf file responsible for the job. to toggle enable/disable - on boot one needs to modify this file. Percona uses the init.d - mechanism and there is no mysql.conf file. Instead, the update-rc.d - command needs to be used to modify the /etc/rc#.d/[S/K]##mysql links - """ LOG.info("Enabling mysql on boot.") - conf = "/etc/init/mysql.conf" - if os.path.isfile(conf): - command = "sudo sed -i '/^manual$/d' %(conf)s" % {'conf': conf} - else: - command = system.MYSQL_CMD_ENABLE - utils.execute_with_timeout(command, shell=True) + try: + mysql_service = operating_system.service_discovery( + MYSQL_SERVICE_CANDIDATES) + utils.execute_with_timeout(mysql_service['cmd_enable'], shell=True) + except KeyError: + raise RuntimeError("Service is not discovered.") def _disable_mysql_on_boot(self): - """ - There is a difference between the init.d mechanism and the upstart - The stock mysql uses the upstart mechanism, therefore, there is a - mysql.conf file responsible for the job. to toggle enable/disable - on boot one needs to modify this file. Percona uses the init.d - mechanism and there is no mysql.conf file. Instead, the update-rc.d - command needs to be used to modify the /etc/rc#.d/[S/K]##mysql links - """ - LOG.info("Disabling mysql on boot.") - conf = "/etc/init/mysql.conf" - if os.path.isfile(conf): - command = "sudo sh -c 'echo manual >> %(conf)s'" % {'conf': conf} - else: - command = system.MYSQL_CMD_DISABLE - utils.execute_with_timeout(command, shell=True) + try: + mysql_service = operating_system.service_discovery( + MYSQL_SERVICE_CANDIDATES) + utils.execute_with_timeout(mysql_service['cmd_disable'], + shell=True) + except KeyError: + raise RuntimeError("Service is not discovered.") def stop_db(self, update_db=False, do_not_start_on_reboot=False): LOG.info(_("Stopping mysql...")) if do_not_start_on_reboot: self._disable_mysql_on_boot() - utils.execute_with_timeout(system.MYSQL_CMD_STOP, shell=True) + try: + mysql_service = operating_system.service_discovery( + MYSQL_SERVICE_CANDIDATES) + utils.execute_with_timeout(mysql_service['cmd_stop'], shell=True) + except KeyError: + raise RuntimeError("Service is not discovered.") if not self.status.wait_for_real_status_to_change_to( rd_instance.ServiceStatuses.SHUTDOWN, self.state_change_wait_time, update_db): @@ -696,13 +728,13 @@ class MySqlApp(object): with open(TMP_MYCNF, 'w') as t: t.write(config_contents) utils.execute_with_timeout("sudo", "mv", TMP_MYCNF, - system.MYSQL_CONFIG) + MYSQL_CONFIG) - self._write_temp_mycnf_with_admin_account(system.MYSQL_CONFIG, + self._write_temp_mycnf_with_admin_account(MYSQL_CONFIG, TMP_MYCNF, admin_password) utils.execute_with_timeout("sudo", "mv", TMP_MYCNF, - system.MYSQL_CONFIG) + MYSQL_CONFIG) self.wipe_ib_logfiles() @@ -715,8 +747,11 @@ class MySqlApp(object): self._enable_mysql_on_boot() try: - utils.execute_with_timeout(system. - MYSQL_CMD_START, shell=True) + mysql_service = operating_system.service_discovery( + MYSQL_SERVICE_CANDIDATES) + utils.execute_with_timeout(mysql_service['cmd_start'], shell=True) + except KeyError: + raise RuntimeError("Service is not discovered.") except exception.ProcessExecutionError: # it seems mysql (percona, at least) might come back with [Fail] # but actually come up ok. we're looking into the timing issue on @@ -756,11 +791,6 @@ class MySqlApp(object): LOG.info(_("Resetting configuration")) self._write_mycnf(None, config_contents) - def is_installed(self): - #(cp16net) could raise an exception, does it need to be handled here? - version = packager.pkg_version(self.MYSQL_PACKAGE_VERSION) - return not version is None - class MySqlRootAccess(object): @classmethod diff --git a/trove/guestagent/datastore/mysql/system.py b/trove/guestagent/datastore/mysql/system.py deleted file mode 100644 index d2217902..00000000 --- a/trove/guestagent/datastore/mysql/system.py +++ /dev/null @@ -1,51 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Determines operating system version and os depended commands. -""" -import os.path -from trove.common import cfg - -CONF = cfg.CONF - -REDHAT = 'redhat' -DEBIAN = 'debian' - -# The default is debian -OS = DEBIAN -MYSQL_CONFIG = "/etc/mysql/my.cnf" -MYSQL_BIN = "/usr/sbin/mysqld" -MYSQL_CMD_ENABLE = "sudo update-rc.d mysql enable" -MYSQL_CMD_DISABLE = "sudo update-rc.d mysql disable" -MYSQL_CMD_START = "sudo service mysql start || /bin/true" -MYSQL_CMD_STOP = "sudo service mysql stop || /bin/true" - -if os.path.isfile("/etc/redhat-release"): - OS = REDHAT - MYSQL_CONFIG = "/etc/my.cnf" - if CONF.service_type == 'percona': - MYSQL_CMD_ENABLE = "sudo chkconfig mysql on" - MYSQL_CMD_DISABLE = "sudo chkconfig mysql off" - MYSQL_CMD_START = "sudo service mysql start" - MYSQL_CMD_STOP = "sudo service mysql stop" - else: - MYSQL_BIN = "/usr/libexec/mysqld" - MYSQL_CMD_ENABLE = "sudo chkconfig mysqld on" - MYSQL_CMD_DISABLE = "sudo chkconfig mysqld off" - MYSQL_CMD_START = "sudo service mysqld start" - MYSQL_CMD_STOP = "sudo service mysqld stop" diff --git a/trove/guestagent/dbaas.py b/trove/guestagent/dbaas.py index 7f330094..4c82509f 100644 --- a/trove/guestagent/dbaas.py +++ b/trove/guestagent/dbaas.py @@ -42,10 +42,10 @@ CONF = cfg.CONF def get_custom_managers(): - return CONF.service_registry_ext + return CONF.datastore_registry_ext -def service_registry(): +def datastore_registry(): return dict(chain(defaults.iteritems(), get_custom_managers().iteritems())) diff --git a/trove/guestagent/pkg.py b/trove/guestagent/pkg.py index 080831e3..01f869fb 100644 --- a/trove/guestagent/pkg.py +++ b/trove/guestagent/pkg.py @@ -20,6 +20,7 @@ Manages packages on the Guest VM. """ import commands import re +from tempfile import NamedTemporaryFile import pexpect @@ -35,6 +36,7 @@ LOG = logging.getLogger(__name__) OK = 0 RUN_DPKG_FIRST = 1 REINSTALL_FIRST = 2 +CONFLICT_REMOVED = 3 class PkgAdminLockError(exception.TroveError): @@ -61,11 +63,15 @@ class PkgScriptletError(exception.TroveError): pass -class PkgTransactionCheckError(exception.TroveError): +class PkgDownloadError(exception.TroveError): pass -class PkgDownloadError(exception.TroveError): +class PkgSignError(exception.TroveError): + pass + + +class PkgBrokenError(exception.TroveError): pass @@ -84,16 +90,29 @@ class BasePackagerMixin: child = pexpect.spawn(cmd, timeout=time_out) try: i = child.expect(output_expects) + match = child.match self.pexpect_wait_and_close_proc(child) except pexpect.TIMEOUT: self.pexpect_kill_proc(child) raise PkgTimeout("Process timeout after %i seconds." % time_out) - return i + return (i, match) class RedhatPackagerMixin(BasePackagerMixin): - def _install(self, package_name, time_out): + def _rpm_remove_nodeps(self, package_name): + """ + Sometimes transaction errors happens, easy way is to remove + conflicted package without dependencies and hope it will replaced + by anoter package + """ + try: + utils.execute("rpm", "-e", "--nodeps", package_name, + run_as_root=True, root_helper="sudo") + except ProcessExecutionError: + LOG.error(_("Error removing conflict %s") % package_name) + + def _install(self, packages, time_out): """Attempts to install a package. Returns OK if the package installs fine or a result code if a @@ -101,27 +120,35 @@ class RedhatPackagerMixin(BasePackagerMixin): Raises an exception if a non-recoverable error or time out occurs. """ - cmd = "sudo yum --color=never -y install %s" % package_name + cmd = "sudo yum --color=never -y install %s" % packages output_expects = ['\[sudo\] password for .*:', - 'No package %s available.' % package_name, - 'Transaction Check Error:', + 'No package (.*) available.', + ('file .* from install of .* conflicts with file' + ' from package (.*?)\r\n'), + 'Error: (.*?) conflicts with .*?\r\n', + 'Processing Conflict: .* conflicts (.*?)\r\n', '.*scriptlet failed*', 'HTTP Error', 'No more mirrors to try.', + 'GPG key retrieval failed:', '.*already installed and latest version', 'Updated:', 'Installed:'] - i = self.pexpect_run(cmd, output_expects, time_out) + LOG.debug("Running package install command: %s" % cmd) + i, match = self.pexpect_run(cmd, output_expects, time_out) if i == 0: raise PkgPermissionError("Invalid permissions.") elif i == 1: - raise PkgNotFoundError("Could not find pkg %s" % package_name) - elif i == 2: - raise PkgTransactionCheckError("Transaction Check Error") - elif i == 3: + raise PkgNotFoundError("Could not find pkg %s" % match.group(1)) + elif i == 2 or i == 3 or i == 4: + self._rpm_remove_nodeps(match.group(1)) + return CONFLICT_REMOVED + elif i == 5: raise PkgScriptletError("Package scriptlet failed") - elif i == 4 or i == 5: + elif i == 6 or i == 7: raise PkgDownloadError("Package download problem") + elif i == 8: + raise PkgSignError("GPG key retrieval failed") return OK def _remove(self, package_name, time_out): @@ -136,18 +163,35 @@ class RedhatPackagerMixin(BasePackagerMixin): output_expects = ['\[sudo\] password for .*:', 'No Packages marked for removal', 'Removed:'] - i = self.pexpect_run(cmd, output_expects, time_out) + i, match = self.pexpect_run(cmd, output_expects, time_out) if i == 0: raise PkgPermissionError("Invalid permissions.") elif i == 1: raise PkgNotFoundError("Could not find pkg %s" % package_name) return OK - def pkg_install(self, package_name, time_out): - result = self._install(package_name, time_out) + def pkg_install(self, packages, config_opts, time_out): + result = self._install(packages, time_out) if result != OK: - raise PkgPackageStateError("Package %s is in a bad state." - % package_name) + while result == CONFLICT_REMOVED: + result = self._install(packages, time_out) + if result != OK: + raise PkgPackageStateError("Cannot install packages.") + + def pkg_is_installed(self, packages): + pkg_list = packages.split() + cmd = "rpm -qa" + p = commands.getstatusoutput(cmd) + std_out = p[1] + for pkg in pkg_list: + found = False + for line in std_out.split("\n"): + if line.find(pkg) != -1: + found = True + break + if not found: + return False + return True def pkg_version(self, package_name): cmd_list = ["rpm", "-qa", "--qf", "'%{VERSION}-%{RELEASE}\n'", @@ -185,33 +229,71 @@ class DebianPackagerMixin(BasePackagerMixin): except ProcessExecutionError: LOG.error(_("Error fixing dpkg")) - def _install(self, package_name, time_out): - """Attempts to install a package. + def _fix_package_selections(self, packages, config_opts): + """ + Sometimes you have to run this command before a pkg will install. + This command sets package selections to configure package. + """ + selections = "" + for package in packages: + m = re.match('(.+)=(.+)', package) + if m: + package_name = m.group(1) + else: + package_name = package + command = "sudo debconf-show %s" % package_name + p = commands.getstatusoutput(command) + std_out = p[1] + for line in std_out.split("\n"): + for selection, value in config_opts.items(): + m = re.match(".* (.*/%s):.*" % selection, line) + if m: + selections += ("%s %s string '%s'\n" % + (package_name, m.group(1), value)) + if selections: + with NamedTemporaryFile(delete=False) as f: + fname = f.name + f.write(selections) + utils.execute("debconf-set-selections %s && dpkg --configure -a" + % fname, run_as_root=True, root_helper="sudo", + shell=True) + os.remove(fname) + + def _install(self, packages, time_out): + """Attempts to install a packages. Returns OK if the package installs fine or a result code if a recoverable-error occurred. Raises an exception if a non-recoverable error or time out occurs. """ - cmd = "sudo -E DEBIAN_FRONTEND=noninteractive " \ - "apt-get -y --allow-unauthenticated install %s" % package_name + cmd = "sudo -E DEBIAN_FRONTEND=noninteractive apt-get -y " \ + "--force-yes --allow-unauthenticated -o " \ + "DPkg::options::=--force-confmiss --reinstall " \ + "install %s" % packages output_expects = ['.*password*', - 'E: Unable to locate package %s' % package_name, - "Couldn't find package % s" % package_name, + 'E: Unable to locate package (.*)', + "Couldn't find package (.*)", + "E: Version '.*' for '(.*)' was not found", ("dpkg was interrupted, you must manually run " "'sudo dpkg --configure -a'"), "Unable to lock the administration directory", - "Setting up %s*" % package_name, + ("E: Unable to correct problems, you have held " + "broken packages."), + "Setting up (.*)", "is already the newest version"] - i = self.pexpect_run(cmd, output_expects, time_out) + LOG.debug("Running package install command: %s" % cmd) + i, match = self.pexpect_run(cmd, output_expects, time_out) if i == 0: raise PkgPermissionError("Invalid permissions.") - elif i == 1 or i == 2: - raise PkgNotFoundError("Could not find apt %s" % package_name) - elif i == 3: - return RUN_DPKG_FIRST + elif i == 1 or i == 2 or i == 3: + raise PkgNotFoundError("Could not find apt %s" % match.group(1)) elif i == 4: + return RUN_DPKG_FIRST + elif i == 5: raise PkgAdminLockError() + elif i == 6: + raise PkgBrokenError() return OK def _remove(self, package_name, time_out): @@ -232,7 +314,7 @@ class DebianPackagerMixin(BasePackagerMixin): "'sudo dpkg --configure -a'"), "Unable to lock the administration directory", "Removing %s*" % package_name] - i = self.pexpect_run(cmd, output_expects, time_out) + i, match = self.pexpect_run(cmd, output_expects, time_out) if i == 0: raise PkgPermissionError("Invalid permissions.") elif i == 1: @@ -245,58 +327,54 @@ class DebianPackagerMixin(BasePackagerMixin): raise PkgAdminLockError() return OK - def pkg_install(self, package_name, time_out): - """Installs a package.""" + def pkg_install(self, packages, config_opts, time_out): + """Installs a packages.""" try: utils.execute("apt-get", "update", run_as_root=True, root_helper="sudo") except ProcessExecutionError: LOG.error(_("Error updating the apt sources")) - result = self._install(package_name, time_out) + result = self._install(packages, time_out) if result != OK: if result == RUN_DPKG_FIRST: self._fix(time_out) - result = self._install(package_name, time_out) + result = self._install(packages, time_out) if result != OK: - raise PkgPackageStateError("Package %s is in a bad state." - % package_name) + raise PkgPackageStateError("Packages is in a bad state.") + # even after successful install, packages can stay unconfigured + # config_opts - is dict with name/value for questions asked by + # interactive configure script + self._fix_package_selections(packages, config_opts) def pkg_version(self, package_name): - cmd_list = ["dpkg", "-l", package_name] - p = commands.getstatusoutput(' '.join(cmd_list)) - # check the command status code - if not p[0] == 0: - return None - # Need to capture the version string - # check the command output + p = commands.getstatusoutput("apt-cache policy %s" % package_name) std_out = p[1] - patterns = ['.*No packages found matching.*', - "\w\w\s+(\S+)\s+(\S+)\s+(.*)$"] for line in std_out.split("\n"): - for p in patterns: - regex = re.compile(p) - matches = regex.match(line) - if matches: - line = matches.group() - parts = line.split() - if not parts: - msg = _("returned nothing") - LOG.error(msg) - raise exception.GuestError(msg) - if len(parts) <= 2: - msg = _("Unexpected output.") - LOG.error(msg) - raise exception.GuestError(msg) - if parts[1] != package_name: - msg = _("Unexpected output:[1] = %s") % str(parts[1]) - LOG.error(msg) - raise exception.GuestError(msg) - if parts[0] == 'un' or parts[2] == '<none>': - return None - return parts[2] - msg = _("version() saw unexpected output from dpkg!") - LOG.error(msg) + m = re.match("\s+Installed: (.*)", line) + if m: + version = m.group(1) + if version == "(none)": + version = None + return version + + def pkg_is_installed(self, packages): + pkg_list = packages.split() + for pkg in pkg_list: + m = re.match('(.+)=(.+)', pkg) + if m: + package_name = m.group(1) + package_version = m.group(2) + else: + package_name = pkg + package_version = None + installed_version = self.pkg_version(package_name) + if ((package_version and installed_version == package_version) or + (installed_version and not package_version)): + LOG.debug(_("Package %s already installed.") % package_name) + else: + return False + return True def pkg_remove(self, package_name, time_out): """Removes a package.""" diff --git a/trove/instance/models.py b/trove/instance/models.py index fdda0095..86bd4b30 100644 --- a/trove/instance/models.py +++ b/trove/instance/models.py @@ -30,6 +30,7 @@ from trove.common import utils from trove.extensions.security_group.models import SecurityGroup from trove.db import get_db_api from trove.db import models as dbmodels +from trove.datastore import models as datastore_models from trove.backup.models import Backup from trove.quota.quota import run_with_quotas from trove.instance.tasks import InstanceTask @@ -121,6 +122,10 @@ class SimpleInstance(object): self.db_info = db_info self.service_status = service_status self.root_pass = root_password + self.ds_version = (datastore_models.DatastoreVersion. + load(self.db_info.datastore_version_id)) + self.ds = (datastore_models.Datastore. + load(self.ds_version.datastore_id)) @property def addresses(self): @@ -227,8 +232,12 @@ class SimpleInstance(object): return self.db_info.volume_size @property - def service_type(self): - return self.db_info.service_type + def datastore_version(self): + return self.ds_version + + @property + def datastore(self): + return self.ds @property def root_password(self): @@ -426,8 +435,8 @@ class Instance(BuiltInstance): """ @classmethod - def create(cls, context, name, flavor_id, image_id, - databases, users, service_type, volume_size, backup_id, + def create(cls, context, name, flavor_id, image_id, databases, users, + datastore, datastore_version, volume_size, backup_id, availability_zone=None): client = create_nova_client(context) @@ -463,7 +472,8 @@ class Instance(BuiltInstance): db_info = DBInstance.create(name=name, flavor_id=flavor_id, tenant_id=context.tenant, volume_size=volume_size, - service_type=service_type, + datastore_version_id= + datastore_version.id, task_status=InstanceTasks.BUILDING) LOG.debug(_("Tenant %(tenant)s created new " "Trove instance %(db)s...") % @@ -485,8 +495,9 @@ class Instance(BuiltInstance): task_api.API(context).create_instance(db_info.id, name, flavor, image_id, databases, users, - service_type, volume_size, - backup_id, + datastore.manager, + datastore_version.packages, + volume_size, backup_id, availability_zone, root_password) @@ -694,7 +705,8 @@ class DBInstance(dbmodels.DatabaseModelBase): _data_fields = ['name', 'created', 'compute_instance_id', 'task_id', 'task_description', 'task_start_time', - 'volume_id', 'deleted', 'tenant_id', 'service_type'] + 'volume_id', 'deleted', 'tenant_id', + 'datastore_version_id'] def __init__(self, task_status, **kwargs): kwargs["task_id"] = task_status.code @@ -719,12 +731,6 @@ class DBInstance(dbmodels.DatabaseModelBase): task_status = property(get_task_status, set_task_status) -class ServiceImage(dbmodels.DatabaseModelBase): - """Defines the status of the service being run.""" - - _data_fields = ['service_name', 'image_id'] - - class InstanceServiceStatus(dbmodels.DatabaseModelBase): _data_fields = ['instance_id', 'status_id', 'status_description', 'updated_at'] @@ -758,7 +764,6 @@ class InstanceServiceStatus(dbmodels.DatabaseModelBase): def persisted_models(): return { 'instance': DBInstance, - 'service_image': ServiceImage, 'service_statuses': InstanceServiceStatus, } diff --git a/trove/instance/service.py b/trove/instance/service.py index b668353c..30c0ddd4 100644 --- a/trove/instance/service.py +++ b/trove/instance/service.py @@ -25,6 +25,7 @@ from trove.common import wsgi from trove.extensions.mysql.common import populate_validated_databases from trove.extensions.mysql.common import populate_users from trove.instance import models, views +from trove.datastore import models as datastore_models from trove.backup.models import Backup as backup_model from trove.backup import views as backup_views from trove.openstack.common import log as logging @@ -178,11 +179,10 @@ class InstanceController(wsgi.Controller): LOG.info(_("req : '%s'\n\n") % req) LOG.info(_("body : '%s'\n\n") % body) context = req.environ[wsgi.CONTEXT_KEY] - # Set the service type to mysql if its not in the request - service_type = (body['instance'].get('service_type') or - CONF.service_type) - service = models.ServiceImage.find_by(service_name=service_type) - image_id = service['image_id'] + datastore_args = body['instance'].get('datastore', {}) + datastore, datastore_version = ( + datastore_models.get_datastore_version(**datastore_args)) + image_id = datastore_version.image_id name = body['instance']['name'] flavor_ref = body['instance']['flavorRef'] flavor_id = utils.get_id_from_href(flavor_ref) @@ -214,8 +214,9 @@ class InstanceController(wsgi.Controller): instance = models.Instance.create(context, name, flavor_id, image_id, databases, users, - service_type, volume_size, - backup_id, availability_zone) + datastore, datastore_version, + volume_size, backup_id, + availability_zone) view = views.InstanceDetailView(instance, req=req) return wsgi.Result(view.data(), 200) diff --git a/trove/instance/views.py b/trove/instance/views.py index b0693418..2786929f 100644 --- a/trove/instance/views.py +++ b/trove/instance/views.py @@ -55,6 +55,7 @@ class InstanceView(object): "status": self.instance.status, "links": self._build_links(), "flavor": self._build_flavor_info(), + "datastore": {"type": self.instance.datastore.name}, } if CONF.trove_volume_support: instance_dict['volume'] = {'size': self.instance.volume_size} @@ -88,6 +89,9 @@ class InstanceDetailView(InstanceView): result['instance']['created'] = self.instance.created result['instance']['updated'] = self.instance.updated + result['instance']['datastore']['version'] = (self.instance. + datastore_version.name) + dns_support = CONF.trove_dns_support if dns_support: result['instance']['hostname'] = self.instance.hostname diff --git a/trove/taskmanager/api.py b/trove/taskmanager/api.py index d56128e9..f96eb4fc 100644 --- a/trove/taskmanager/api.py +++ b/trove/taskmanager/api.py @@ -99,10 +99,10 @@ class API(proxy.RpcProxy): self.cast(self.context, self.make_msg("delete_backup", backup_id=backup_id)) - def create_instance(self, instance_id, name, flavor, image_id, - databases, users, service_type, volume_size, - backup_id=None, availability_zone=None, - root_password=None): + def create_instance(self, instance_id, name, flavor, + image_id, databases, users, datastore_manager, + packages, volume_size, backup_id=None, + availability_zone=None, root_password=None): LOG.debug("Making async call to create instance %s " % instance_id) self.cast(self.context, self.make_msg("create_instance", @@ -111,7 +111,8 @@ class API(proxy.RpcProxy): image_id=image_id, databases=databases, users=users, - service_type=service_type, + datastore_manager=datastore_manager, + packages=packages, volume_size=volume_size, backup_id=backup_id, availability_zone=availability_zone, diff --git a/trove/taskmanager/manager.py b/trove/taskmanager/manager.py index 5bba31ca..0b7038cf 100644 --- a/trove/taskmanager/manager.py +++ b/trove/taskmanager/manager.py @@ -81,12 +81,13 @@ class Manager(periodic_task.PeriodicTasks): instance_tasks.create_backup(backup_id) def create_instance(self, context, instance_id, name, flavor, - image_id, databases, users, service_type, - volume_size, backup_id, availability_zone, + image_id, databases, users, datastore_manager, + packages, volume_size, backup_id, availability_zone, root_password): instance_tasks = FreshInstanceTasks.load(context, instance_id) instance_tasks.create_instance(flavor, image_id, databases, users, - service_type, volume_size, backup_id, + datastore_manager, packages, + volume_size, backup_id, availability_zone, root_password) if CONF.exists_notification_transformer: diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index c7dd8b50..eb3665b9 100644 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -69,14 +69,14 @@ class NotifyMixin(object): This adds the ability to send usage events to an Instance object. """ - def _get_service_id(self, service_type, id_map): - if service_type in id_map: - service_type_id = id_map[service_type] + def _get_service_id(self, datastore_manager, id_map): + if datastore_manager in id_map: + datastore_manager_id = id_map[datastore_manager] else: - service_type_id = cfg.UNKNOWN_SERVICE_ID - LOG.error(_("Service ID for Type (%s) is not configured") - % service_type) - return service_type_id + datastore_manager_id = cfg.UNKNOWN_SERVICE_ID + LOG.error("Datastore ID for Manager (%s) is not configured" + % datastore_manager) + return datastore_manager_id def send_usage_event(self, event_type, **kwargs): event_type = 'trove.instance.%s' % event_type @@ -117,7 +117,7 @@ class NotifyMixin(object): }) payload['service_id'] = self._get_service_id( - self.service_type, CONF.notification_service_id) + self.datastore.manager, CONF.notification_service_id) # Update payload with all other kwargs payload.update(kwargs) @@ -133,17 +133,17 @@ class ConfigurationMixin(object): Configuration related tasks for instances and resizes. """ - def _render_config(self, service_type, flavor, instance_id): + def _render_config(self, datastore_manager, flavor, instance_id): config = template.SingleInstanceConfigTemplate( - service_type, flavor, instance_id) + datastore_manager, flavor, instance_id) config.render() return config class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): def create_instance(self, flavor, image_id, databases, users, - service_type, volume_size, backup_id, - availability_zone, root_password): + datastore_manager, packages, volume_size, + backup_id, availability_zone, root_password): LOG.debug(_("begin create_instance for id: %s") % self.id) security_groups = None @@ -170,7 +170,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): flavor, image_id, security_groups, - service_type, + datastore_manager, volume_size, availability_zone) elif use_nova_server_volume: @@ -178,7 +178,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): flavor['id'], image_id, security_groups, - service_type, + datastore_manager, volume_size, availability_zone) else: @@ -186,15 +186,15 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): flavor['id'], image_id, security_groups, - service_type, + datastore_manager, volume_size, availability_zone) - config = self._render_config(service_type, flavor, self.id) + config = self._render_config(datastore_manager, flavor, self.id) if server: self._guest_prepare(server, flavor['ram'], volume_info, - databases, users, backup_id, + packages, databases, users, backup_id, config.config_contents, root_password) if not self.db_info.task_status.is_error: @@ -285,15 +285,15 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): return False def _create_server_volume(self, flavor_id, image_id, security_groups, - service_type, volume_size, + datastore_manager, volume_size, availability_zone): LOG.debug(_("begin _create_server_volume for id: %s") % self.id) server = None try: - files = {"/etc/guest_info": ("[DEFAULT]\n--guest_id=%s\n" - "--service_type=%s\n" + files = {"/etc/guest_info": ("[DEFAULT]\n--guest_id=" + "%s\n--datastore_manager=%s\n" "--tenant_id=%s\n" % - (self.id, service_type, + (self.id, datastore, self.tenant_id))} name = self.hostname or self.name volume_desc = ("mysql volume for %s" % self.id) @@ -332,14 +332,14 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): return server, volume_info def _create_server_volume_heat(self, flavor, image_id, - security_groups, service_type, + security_groups, datastore_manager, volume_size, availability_zone): LOG.debug(_("begin _create_server_volume_heat for id: %s") % self.id) client = create_heat_client(self.context) novaclient = create_nova_client(self.context) cinderclient = create_cinder_client(self.context) - template_obj = template.load_heat_template(service_type) + template_obj = template.load_heat_template(datastore_manager) heat_template_unicode = template_obj.render() try: heat_template = heat_template_unicode.encode('ascii') @@ -351,6 +351,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): "VolumeSize": volume_size, "InstanceId": self.id, "ImageId": image_id, + "DatastoreManager": datastore_manager, "AvailabilityZone": availability_zone} stack_name = 'trove-%s' % self.id client.stacks.create(stack_name=stack_name, @@ -377,7 +378,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): return server, volume_info def _create_server_volume_individually(self, flavor_id, image_id, - security_groups, service_type, + security_groups, datastore_manager, volume_size, availability_zone): LOG.debug(_("begin _create_server_volume_individually for id: %s") % @@ -387,7 +388,8 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): block_device_mapping = volume_info['block_device'] try: server = self._create_server(flavor_id, image_id, security_groups, - service_type, block_device_mapping, + datastore_manager, + block_device_mapping, availability_zone) server_id = server.id # Save server ID. @@ -477,17 +479,19 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): return volume_info def _create_server(self, flavor_id, image_id, security_groups, - service_type, block_device_mapping, + datastore_manager, block_device_mapping, availability_zone): files = {"/etc/guest_info": ("[DEFAULT]\nguest_id=%s\n" - "service_type=%s\n" "tenant_id=%s\n" % - (self.id, service_type, self.tenant_id))} + "datastore_manager=%s\n" + "tenant_id=%s\n" % + (self.id, datastore_manager, + self.tenant_id))} if os.path.isfile(CONF.get('guest_config')): with open(CONF.get('guest_config'), "r") as f: files["/etc/trove-guestagent.conf"] = f.read() userdata = None cloudinit = os.path.join(CONF.get('cloudinit_location'), - "%s.cloudinit" % service_type) + "%s.cloudinit" % datastore_manager) if os.path.isfile(cloudinit): with open(cloudinit, "r") as f: userdata = f.read() @@ -503,11 +507,11 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): return server def _guest_prepare(self, server, flavor_ram, volume_info, - databases, users, backup_id=None, + packages, databases, users, backup_id=None, config_contents=None, root_password=None): LOG.info(_("Entering guest_prepare")) # Now wait for the response from the create to do additional work - self.guest.prepare(flavor_ram, databases, users, + self.guest.prepare(flavor_ram, packages, databases, users, device_path=volume_info['device_path'], mount_point=volume_info['mount_point'], backup_id=backup_id, @@ -1007,7 +1011,7 @@ class ResizeAction(ResizeActionBase): % self.instance.id) LOG.debug(_("Repairing config.")) try: - config = self._render_config(self.instance.service_type, + config = self._render_config(self.instance.datastore.manager, self.old_flavor, self.instance.id) config = {'config_contents': config.config_contents} self.instance.guest.reset_configuration(config) @@ -1028,7 +1032,7 @@ class ResizeAction(ResizeActionBase): modify_at=timeutils.isotime(self.instance.updated)) def _start_mysql(self): - config = self._render_config(self.instance.service_type, + config = self._render_config(self.instance.datastore.manager, self.new_flavor, self.instance.id) self.instance.guest.start_db_with_conf_changes(config.config_contents) diff --git a/trove/templates/mysql/heat.template b/trove/templates/mysql/heat.template index 1bb2fc3c..d09ec9ac 100644 --- a/trove/templates/mysql/heat.template +++ b/trove/templates/mysql/heat.template @@ -10,6 +10,8 @@ Parameters: Type: String ImageId: Type: String + DatastoreManager: + Type: String AvailabilityZone: Type: String Default: nova @@ -25,7 +27,7 @@ Resources: Fn::Join: - '' - ["[DEFAULT]\nguest_id=", {Ref: InstanceId}, - "\nservice_type=mysql"] + "\\ndatastore_manager=", {Ref: DatastoreManager}] mode: '000644' owner: root group: root @@ -69,4 +71,4 @@ Resources: Type: AWS::EC2::EIPAssociation Properties: InstanceId: {Ref: BaseInstance} - EIP: {Ref: DatabaseIPAddress}
\ No newline at end of file + EIP: {Ref: DatabaseIPAddress} diff --git a/trove/tests/api/datastores.py b/trove/tests/api/datastores.py new file mode 100644 index 00000000..c8fcb4aa --- /dev/null +++ b/trove/tests/api/datastores.py @@ -0,0 +1,114 @@ +# Copyright (c) 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from nose.tools import assert_equal +from nose.tools import assert_false +from nose.tools import assert_true +from troveclient.compat import exceptions + +from proboscis import before_class +from proboscis import test +from proboscis.asserts import assert_raises +from proboscis import SkipTest + +from trove import tests +from trove.tests.util import create_dbaas_client +from trove.tests.util import test_config +from trove.tests.util.users import Requirements +from trove.tests.util.check import TypeCheck + +GROUP = "dbaas.api.datastores" +NAME = "nonexistent" + + +@test(groups=[tests.DBAAS_API, GROUP, tests.PRE_INSTANCES], + depends_on_groups=["services.initialize"]) +class Datastores(object): + + @before_class + def setUp(self): + rd_user = test_config.users.find_user( + Requirements(is_admin=False, services=["trove"])) + self.rd_client = create_dbaas_client(rd_user) + + @test + def test_datastore_list_attrs(self): + datastores = self.rd_client.datastores.list() + for datastore in datastores: + with TypeCheck('Datastore', datastore) as check: + check.has_field("id", basestring) + check.has_field("name", basestring) + check.has_field("links", list) + + @test + def test_datastore_get_attrs(self): + datastore = self.rd_client.datastores.get(test_config. + dbaas_datastore) + with TypeCheck('Datastore', datastore) as check: + check.has_field("id", basestring) + check.has_field("name", basestring) + check.has_field("links", list) + assert_equal(datastore.name, test_config.dbaas_datastore) + + @test + def test_datastore_not_found(self): + try: + assert_raises(exceptions.NotFound, + self.rd_client.datastores.get, NAME) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Datastore '%s' cannot be found." % NAME) + + @test + def test_datastore_version_list_attrs(self): + versions = self.rd_client.datastore_versions.list(test_config. + dbaas_datastore) + for version in versions: + with TypeCheck('DatastoreVersion', version) as check: + check.has_field("id", basestring) + check.has_field("name", basestring) + check.has_field("links", list) + + @test + def test_datastore_version_get_attrs(self): + version = self.rd_client.datastore_versions.get( + test_config.dbaas_datastore, test_config.dbaas_datastore_version) + with TypeCheck('DatastoreVersion', version) as check: + check.has_field("id", basestring) + check.has_field("name", basestring) + check.has_field("links", list) + assert_equal(version.name, test_config.dbaas_datastore_version) + + @test + def test_datastore_version_datastore_not_found(self): + try: + assert_raises(exceptions.NotFound, + self.rd_client.datastore_versions.get, + NAME, NAME) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Datastore '%s' cannot be found." % NAME) + + @test + def test_datastore_version_not_found(self): + try: + assert_raises(exceptions.NotFound, + self.rd_client.datastore_versions.get, + test_config.dbaas_datastore, NAME) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Datastore version '%s' cannot be found." % NAME) diff --git a/trove/tests/api/instances.py b/trove/tests/api/instances.py index cfa5cd6c..03668b20 100644 --- a/trove/tests/api/instances.py +++ b/trove/tests/api/instances.py @@ -36,6 +36,7 @@ GROUP_SECURITY_GROUPS = "dbaas.api.security_groups" from datetime import datetime from time import sleep +from trove.datastore import models as datastore_models from trove.common import exception as rd_exceptions from troveclient.compat import exceptions @@ -65,6 +66,9 @@ from trove.tests.util import string_in_list from trove.common.utils import poll_until from trove.tests.util.check import AttrCheck from trove.tests.util.check import TypeCheck +from trove.tests.util import test_config + +FAKE = test_config.values['fake_mode'] class InstanceTestInfo(object): @@ -75,8 +79,8 @@ class InstanceTestInfo(object): self.dbaas_admin = None # The rich client with admin access. self.dbaas_flavor = None # The flavor object of the instance. self.dbaas_flavor_href = None # The flavor of the instance. - self.dbaas_image = None # The image used to create the instance. - self.dbaas_image_href = None # The link of the image. + self.dbaas_datastore = None # The datastore id + self.dbaas_datastore_version = None # The datastore version id self.id = None # The ID of the instance in the database. self.local_id = None self.address = None @@ -162,7 +166,7 @@ def clear_messages_off_queue(): class InstanceSetup(object): """Makes sure the client can hit the ReST service. - This test also uses the API to find the image and flavor to use. + This test also uses the API to find the flavor to use. """ @@ -223,6 +227,7 @@ class CreateInstanceQuotaTest(unittest.TestCase): import copy self.test_info = copy.deepcopy(instance_info) + self.test_info.dbaas_datastore = CONFIG.dbaas_datastore def tearDown(self): quota_dict = {'instances': CONFIG.trove_max_instances_per_user} @@ -323,6 +328,7 @@ class CreateInstance(object): users.append({"name": "lite", "password": "litepass", "databases": [{"name": "firstdb"}]}) instance_info.users = users + instance_info.dbaas_datastore = CONFIG.dbaas_datastore if VOLUME_SUPPORT: instance_info.volume = {'size': 1} else: @@ -335,7 +341,9 @@ class CreateInstance(object): instance_info.volume, databases, users, - availability_zone="nova") + availability_zone="nova", + datastore=instance_info.dbaas_datastore, + datastore_version=instance_info.dbaas_datastore_version) assert_equal(200, dbaas.last_http_code) else: id = existing_instance() @@ -355,7 +363,7 @@ class CreateInstance(object): # Check these attrs only are returned in create response expected_attrs = ['created', 'flavor', 'addresses', 'id', 'links', - 'name', 'status', 'updated'] + 'name', 'status', 'updated', 'datastore'] if ROOT_ON_CREATE: expected_attrs.append('password') if VOLUME_SUPPORT: @@ -369,6 +377,7 @@ class CreateInstance(object): msg="Create response") # Don't CheckInstance if the instance already exists. check.flavor() + check.datastore() check.links(result._info['links']) if VOLUME_SUPPORT: check.volume() @@ -454,7 +463,7 @@ class CreateInstance(object): result = dbaas_admin.management.show(instance_info.id) expected_attrs = ['account_id', 'addresses', 'created', 'databases', 'flavor', 'guest_status', 'host', - 'hostname', 'id', 'name', + 'hostname', 'id', 'name', 'datastore', 'server_state_description', 'status', 'updated', 'users', 'volume', 'root_enabled_at', 'root_enabled_by'] @@ -462,8 +471,122 @@ class CreateInstance(object): check.attrs_exist(result._info, expected_attrs, msg="Mgmt get instance") check.flavor() + check.datastore() check.guest_status() + @test + def test_create_failure_with_datastore_default_notfound(self): + if not FAKE: + raise SkipTest("This test only for fake mode.") + if VOLUME_SUPPORT: + volume = {'size': 1} + else: + volume = None + instance_name = "datastore_default_notfound" + databases = [] + users = [] + origin_default_datastore = (datastore_models.CONF. + default_datastore) + datastore_models.CONF.default_datastore = "" + try: + assert_raises(exceptions.NotFound, + dbaas.instances.create, instance_name, + instance_info.dbaas_flavor_href, + volume, databases, users) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Please specify datastore.") + datastore_models.CONF.default_datastore = \ + origin_default_datastore + + @test + def test_create_failure_with_datastore_default_version_notfound(self): + if VOLUME_SUPPORT: + volume = {'size': 1} + else: + volume = None + instance_name = "datastore_default_version_notfound" + databases = [] + users = [] + datastore = "Test_Datastore_1" + try: + assert_raises(exceptions.NotFound, + dbaas.instances.create, instance_name, + instance_info.dbaas_flavor_href, + volume, databases, users, + datastore=datastore) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Default version for datastore '%s' not found." % + datastore) + + @test + def test_create_failure_with_datastore_notfound(self): + if VOLUME_SUPPORT: + volume = {'size': 1} + else: + volume = None + instance_name = "datastore_notfound" + databases = [] + users = [] + datastore = "nonexistent" + try: + assert_raises(exceptions.NotFound, + dbaas.instances.create, instance_name, + instance_info.dbaas_flavor_href, + volume, databases, users, + datastore=datastore) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Datastore '%s' cannot be found." % + datastore) + + @test + def test_create_failure_with_datastore_version_notfound(self): + if VOLUME_SUPPORT: + volume = {'size': 1} + else: + volume = None + instance_name = "datastore_version_notfound" + databases = [] + users = [] + datastore = "Test_Mysql" + datastore_version = "nonexistent" + try: + assert_raises(exceptions.NotFound, + dbaas.instances.create, instance_name, + instance_info.dbaas_flavor_href, + volume, databases, users, + datastore=datastore, + datastore_version=datastore_version) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Datastore version '%s' cannot be found." % + datastore_version) + + @test + def test_create_failure_with_datastore_version_inactive(self): + if VOLUME_SUPPORT: + volume = {'size': 1} + else: + volume = None + instance_name = "datastore_version_inactive" + databases = [] + users = [] + datastore = "Test_Mysql" + datastore_version = "mysql_inactive_version" + try: + assert_raises(exceptions.NotFound, + dbaas.instances.create, instance_name, + instance_info.dbaas_flavor_href, + volume, databases, users, + datastore=datastore, + datastore_version=datastore_version) + except exceptions.BadRequest as e: + assert_equal(e.message, + "Datastore version '%s' is not active." % + datastore_version) + def assert_unprocessable(func, *args): try: @@ -730,7 +853,8 @@ class TestInstanceListing(object): @test def test_index_list(self): - expected_attrs = ['id', 'links', 'name', 'status', 'flavor'] + expected_attrs = ['id', 'links', 'name', 'status', 'flavor', + 'datastore'] if VOLUME_SUPPORT: expected_attrs.append('volume') instances = dbaas.instances.list() @@ -743,12 +867,14 @@ class TestInstanceListing(object): msg="Instance Index") check.links(instance_dict['links']) check.flavor() + check.datastore() check.volume() @test def test_get_instance(self): expected_attrs = ['created', 'databases', 'flavor', 'hostname', 'id', - 'links', 'name', 'status', 'updated', 'ip'] + 'links', 'name', 'status', 'updated', 'ip', + 'datastore'] if VOLUME_SUPPORT: expected_attrs.append('volume') else: @@ -761,6 +887,7 @@ class TestInstanceListing(object): check.attrs_exist(instance_dict, expected_attrs, msg="Get Instance") check.flavor() + check.datastore() check.links(instance_dict['links']) check.used_volume() @@ -835,12 +962,13 @@ class TestInstanceListing(object): expected_attrs = ['account_id', 'addresses', 'created', 'databases', 'flavor', 'guest_status', 'host', 'hostname', 'id', 'name', 'root_enabled_at', 'root_enabled_by', - 'server_state_description', 'status', + 'server_state_description', 'status', 'datastore', 'updated', 'users', 'volume'] with CheckInstance(result._info) as check: check.attrs_exist(result._info, expected_attrs, msg="Mgmt get instance") check.flavor() + check.datastore() check.guest_status() check.addresses() check.volume_mgmt() @@ -1063,6 +1191,14 @@ class CheckInstance(AttrCheck): msg="Flavor") self.links(self.instance['flavor']['links']) + def datastore(self): + if 'datastore' not in self.instance: + self.fail("'datastore' not found in instance.") + else: + expected_attrs = ['type', 'version'] + self.attrs_exist(self.instance['datastore'], expected_attrs, + msg="datastore") + def volume_key_exists(self): if 'volume' not in self.instance: self.fail("'volume' not found in instance.") diff --git a/trove/tests/api/instances_resize.py b/trove/tests/api/instances_resize.py index 89e69d01..cf15df68 100644 --- a/trove/tests/api/instances_resize.py +++ b/trove/tests/api/instances_resize.py @@ -31,6 +31,7 @@ from trove.instance.tasks import InstanceTasks from trove.openstack.common.rpc.common import RPCException from trove.taskmanager import models as models from trove.tests.fakes import nova +from trove.tests.util import test_config GROUP = 'dbaas.api.instances.resize' @@ -51,7 +52,7 @@ class ResizeTestBase(TestCase): flavor_id=OLD_FLAVOR_ID, tenant_id=999, volume_size=None, - service_type='mysql', + datastore_version_id=test_config.dbaas_datastore_version, task_status=InstanceTasks.RESIZING) self.server = self.mock.CreateMock(Server) self.instance = models.BuiltInstanceTasks(context, diff --git a/trove/tests/api/mgmt/instances.py b/trove/tests/api/mgmt/instances.py index b83cd598..76b4fd60 100644 --- a/trove/tests/api/mgmt/instances.py +++ b/trove/tests/api/mgmt/instances.py @@ -53,6 +53,12 @@ def flavor_check(flavor): check.has_element("links", list) +def datastore_check(datastore): + with CollectionCheck("datastore", datastore) as check: + check.has_element("type", basestring) + check.has_element("version", basestring) + + def guest_status_check(guest_status): with CollectionCheck("guest_status", guest_status) as check: check.has_element("state_description", basestring) @@ -87,6 +93,7 @@ def mgmt_instance_get(): # lets avoid creating more ordering work. instance.has_field('deleted_at', (basestring, None)) instance.has_field('flavor', dict, flavor_check) + instance.has_field('datastore', dict, datastore_check) instance.has_field('guest_status', dict, guest_status_check) instance.has_field('id', basestring) instance.has_field('links', list) @@ -175,6 +182,7 @@ class WhenMgmtInstanceGetIsCalledButServerIsNotReady(object): # lets avoid creating more ordering work. instance.has_field('deleted_at', (basestring, None)) instance.has_field('flavor', dict, flavor_check) + instance.has_field('datastore', dict, datastore_check) instance.has_field('guest_status', dict, guest_status_check) instance.has_field('id', basestring) instance.has_field('links', list) @@ -211,6 +219,7 @@ class MgmtInstancesIndex(object): 'deleted', 'deleted_at', 'flavor', + 'datastore', 'id', 'links', 'name', diff --git a/trove/tests/api/mgmt/instances_actions.py b/trove/tests/api/mgmt/instances_actions.py index 4f3eb6b0..4f6fcbf2 100644 --- a/trove/tests/api/mgmt/instances_actions.py +++ b/trove/tests/api/mgmt/instances_actions.py @@ -52,9 +52,9 @@ class MgmtInstanceBase(object): self.db_info = DBInstance.create( name="instance", flavor_id=1, + datastore_version_id=test_config.dbaas_datastore_version, tenant_id=self.tenant_id, volume_size=None, - service_type='mysql', task_status=InstanceTasks.NONE) self.server = self.mock.CreateMock(Server) self.instance = imodels.Instance(self.context, diff --git a/trove/tests/api/mgmt/malformed_json.py b/trove/tests/api/mgmt/malformed_json.py index 00fbf5b0..ef1e4e50 100644 --- a/trove/tests/api/mgmt/malformed_json.py +++ b/trove/tests/api/mgmt/malformed_json.py @@ -41,8 +41,7 @@ class MalformedJson(object): users = "bar" try: self.dbaas.instances.create("bad_instance", 3, 3, - databases=databases, - users=users) + databases=databases, users=users) except Exception as e: resp, body = self.dbaas.client.last_response httpCode = resp.status @@ -261,6 +260,33 @@ class MalformedJson(object): (flavorId, flavorId, flavorId, flavorId)) @test + def test_bad_body_datastore_create_instance(self): + tests_utils.skip_if_xml() + + datastore = "*" + datastore_version = "*" + try: + self.dbaas.instances.create("test_instance", + 3, {"size": 2}, + datastore=datastore, + datastore_version=datastore_version) + except Exception as e: + resp, body = self.dbaas.client.last_response + httpCode = resp.status + assert_equal(httpCode, 400, + "Create instance failed with code %s, exception %s" % + (httpCode, e)) + + if not isinstance(self.dbaas.client, + troveclient.compat.xml.TroveXmlClient): + assert_equal(e.message, + "Validation error: instance['datastore']['type']" + " u'%s' does not match '^.*[0-9a-zA-Z]+.*$'; " + "instance['datastore']['version'] u'%s' does not" + " match '^.*[0-9a-zA-Z]+.*$'" % + (datastore, datastore_version)) + + @test def test_bad_body_volsize_create_instance(self): volsize = "h3ll0" try: diff --git a/trove/tests/config.py b/trove/tests/config.py index 79b57f62..bc64cf03 100644 --- a/trove/tests/config.py +++ b/trove/tests/config.py @@ -70,8 +70,9 @@ class TestConfig(object): 'dbaas_url': "http://localhost:8775/v1.0/dbaas", 'version_url': "http://localhost:8775/", 'nova_url': "http://localhost:8774/v1.1", + 'dbaas_datastore': "Test_Mysql", + 'dbaas_datastore_version': "mysql_test_version", 'instance_create_time': 16 * 60, - 'dbaas_image': None, 'mysql_connection_method': {"type": "direct"}, 'typical_nova_image_name': None, 'white_box': os.environ.get("WHITE_BOX", "False") == "True", diff --git a/trove/tests/fakes/guestagent.py b/trove/tests/fakes/guestagent.py index 550bc9e9..e2eeffb4 100644 --- a/trove/tests/fakes/guestagent.py +++ b/trove/tests/fakes/guestagent.py @@ -207,7 +207,7 @@ class FakeGuest(object): % (username, hostname)) return self.users.get((username, hostname), None) - def prepare(self, memory_mb, databases, users, device_path=None, + def prepare(self, memory_mb, packages, databases, users, device_path=None, mount_point=None, backup_id=None, config_contents=None, root_password=None): from trove.instance.models import DBInstance diff --git a/trove/tests/unittests/guestagent/test_api.py b/trove/tests/unittests/guestagent/test_api.py index 41c1dc2d..31195b29 100644 --- a/trove/tests/unittests/guestagent/test_api.py +++ b/trove/tests/unittests/guestagent/test_api.py @@ -252,14 +252,15 @@ class ApiTest(testtools.TestCase): mock_conn = mock() when(rpc).create_connection(new=True).thenReturn(mock_conn) when(mock_conn).create_consumer(any(), any(), any()).thenReturn(None) - exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users', - 'device_path', 'mount_point', 'backup_id', - 'config_contents', 'root_password') + exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'packages', + 'databases', 'users', 'device_path', + 'mount_point', 'backup_id', 'config_contents', + 'root_password') when(rpc).cast(any(), any(), exp_msg).thenReturn(None) - self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt', - 'bkup-1232', 'cont', '1-2-3-4') + self.api.prepare('2048', 'package1', 'db1', 'user1', '/dev/vdt', + '/mnt/opt', 'bkup-1232', 'cont', '1-2-3-4') self._verify_rpc_connection_and_cast(rpc, mock_conn, exp_msg) @@ -267,13 +268,14 @@ class ApiTest(testtools.TestCase): mock_conn = mock() when(rpc).create_connection(new=True).thenReturn(mock_conn) when(mock_conn).create_consumer(any(), any(), any()).thenReturn(None) - exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users', - 'device_path', 'mount_point', 'backup_id', - 'config_contents', 'root_password') + exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'packages', + 'databases', 'users', 'device_path', + 'mount_point', 'backup_id', 'config_contents', + 'root_password') when(rpc).cast(any(), any(), exp_msg).thenReturn(None) - self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt', - 'backup_id_123', 'cont', '1-2-3-4') + self.api.prepare('2048', 'package1', 'db1', 'user1', '/dev/vdt', + '/mnt/opt', 'backup_id_123', 'cont', '1-2-3-4') self._verify_rpc_connection_and_cast(rpc, mock_conn, exp_msg) @@ -291,11 +293,13 @@ class ApiTest(testtools.TestCase): def test_rpc_cast_with_consumer_exception(self): mock_conn = mock() when(rpc).create_connection(new=True).thenRaise(IOError('host down')) - exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users', - 'device_path', 'mount_point') + exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'packages', + 'databases', 'users', 'device_path', + 'mount_point') with testtools.ExpectedException(exception.GuestError, '.* host down'): - self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt') + self.api.prepare('2048', 'package1', 'db1', 'user1', '/dev/vdt', + '/mnt/opt') verify(rpc).create_connection(new=True) verifyZeroInteractions(mock_conn) diff --git a/trove/tests/unittests/guestagent/test_dbaas.py b/trove/tests/unittests/guestagent/test_dbaas.py index aa5e81c3..043f8697 100644 --- a/trove/tests/unittests/guestagent/test_dbaas.py +++ b/trove/tests/unittests/guestagent/test_dbaas.py @@ -39,6 +39,7 @@ from trove.common import utils from trove.common import instance as rd_instance import trove.guestagent.datastore.mysql.service as dbaas from trove.guestagent import dbaas as dbaas_sr +from trove.guestagent import pkg from trove.guestagent.dbaas import to_gb from trove.guestagent.dbaas import get_filesystem_volume_stats from trove.guestagent.datastore.service import BaseDbStatus @@ -108,11 +109,18 @@ class DbaasTest(testtools.TestCase): self.assertRaises(RuntimeError, dbaas.get_auth_password) + def test_service_discovery(self): + when(os.path).isfile(any()).thenReturn(True) + mysql_service = dbaas.operating_system.service_discovery(["mysql"]) + self.assertIsNotNone(mysql_service['cmd_start']) + self.assertIsNotNone(mysql_service['cmd_enable']) + def test_load_mysqld_options(self): output = "mysqld would've been started with the these args:\n"\ "--user=mysql --port=3306 --basedir=/usr "\ "--tmpdir=/tmp --skip-external-locking" + when(os.path).isfile(any()).thenReturn(True) dbaas.utils.execute = Mock(return_value=(output, None)) options = dbaas.load_mysqld_options() @@ -453,6 +461,13 @@ class MySqlAppTest(testtools.TestCase): self.appStatus = FakeAppStatus(self.FAKE_ID, rd_instance.ServiceStatuses.NEW) self.mySqlApp = MySqlApp(self.appStatus) + mysql_service = {'cmd_start': Mock(), + 'cmd_stop': Mock(), + 'cmd_enable': Mock(), + 'cmd_disable': Mock(), + 'bin': Mock()} + dbaas.operating_system.service_discovery = Mock(return_value= + mysql_service) dbaas.time.sleep = Mock() def tearDown(self): @@ -564,13 +579,14 @@ class MySqlAppTest(testtools.TestCase): dbaas.utils.execute_with_timeout = Mock() self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING) - + self.mySqlApp._enable_mysql_on_boot = Mock() self.mySqlApp.start_mysql() self.assert_reported_status(rd_instance.ServiceStatuses.NEW) def test_start_mysql_with_db_update(self): dbaas.utils.execute_with_timeout = Mock() + self.mySqlApp._enable_mysql_on_boot = Mock() self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING) self.mySqlApp.start_mysql(True) @@ -579,6 +595,7 @@ class MySqlAppTest(testtools.TestCase): def test_start_mysql_runs_forever(self): dbaas.utils.execute_with_timeout = Mock() + self.mySqlApp._enable_mysql_on_boot = Mock() self.mySqlApp.state_change_wait_time = 1 self.appStatus.set_next_status(rd_instance.ServiceStatuses.SHUTDOWN) @@ -634,13 +651,19 @@ class MySqlAppInstallTest(MySqlAppTest): def test_install(self): self.mySqlApp._install_mysql = Mock() - self.mySqlApp.is_installed = Mock(return_value=False) - self.mySqlApp.install_if_needed() - self.assertTrue(self.mySqlApp._install_mysql.called) + pkg.Package.pkg_is_installed = Mock(return_value=False) + utils.execute_with_timeout = Mock() + pkg.Package.pkg_install = Mock() + self.mySqlApp._clear_mysql_config = Mock() + self.mySqlApp._create_mysql_confd_dir = Mock() + self.mySqlApp.start_mysql = Mock() + self.mySqlApp.install_if_needed(["package"]) + self.assertTrue(pkg.Package.pkg_install.called) self.assert_reported_status(rd_instance.ServiceStatuses.NEW) def test_secure(self): + dbaas.clear_expired_password = Mock() self.mySqlApp.start_mysql = Mock() self.mySqlApp.stop_db = Mock() self.mySqlApp._write_mycnf = Mock() @@ -660,17 +683,20 @@ class MySqlAppInstallTest(MySqlAppTest): from trove.guestagent import pkg self.mySqlApp.start_mysql = Mock() self.mySqlApp.stop_db = Mock() - self.mySqlApp.is_installed = Mock(return_value=False) - self.mySqlApp._install_mysql = Mock( - side_effect=pkg.PkgPackageStateError("Install error")) + pkg.Package.pkg_is_installed = Mock(return_value=False) + self.mySqlApp._clear_mysql_config = Mock() + self.mySqlApp._create_mysql_confd_dir = Mock() + pkg.Package.pkg_install = \ + Mock(side_effect=pkg.PkgPackageStateError("Install error")) self.assertRaises(pkg.PkgPackageStateError, - self.mySqlApp.install_if_needed) + self.mySqlApp.install_if_needed, ["package"]) self.assert_reported_status(rd_instance.ServiceStatuses.NEW) def test_secure_write_conf_error(self): + dbaas.clear_expired_password = Mock() self.mySqlApp.start_mysql = Mock() self.mySqlApp.stop_db = Mock() self.mySqlApp._write_mycnf = Mock( @@ -686,18 +712,6 @@ class MySqlAppInstallTest(MySqlAppTest): self.assertFalse(self.mySqlApp.start_mysql.called) self.assert_reported_status(rd_instance.ServiceStatuses.NEW) - def test_is_installed(self): - - dbaas.packager.pkg_version = Mock(return_value=True) - - self.assertTrue(self.mySqlApp.is_installed()) - - def test_is_installed_not(self): - - dbaas.packager.pkg_version = Mock(return_value=None) - - self.assertFalse(self.mySqlApp.is_installed()) - class TextClauseMatcher(matchers.Matcher): def __init__(self, text): @@ -772,8 +786,11 @@ class MySqlAppMockTest(testtools.TestCase): mock_status = mock() when(mock_status).wait_for_real_status_to_change_to( any(), any(), any()).thenReturn(True) + when(dbaas).clear_expired_password().thenReturn(None) app = MySqlApp(mock_status) when(app)._write_mycnf(any(), any()).thenReturn(True) + when(app).start_mysql().thenReturn(None) + when(app).stop_db().thenReturn(None) app.secure('foo') verify(mock_conn, never).execute(TextClauseMatcher('root')) @@ -883,16 +900,16 @@ class ServiceRegistryTest(testtools.TestCase): def tearDown(self): super(ServiceRegistryTest, self).tearDown() - def test_service_registry_with_extra_manager(self): - service_registry_ext_test = { + def test_datastore_registry_with_extra_manager(self): + datastore_registry_ext_test = { 'test': 'trove.guestagent.datastore.test.manager.Manager', } dbaas_sr.get_custom_managers = Mock(return_value= - service_registry_ext_test) - test_dict = dbaas_sr.service_registry() + datastore_registry_ext_test) + test_dict = dbaas_sr.datastore_registry() self.assertEqual(3, len(test_dict)) self.assertEqual(test_dict.get('test'), - service_registry_ext_test.get('test', None)) + datastore_registry_ext_test.get('test', None)) self.assertEqual(test_dict.get('mysql'), 'trove.guestagent.datastore.mysql.' 'manager.Manager') @@ -900,14 +917,14 @@ class ServiceRegistryTest(testtools.TestCase): 'trove.guestagent.datastore.mysql.' 'manager.Manager') - def test_service_registry_with_existing_manager(self): - service_registry_ext_test = { + def test_datastore_registry_with_existing_manager(self): + datastore_registry_ext_test = { 'mysql': 'trove.guestagent.datastore.mysql.' 'manager.Manager123', } dbaas_sr.get_custom_managers = Mock(return_value= - service_registry_ext_test) - test_dict = dbaas_sr.service_registry() + datastore_registry_ext_test) + test_dict = dbaas_sr.datastore_registry() self.assertEqual(2, len(test_dict)) self.assertEqual(test_dict.get('mysql'), 'trove.guestagent.datastore.mysql.' @@ -916,11 +933,11 @@ class ServiceRegistryTest(testtools.TestCase): 'trove.guestagent.datastore.mysql.' 'manager.Manager') - def test_service_registry_with_blank_dict(self): - service_registry_ext_test = dict() + def test_datastore_registry_with_blank_dict(self): + datastore_registry_ext_test = dict() dbaas_sr.get_custom_managers = Mock(return_value= - service_registry_ext_test) - test_dict = dbaas_sr.service_registry() + datastore_registry_ext_test) + test_dict = dbaas_sr.datastore_registry() self.assertEqual(2, len(test_dict)) self.assertEqual(test_dict.get('mysql'), 'trove.guestagent.datastore.mysql.' diff --git a/trove/tests/unittests/guestagent/test_manager.py b/trove/tests/unittests/guestagent/test_manager.py index 3c2a4690..ca86790c 100644 --- a/trove/tests/unittests/guestagent/test_manager.py +++ b/trove/tests/unittests/guestagent/test_manager.py @@ -24,6 +24,7 @@ from trove.guestagent.datastore.mysql.manager import Manager import trove.guestagent.datastore.mysql.service as dbaas from trove.guestagent import backup from trove.guestagent.volume import VolumeDevice +from trove.guestagent import pkg class GuestAgentManagerTest(testtools.TestCase): @@ -37,10 +38,10 @@ class GuestAgentManagerTest(testtools.TestCase): self.origin_format = volume.VolumeDevice.format self.origin_migrate_data = volume.VolumeDevice.migrate_data self.origin_mount = volume.VolumeDevice.mount - self.origin_is_installed = dbaas.MySqlApp.is_installed self.origin_stop_mysql = dbaas.MySqlApp.stop_db self.origin_start_mysql = dbaas.MySqlApp.start_mysql - self.origin_install_mysql = dbaas.MySqlApp._install_mysql + self.origin_pkg_is_installed = pkg.Package.pkg_is_installed + self.origin_os_path_exists = os.path.exists def tearDown(self): super(GuestAgentManagerTest, self).tearDown() @@ -49,10 +50,10 @@ class GuestAgentManagerTest(testtools.TestCase): volume.VolumeDevice.format = self.origin_format volume.VolumeDevice.migrate_data = self.origin_migrate_data volume.VolumeDevice.mount = self.origin_mount - dbaas.MySqlApp.is_installed = self.origin_is_installed dbaas.MySqlApp.stop_db = self.origin_stop_mysql dbaas.MySqlApp.start_mysql = self.origin_start_mysql - dbaas.MySqlApp._install_mysql = self.origin_install_mysql + pkg.Package.pkg_is_installed = self.origin_pkg_is_installed + os.path.exists = self.origin_os_path_exists unstub() def test_update_status(self): @@ -139,8 +140,6 @@ class GuestAgentManagerTest(testtools.TestCase): # covering all outcomes is starting to cause trouble here COUNT = 1 if device_path else 0 - SEC_COUNT = 1 if is_mysql_installed else 0 - migrate_count = 1 * COUNT if not backup_id else 0 # TODO(juice): this should stub an instance of the MySqlAppStatus mock_status = mock() @@ -155,16 +154,18 @@ class GuestAgentManagerTest(testtools.TestCase): when(backup).restore(self.context, backup_id).thenReturn(None) when(dbaas.MySqlApp).secure(any()).thenReturn(None) when(dbaas.MySqlApp).secure_root(any()).thenReturn(None) - when(dbaas.MySqlApp).is_installed().thenReturn(is_mysql_installed) + (when(pkg.Package).pkg_is_installed(any()). + thenReturn(is_mysql_installed)) when(dbaas.MySqlAdmin).is_root_enabled().thenReturn(is_root_enabled) when(dbaas.MySqlAdmin).create_user().thenReturn(None) when(dbaas.MySqlAdmin).create_database().thenReturn(None) when(dbaas.MySqlAdmin).report_root_enabled(self.context).thenReturn( None) - when(os.path).exists(any()).thenReturn(is_mysql_installed) + when(os.path).exists(any()).thenReturn(True) # invocation - self.manager.prepare(context=self.context, databases=None, + self.manager.prepare(context=self.context, packages=None, + databases=None, memory_mb='2048', users=None, device_path=device_path, mount_point='/var/lib/mysql', @@ -173,12 +174,11 @@ class GuestAgentManagerTest(testtools.TestCase): verify(mock_status).begin_install() verify(VolumeDevice, times=COUNT).format() - verify(dbaas.MySqlApp, times=(COUNT * SEC_COUNT)).stop_db() - verify(VolumeDevice, times=(migrate_count * SEC_COUNT)).migrate_data( + verify(dbaas.MySqlApp, times=COUNT).stop_db() + verify(VolumeDevice, times=COUNT).migrate_data( any()) if backup_id: verify(backup).restore(self.context, backup_id, '/var/lib/mysql') - verify(dbaas.MySqlApp).install_if_needed() # We dont need to make sure the exact contents are there verify(dbaas.MySqlApp).secure(any()) verify(dbaas.MySqlAdmin, never).create_database() diff --git a/trove/tests/unittests/guestagent/test_pkg.py b/trove/tests/unittests/guestagent/test_pkg.py index 48aafc8c..f703eef3 100644 --- a/trove/tests/unittests/guestagent/test_pkg.py +++ b/trove/tests/unittests/guestagent/test_pkg.py @@ -17,6 +17,7 @@ import testtools from mock import Mock +from mockito import when, any import pexpect from trove.common import utils from trove.common import exception @@ -38,10 +39,12 @@ class PkgDEBInstallTestCase(testtools.TestCase): self.pexpect_spawn_closed = pexpect.spawn.close self.pkg = pkg.DebianPackagerMixin() self.pkg_fix = self.pkg._fix + self.pkg_fix_package_selections = self.pkg._fix_package_selections utils.execute = Mock() pexpect.spawn.__init__ = Mock(return_value=None) pexpect.spawn.closed = Mock(return_value=None) self.pkg._fix = Mock(return_value=None) + self.pkg._fix_package_selections = Mock(return_value=None) self.pkgName = 'packageName' def tearDown(self): @@ -50,53 +53,78 @@ class PkgDEBInstallTestCase(testtools.TestCase): pexpect.spawn.__init__ = self.pexpect_spawn_init pexpect.spawn.close = self.pexpect_spawn_closed self.pkg._fix = self.pkg_fix + self.pkg._fix_package_selections = self.pkg_fix_package_selections + + def test_pkg_is_instaled_no_packages(self): + packages = "" + self.assertTrue(self.pkg.pkg_is_installed(packages)) + + def test_pkg_is_instaled_yes(self): + packages = "package1=1.0 package2" + when(self.pkg).pkg_version("package1").thenReturn("1.0") + when(self.pkg).pkg_version("package2").thenReturn("2.0") + self.assertTrue(self.pkg.pkg_is_installed(packages)) + + def test_pkg_is_instaled_no(self): + packages = "package1=1.0 package2 package3=3.1" + when(self.pkg).pkg_version("package1").thenReturn("1.0") + when(self.pkg).pkg_version("package2").thenReturn("2.0") + when(self.pkg).pkg_version("package3").thenReturn("3.0") + self.assertFalse(self.pkg.pkg_is_installed(packages)) def test_success_install(self): # test - pexpect.spawn.expect = Mock(return_value=5) - self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None) - # verify - - def test_already_instaled(self): - # test happy path - pexpect.spawn.expect = Mock(return_value=6) - self.pkg.pkg_install(self.pkgName, 5000) + pexpect.spawn.expect = Mock(return_value=7) + pexpect.spawn.match = False + self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None) def test_permission_error(self): # test pexpect.spawn.expect = Mock(return_value=0) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgPermissionError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) def test_package_not_found_1(self): # test pexpect.spawn.expect = Mock(return_value=1) + pexpect.spawn.match = re.match('(.*)', self.pkgName) # test and verify self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) def test_package_not_found_2(self): # test pexpect.spawn.expect = Mock(return_value=2) + pexpect.spawn.match = re.match('(.*)', self.pkgName) # test and verify self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) def test_run_DPKG_bad_State(self): # test _fix method is called and PackageStateError is thrown - pexpect.spawn.expect = Mock(return_value=3) + pexpect.spawn.expect = Mock(return_value=4) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) self.assertTrue(self.pkg._fix.called) def test_admin_lock_error(self): # test 'Unable to lock the administration directory' error - pexpect.spawn.expect = Mock(return_value=4) + pexpect.spawn.expect = Mock(return_value=5) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgAdminLockError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) + + def test_package_broken_error(self): + pexpect.spawn.expect = Mock(return_value=6) + pexpect.spawn.match = False + # test and verify + self.assertRaises(pkg.PkgBrokenError, self.pkg.pkg_install, + self.pkgName, {}, 5000) def test_timeout_error(self): # test timeout error @@ -104,7 +132,7 @@ class PkgDEBInstallTestCase(testtools.TestCase): TIMEOUT('timeout error')) # test and verify self.assertRaises(pkg.PkgTimeout, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) class PkgDEBRemoveTestCase(testtools.TestCase): @@ -140,11 +168,13 @@ class PkgDEBRemoveTestCase(testtools.TestCase): def test_success_remove(self): # test pexpect.spawn.expect = Mock(return_value=6) + pexpect.spawn.match = False self.assertTrue(self.pkg.pkg_remove(self.pkgName, 5000) is None) def test_permission_error(self): # test pexpect.spawn.expect = Mock(return_value=0) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgPermissionError, self.pkg.pkg_remove, self.pkgName, 5000) @@ -152,6 +182,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase): def test_package_not_found(self): # test pexpect.spawn.expect = Mock(return_value=1) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_remove, self.pkgName, 5000) @@ -159,6 +190,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase): def test_package_reinstall_first_1(self): # test pexpect.spawn.expect = Mock(return_value=2) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_remove, self.pkgName, 5000) @@ -168,6 +200,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase): def test_package_reinstall_first_2(self): # test pexpect.spawn.expect = Mock(return_value=3) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_remove, self.pkgName, 5000) @@ -177,6 +210,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase): def test_package_DPKG_first(self): # test pexpect.spawn.expect = Mock(return_value=4) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_remove, self.pkgName, 5000) @@ -186,6 +220,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase): def test_admin_lock_error(self): # test 'Unable to lock the administration directory' error pexpect.spawn.expect = Mock(return_value=5) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgAdminLockError, self.pkg.pkg_remove, self.pkgName, 5000) @@ -201,22 +236,6 @@ class PkgDEBRemoveTestCase(testtools.TestCase): class PkgDEBVersionTestCase(testtools.TestCase): - @staticmethod - def build_output(packageName, packageVersion, parts=None): - if parts is None: - parts = "ii " + packageName + " " + packageVersion + \ - " MySQL database server binaries "\ - "and system database setup \n" - cmd_out = "Desired=Unknown/Install/Remove/Purge/Hold\n" \ - "| Status=Not/Inst/Conf-files/Unpacked/halF-conf/"\ - "Half-inst/trig-aWait/Trig-pend\n" \ - "|/ Err?=(none)/Reinst-required "\ - "(Status,Err: uppercase=bad)\n"\ - "||/ Name Version Description\n" \ - "+++-==============-================-=============\n" \ - "=================================\n" + parts - return cmd_out - def setUp(self): super(PkgDEBVersionTestCase, self).setUp() self.pkgName = 'mysql-server-5.5' @@ -228,43 +247,19 @@ class PkgDEBVersionTestCase(testtools.TestCase): commands.getstatusoutput = self.commands_output def test_version_success(self): - cmd_out = self.build_output(self.pkgName, self.pkgVersion) + cmd_out = "%s:\n Installed: %s\n" % (self.pkgName, self.pkgVersion) commands.getstatusoutput = Mock(return_value=(0, cmd_out)) version = pkg.DebianPackagerMixin().pkg_version(self.pkgName) self.assertTrue(version) self.assertEqual(self.pkgVersion, version) - def test_version_status_error(self): - cmd_out = self.build_output(self.pkgName, self.pkgVersion) - commands.getstatusoutput = Mock(return_value=(1, cmd_out)) - self.assertFalse(pkg.DebianPackagerMixin().pkg_version(self.pkgName)) - - def test_version_no_output(self): - cmd_out = self.build_output(self.pkgName, self.pkgVersion, "") - commands.getstatusoutput = Mock(return_value=(0, cmd_out)) - self.assertIsNone(pkg.DebianPackagerMixin().pkg_version(self.pkgName)) - - def test_version_unexpected_parts(self): - unexp_parts = "ii 123" - cmd_out = self.build_output(self.pkgName, self.pkgVersion, unexp_parts) - commands.getstatusoutput = Mock(return_value=(0, cmd_out)) - self.assertIsNone(pkg.DebianPackagerMixin().pkg_version(self.pkgName)) - - def test_version_wrong_package(self): - invalid_pkg = "package_invalid_001" - cmd_out = self.build_output(invalid_pkg, self.pkgVersion) - commands.getstatusoutput = Mock(return_value=(0, cmd_out)) - self.assertRaises(exception.GuestError, - pkg.DebianPackagerMixin().pkg_version, self.pkgName) - def test_version_unknown_package(self): - unk_parts = "un " + self.pkgName + " " + self.pkgVersion + " \n" - cmd_out = self.build_output(self.pkgName, self.pkgVersion, unk_parts) + cmd_out = "N: Unable to locate package %s" % self.pkgName commands.getstatusoutput = Mock(return_value=(0, cmd_out)) self.assertFalse(pkg.DebianPackagerMixin().pkg_version(self.pkgName)) def test_version_no_version(self): - cmd_out = self.build_output(self.pkgName, '<none>') + cmd_out = "%s:\n Installed: %s\n" % (self.pkgName, "(none)") commands.getstatusoutput = Mock(return_value=(0, cmd_out)) self.assertFalse(pkg.DebianPackagerMixin().pkg_version(self.pkgName)) @@ -313,73 +308,107 @@ class PkgRPMInstallTestCase(testtools.TestCase): pexpect.spawn.__init__ = self.pexpect_spawn_init pexpect.spawn.close = self.pexpect_spawn_closed + def test_pkg_is_instaled_no_packages(self): + packages = "" + self.assertTrue(self.pkg.pkg_is_installed(packages)) + + def test_pkg_is_instaled_yes(self): + packages = "package1=1.0 package2" + when(commands).getstatusoutput(any()).thenReturn({1: "package1=1.0\n" + "package2=2.0"}) + self.assertTrue(self.pkg.pkg_is_installed(packages)) + + def test_pkg_is_instaled_no(self): + packages = "package1=1.0 package2 package3=3.0" + when(commands).getstatusoutput({1: "package1=1.0\npackage2=2.0"}) + self.assertFalse(self.pkg.pkg_is_installed(packages)) + def test_permission_error(self): # test pexpect.spawn.expect = Mock(return_value=0) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgPermissionError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) def test_package_not_found(self): # test pexpect.spawn.expect = Mock(return_value=1) + pexpect.spawn.match = re.match('(.*)', self.pkgName) # test and verify self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) - def test_transaction_check_error(self): + def test_package_conflict_remove(self): # test pexpect.spawn.expect = Mock(return_value=2) + pexpect.spawn.match = re.match('(.*)', self.pkgName) + self.pkg._rpm_remove_nodeps = Mock() # test and verify - self.assertRaises(pkg.PkgTransactionCheckError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkg._install(self.pkgName, 5000) + self.assertTrue(self.pkg._rpm_remove_nodeps.called) def test_package_scriptlet_error(self): # test - pexpect.spawn.expect = Mock(return_value=3) + pexpect.spawn.expect = Mock(return_value=5) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgScriptletError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) def test_package_http_error(self): # test - pexpect.spawn.expect = Mock(return_value=4) + pexpect.spawn.expect = Mock(return_value=6) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgDownloadError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) def test_package_nomirrors_error(self): # test - pexpect.spawn.expect = Mock(return_value=5) + pexpect.spawn.expect = Mock(return_value=7) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgDownloadError, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) + + def test_package_sign_error(self): + # test + pexpect.spawn.expect = Mock(return_value=8) + pexpect.spawn.match = False + # test and verify + self.assertRaises(pkg.PkgSignError, self.pkg.pkg_install, + self.pkgName, {}, 5000) def test_package_already_installed(self): # test - pexpect.spawn.expect = Mock(return_value=6) + pexpect.spawn.expect = Mock(return_value=9) + pexpect.spawn.match = False # test and verify - self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None) + self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None) def test_package_success_updated(self): # test - pexpect.spawn.expect = Mock(return_value=7) + pexpect.spawn.expect = Mock(return_value=10) + pexpect.spawn.match = False # test and verify - self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None) + self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None) def test_package_success_installed(self): # test - pexpect.spawn.expect = Mock(return_value=8) + pexpect.spawn.expect = Mock(return_value=11) + pexpect.spawn.match = False # test and verify - self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None) + self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None) def test_timeout_error(self): # test timeout error pexpect.spawn.expect = Mock(side_effect=pexpect. TIMEOUT('timeout error')) + pexpect.spawn.match = False # test and verify self.assertRaises(pkg.PkgTimeout, self.pkg.pkg_install, - self.pkgName, 5000) + self.pkgName, {}, 5000) class PkgRPMRemoveTestCase(testtools.TestCase): diff --git a/trove/tests/unittests/mgmt/test_models.py b/trove/tests/unittests/mgmt/test_models.py index c850bad9..7d129bd7 100644 --- a/trove/tests/unittests/mgmt/test_models.py +++ b/trove/tests/unittests/mgmt/test_models.py @@ -24,6 +24,7 @@ from oslo.config.cfg import ConfigOpts from trove.backup.models import Backup from trove.common.context import TroveContext from trove.common import instance as rd_instance +from trove.datastore import models as datastore_models from trove.db.models import DatabaseModelBase from trove.instance.models import DBInstance from trove.instance.models import InstanceServiceStatus @@ -31,6 +32,7 @@ from trove.instance.tasks import InstanceTasks import trove.extensions.mgmt.instances.models as mgmtmodels from trove.openstack.common.notifier import api as notifier from trove.common import remote +from trove.tests.util import test_config class MockMgmtInstanceTest(TestCase): @@ -62,11 +64,12 @@ class MockMgmtInstanceTest(TestCase): name='test_name', id='1', flavor_id='flavor_1', + datastore_version_id= + test_config.dbaas_datastore_version, compute_instance_id='compute_id_1', server_id='server_id_1', tenant_id='tenant_id_1', - server_status=status, - service_type='mysql') + server_status=status) class TestNotificationTransformer(MockMgmtInstanceTest): @@ -78,6 +81,10 @@ class TestNotificationTransformer(MockMgmtInstanceTest): when(DatabaseModelBase).find_all(deleted=False).thenReturn( [db_instance]) + stub_datastore = mock() + stub_datastore.datastore_id = "stub" + stub_datastore.manager = "mysql" + when(DatabaseModelBase).find_by(id=any()).thenReturn(stub_datastore) when(DatabaseModelBase).find_by(instance_id='1').thenReturn( InstanceServiceStatus(rd_instance.ServiceStatuses.BUILDING)) @@ -165,14 +172,17 @@ class TestNovaNotificationTransformer(MockMgmtInstanceTest): self.assertThat(payload['user_id'], Equals('test_user_id')) self.assertThat(payload['service_id'], Equals('123')) - def test_tranformer_invalid_service_type(self): + def test_tranformer_invalid_datastore_manager(self): status = rd_instance.ServiceStatuses.BUILDING.api_status db_instance = MockMgmtInstanceTest.build_db_instance( status, task_status=InstanceTasks.BUILDING) - db_instance.service_type = 'm0ng0' server = mock(Server) server.user_id = 'test_user_id' + stub_datastore = mock() + stub_datastore.manager = "m0ng0" + when(datastore_models. + Datastore).load(any()).thenReturn(stub_datastore) mgmt_instance = mgmtmodels.SimpleMgmtInstance(self.context, db_instance, server, diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py index a887dfe5..9740f4ed 100644 --- a/trove/tests/unittests/taskmanager/test_models.py +++ b/trove/tests/unittests/taskmanager/test_models.py @@ -15,6 +15,7 @@ import testtools from mock import Mock from testtools.matchers import Equals from mockito import mock, when, unstub, any, verify, never +from trove.datastore import models as datastore_models from trove.taskmanager import models as taskmanager_models import trove.common.remote as remote from trove.common.instance import ServiceStatuses @@ -138,6 +139,10 @@ class FreshInstanceTasksTest(testtools.TestCase): "hostname") when(taskmanager_models.FreshInstanceTasks).name().thenReturn( 'name') + when(datastore_models. + DatastoreVersion).load(any()).thenReturn(mock()) + when(datastore_models. + Datastore).load(any()).thenReturn(mock()) taskmanager_models.FreshInstanceTasks.nova_client = fake_nova_client() taskmanager_models.CONF = mock() when(taskmanager_models.CONF).get(any()).thenReturn('') @@ -152,7 +157,7 @@ class FreshInstanceTasksTest(testtools.TestCase): self.guestconfig = f.name f.write(self.guestconfig_content) self.freshinstancetasks = taskmanager_models.FreshInstanceTasks( - None, None, None, None) + None, mock(), None, None) def tearDown(self): super(FreshInstanceTasksTest, self).tearDown() @@ -164,11 +169,12 @@ class FreshInstanceTasksTest(testtools.TestCase): def test_create_instance_userdata(self): cloudinit_location = os.path.dirname(self.cloudinit) - service_type = os.path.splitext(os.path.basename(self.cloudinit))[0] + datastore_manager = os.path.splitext(os.path.basename(self. + cloudinit))[0] when(taskmanager_models.CONF).get("cloudinit_location").thenReturn( cloudinit_location) server = self.freshinstancetasks._create_server( - None, None, None, service_type, None, None) + None, None, None, datastore_manager, None, None) self.assertEqual(server.userdata, self.userdata) def test_create_instance_guestconfig(self): @@ -181,23 +187,20 @@ class FreshInstanceTasksTest(testtools.TestCase): self.guestconfig_content) def test_create_instance_with_az_kwarg(self): - service_type = 'mysql' server = self.freshinstancetasks._create_server( - None, None, None, service_type, None, availability_zone='nova') + None, None, None, None, None, availability_zone='nova') self.assertIsNotNone(server) def test_create_instance_with_az(self): - service_type = 'mysql' server = self.freshinstancetasks._create_server( - None, None, None, service_type, None, 'nova') + None, None, None, None, None, 'nova') self.assertIsNotNone(server) def test_create_instance_with_az_none(self): - service_type = 'mysql' server = self.freshinstancetasks._create_server( - None, None, None, service_type, None, None) + None, None, None, None, None, None) self.assertIsNotNone(server) diff --git a/trove/tests/util/client.py b/trove/tests/util/client.py index abc8108b..b15e428d 100644 --- a/trove/tests/util/client.py +++ b/trove/tests/util/client.py @@ -102,20 +102,6 @@ class TestClient(object): flavor_href = self.find_flavor_self_href(flavor) return flavor, flavor_href - def find_image_and_self_href(self, image_id): - """Given an ID, returns tuple with image and its self href.""" - assert_false(image_id is None) - image = self.images.get(image_id) - assert_true(image is not None) - self_links = [link['href'] for link in image.links - if link['rel'] == 'self'] - assert_true(len(self_links) > 0, - "Found image with ID %s but it had no self link!" % - str(image_id)) - image_href = self_links[0] - assert_false(image_href is None, "Image link self href missing.") - return image, image_href - def __getattr__(self, item): return getattr(self.real_client, item) |