diff options
23 files changed, 108 insertions, 84 deletions
diff --git a/integration/tests/integration/int_tests.py b/integration/tests/integration/int_tests.py index 54118f20..de5417ef 100644 --- a/integration/tests/integration/int_tests.py +++ b/integration/tests/integration/int_tests.py @@ -71,10 +71,10 @@ def add_support_for_localization(): if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) - if six.PY2: - gettext.install('nova', unicode=1) - else: + if six.PY3: gettext.install('nova') + else: + gettext.install('nova', unicode=True) MAIN_RUNNER = None diff --git a/integration/tests/integration/tests/colorizer.py b/integration/tests/integration/tests/colorizer.py index 7defd0c2..9f8dda27 100644 --- a/integration/tests/integration/tests/colorizer.py +++ b/integration/tests/integration/tests/colorizer.py @@ -63,10 +63,10 @@ import six import sys import time -if six.PY2: - gettext.install('nova', unicode=1) -else: +if six.PY3: gettext.install('nova') +else: + gettext.install('nova', unicode=True) from nose import config from nose import core diff --git a/run_tests.py b/run_tests.py index eb00e032..95c2f3b8 100644 --- a/run_tests.py +++ b/run_tests.py @@ -57,10 +57,10 @@ def add_support_for_localization(): if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) - if six.PY2: - gettext.install('nova', unicode=1) - else: + if six.PY3: gettext.install('nova') + else: + gettext.install('nova', unicode=True) def initialize_trove(config_file): @@ -29,22 +29,25 @@ whitelist_externals = find sitepackages = True [testenv:pep8] -basepython = python2.7 +basepython = python3 commands = flake8 doc8 {posargs} [testenv:apiexamples] +basepython = python3 commands = {envpython} generate_examples.py [testenv:fakemodetests] +basepython = python3 commands = {envpython} run_tests.py [testenv:debug] +basepython = python3 commands = oslo_debug_helper {posargs} [testenv:cover] -basepython = python2.7 +basepython = python3 commands = {[testenv]commands} coverage erase @@ -55,6 +58,7 @@ commands = coverage report [testenv:venv] +basepython = python3 commands = {posargs} [doc8] @@ -77,6 +81,7 @@ import_exceptions = trove.common.i18n local-check-factory = trove.hacking.checks.factory [testenv:api-ref] +basepython = python3 # This environment is called from CI scripts to test and publish # the API Ref to developer.openstack.org. commands = @@ -84,19 +89,24 @@ commands = sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html [testenv:releasenotes] +basepython = python3 commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:bandit] +basepython = python3 commands = bandit -r trove -n5 -x tests [testenv:bandit-baseline] +basepython = python3 envdir = {toxworkdir}/bandit commands = bandit-baseline -r trove -n5 -x tests -ii -ll [testenv:genpolicy] +basepython = python3 commands = oslopolicy-sample-generator --config-file=tools/trove-policy-generator.conf [testenv:pylint] +basepython = python3 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = {[testenv]commands} diff --git a/trove/cmd/manage.py b/trove/cmd/manage.py index dc44d0f5..9154b598 100644 --- a/trove/cmd/manage.py +++ b/trove/cmd/manage.py @@ -147,9 +147,9 @@ class Commands(object): datastore_name, datastore_version_name) if vtlist.count() > 0: for volume_type in vtlist: - print ("Datastore: %s, Version: %s, Volume Type: %s" % - (datastore_name, datastore_version_name, - volume_type.value)) + print("Datastore: %s, Version: %s, Volume Type: %s" % + (datastore_name, datastore_version_name, + volume_type.value)) else: print("No Volume Type Associations found for Datastore: %s, " "Version: %s." % diff --git a/trove/common/base_wsgi.py b/trove/common/base_wsgi.py index da0a0275..fd82972b 100644 --- a/trove/common/base_wsgi.py +++ b/trove/common/base_wsgi.py @@ -455,8 +455,7 @@ class JSONDictSerializer(DictSerializer): _dtime = obj - datetime.timedelta(microseconds=obj.microsecond) return _dtime.isoformat() return obj -# return six.text_type(obj) - return jsonutils.dumps(data, default=sanitizer) + return jsonutils.dump_as_bytes(data, default=sanitizer) class XMLDictSerializer(DictSerializer): diff --git a/trove/common/db/models.py b/trove/common/db/models.py index 0d0b76ff..ebea8e89 100644 --- a/trove/common/db/models.py +++ b/trove/common/db/models.py @@ -14,6 +14,7 @@ # under the License. import abc +import six from trove.common import cfg from trove.common.i18n import _ @@ -84,8 +85,7 @@ class DatastoreModelsBase(object): :param desc: Description for exception message. :raises: ValueError if not a string/unicode. """ - if not (isinstance(value, str) or - isinstance(value, unicode)): + if not isinstance(value, six.string_types): raise ValueError(_("%(desc)s is not a string. Type = %(t)s.") % {'desc': desc, 't': type(value)}) diff --git a/trove/common/wsgi.py b/trove/common/wsgi.py index 366f3f0f..3be631f7 100644 --- a/trove/common/wsgi.py +++ b/trove/common/wsgi.py @@ -509,7 +509,7 @@ class Fault(webob.exc.HTTPException): content_type = req.best_match_content_type() serializer = { - 'application/json': base_wsgi.JSONDictSerializer(), + 'application/json': JSONDictSerializer(), }[content_type] self.wrapped_exc.body = serializer.serialize(fault_data, content_type) @@ -657,10 +657,3 @@ class DictSerializer(ActionDispatcher): def default(self, data): return "" - - -class JSONDictSerializer(DictSerializer): - """Default JSON request body serialization.""" - - def default(self, data): - return jsonutils.dump_as_bytes(data) diff --git a/trove/extensions/mgmt/quota/service.py b/trove/extensions/mgmt/quota/service.py index 1879f165..a16f8716 100644 --- a/trove/extensions/mgmt/quota/service.py +++ b/trove/extensions/mgmt/quota/service.py @@ -37,8 +37,8 @@ class QuotaController(wsgi.Controller): usages = quota_engine.get_all_quota_usages_by_tenant(id) limits = quota_engine.get_all_quotas_by_tenant(id) - map(lambda r: setattr(usages[r], "limit", limits[r].hard_limit), - usages.keys()) + for key in usages.keys(): + setattr(usages[key], "limit", limits[key].hard_limit) return wsgi.Result(views.QuotaUsageView(usages).data(), 200) @admin_context diff --git a/trove/guestagent/datastore/experimental/couchdb/service.py b/trove/guestagent/datastore/experimental/couchdb/service.py index 3dbb9e94..6d1f6286 100644 --- a/trove/guestagent/datastore/experimental/couchdb/service.py +++ b/trove/guestagent/datastore/experimental/couchdb/service.py @@ -231,7 +231,6 @@ class CouchDBAdmin(object): shell=True) except exception.ProcessExecutionError as pe: LOG.exception("Error creating user: %s.", user.name) - pass for database in user.databases: mydb = models.CouchDBSchema.deserialize(database) @@ -251,11 +250,8 @@ class CouchDBAdmin(object): "database: %(db)s.", {'user': user.name, 'db': mydb.name}) LOG.debug(pe) - pass except exception.ProcessExecutionError as pe: - LOG.exception("An error occurred creating users: %s.", - pe.message) - pass + LOG.exception("An error occurred creating users: %s.", str(pe)) def delete_user(self, user): LOG.debug("Delete a given CouchDB user.") diff --git a/trove/guestagent/datastore/experimental/db2/service.py b/trove/guestagent/datastore/experimental/db2/service.py index 94b9e34b..688b7fef 100644 --- a/trove/guestagent/datastore/experimental/db2/service.py +++ b/trove/guestagent/datastore/experimental/db2/service.py @@ -280,7 +280,7 @@ class DB2App(object): def _reset_config(self, config): try: - for k, v in config.iteritems(): + for k, v in config.items(): default_cfg_value = self.dbm_default_config[k] self._update_dbm_config(k, default_cfg_value) except Exception: @@ -356,7 +356,6 @@ class DB2Admin(object): LOG.exception( "There was an error creating database: %s.", dbName) db_create_failed.append(dbName) - pass ''' Configure each database to do archive logging for online @@ -438,7 +437,6 @@ class DB2Admin(object): err_msg = encodeutils.exception_to_unicode(pe) LOG.exception("An error occurred listing databases: %s.", err_msg) - pass return databases, next_marker def create_user(self, users): @@ -470,11 +468,8 @@ class DB2Admin(object): "database: %(db)s.", {'user': user.name, 'db': mydb.name}) LOG.debug(pe) - pass except exception.ProcessExecutionError as pe: - LOG.exception("An error occurred creating users: %s.", - pe.message) - pass + LOG.exception("An error occurred creating users: %s.", str(pe)) def delete_user(self, user): LOG.debug("Delete a given user.") @@ -503,7 +498,6 @@ class DB2Admin(object): except exception.ProcessExecutionError as pe: LOG.debug("Error occurred while revoking access to %s.", mydb.name) - pass try: utils.execute_with_timeout(system.DELETE_USER_COMMAND % { 'login': db2_user.name.lower()}, shell=True) diff --git a/trove/guestagent/datastore/experimental/redis/service.py b/trove/guestagent/datastore/experimental/redis/service.py index 2b40d6c4..f048f0db 100644 --- a/trove/guestagent/datastore/experimental/redis/service.py +++ b/trove/guestagent/datastore/experimental/redis/service.py @@ -362,8 +362,10 @@ class RedisApp(object): def cluster_addslots(self, first_slot, last_slot): try: - slots = map(str, range(first_slot, last_slot + 1)) group_size = 200 + # Create list of slots represented in strings + # eg. ['10', '11', '12', '13'] + slots = list(map(str, range(first_slot, last_slot + 1))) while slots: cmd = (['redis-cli', 'cluster', 'addslots'] + slots[0:group_size]) diff --git a/trove/guestagent/datastore/experimental/vertica/service.py b/trove/guestagent/datastore/experimental/vertica/service.py index 00bdcdfe..f316f6c2 100644 --- a/trove/guestagent/datastore/experimental/vertica/service.py +++ b/trove/guestagent/datastore/experimental/vertica/service.py @@ -107,7 +107,7 @@ class VerticaApp(object): def _reset_config(self, config): try: db_password = self._get_database_password() - for k, v in config.iteritems(): + for k, v in config.items(): alter_db_cmd = system.ALTER_DB_RESET_CFG % (DB_NAME, str(k)) out, err = system.exec_vsql_command(db_password, alter_db_cmd) if err: @@ -125,7 +125,7 @@ class VerticaApp(object): def _apply_config(self, config): try: db_password = self._get_database_password() - for k, v in config.iteritems(): + for k, v in config.items(): alter_db_cmd = system.ALTER_DB_CFG % (DB_NAME, str(k), str(v)) out, err = system.exec_vsql_command(db_password, alter_db_cmd) if err: diff --git a/trove/guestagent/datastore/manager.py b/trove/guestagent/datastore/manager.py index 9f9dbcdb..3c35b265 100644 --- a/trove/guestagent/datastore/manager.py +++ b/trove/guestagent/datastore/manager.py @@ -111,7 +111,7 @@ class Manager(periodic_task.PeriodicTasks): return repl_strategy.get_instance(self.manager) except Exception as ex: LOG.debug("Cannot get replication instance for '%(manager)s': " - "%(msg)s", {'manager': self.manager, 'msg': ex.message}) + "%(msg)s", {'manager': self.manager, 'msg': str(ex)}) return None @@ -122,7 +122,7 @@ class Manager(periodic_task.PeriodicTasks): return repl_strategy.get_strategy(self.manager) except Exception as ex: LOG.debug("Cannot get replication strategy for '%(manager)s': " - "%(msg)s", {'manager': self.manager, 'msg': ex.message}) + "%(msg)s", {'manager': self.manager, 'msg': str(ex)}) return None @@ -313,7 +313,7 @@ class Manager(periodic_task.PeriodicTasks): LOG.info('Module apply completed.') except Exception as ex: LOG.exception("An error occurred applying modules: " - "%s", ex.message) + "%s", str(ex)) # The following block performs single-instance initialization. # Failures will be recorded, but won't stop the provisioning # or change the instance state. @@ -325,7 +325,7 @@ class Manager(periodic_task.PeriodicTasks): LOG.info('Databases created successfully.') except Exception as ex: LOG.exception("An error occurred creating databases: " - "%s", ex.message) + "%s", str(ex)) try: if users: LOG.info("Creating users (called from 'prepare')") @@ -333,7 +333,7 @@ class Manager(periodic_task.PeriodicTasks): LOG.info('Users created successfully.') except Exception as ex: LOG.exception("An error occurred creating users: " - "%s", ex.message) + "%s", str(ex)) # We only enable-root automatically if not restoring a backup # that may already have root enabled in which case we keep it @@ -345,7 +345,7 @@ class Manager(periodic_task.PeriodicTasks): LOG.info('Root enabled successfully.') except Exception as ex: LOG.exception("An error occurred enabling root user: " - "%s", ex.message) + "%s", str(ex)) try: LOG.info("Calling post_prepare for '%s' datastore.", @@ -358,7 +358,7 @@ class Manager(periodic_task.PeriodicTasks): self.manager) except Exception as ex: LOG.exception("An error occurred in post prepare: %s", - ex.message) + str(ex)) raise def apply_overrides_on_prepare(self, context, overrides): diff --git a/trove/guestagent/datastore/mysql_common/service.py b/trove/guestagent/datastore/mysql_common/service.py index ec27d606..28931b03 100644 --- a/trove/guestagent/datastore/mysql_common/service.py +++ b/trove/guestagent/datastore/mysql_common/service.py @@ -573,7 +573,7 @@ class BaseKeepAliveConnection(interfaces.PoolListener): # way than MySQL and PXC, which manifests itself as # an invalid packet sequence. Handle it as well. except pymysql_err.InternalError as ex: - if "Packet sequence number wrong" in ex.message: + if "Packet sequence number wrong" in str(ex): raise exc.DisconnectionError() else: raise diff --git a/trove/guestagent/module/drivers/module_driver.py b/trove/guestagent/module/drivers/module_driver.py index 851e7b53..ede44661 100644 --- a/trove/guestagent/module/drivers/module_driver.py +++ b/trove/guestagent/module/drivers/module_driver.py @@ -196,7 +196,7 @@ def output(log_message=None, success_message=None, LOG.exception(message) except Exception as ex: message = ("%(msg)s: %(err)s" % - {'msg': fail_msg, 'err': ex.message}) + {'msg': fail_msg, 'err': str(ex)}) LOG.exception(message) return success, message diff --git a/trove/guestagent/module/module_manager.py b/trove/guestagent/module/module_manager.py index 52237334..4c8fd1d4 100644 --- a/trove/guestagent/module/module_manager.py +++ b/trove/guestagent/module/module_manager.py @@ -68,7 +68,7 @@ class ModuleManager(object): except Exception as ex: LOG.exception("Could not apply module '%s'", name) applied = False - message = ex.message + message = str(ex) finally: status = 'OK' if applied else 'ERROR' result['removed'] = None diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index 87b14d5c..d97599fa 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -1396,9 +1396,9 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin): sleep_time=2, time_out=600) if not self.server_status_matches(['ACTIVE']): raise TroveError(_("Instance %(instance)s failed to " - "upgrade to %(datastore_version)s") % - {'instance': self, - 'datastore_version': datastore_version}) + "upgrade to %(datastore_version)s"), + instance=self, + datastore_version=datastore_version) self.guest.post_upgrade(upgrade_info) diff --git a/trove/tests/api/configurations.py b/trove/tests/api/configurations.py index 40a5023e..1a7dbaa8 100644 --- a/trove/tests/api/configurations.py +++ b/trove/tests/api/configurations.py @@ -192,7 +192,7 @@ class CreateConfigurations(ConfigurationsTestBase): instance_info.dbaas_datastore_version) resp, body = instance_info.dbaas.client.last_response attrcheck = AttrCheck() - config_parameters_dict = json.loads(body) + config_parameters_dict = json.loads(body.decode()) attrcheck.contains_allowed_attrs( config_parameters_dict, allowed_attrs, msg="Configurations parameters") @@ -226,7 +226,7 @@ class CreateConfigurations(ConfigurationsTestBase): print("resp: %s" % resp) print("body: %s" % body) attrcheck = AttrCheck() - config_parameter_dict = json.loads(body) + config_parameter_dict = json.loads(body.decode()) print("config_parameter_dict: %s" % config_parameter_dict) attrcheck.contains_allowed_attrs( config_parameter_dict, @@ -659,7 +659,7 @@ class DeleteConfigurations(ConfigurationsTestBase): resp, body = instance_info.dbaas.client.last_response print(resp) print(body) - self.config_parameter_dict = json.loads(body) + self.config_parameter_dict = json.loads(body.decode()) @after_class(always_run=True) def tearDown(self): diff --git a/trove/tests/api/mgmt/malformed_json.py b/trove/tests/api/mgmt/malformed_json.py index bc7e28c6..2967e57b 100644 --- a/trove/tests/api/mgmt/malformed_json.py +++ b/trove/tests/api/mgmt/malformed_json.py @@ -17,6 +17,7 @@ # from collections import deque +import six from proboscis import after_class from proboscis import asserts @@ -68,10 +69,14 @@ class MalformedJson(object): asserts.assert_equal(httpCode, 400, "Create instance failed with code %s," " exception %s" % (httpCode, e)) - databases = "u'foo'" - users = "u'bar'" + if six.PY3: + databases = "'%s'" % databases + users = "'%s'" % users + else: + databases = "u'%s'" % databases + users = "u'%s'" % users assert_contains( - e.message, + str(e), ["Validation error:", "instance['databases'] %s is not of type 'array'" % databases, "instance['users'] %s is not of type 'array'" % users, @@ -88,11 +93,15 @@ class MalformedJson(object): asserts.assert_equal(httpCode, 400, "Create database failed with code %s, " "exception %s" % (httpCode, e)) - _bad_db_data = "u'{foo}'" - asserts.assert_equal(e.message, - "Validation error: " - "databases %s is not of type 'array'" % - _bad_db_data) + if six.PY3: + _bad_db_data = "'%s'" % _bad_db_data + else: + _bad_db_data = "u'%s'" % _bad_db_data + asserts.assert_equal( + str(e), + "Validation error: " + "databases %s is not of type 'array' (HTTP 400)" % + _bad_db_data) @test def test_bad_user_data(self): @@ -117,7 +126,7 @@ class MalformedJson(object): "exception %s" % (httpCode, e)) err_1 = format_path(deque(('users', 0))) assert_contains( - e.message, + str(e), ["Validation error:", "%(err_1)s 'name' is a required property" % {'err_1': err_1}, "%(err_1)s 'password' is a required property" @@ -161,9 +170,12 @@ class MalformedJson(object): asserts.assert_equal(httpCode, 400, "Resize instance failed with code %s, " "exception %s" % (httpCode, e)) - data = "u'bad data'" + if six.PY3: + data = "'bad data'" + else: + data = "u'bad data'" assert_contains( - e.message, + str(e), ["Validation error:", "resize['volume']['size'] %s is not valid under " "any of the given schemas" % data, @@ -191,9 +203,12 @@ class MalformedJson(object): asserts.assert_equal(httpCode, 400, "Change usr/passwd failed with code %s, " "exception %s" % (httpCode, e)) - password = "u''" + if six.PY3: + password = "'%s'" % password + else: + password = "u'%s'" % password assert_contains( - e.message, + str(e), ["Validation error: users[0] 'password' " "is a required property", "users[0]['name'] %s is too short" % password, @@ -241,7 +256,8 @@ class MalformedJson(object): asserts.assert_equal(httpCode, 404, "Revoke user access failed w/code %s, " "exception %s" % (httpCode, e)) - asserts.assert_equal(e.message, "The resource could not be found.") + asserts.assert_equal(str(e), "The resource could not be found." + " (HTTP 404)") @test def test_bad_body_flavorid_create_instance(self): @@ -259,7 +275,7 @@ class MalformedJson(object): "exception %s" % (httpCode, e)) flavorId = [u'?'] assert_contains( - e.message, + str(e), ["Validation error:", "instance['flavorRef'] %s is not valid " "under any of the given schemas" % flavorId, @@ -284,13 +300,19 @@ class MalformedJson(object): asserts.assert_equal(httpCode, 400, "Create instance failed with code %s, " "exception %s" % (httpCode, e)) + if six.PY3: + datastore = "'%s'" % datastore + datastore_version = "'%s'" % datastore_version + else: + datastore = "u'%s'" % datastore + datastore_version = "u'%s'" % datastore_version assert_contains( - e.message, + str(e), ["Validation error:", "instance['datastore']['type']" - " u'%s' does not match" + " %s does not match" " '^.*[0-9a-zA-Z]+.*$'" % datastore, - "instance['datastore']['version'] u'%s' " + "instance['datastore']['version'] %s " "does not match '^.*[0-9a-zA-Z]+.*$'" % datastore_version]) @test @@ -306,8 +328,12 @@ class MalformedJson(object): asserts.assert_equal(httpCode, 400, "Create instance failed with code %s, " "exception %s" % (httpCode, e)) - volsize = "u'h3ll0'" - asserts.assert_equal(e.message, + if six.PY3: + volsize = "'%s'" % volsize + else: + volsize = "u'%s'" % volsize + print("DEBUG DEV: %s" % str(e)) + asserts.assert_equal(str(e), "Validation error: " "instance['volume'] %s is not of " - "type 'object'" % volsize) + "type 'object' (HTTP 400)" % volsize) diff --git a/trove/tests/examples/client.py b/trove/tests/examples/client.py index 57835e35..06c99f56 100644 --- a/trove/tests/examples/client.py +++ b/trove/tests/examples/client.py @@ -288,6 +288,6 @@ class JsonClient(TroveHTTPClient): self.pretty_log(args, kwargs, resp, body) def write_snippet(): - return write_to_snippet(self, args, kwargs, resp, body) + return write_to_snippet(self, args, kwargs, resp, body.decode()) self.write_snippet = write_snippet diff --git a/trove/tests/scenario/runners/configuration_runners.py b/trove/tests/scenario/runners/configuration_runners.py index 8591286f..40fba068 100644 --- a/trove/tests/scenario/runners/configuration_runners.py +++ b/trove/tests/scenario/runners/configuration_runners.py @@ -233,7 +233,7 @@ class ConfigurationRunner(TestRunner): with CollectionCheck("configuration_values", result.values) as check: # check each item has the correct type according to the rules - for (item_key, item_val) in result.values.iteritems(): + for (item_key, item_val) in result.values.items(): print("item_key: %s" % item_key) print("item_val: %s" % item_val) param = ( diff --git a/trove/tests/util/client.py b/trove/tests/util/client.py index 163e62a1..106da2b2 100644 --- a/trove/tests/util/client.py +++ b/trove/tests/util/client.py @@ -99,4 +99,8 @@ class TestClient(object): return flavor, flavor_href def __getattr__(self, item): - return getattr(self.real_client, item) + if item == "__setstate__": + raise AttributeError(item) + if hasattr(self.real_client, item): + return getattr(self.real_client, item) + raise AttributeError(item) |