diff options
-rw-r--r-- | ironic/api/app.py | 3 | ||||
-rw-r--r-- | ironic/api/hooks.py | 34 | ||||
-rw-r--r-- | ironic/common/hash_ring.py | 85 | ||||
-rw-r--r-- | ironic/db/sqlalchemy/migrate_repo/versions/014_add_address_uc_sqlite.py | 37 | ||||
-rw-r--r-- | ironic/openstack/common/context.py | 7 | ||||
-rw-r--r-- | ironic/openstack/common/uuidutils.py | 39 | ||||
-rw-r--r-- | ironic/tests/api/test_hooks.py | 88 | ||||
-rw-r--r-- | ironic/tests/api/test_nodes.py | 3 | ||||
-rw-r--r-- | ironic/tests/api/test_ports.py | 6 | ||||
-rw-r--r-- | ironic/tests/db/sqlalchemy/test_migrations.py | 21 | ||||
-rw-r--r-- | ironic/tests/db/test_ports.py | 3 | ||||
-rw-r--r-- | ironic/tests/test_hash_ring.py | 107 | ||||
-rw-r--r-- | ironic/tests/test_utils.py | 20 | ||||
-rw-r--r-- | openstack-common.conf | 1 |
14 files changed, 404 insertions, 50 deletions
diff --git a/ironic/api/app.py b/ironic/api/app.py index ec42b6c54..b3478873d 100644 --- a/ironic/api/app.py +++ b/ironic/api/app.py @@ -47,7 +47,8 @@ def setup_app(pecan_config=None, extra_hooks=None): app_hooks = [hooks.ConfigHook(), hooks.DBHook(), hooks.ContextHook(pecan_config.app.acl_public_routes), - hooks.RPCHook()] + hooks.RPCHook(), + hooks.NoExceptionTracebackHook()] if extra_hooks: app_hooks.extend(extra_hooks) diff --git a/ironic/api/hooks.py b/ironic/api/hooks.py index 20739a948..3513c0183 100644 --- a/ironic/api/hooks.py +++ b/ironic/api/hooks.py @@ -110,3 +110,37 @@ class AdminAuthHook(hooks.PecanHook): if not is_admin_api and not ctx.is_public_api: raise exc.HTTPForbidden() + + +class NoExceptionTracebackHook(hooks.PecanHook): + """Ensures that no exception traceback is built into the faultstring + (it's a workaround for the rpc.common: deserialize_remote_exception + behavior) + """ + + # NOTE(max_lobur): 'after' hook used instead of 'on_error' because + # 'on_error' never fired for wsme+pecan pair. wsme @wsexpose decorator + # catches and handles all the errors, so 'on_error' dedicated for unhandled + # exceptions never fired. + def after(self, state): + # Do not remove traceback when server in debug mode. + if cfg.CONF.debug: + return + # Do nothing if there is no error. + if 200 <= state.response.status_int < 400: + return + # Omit empty body. Some errors may not have body at this level yet. + if not state.response.body: + return + + json_body = state.response.json + faultsting = json_body.get('faultstring') + traceback_marker = 'Traceback (most recent call last):' + if faultsting and (traceback_marker in faultsting): + # Cut-off traceback. + faultsting = faultsting.split(traceback_marker, 1)[0] + # Remove trailing newlines and spaces if any. + json_body['faultstring'] = faultsting.rstrip() + # Replace the whole json. Cannot change original one beacause it's + # generated on the fly. + state.response.json = json_body diff --git a/ironic/common/hash_ring.py b/ironic/common/hash_ring.py new file mode 100644 index 000000000..008ee9781 --- /dev/null +++ b/ironic/common/hash_ring.py @@ -0,0 +1,85 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import array +import hashlib +import struct + +from oslo.config import cfg + +hash_opts = [ + cfg.IntOpt('hash_partition_exponent', + default=16, + help='Exponent to determine number of hash partitions to use ' + 'when distributing load across conductors.'), +] + +CONF = cfg.CONF +CONF.register_opts(hash_opts) + + +class HashRing(object): + + def __init__(self, hosts, replicas=1): + """Create a new hash ring across the specified hosts. + + :param hosts: array of hosts which will be mapped + :param replicas: number of replicas. Default: 1 + + """ + self.hosts = hosts + self.replicas = replicas + self.partition_shift = 32 - CONF.hash_partition_exponent + self.part2host = array.array('H') + for p in xrange(2 ** CONF.hash_partition_exponent): + self.part2host.append(p % len(hosts)) + + def _get_partition(self, data): + return (struct.unpack_from('>I', hashlib.md5(data).digest())[0] + >> self.partition_shift) + + def get_hosts(self, data, ignore_hosts=None): + """Get the list of hosts which the supplied data maps onto. + + :param data: A string identifier to be mapped across the ring. + :param ignore_hosts: A list of hosts to skip when performing the hash. + Useful to temporarily skip down hosts without + performing a full rebalance. + Default: None. + :returns: a list of hosts. + The length of this list depends on the number of replicas + this `HashRing` was created with. It may be less than this + if ignore_hosts is not None. + """ + host_ids = [] + if ignore_hosts is None: + ignore_host_ids = [] + else: + ignore_host_ids = [self.hosts.index(h) + for h in ignore_hosts if h in self.hosts] + + partition = self._get_partition(data) + for replica in xrange(0, self.replicas): + if len(host_ids + ignore_host_ids) == len(self.hosts): + # prevent infinite loop + break + while self.part2host[partition] in host_ids + ignore_host_ids: + partition += 1 + if partition >= len(self.part2host): + partition = 0 + host_ids.append(self.part2host[partition]) + return [self.hosts[h] for h in host_ids] diff --git a/ironic/db/sqlalchemy/migrate_repo/versions/014_add_address_uc_sqlite.py b/ironic/db/sqlalchemy/migrate_repo/versions/014_add_address_uc_sqlite.py new file mode 100644 index 000000000..3557e9e53 --- /dev/null +++ b/ironic/db/sqlalchemy/migrate_repo/versions/014_add_address_uc_sqlite.py @@ -0,0 +1,37 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# -*- encoding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from migrate.changeset import UniqueConstraint +from sqlalchemy import MetaData, Table + + +def upgrade(migrate_engine): + if migrate_engine.name == 'sqlite': + meta = MetaData(bind=migrate_engine) + ports = Table('ports', meta, autoload=True) + + uniques = ( + UniqueConstraint('address', table=ports, name='iface_address_ux'), + # NOTE(yuriyz): this migration can drop first UC in 'ports' table + # for sqlite backend (sqlalchemy-migrate bug), recreate it + UniqueConstraint('uuid', table=ports, name='port_uuid_ux') + ) + + for uc in uniques: + uc.create() + + +def downgrade(migrate_engine): + raise NotImplementedError('Downgrade from version 014 is unsupported.') diff --git a/ironic/openstack/common/context.py b/ironic/openstack/common/context.py index 8dd7936a6..2e46d7024 100644 --- a/ironic/openstack/common/context.py +++ b/ironic/openstack/common/context.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -23,12 +21,11 @@ context or provide additional information in their specific WSGI pipeline. """ import itertools - -from ironic.openstack.common import uuidutils +import uuid def generate_request_id(): - return 'req-%s' % uuidutils.generate_uuid() + return 'req-%s' % str(uuid.uuid4()) class RequestContext(object): diff --git a/ironic/openstack/common/uuidutils.py b/ironic/openstack/common/uuidutils.py deleted file mode 100644 index 7608acb94..000000000 --- a/ironic/openstack/common/uuidutils.py +++ /dev/null @@ -1,39 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2012 Intel Corporation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -UUID related utilities and helper functions. -""" - -import uuid - - -def generate_uuid(): - return str(uuid.uuid4()) - - -def is_uuid_like(val): - """Returns validation of a value as a UUID. - - For our purposes, a UUID is a canonical form string: - aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa - - """ - try: - return str(uuid.UUID(val)) == val - except (TypeError, ValueError, AttributeError): - return False diff --git a/ironic/tests/api/test_hooks.py b/ironic/tests/api/test_hooks.py new file mode 100644 index 000000000..3db5c0913 --- /dev/null +++ b/ironic/tests/api/test_hooks.py @@ -0,0 +1,88 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# -*- encoding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests for the Pecan API hooks. +""" + +import json +import mock +from oslo.config import cfg + +from ironic.api.controllers import root +from ironic.openstack.common.rpc import common as rpc_common +from ironic.tests.api import base + + +class TestNoExceptionTracebackHook(base.FunctionalTest): + + TRACE = [u'Traceback (most recent call last):', + u' File "/opt/stack/ironic/ironic/openstack/common/rpc/amqp.py",' + ' line 434, in _process_data\\n **args)', + u' File "/opt/stack/ironic/ironic/openstack/common/rpc/' + 'dispatcher.py", line 172, in dispatch\\n result =' + ' getattr(proxyobj, method)(ctxt, **kwargs)'] + MSG_WITHOUT_TRACE = "Test exception message." + MSG_WITH_TRACE = MSG_WITHOUT_TRACE + "\n" + "\n".join(TRACE) + + def setUp(self): + super(TestNoExceptionTracebackHook, self).setUp() + p = mock.patch.object(root.Root, 'convert') + self.root_convert_mock = p.start() + self.addCleanup(p.stop) + + def test_hook_exception_success(self): + self.root_convert_mock.side_effect = Exception(self.MSG_WITH_TRACE) + + response = self.get_json('/', path_prefix='', expect_errors=True) + + actual_msg = json.loads(response.json['error_message'])['faultstring'] + self.assertEqual(self.MSG_WITHOUT_TRACE, actual_msg) + + def test_hook_remote_error_success(self): + test_exc_type = 'TestException' + self.root_convert_mock.side_effect = rpc_common.RemoteError( + test_exc_type, self.MSG_WITHOUT_TRACE, self.TRACE) + + response = self.get_json('/', path_prefix='', expect_errors=True) + + # NOTE(max_lobur): For RemoteError the client message will still have + # some garbage because in RemoteError traceback is serialized as a list + # instead of'\n'.join(trace). But since RemoteError is kind of very + # rare thing (happens due to wrong deserialization settings etc.) + # we don't care about this garbage. + expected_msg = "Remote error: %s %s" \ + % (test_exc_type, self.MSG_WITHOUT_TRACE) + "\n[u'" + actual_msg = json.loads(response.json['error_message'])['faultstring'] + self.assertEqual(expected_msg, actual_msg) + + def test_hook_without_traceback(self): + msg = "Error message without traceback \n but \n multiline" + self.root_convert_mock.side_effect = Exception(msg) + + response = self.get_json('/', path_prefix='', expect_errors=True) + + actual_msg = json.loads(response.json['error_message'])['faultstring'] + self.assertEqual(msg, actual_msg) + + def test_hook_server_debug_on(self): + cfg.CONF.set_override('debug', True) + self.root_convert_mock.side_effect = Exception(self.MSG_WITH_TRACE) + + response = self.get_json('/', path_prefix='', expect_errors=True) + + actual_msg = json.loads( + response.json['error_message'])['faultstring'] + self.assertEqual(self.MSG_WITH_TRACE, actual_msg) diff --git a/ironic/tests/api/test_nodes.py b/ironic/tests/api/test_nodes.py index 2c787f5f1..0209fdcdd 100644 --- a/ironic/tests/api/test_nodes.py +++ b/ironic/tests/api/test_nodes.py @@ -141,7 +141,8 @@ class TestListNodes(base.FunctionalTest): for id in xrange(2): pdict = dbutils.get_test_port(id=id, node_id=ndict['id'], - uuid=utils.generate_uuid()) + uuid=utils.generate_uuid(), + address='52:54:00:cf:2d:3%s' % id) self.dbapi.create_port(pdict) data = self.get_json('/nodes/%s/ports' % ndict['uuid']) diff --git a/ironic/tests/api/test_ports.py b/ironic/tests/api/test_ports.py index bfadd3cc4..123018ede 100644 --- a/ironic/tests/api/test_ports.py +++ b/ironic/tests/api/test_ports.py @@ -59,7 +59,8 @@ class TestListPorts(base.FunctionalTest): ports = [] for id in xrange(5): ndict = dbutils.get_test_port(id=id, - uuid=utils.generate_uuid()) + uuid=utils.generate_uuid(), + address='52:54:00:cf:2d:3%s' % id) port = self.dbapi.create_port(ndict) ports.append(port['uuid']) data = self.get_json('/ports') @@ -83,7 +84,8 @@ class TestListPorts(base.FunctionalTest): ports = [] for id in xrange(5): ndict = dbutils.get_test_port(id=id, - uuid=utils.generate_uuid()) + uuid=utils.generate_uuid(), + address='52:54:00:cf:2d:3%s' % id) port = self.dbapi.create_port(ndict) ports.append(port['uuid']) data = self.get_json('/ports/?limit=3') diff --git a/ironic/tests/db/sqlalchemy/test_migrations.py b/ironic/tests/db/sqlalchemy/test_migrations.py index 6fd19f9ba..da7873efc 100644 --- a/ironic/tests/db/sqlalchemy/test_migrations.py +++ b/ironic/tests/db/sqlalchemy/test_migrations.py @@ -736,3 +736,24 @@ class TestMigrations(BaseMigrationTestCase, WalkVersionsMixin): self.assertEqual(len(col_names_pre), len(col_names) - 1) self.assertTrue(isinstance(nodes.c['last_error'].type, getattr(sqlalchemy.types, 'Text'))) + + def _check_014(self, engine, data): + if engine.name == 'sqlite': + ports = db_utils.get_table(engine, 'ports') + ports_data = {'address': 'BB:BB:AA:AA:AA:AA', 'extra': 'extra1'} + ports.insert().values(ports_data).execute() + self.assertRaises(sqlalchemy.exc.IntegrityError, + ports.insert().execute, + {'address': 'BB:BB:AA:AA:AA:AA', + 'extra': 'extra2'}) + # test recreate old UC + ports_data = { + 'address': 'BB:BB:AA:AA:AA:BB', + 'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c781', + 'extra': 'extra2'} + ports.insert().values(ports_data).execute() + self.assertRaises(sqlalchemy.exc.IntegrityError, + ports.insert().execute, + {'address': 'CC:BB:AA:AA:AA:CC', + 'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c781', + 'extra': 'extra3'}) diff --git a/ironic/tests/db/test_ports.py b/ironic/tests/db/test_ports.py index 7850c4f10..a5dc3e7fc 100644 --- a/ironic/tests/db/test_ports.py +++ b/ironic/tests/db/test_ports.py @@ -51,7 +51,8 @@ class DbPortTestCase(base.DbTestCase): def test_get_port_list(self): uuids = [] for i in xrange(1, 6): - n = utils.get_test_port(id=i, uuid=ironic_utils.generate_uuid()) + n = utils.get_test_port(id=i, uuid=ironic_utils.generate_uuid(), + address='52:54:00:cf:2d:3%s' % i) self.dbapi.create_port(n) uuids.append(six.text_type(n['uuid'])) res = self.dbapi.get_port_list() diff --git a/ironic/tests/test_hash_ring.py b/ironic/tests/test_hash_ring.py new file mode 100644 index 000000000..3bf407def --- /dev/null +++ b/ironic/tests/test_hash_ring.py @@ -0,0 +1,107 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from ironic.common import hash_ring as hash +from ironic.tests import base + +CONF = cfg.CONF + + +class HashRingTestCase(base.TestCase): + + # NOTE(deva): the mapping used in these tests is as follows: + # if hosts = [foo, bar]: + # fake -> foo, bar + # if hosts = [foo, bar, baz]: + # fake -> foo, bar, baz + # fake-again -> bar, baz, foo + + def test_create_ring(self): + hosts = ['foo', 'bar'] + replicas = 2 + ring = hash.HashRing(hosts, replicas=replicas) + self.assertEqual(hosts, ring.hosts) + self.assertEqual(replicas, ring.replicas) + + def test_create_with_different_partition_counts(self): + hosts = ['foo', 'bar'] + CONF.set_override('hash_partition_exponent', 2) + ring = hash.HashRing(hosts) + self.assertEqual(2 ** 2, len(ring.part2host)) + + CONF.set_override('hash_partition_exponent', 8) + ring = hash.HashRing(hosts) + self.assertEqual(2 ** 8, len(ring.part2host)) + + CONF.set_override('hash_partition_exponent', 16) + ring = hash.HashRing(hosts) + self.assertEqual(2 ** 16, len(ring.part2host)) + + def test_distribution_one_replica(self): + hosts = ['foo', 'bar', 'baz'] + ring = hash.HashRing(hosts, replicas=1) + self.assertEqual(['foo'], ring.get_hosts('fake')) + self.assertEqual(['bar'], ring.get_hosts('fake-again')) + + def test_distribution_two_replicas(self): + hosts = ['foo', 'bar', 'baz'] + ring = hash.HashRing(hosts, replicas=2) + self.assertEqual(['foo', 'bar'], ring.get_hosts('fake')) + self.assertEqual(['bar', 'baz'], ring.get_hosts('fake-again')) + + def test_distribution_three_replicas(self): + hosts = ['foo', 'bar', 'baz'] + ring = hash.HashRing(hosts, replicas=3) + self.assertEqual(['foo', 'bar', 'baz'], ring.get_hosts('fake')) + self.assertEqual(['bar', 'baz', 'foo'], ring.get_hosts('fake-again')) + + def test_ignore_hosts(self): + hosts = ['foo', 'bar', 'baz'] + ring = hash.HashRing(hosts) + self.assertEqual(['bar'], ring.get_hosts('fake', + ignore_hosts=['foo'])) + self.assertEqual(['baz'], ring.get_hosts('fake', + ignore_hosts=['foo', 'bar'])) + self.assertEqual([], ring.get_hosts('fake', + ignore_hosts=hosts)) + + def test_ignore_hosts_with_replicas(self): + hosts = ['foo', 'bar', 'baz'] + ring = hash.HashRing(hosts, replicas=2) + self.assertEqual(['bar', 'baz'], ring.get_hosts('fake', + ignore_hosts=['foo'])) + self.assertEqual(['baz'], ring.get_hosts('fake', + ignore_hosts=['foo', 'bar'])) + self.assertEqual(['baz', 'foo'], ring.get_hosts('fake-again', + ignore_hosts=['bar'])) + self.assertEqual(['foo'], ring.get_hosts('fake-again', + ignore_hosts=['bar', 'baz'])) + self.assertEqual([], ring.get_hosts('fake', + ignore_hosts=hosts)) + + def test_more_replicas_than_hosts(self): + hosts = ['foo', 'bar'] + ring = hash.HashRing(hosts, replicas=10) + self.assertEqual(hosts, ring.get_hosts('fake')) + + def test_ignore_non_existent_host(self): + hosts = ['foo', 'bar'] + ring = hash.HashRing(hosts) + self.assertEqual(['foo'], ring.get_hosts('fake', + ignore_hosts=['baz'])) diff --git a/ironic/tests/test_utils.py b/ironic/tests/test_utils.py index 8ca890807..d5539be17 100644 --- a/ironic/tests/test_utils.py +++ b/ironic/tests/test_utils.py @@ -22,6 +22,7 @@ import os import os.path import StringIO import tempfile +import uuid import mock import netaddr @@ -369,3 +370,22 @@ class IntLikeTestCase(base.TestCase): self.assertFalse( utils.is_int_like("0cc3346e-9fef-4445-abe6-5d2b2690ec64")) self.assertFalse(utils.is_int_like("a1")) + + +class UUIDTestCase(base.TestCase): + + def test_generate_uuid(self): + uuid_string = utils.generate_uuid() + self.assertTrue(isinstance(uuid_string, str)) + self.assertEqual(len(uuid_string), 36) + # make sure there are 4 dashes + self.assertEqual(len(uuid_string.replace('-', '')), 32) + + def test_is_uuid_like(self): + self.assertTrue(utils.is_uuid_like(str(uuid.uuid4()))) + + def test_id_is_uuid_like(self): + self.assertFalse(utils.is_uuid_like(1234567)) + + def test_name_is_uuid_like(self): + self.assertFalse(utils.is_uuid_like('zhongyueluo')) diff --git a/openstack-common.conf b/openstack-common.conf index 2782abd43..151319957 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -27,7 +27,6 @@ module=rpc module=setup module=strutils module=timeutils -module=uuidutils module=version # The base module to hold the copy of openstack.common |