diff options
| -rw-r--r-- | .zuul.yaml | 13 | ||||
| -rw-r--r-- | devstack/lib/heat | 4 | ||||
| -rwxr-xr-x | devstack/upgrade/resources.sh | 31 | ||||
| -rw-r--r-- | doc/source/getting_started/on_devstack.rst | 4 | ||||
| -rw-r--r-- | heat/engine/resources/openstack/nova/server_group.py | 10 | ||||
| -rw-r--r-- | heat/engine/stack.py | 11 | ||||
| -rw-r--r-- | heat/tests/openstack/nova/test_server_group.py | 35 | ||||
| -rw-r--r-- | heat/tests/test_convg_stack.py | 28 | ||||
| -rwxr-xr-x | heat_integrationtests/cleanup_test_env.sh | 2 | ||||
| -rwxr-xr-x | heat_integrationtests/pre_test_hook.sh | 2 | ||||
| -rwxr-xr-x | heat_integrationtests/prepare_test_env.sh | 4 | ||||
| -rw-r--r-- | lower-constraints.txt | 161 | ||||
| -rw-r--r-- | requirements.txt | 4 | ||||
| -rwxr-xr-x | tools/test-setup.sh | 4 | ||||
| -rw-r--r-- | tox.ini | 8 |
15 files changed, 103 insertions, 218 deletions
diff --git a/.zuul.yaml b/.zuul.yaml index 992275c5f..fc6cbe0fe 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -65,7 +65,6 @@ - job: name: grenade-heat parent: grenade - voting: false required-projects: - opendev.org/openstack/heat - opendev.org/openstack/heat-tempest-plugin @@ -78,7 +77,7 @@ h-eng: true heat: true # We do run a list of tests after upgrade. This is just to bypass the req from parent. - tempest_test_regex: ^heat_tempest_plugin\.tests\.functional\.test_nova_server_networks + tempest_test_regex: ^tempest\.api\.identity\.v3\.test_tokens tox_envlist: all devstack_plugins: heat: https://opendev.org/openstack/heat @@ -89,11 +88,12 @@ name: grenade-heat-multinode parent: grenade-heat nodeset: openstack-two-node-bionic + voting: false - project: + queue: heat templates: - openstack-cover-jobs - - openstack-lower-constraints-jobs - openstack-python-jobs - openstack-python3-train-jobs - periodic-stable-jobs @@ -111,13 +111,6 @@ - heat-functional-convg-mysql-lbaasv2-non-apache - heat-functional-convg-mysql-lbaasv2-py3 gate: - queue: heat jobs: - grenade-heat - - grenade-heat-multinode - - heat-functional-orig-mysql-lbaasv2 - - heat-functional-convg-mysql-lbaasv2 - heat-functional-convg-mysql-lbaasv2-py3 - experimental: - jobs: - - tripleo-ci-centos-7-scenario002-standalone diff --git a/devstack/lib/heat b/devstack/lib/heat index 4a31d4acc..54c780e71 100644 --- a/devstack/lib/heat +++ b/devstack/lib/heat @@ -78,6 +78,7 @@ else fi HEAT_PLUGIN_DIR=${HEAT_PLUGIN_DIR:-$DATA_DIR/heat/plugins} ENABLE_HEAT_PLUGINS=${ENABLE_HEAT_PLUGINS:-} +HEAT_ENGINE_WORKERS=${HEAT_ENGINE_WORKERS:=$(( ($(nproc)/4)<2 ? 2 : ($(nproc)/4) ))} # Functions # --------- @@ -135,6 +136,9 @@ function configure_heat { # logging iniset $HEAT_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL + # reduce Heat engine workers + iniset $HEAT_CONF DEFAULT num_engine_workers "$HEAT_ENGINE_WORKERS" + local no_format="False" if [[ "$HEAT_USE_APACHE" == "True" && "$WSGI_MODE" != "uwsgi" ]]; then no_format="True" diff --git a/devstack/upgrade/resources.sh b/devstack/upgrade/resources.sh index bfa276107..65d87a83a 100755 --- a/devstack/upgrade/resources.sh +++ b/devstack/upgrade/resources.sh @@ -65,26 +65,11 @@ function _run_heat_integrationtests { # Run set of specified functional tests UPGRADE_TESTS=upgrade_tests.list _write_heat_integrationtests $UPGRADE_TESTS - - # NOTE(gmann): If devstack is pinned to use the non master - # Tempest and constraints for Tempest venv then use the same - # while running the tests too otherwise, it will recreate - # the Tempest venv due to constraints mismatch. - # recreation of Tempest venv can flush the initially installed - # tempest plugins and their deps. - if [[ $use_stable_constraints == "True" ]]; then - echo "Using $DEST/requirements/upper-constraints.txt constraints in Tempest venv." - # NOTE: setting both tox env var and once Tempest start using new var - # TOX_CONSTRAINTS_FILE then we can remove the old one. - export UPPER_CONSTRAINTS_FILE=$DEST/requirements/upper-constraints.txt - export TOX_CONSTRAINTS_FILE=$UPPER_CONSTRAINTS_FILE - else - echo "Using master constraints in Tempest venv." - # NOTE: setting both tox env var and once Tempest start using new var - # TOX_CONSTRAINTS_FILE then we can remove the old one. - export UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/master - export TOX_CONSTRAINTS_FILE=$UPPER_CONSTRAINTS_FILE - fi + export UPPER_CONSTRAINTS_FILE=$DEST/requirements/upper-constraints.txt + export TOX_CONSTRAINTS_FILE=$UPPER_CONSTRAINTS_FILE + export HEAT_TEMPEST_PLUGIN=$DEST/heat-tempest-plugin + sudo git config --system --add safe.directory $HEAT_TEMPEST_PLUGIN + tox -evenv-tempest -- pip install -c$UPPER_CONSTRAINTS_FILE $HEAT_TEMPEST_PLUGIN tox -evenv-tempest -- stestr --test-path=$DEST/heat/heat_integrationtests --top-dir=$DEST/heat \ --group_regex='heat_tempest_plugin\.tests\.api\.test_heat_api[._]([^_]+)' \ run --whitelist-file $UPGRADE_TESTS @@ -125,7 +110,7 @@ function create { local stack_name='grenadine' resource_save heat stack_name $stack_name local loc=`dirname $BASH_SOURCE` - heat stack-create -f $loc/templates/random_string.yaml $stack_name + openstack stack create -t $loc/templates/random_string.yaml $stack_name } function verify { @@ -137,7 +122,7 @@ function verify { fi fi stack_name=$(resource_get heat stack_name) - heat stack-show $stack_name + openstack stack show $stack_name # TODO(sirushtim): Create more granular checks for Heat. } @@ -149,7 +134,7 @@ function verify_noapi { function destroy { _heat_set_user - heat stack-delete $(resource_get heat stack_name) + openstack stack delete -y $(resource_get heat stack_name) source $TOP_DIR/openrc admin admin local user_id=$(resource_get heat user_id) diff --git a/doc/source/getting_started/on_devstack.rst b/doc/source/getting_started/on_devstack.rst index af73c49e6..760381c34 100644 --- a/doc/source/getting_started/on_devstack.rst +++ b/doc/source/getting_started/on_devstack.rst @@ -51,8 +51,8 @@ a VM image that heat can launch. To do that add the following to `[[local|localrc]]` section of `local.conf`:: IMAGE_URL_SITE="https://download.fedoraproject.org" - IMAGE_URL_PATH="/pub/fedora/linux/releases/33/Cloud/x86_64/images/" - IMAGE_URL_FILE="Fedora-Cloud-Base-33-1.2.x86_64.qcow2" + IMAGE_URL_PATH="/pub/fedora/linux/releases/36/Cloud/x86_64/images/" + IMAGE_URL_FILE="Fedora-Cloud-Base-36-1.5.x86_64.qcow2" IMAGE_URLS+=","$IMAGE_URL_SITE$IMAGE_URL_PATH$IMAGE_URL_FILE URLs for any cloud image may be specified, but fedora images from F20 contain diff --git a/heat/engine/resources/openstack/nova/server_group.py b/heat/engine/resources/openstack/nova/server_group.py index 5b5a4c3ff..7d8049598 100644 --- a/heat/engine/resources/openstack/nova/server_group.py +++ b/heat/engine/resources/openstack/nova/server_group.py @@ -80,6 +80,16 @@ class ServerGroup(resource.Resource): policies=policies) self.resource_id_set(server_group.id) + def needs_replace_failed(self): + if not self.resource_id: + return True + + with self.client_plugin().ignore_not_found: + self._show_resource() + return False + + return True + def physical_resource_name(self): name = self.properties[self.NAME] if name: diff --git a/heat/engine/stack.py b/heat/engine/stack.py index f0887bbd6..0519b5b1f 100644 --- a/heat/engine/stack.py +++ b/heat/engine/stack.py @@ -1491,7 +1491,10 @@ class Stack(collections.Mapping): # Rolling back to previous resource score += 10 - return score, ext_rsrc.updated_at + last_changed_at = ext_rsrc.updated_at + if last_changed_at is None: + last_changed_at = ext_rsrc.created_at + return score, last_changed_at candidates = sorted((r for r in self.ext_rsrcs_db.values() if r.name == rsrc_name), @@ -1831,11 +1834,13 @@ class Stack(collections.Mapping): def _try_get_user_creds(self): # There are cases where the user_creds cannot be returned # due to credentials truncated when being saved to DB. - # Ignore this error instead of blocking stack deletion. + # Also, there are cases where auth_encryption_key has + # changed for some reason. + # Ignore these errors instead of blocking stack deletion. try: return ucreds_object.UserCreds.get_by_id(self.context, self.user_creds_id) - except exception.Error: + except (exception.Error, exception.InvalidEncryptionKey): LOG.exception("Failed to retrieve user_creds") return None diff --git a/heat/tests/openstack/nova/test_server_group.py b/heat/tests/openstack/nova/test_server_group.py index a8129fa9e..354328151 100644 --- a/heat/tests/openstack/nova/test_server_group.py +++ b/heat/tests/openstack/nova/test_server_group.py @@ -15,6 +15,9 @@ import json import mock +from novaclient import exceptions +from oslo_utils import excutils + from heat.common import template_format from heat.engine.clients.os import nova from heat.engine import scheduler @@ -54,6 +57,20 @@ class NovaServerGroupTest(common.HeatTestCase): # create mock clients and objects nova = mock.MagicMock() self.sg.client = mock.MagicMock(return_value=nova) + + class FakeNovaPlugin(object): + + @excutils.exception_filter + def ignore_not_found(self, ex): + if not isinstance(ex, exceptions.NotFound): + raise ex + + def is_conflict(self, ex): + return False + + self.patchobject(excutils.exception_filter, '__exit__') + self.patchobject(self.sg, 'client_plugin', + return_value=FakeNovaPlugin()) self.sg_mgr = nova.server_groups def _create_sg(self, name): @@ -103,3 +120,21 @@ class NovaServerGroupTest(common.HeatTestCase): self.sg.client().server_groups = s_groups self.assertEqual({'server_gr': 'info'}, self.sg.FnGetAtt('show')) s_groups.get.assert_called_once_with('test') + + def test_needs_replace_failed(self): + self._create_sg('test') + self.sg.state_set(self.sg.CREATE, self.sg.FAILED) + mock_show_resource = self.patchobject(self.sg, '_show_resource') + mock_show_resource.side_effect = [exceptions.NotFound(404), None] + + self.sg.resource_id = None + self.assertTrue(self.sg.needs_replace_failed()) + self.assertEqual(0, mock_show_resource.call_count) + + self.sg.resource_id = 'sg_id' + self.assertTrue(self.sg.needs_replace_failed()) + self.assertEqual(1, mock_show_resource.call_count) + + mock_show_resource.return_value = None + self.assertFalse(self.sg.needs_replace_failed()) + self.assertEqual(2, mock_show_resource.call_count) diff --git a/heat/tests/test_convg_stack.py b/heat/tests/test_convg_stack.py index a642b51cc..2754ec943 100644 --- a/heat/tests/test_convg_stack.py +++ b/heat/tests/test_convg_stack.py @@ -11,7 +11,10 @@ # License for the specific language governing permissions and limitations # under the License. +from datetime import datetime +from datetime import timedelta import mock + from oslo_config import cfg from heat.common import template_format @@ -428,22 +431,32 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase): stack.prev_raw_template_id = 2 stack.t.id = 3 - def db_resource(current_template_id): + def db_resource(current_template_id, + created_at=None, + updated_at=None): db_res = resource_objects.Resource(stack.context) db_res['id'] = current_template_id db_res['name'] = 'A' db_res['current_template_id'] = current_template_id - db_res['action'] = 'CREATE' + db_res['action'] = 'UPDATE' if updated_at else 'CREATE' db_res['status'] = 'COMPLETE' - db_res['updated_at'] = None + db_res['updated_at'] = updated_at + db_res['created_at'] = created_at db_res['replaced_by'] = None return db_res + start_time = datetime.utcfromtimestamp(0) + + def t(minutes): + return start_time + timedelta(minutes=minutes) + a_res_2 = db_resource(2) a_res_3 = db_resource(3) - a_res_1 = db_resource(1) + a_res_0 = db_resource(0, created_at=t(0), updated_at=t(1)) + a_res_1 = db_resource(1, created_at=t(2)) existing_res = {a_res_2.id: a_res_2, a_res_3.id: a_res_3, + a_res_0.id: a_res_0, a_res_1.id: a_res_1} stack.ext_rsrcs_db = existing_res best_res = stack._get_best_existing_rsrc_db('A') @@ -459,9 +472,14 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase): # no resource with current template id as 3 or 2 del existing_res[2] best_res = stack._get_best_existing_rsrc_db('A') - # should return resource with template id 1 existing in DB + # should return resource with template id 1 which is the newest self.assertEqual(a_res_1.id, best_res.id) + del existing_res[1] + best_res = stack._get_best_existing_rsrc_db('A') + # should return resource with template id 0 existing in the db + self.assertEqual(a_res_0.id, best_res.id) + @mock.patch.object(parser.Stack, '_converge_create_or_update') def test_updated_time_stack_create(self, mock_ccu, mock_cr): stack = parser.Stack(utils.dummy_context(), 'convg_updated_time_test', diff --git a/heat_integrationtests/cleanup_test_env.sh b/heat_integrationtests/cleanup_test_env.sh index 55e235c4c..95741b803 100755 --- a/heat_integrationtests/cleanup_test_env.sh +++ b/heat_integrationtests/cleanup_test_env.sh @@ -30,4 +30,4 @@ openstack flavor delete m1.heat_int openstack flavor delete m1.heat_micro # delete the image created -openstack image delete Fedora-Cloud-Base-33-1.2.x86_64 +openstack image delete Fedora-Cloud-Base-36-1.5.x86_64 diff --git a/heat_integrationtests/pre_test_hook.sh b/heat_integrationtests/pre_test_hook.sh index 444c7f853..c470ba62b 100755 --- a/heat_integrationtests/pre_test_hook.sh +++ b/heat_integrationtests/pre_test_hook.sh @@ -46,7 +46,7 @@ if [[ -e /etc/ci/mirror_info.sh ]]; then source /etc/ci/mirror_info.sh fi HEAT_TEST_FEDORA_IMAGE_UPSTREAM=https://download.fedoraproject.org/pub/fedora/linux -HEAT_TEST_FEDORA_IMAGE_PATH=releases/33/Cloud/x86_64/images/Fedora-Cloud-Base-33-1.2.x86_64.qcow2 +HEAT_TEST_FEDORA_IMAGE_PATH=releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2 if curl --output /dev/null --silent --head --fail "${NODEPOOL_FEDORA_MIRROR}/${HEAT_TEST_FEDORA_IMAGE_PATH}"; then echo "IMAGE_URLS+=${NODEPOOL_FEDORA_MIRROR}/${HEAT_TEST_FEDORA_IMAGE_PATH}" >> $localconf else diff --git a/heat_integrationtests/prepare_test_env.sh b/heat_integrationtests/prepare_test_env.sh index eb26986f1..f129a211d 100755 --- a/heat_integrationtests/prepare_test_env.sh +++ b/heat_integrationtests/prepare_test_env.sh @@ -46,7 +46,7 @@ function _config_iniset { iniset $conf_file heat_plugin instance_type m1.heat_int iniset $conf_file heat_plugin minimal_instance_type m1.heat_micro - iniset $conf_file heat_plugin image_ref Fedora-Cloud-Base-33-1.2.x86_64 + iniset $conf_file heat_plugin image_ref Fedora-Cloud-Base-36-1.5.x86_64 iniset $conf_file heat_plugin minimal_image_ref $default_image_name iniset $conf_file heat_plugin hidden_stack_tag hidden @@ -110,5 +110,5 @@ _config_tempest_plugin openstack flavor show m1.heat_int && openstack flavor delete m1.heat_int openstack flavor show m1.heat_micro && openstack flavor delete m1.heat_micro -openstack flavor show m1.heat_int || openstack flavor create m1.heat_int --ram 512 --disk 4 +openstack flavor show m1.heat_int || openstack flavor create m1.heat_int --ram 512 --disk 10 openstack flavor show m1.heat_micro || openstack flavor create m1.heat_micro --ram 128 --disk 1 diff --git a/lower-constraints.txt b/lower-constraints.txt deleted file mode 100644 index 8027cb242..000000000 --- a/lower-constraints.txt +++ /dev/null @@ -1,161 +0,0 @@ -alembic==0.9.8 -amqp==2.2.2 -aodhclient==0.9.0 -appdirs==1.4.3 -asn1crypto==0.24.0 -Babel==2.3.4 -bandit==1.1.0 -bcrypt==3.1.4 -cachetools==2.0.1 -certifi==2018.1.18 -cffi==1.11.5 -chardet==3.0.4 -cliff==2.11.0 -cmd2==0.8.1 -contextlib2==0.5.5 -coverage==4.0 -croniter==0.3.4 -cryptography==2.1.4 -debtcollector==1.19.0 -decorator==4.2.1 -deprecation==2.0 -doc8==0.6.0 -docker-pycreds==0.2.2 -docker==3.1.1 -dogpile.cache==0.6.5 -enum-compat==0.0.2 -eventlet==0.18.2 -extras==1.0.0 -fasteners==0.14.1 -fixtures==3.0.0 -flake8==2.5.5 -future==0.16.0 -futurist==1.6.0 -gitdb2==2.0.3 -GitPython==2.1.8 -greenlet==0.4.13 -hacking==0.12.0 -idna==2.6 -iso8601==0.1.12 -Jinja2==2.10 -jmespath==0.9.3 -jsonpatch==1.21 -jsonpointer==2.0 -jsonschema==2.6.0 -keystoneauth1==3.4.0 -keystonemiddleware==4.17.0 -kombu==4.0.0 -linecache2==1.0.0 -lxml==3.4.1 -Mako==1.0.7 -MarkupSafe==1.0 -mccabe==0.2.1 -mock==2.0.0 -monotonic==1.4 -msgpack==0.5.6 -munch==2.2.0 -netaddr==0.7.18 -netifaces==0.10.6 -neutron-lib==1.14.0 -openstacksdk==0.11.2 -os-client-config==1.29.0 -os-service-types==1.2.0 -osc-lib==1.10.0 -oslo.cache==1.26.0 -oslo.concurrency==3.26.0 -oslo.config==5.2.0 -oslo.context==2.19.2 -oslo.db==4.27.0 -oslo.i18n==3.15.3 -oslo.log==3.36.0 -oslo.messaging==5.29.0 -oslo.middleware==3.31.0 -oslo.policy==1.30.0 -oslo.reports==1.18.0 -oslo.serialization==2.18.0 -oslo.service==1.24.0 -oslo.upgradecheck==0.1.0 -oslo.utils==3.37.0 -oslo.versionedobjects==1.31.2 -oslotest==3.2.0 -osprofiler==1.4.0 -packaging==17.1 -paramiko==2.4.1 -Paste==2.0.3 -PasteDeploy==1.5.0 -pbr==2.0.0 -pep8==1.5.7 -pika-pool==0.1.3 -pika==0.10.0 -ply==3.11 -prettytable==0.7.2 -psutil==5.4.3 -psycopg2==2.7 -pyasn1==0.4.2 -pycadf==2.7.0 -pycparser==2.18 -pyflakes==0.8.1 -Pygments==2.2.0 -pyinotify==0.9.6 -PyMySQL==0.7.6 -PyNaCl==1.2.1 -pyOpenSSL==17.5.0 -pyparsing==2.2.0 -pyperclip==1.6.0 -python-barbicanclient==4.5.2 -python-blazarclient===1.0.1 -python-ceilometerclient==2.5.0 -python-cinderclient==3.3.0 -python-dateutil==2.7.0 -python-designateclient==2.7.0 -python-editor==1.0.3 -python-glanceclient==2.8.0 -python-heatclient==1.10.0 -python-keystoneclient==3.8.0 -python-magnumclient==2.3.0 -python-manilaclient==1.16.0 -python-mimeparse==1.6.0 -python-mistralclient==3.1.0 -python-monascaclient==1.12.0 -python-neutronclient==6.7.0 -python-novaclient==9.1.0 -python-octaviaclient==1.3.0 -python-openstackclient==3.12.0 -python-saharaclient==1.4.0 -python-subunit==1.2.0 -python-swiftclient==3.2.0 -python-troveclient==2.2.0 -python-zaqarclient==1.3.0 -python-zunclient==2.0.0 -pytz==2013.6 -PyYAML==3.12 -repoze.lru==0.7 -requests==2.14.2 -requestsexceptions==1.4.0 -rfc3986==1.1.0 -Routes==2.3.1 -simplejson==3.13.2 -six==1.10.0 -smmap2==2.0.3 -sqlalchemy-migrate==0.11.0 -SQLAlchemy==1.0.10 -sqlparse==0.2.4 -statsd==3.2.2 -stestr==2.0.0 -stevedore==1.20.0 -tempest==17.1.0 -Tempita==0.5.2 -tenacity==4.4.0 -testresources==2.0.0 -testscenarios==0.4 -testtools==2.2.0 -traceback2==1.4.0 -unittest2==1.1.0 -urllib3==1.22 -vine==1.1.4 -voluptuous==0.11.1 -warlock==1.2.0 -WebOb==1.7.1 -websocket-client==0.47.0 -wrapt==1.10.11 -yaql==1.1.3 diff --git a/requirements.txt b/requirements.txt index f3b7de7fa..d079c5b74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# Requirements lower bounds listed here are our best effort to keep them up to +# date but we do not test them so no guarantee of having them all correct. If +# you find any incorrect lower bounds, let us know or propose a fix. + # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 07a078547..505a58cb7 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -23,8 +23,8 @@ sudo -H mysqladmin -u root password $DB_ROOT_PW sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " DELETE FROM mysql.user WHERE User=''; FLUSH PRIVILEGES; - GRANT ALL PRIVILEGES ON *.* - TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;" + CREATE USER '$DB_USER'@'%' IDENTIFIED BY '$DB_PW'; + GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' WITH GRANT OPTION;" # Now create our database. mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " @@ -144,11 +144,3 @@ basepython = python3 deps = bindep commands = bindep test usedevelop = False - -[testenv:lower-constraints] -basepython = python3 -install_command = pip install {opts} {packages} -deps = - -c{toxinidir}/lower-constraints.txt - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt |