From 25afea2adb7c23fafcf1ea6261ba26f98010b0d3 Mon Sep 17 00:00:00 2001 From: Takashi Natsume Date: Mon, 4 May 2020 13:50:07 +0900 Subject: Switch legacy Zuul jobs to native Zuul v3 jobs Replace the 'novaclient-dsvm-functional' legacy Zuul job with the 'python-novaclient-functional' native Zuul v3 job. See https://governance.openstack.org/tc/goals/selected/victoria/native-zuulv3-jobs.html for more details. Change-Id: I35b5699b8b0814f2d17a37e024286cd440047ec8 Signed-off-by: Takashi Natsume Co-Authored-By: Andrey Kurilin (cherry picked from commit b7ae8c75cb8c34e2c6ab81a4f6b60fd937ec7f98) --- .zuul.yaml | 17 ++--- novaclient/tests/functional/base.py | 8 ++- .../tests/functional/hooks/post_test_hook.sh | 46 ------------- novaclient/tests/functional/test_auth.py | 54 +++++++++++---- .../legacy/novaclient-dsvm-functional/post.yaml | 80 ---------------------- .../legacy/novaclient-dsvm-functional/run.yaml | 54 --------------- playbooks/post.yaml | 6 ++ playbooks/python-novaclient-functional.yaml | 14 ++++ roles/get-os-environment/defaults/main.yaml | 2 + roles/get-os-environment/tasks/main.yaml | 12 ++++ tox.ini | 2 +- 11 files changed, 92 insertions(+), 203 deletions(-) delete mode 100755 novaclient/tests/functional/hooks/post_test_hook.sh delete mode 100644 playbooks/legacy/novaclient-dsvm-functional/post.yaml delete mode 100644 playbooks/legacy/novaclient-dsvm-functional/run.yaml create mode 100644 playbooks/post.yaml create mode 100644 playbooks/python-novaclient-functional.yaml create mode 100644 roles/get-os-environment/defaults/main.yaml create mode 100644 roles/get-os-environment/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 95eabd35..c7546eb2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,14 +1,15 @@ - job: - # TODO(efried): Cut over to zuulv3 - name: novaclient-dsvm-functional - parent: legacy-dsvm-base - run: playbooks/legacy/novaclient-dsvm-functional/run.yaml - post-run: playbooks/legacy/novaclient-dsvm-functional/post.yaml + name: python-novaclient-functional + parent: devstack + run: playbooks/python-novaclient-functional.yaml + post-run: playbooks/post.yaml timeout: 7200 required-projects: - - openstack/devstack-gate - openstack/nova - openstack/python-novaclient + vars: + devstack_localrc: + USE_PYTHON3: true irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -25,7 +26,7 @@ - release-notes-jobs-python3 check: jobs: - - novaclient-dsvm-functional + - python-novaclient-functional gate: jobs: - - novaclient-dsvm-functional + - python-novaclient-functional diff --git a/novaclient/tests/functional/base.py b/novaclient/tests/functional/base.py index f5c758be..9497389f 100644 --- a/novaclient/tests/functional/base.py +++ b/novaclient/tests/functional/base.py @@ -206,6 +206,8 @@ class ClientTestBase(testtools.TestCase): self.insecure = cloud_config.config['insecure'] else: self.insecure = False + self.cacert = cloud_config.config['cacert'] + self.cert = cloud_config.config['cert'] auth = identity.Password(username=user, password=passwd, @@ -213,7 +215,11 @@ class ClientTestBase(testtools.TestCase): auth_url=auth_url, project_domain_id=self.project_domain_id, user_domain_id=user_domain_id) - session = ksession.Session(auth=auth, verify=(not self.insecure)) + session = ksession.Session( + cert=self.cert, + auth=auth, + verify=(self.cacert or not self.insecure) + ) self.client = self._get_novaclient(session) diff --git a/novaclient/tests/functional/hooks/post_test_hook.sh b/novaclient/tests/functional/hooks/post_test_hook.sh deleted file mode 100755 index 473393eb..00000000 --- a/novaclient/tests/functional/hooks/post_test_hook.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -xe - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This script is executed inside post_test_hook function in devstack gate. - -function generate_testr_results { - if [ -f .testrepository/0 ]; then - sudo .tox/functional/bin/testr last --subunit > $WORKSPACE/testrepository.subunit - sudo mv $WORKSPACE/testrepository.subunit $BASE/logs/testrepository.subunit - sudo /usr/os-testr-env/bin/subunit2html $BASE/logs/testrepository.subunit $BASE/logs/testr_results.html - sudo gzip -9 $BASE/logs/testrepository.subunit - sudo gzip -9 $BASE/logs/testr_results.html - sudo chown $USER:$USER $BASE/logs/testrepository.subunit.gz $BASE/logs/testr_results.html.gz - sudo chmod a+r $BASE/logs/testrepository.subunit.gz $BASE/logs/testr_results.html.gz - fi -} - -export NOVACLIENT_DIR="$BASE/new/python-novaclient" - -sudo chown -R $USER:stack $NOVACLIENT_DIR - -# Go to the novaclient dir -cd $NOVACLIENT_DIR - -# Run tests -echo "Running novaclient functional test suite" -set +e -# Preserve env for OS_ credentials -sudo -E -H -u $USER tox -e ${TOX_ENV:-functional} -EXIT_CODE=$? -set -e - -# Collect and parse result -generate_testr_results -exit $EXIT_CODE diff --git a/novaclient/tests/functional/test_auth.py b/novaclient/tests/functional/test_auth.py index 8987c728..c8d2396a 100644 --- a/novaclient/tests/functional/test_auth.py +++ b/novaclient/tests/functional/test_auth.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import os from urllib import parse import tempest.lib.cli.base @@ -28,14 +29,19 @@ class TestAuthentication(base.ClientTestBase): url.fragment)) def nova_auth_with_password(self, action, identity_api_version): - flags = ('--os-username %s --os-tenant-name %s --os-password %s ' - '--os-auth-url %s --os-endpoint-type publicURL' % ( - self.cli_clients.username, - self.cli_clients.tenant_name, - self.cli_clients.password, - self._get_url(identity_api_version))) + flags = ( + f'--os-username {self.cli_clients.username} ' + f'--os-tenant-name {self.cli_clients.tenant_name} ' + f'--os-password {self.cli_clients.password} ' + f'--os-auth-url {self._get_url(identity_api_version)} ' + f'--os-endpoint-type publicURL' + ) + if self.cacert: + flags = f'{flags} --os-cacert {self.cacert}' + if self.cert: + flags = f'{flags} --os-cert {self.cert}' if self.cli_clients.insecure: - flags += ' --insecure ' + flags = f'{flags} --insecure' return tempest.lib.cli.base.execute( "nova", action, flags, cli_dir=self.cli_clients.cli_dir) @@ -49,15 +55,37 @@ class TestAuthentication(base.ClientTestBase): if identity_api_version == "3": kw["project_domain_id"] = self.project_domain_id nova = client.Client("2", auth_token=token, auth_url=auth_url, - project_name=self.project_name, **kw) + project_name=self.project_name, + cacert=self.cacert, cert=self.cert, + **kw) nova.servers.list() - flags = ('--os-tenant-name %(project_name)s --os-token %(token)s ' - '--os-auth-url %(auth_url)s --os-endpoint-type publicURL' - % {"project_name": self.project_name, - "token": token, "auth_url": auth_url}) + # NOTE(andreykurilin): tempest.lib.cli.base.execute doesn't allow to + # pass 'env' argument to subprocess.Popen for overriding the current + # process' environment. + # When one of OS_AUTH_TYPE or OS_AUTH_PLUGIN environment variables + # presents, keystoneauth1 can load the wrong auth plugin with wrong + # expected cli arguments. To avoid this case, we need to modify + # current environment. + # TODO(andreykurilin): tempest.lib.cli.base.execute is quite simple + # method that can be replaced by subprocess.check_output direct call + # with passing env argument to avoid modifying the current process + # environment. or we probably can propose a change to tempest. + os.environ.pop("OS_AUTH_TYPE", None) + os.environ.pop("OS_AUTH_PLUGIN", None) + + flags = ( + f'--os-tenant-name {self.project_name} ' + f'--os-token {token} ' + f'--os-auth-url {auth_url} ' + f'--os-endpoint-type publicURL' + ) + if self.cacert: + flags = f'{flags} --os-cacert {self.cacert}' + if self.cert: + flags = f'{flags} --os-cert {self.cert}' if self.cli_clients.insecure: - flags += ' --insecure ' + flags = f'{flags} --insecure' tempest.lib.cli.base.execute( "nova", "list", flags, cli_dir=self.cli_clients.cli_dir) diff --git a/playbooks/legacy/novaclient-dsvm-functional/post.yaml b/playbooks/legacy/novaclient-dsvm-functional/post.yaml deleted file mode 100644 index dac87534..00000000 --- a/playbooks/legacy/novaclient-dsvm-functional/post.yaml +++ /dev/null @@ -1,80 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*nose_results.html - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testr_results.html.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.testrepository/tmp* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testrepository.subunit.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}/tox' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.tox/*/log/* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/novaclient-dsvm-functional/run.yaml b/playbooks/legacy/novaclient-dsvm-functional/run.yaml deleted file mode 100644 index 183d3989..00000000 --- a/playbooks/legacy/novaclient-dsvm-functional/run.yaml +++ /dev/null @@ -1,54 +0,0 @@ -- hosts: all - name: novaclient-dsvm-functional job with identity v3 and neutron - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - https://opendev.org \ - openstack/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export DEVSTACK_GATE_USE_PYTHON3=true - export PYTHONUNBUFFERED=true - export BRANCH_OVERRIDE=default - export DEVSTACK_PROJECT_FROM_GIT=python-novaclient - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - # This ensures that if we set override branch to something - # else, we still take python-novaclient from the zuul branch - # name. So override branch can be 'stable/mitaka' but we can - # test master changes. - uc_project=`echo $DEVSTACK_PROJECT_FROM_GIT | tr [:lower:] [:upper:] | tr '-' '_' | sed 's/[^A-Z_]//'` - export "OVERRIDE_"$uc_project"_PROJECT_BRANCH"=$ZUUL_BRANCH - - function post_test_hook { - # Configure and run functional tests - $BASE/new/python-novaclient/novaclient/tests/functional/hooks/post_test_hook.sh - } - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/post.yaml b/playbooks/post.yaml new file mode 100644 index 00000000..9860e2a9 --- /dev/null +++ b/playbooks/post.yaml @@ -0,0 +1,6 @@ +- hosts: all + vars: + tox_envlist: functional + roles: + - fetch-tox-output + - fetch-subunit-output diff --git a/playbooks/python-novaclient-functional.yaml b/playbooks/python-novaclient-functional.yaml new file mode 100644 index 00000000..ea7d2db2 --- /dev/null +++ b/playbooks/python-novaclient-functional.yaml @@ -0,0 +1,14 @@ +- hosts: all + roles: + - run-devstack + # Run bindep and test-setup after devstack so that they won't interfere + - role: bindep + bindep_profile: test + bindep_dir: "{{ zuul_work_dir }}" + - test-setup + - get-os-environment + - ensure-tox + - role: tox + tox_envlist: functional + tox_install_siblings: false + environment: "{{ os_env_vars }}" diff --git a/roles/get-os-environment/defaults/main.yaml b/roles/get-os-environment/defaults/main.yaml new file mode 100644 index 00000000..91190b32 --- /dev/null +++ b/roles/get-os-environment/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +openrc_file: "{{ devstack_base_dir|default('/opt/stack') }}/devstack/openrc" diff --git a/roles/get-os-environment/tasks/main.yaml b/roles/get-os-environment/tasks/main.yaml new file mode 100644 index 00000000..b3f457bb --- /dev/null +++ b/roles/get-os-environment/tasks/main.yaml @@ -0,0 +1,12 @@ +- name: Extract the OS_ environment variables + shell: + cmd: | + source {{ openrc_file }} admin admin &>/dev/null + env | awk -F= 'BEGIN {print "---" } /^OS_/ { print " "$1": \""$2"\""} ' + args: + executable: "/bin/bash" + register: env_os + +- name: Save the OS_ environment variables as a fact + set_fact: + os_env_vars: "{{ env_os.stdout|from_yaml }}" diff --git a/tox.ini b/tox.ini index 70646a4e..f796f7f7 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,7 @@ commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:functional] -passenv = OS_NOVACLIENT_TEST_NETWORK +passenv = OS_* commands = stestr --test-path=./novaclient/tests/functional run --concurrency=1 {posargs} python novaclient/tests/functional/hooks/check_resources.py -- cgit v1.2.1