diff options
21 files changed, 350 insertions, 183 deletions
diff --git a/doc/source/data/tempest-blacklisted-plugins-registry.header b/doc/source/data/tempest-non-active-plugins-registry.header index 6b6af1147..06d8eaa93 100644 --- a/doc/source/data/tempest-blacklisted-plugins-registry.header +++ b/doc/source/data/tempest-non-active-plugins-registry.header @@ -1,7 +1,7 @@ -Blacklisted Plugins +Non Active Plugins =================== List of Tempest plugin projects that are stale or unmaintained for a long -time (6 months or more). They can be moved out of blacklist state once one +time (6 months or more). They can be moved out of nonactivelist state once one of the relevant patches gets merged: https://review.opendev.org/#/q/topic:tempest-sanity-gate+%28status:open%29 diff --git a/doc/source/overview.rst b/doc/source/overview.rst index e51b90b96..2eaf72f18 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -113,7 +113,7 @@ as it is simpler, and quicker to work with. There is also the option to use `stestr`_ directly. For example, from the workspace dir run:: - $ stestr run --black-regex '\[.*\bslow\b.*\]' '^tempest\.(api|scenario)' + $ stestr run --exclude-regex '\[.*\bslow\b.*\]' '^tempest\.(api|scenario)' will run the same set of tests as the default gate jobs. Or you can use `unittest`_ compatible test runners such as `stestr`_, `pytest`_ etc. diff --git a/doc/source/stable_branch_support_policy.rst b/doc/source/stable_branch_support_policy.rst index 87e3ad1e8..9c2d1ed66 100644 --- a/doc/source/stable_branch_support_policy.rst +++ b/doc/source/stable_branch_support_policy.rst @@ -20,7 +20,7 @@ Tempest from master as long as possible. But, because we won't be actively testing branches in these phases, it's possible that we'll introduce changes to Tempest on master which will break support on *Extended Maintenance* phase branches. When this happens the expectation for those branches is to either -switch to running Tempest from a tag with support for the branch, or blacklist +switch to running Tempest from a tag with support for the branch, or exclude a newly introduced test (if that is the cause of the issue). Tempest will not be creating stable branches to support *Extended Maintenance* phase branches, as the burden is on the *Extended Maintenance* phase branche maintainers, not the Tempest diff --git a/etc/whitelist.yaml b/etc/allow-list.yaml index e69de29bb..e69de29bb 100644 --- a/etc/whitelist.yaml +++ b/etc/allow-list.yaml diff --git a/releasenotes/notes/Inclusive-jargon-17621346744f0cf4.yaml b/releasenotes/notes/Inclusive-jargon-17621346744f0cf4.yaml new file mode 100644 index 000000000..089569e19 --- /dev/null +++ b/releasenotes/notes/Inclusive-jargon-17621346744f0cf4.yaml @@ -0,0 +1,13 @@ +--- +deprecations: + - | + In this release the following tempest arguments are deprecated and + replaced by new ones which are functionally equivalent: + + * --black-regex is replaced by --exclude-regex + * --blacklist-file is replaced by --exclude-list + * --whitelist-file is replaced by --include-list + + For now Tempest supports both (new and old ones) in order to make the + transition for all consumers smoother. However, that's just a temporary + case and the old options will be removed soon. diff --git a/roles/run-tempest/README.rst b/roles/run-tempest/README.rst index 3643edb4e..f9fcf28d2 100644 --- a/roles/run-tempest/README.rst +++ b/roles/run-tempest/README.rst @@ -32,7 +32,11 @@ variable (through the `register` statement). .. zuul:rolevar:: tempest_test_blacklist - Specifies a blacklist file to skip tests that are not needed. + DEPRECATED option, please use tempest_test_exclude_list instead. + +.. zuul:rolevar:: tempest_test_exclude_list + + Specifies an excludelist file to skip tests that are not needed. Pass a full path to the file. @@ -44,6 +48,11 @@ variable (through the `register` statement). .. zuul:rolevar:: tempest_black_regex :default: '' + DEPRECATED option, please use tempest_exclude_regex instead. + +.. zuul:rolevar:: tempest_exclude_regex + :default: '' + A regular expression used to skip the tests. It works only when used with some specific tox environments @@ -51,7 +60,7 @@ variable (through the `register` statement). :: vars: - tempest_black_regex: (tempest.api.identity).*$ + tempest_exclude_regex: (tempest.api.identity).*$ .. zuul:rolevar:: tox_extra_args :default: '' diff --git a/roles/run-tempest/defaults/main.yaml b/roles/run-tempest/defaults/main.yaml index 5867b6cf8..e7a1cc6de 100644 --- a/roles/run-tempest/defaults/main.yaml +++ b/roles/run-tempest/defaults/main.yaml @@ -1,7 +1,10 @@ devstack_base_dir: /opt/stack tempest_test_regex: '' tox_envlist: smoke +# TODO(kopecmartin) remove tempest_black_regex once all consumers of this +# role have switched to the tempest_exclude_regex option. tempest_black_regex: '' +tempest_exclude_regex: '' tox_extra_args: '' tempest_test_timeout: '' stable_constraints_file: "{{ devstack_base_dir }}/requirements/upper-constraints.txt" diff --git a/roles/run-tempest/tasks/main.yaml b/roles/run-tempest/tasks/main.yaml index 1de372518..e9c8e9260 100644 --- a/roles/run-tempest/tasks/main.yaml +++ b/roles/run-tempest/tasks/main.yaml @@ -36,6 +36,9 @@ tempest_tox_environment: "{{ tempest_tox_environment | combine({'OS_TEST_TIMEOUT': tempest_test_timeout}) }}" when: tempest_test_timeout != '' +# TODO(kopecmartin) remove the following 'when block' after all consumers of +# the role have switched to tempest_test_exclude_list option, until then it's +# kept here for backward compatibility - when: - tempest_test_blacklist is defined block: @@ -50,10 +53,48 @@ blacklist_option: "--blacklist-file={{ tempest_test_blacklist|quote }}" when: blacklist_stat.stat.exists +- when: + - tempest_test_exclude_list is defined + block: + - name: Check for test exclude list file + stat: + path: "{{ tempest_test_exclude_list }}" + register: + exclude_list_stat + + - name: Build exclude list option + set_fact: + exclude_list_option: "--exclude-list={{ tempest_test_exclude_list|quote }}" + when: exclude_list_stat.stat.exists + +# TODO(kopecmartin) remove this after all consumers of the role have switched +# to tempest_exclude_regex option, until then it's kept here for the backward +# compatibility +- name: Set tempest_exclude_regex + set_fact: + tempest_exclude_regex: "{{ tempest_black_regex }}" + when: + - tempest_black_regex is defined + - tempest_exclude_regex is not defined + +- name: Build exclude regex (old param) + set_fact: + tempest_exclude_regex: "--black-regex={{tempest_black_regex|quote}}" + when: + - tempest_black_regex is defined + +- name: Build exclude regex (new param) + set_fact: + tempest_exclude_regex: "--exclude-regex={{tempest_exclude_regex|quote}}" + when: + - tempest_black_regex is not defined + - tempest_exclude_regex is defined + - name: Run Tempest - command: tox -e {{tox_envlist}} {{tox_extra_args}} -- {{tempest_test_regex|quote}} {{blacklist_option|default('')}} \ + command: tox -e {{tox_envlist}} {{tox_extra_args}} -- {{tempest_test_regex|quote}} \ + {{blacklist_option|default('')}} {{exclude_list_option|default('')}} \ --concurrency={{tempest_concurrency|default(default_concurrency)}} \ - --black-regex={{tempest_black_regex|quote}} + {{tempest_exclude_regex|default('')}} args: chdir: "{{devstack_base_dir}}/tempest" register: tempest_run_result diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py index 8bebce2c9..2669ff774 100644 --- a/tempest/cmd/run.py +++ b/tempest/cmd/run.py @@ -22,10 +22,10 @@ Tempest run has several options: * ``--regex/-r``: This is a selection regex like what stestr uses. It will run any tests that match on re.match() with the regex * ``--smoke/-s``: Run all the tests tagged as smoke -* ``--black-regex``: It allows to do simple test exclusion via passing a - rejection/black regexp +* ``--exclude-regex``: It allows to do simple test exclusion via passing a + rejection/exclude regexp -There are also the ``--blacklist-file`` and ``--whitelist-file`` options that +There are also the ``--exclude-list`` and ``--include-list`` options that let you pass a filepath to tempest run with the file format being a line separated regex, with '#' used to signify the start of a comment on a line. For example:: @@ -128,6 +128,7 @@ import os import sys from cliff import command +from oslo_log import log from oslo_serialization import jsonutils as json from stestr import commands @@ -141,6 +142,8 @@ from tempest import config CONF = config.CONF SAVED_STATE_JSON = "saved_state.json" +LOG = log.getLogger(__name__) + class TempestRun(command.Command): @@ -201,23 +204,71 @@ class TempestRun(command.Command): self._init_state() regex = self._build_regex(parsed_args) + + # temporary method for parsing deprecated and new stestr options + # and showing warning messages in order to make the transition + # smoother for all tempest consumers + # TODO(kopecmartin) remove this after stestr>=3.1.0 is used + # in all supported OpenStack releases + def parse_dep(old_o, old_v, new_o, new_v): + ret = '' + if old_v: + LOG.warning("'%s' option is deprecated, use '%s' instead " + "which is functionally equivalent. Right now " + "Tempest still supports this option for " + "backward compatibility, however, it will be " + "removed soon.", + old_o, new_o) + ret = old_v + if old_v and new_v: + # both options are specified + LOG.warning("'%s' and '%s' are specified at the same time, " + "'%s' takes precedence over '%s'", + new_o, old_o, new_o, old_o) + if new_v: + ret = new_v + return ret + ex_regex = parse_dep('--black-regex', parsed_args.black_regex, + '--exclude-regex', parsed_args.exclude_regex) + ex_list = parse_dep('--blacklist-file', parsed_args.blacklist_file, + '--exclude-list', parsed_args.exclude_list) + in_list = parse_dep('--whitelist-file', parsed_args.whitelist_file, + '--include-list', parsed_args.include_list) + return_code = 0 if parsed_args.list_tests: - return_code = commands.list_command( - filters=regex, whitelist_file=parsed_args.whitelist_file, - blacklist_file=parsed_args.blacklist_file, - black_regex=parsed_args.black_regex) + try: + return_code = commands.list_command( + filters=regex, include_list=in_list, + exclude_list=ex_list, exclude_regex=ex_regex) + except TypeError: + # exclude_list, include_list and exclude_regex are defined only + # in stestr >= 3.1.0, this except block catches the case when + # tempest is executed with an older stestr + return_code = commands.list_command( + filters=regex, whitelist_file=in_list, + blacklist_file=ex_list, black_regex=ex_regex) else: serial = not parsed_args.parallel - return_code = commands.run_command( - filters=regex, subunit_out=parsed_args.subunit, - serial=serial, concurrency=parsed_args.concurrency, - blacklist_file=parsed_args.blacklist_file, - whitelist_file=parsed_args.whitelist_file, - black_regex=parsed_args.black_regex, - worker_path=parsed_args.worker_file, - load_list=parsed_args.load_list, combine=parsed_args.combine) + params = { + 'filters': regex, 'subunit_out': parsed_args.subunit, + 'serial': serial, 'concurrency': parsed_args.concurrency, + 'worker_path': parsed_args.worker_file, + 'load_list': parsed_args.load_list, + 'combine': parsed_args.combine + } + try: + return_code = commands.run_command( + **params, exclude_list=ex_list, + include_list=in_list, exclude_regex=ex_regex) + except TypeError: + # exclude_list, include_list and exclude_regex are defined only + # in stestr >= 3.1.0, this except block catches the case when + # tempest is executed with an older stestr + return_code = commands.run_command( + **params, blacklist_file=ex_list, + whitelist_file=in_list, black_regex=ex_regex) if return_code > 0: sys.exit(return_code) return return_code @@ -271,15 +322,38 @@ class TempestRun(command.Command): help='A normal stestr selection regex used to ' 'specify a subset of tests to run') parser.add_argument('--black-regex', dest='black_regex', + help='DEPRECATED: This option is deprecated and ' + 'will be removed soon, use --exclude-regex ' + 'which is functionally equivalent. If this ' + 'is specified at the same time as ' + '--exclude-regex, this flag will be ignored ' + 'and --exclude-regex will be used') + parser.add_argument('--exclude-regex', dest='exclude_regex', help='A regex to exclude tests that match it') parser.add_argument('--whitelist-file', '--whitelist_file', - help="Path to a whitelist file, this file " - "contains a separate regex on each " - "newline.") + help='DEPRECATED: This option is deprecated and ' + 'will be removed soon, use --include-list ' + 'which is functionally equivalent. If this ' + 'is specified at the same time as ' + '--include-list, this flag will be ignored ' + 'and --include-list will be used') + parser.add_argument('--include-list', '--include_list', + help="Path to an include file which contains the " + "regex for tests to be included in tempest " + "run, this file contains a separate regex on " + "each newline.") parser.add_argument('--blacklist-file', '--blacklist_file', - help='Path to a blacklist file, this file ' - 'contains a separate regex exclude on ' - 'each newline') + help='DEPRECATED: This option is deprecated and ' + 'will be removed soon, use --exclude-list ' + 'which is functionally equivalent. If this ' + 'is specified at the same time as ' + '--exclude-list, this flag will be ignored ' + 'and --exclude-list will be used') + parser.add_argument('--exclude-list', '--exclude_list', + help='Path to an exclude file which contains the ' + 'regex for tests to be excluded in tempest ' + 'run, this file contains a separate regex on ' + 'each newline.') parser.add_argument('--load-list', '--load_list', help='Path to a non-regex whitelist file, ' 'this file contains a separate test ' diff --git a/tempest/tests/cmd/test_run.py b/tempest/tests/cmd/test_run.py index 3c99bbe57..ec7b76092 100644 --- a/tempest/tests/cmd/test_run.py +++ b/tempest/tests/cmd/test_run.py @@ -68,6 +68,11 @@ class TestTempestRun(base.TestCase): class TestRunReturnCode(base.TestCase): + + exclude_regex = '--exclude-regex' + exclude_list = '--exclude-list' + include_list = '--include-list' + def setUp(self): super(TestRunReturnCode, self).setUp() # Setup test dirs @@ -92,6 +97,14 @@ class TestRunReturnCode(base.TestCase): self.addCleanup(os.chdir, os.path.abspath(os.curdir)) os.chdir(self.directory) + def _get_test_list_file(self, content): + fd, path = tempfile.mkstemp() + self.addCleanup(os.remove, path) + test_file = os.fdopen(fd, 'wb', 0) + self.addCleanup(test_file.close) + test_file.write(content.encode('utf-8')) + return path + def assertRunExit(self, cmd, expected): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -115,19 +128,23 @@ class TestRunReturnCode(base.TestCase): subprocess.call(['stestr', 'init']) self.assertRunExit(['tempest', 'run', '--regex', 'failing'], 1) - def test_tempest_run_blackregex_failing(self): - self.assertRunExit(['tempest', 'run', '--black-regex', 'failing'], 0) + def test_tempest_run_exclude_regex_failing(self): + self.assertRunExit(['tempest', 'run', + self.exclude_regex, 'failing'], 0) - def test_tempest_run_blackregex_failing_with_stestr_repository(self): + def test_tempest_run_exclude_regex_failing_with_stestr_repository(self): subprocess.call(['stestr', 'init']) - self.assertRunExit(['tempest', 'run', '--black-regex', 'failing'], 0) + self.assertRunExit(['tempest', 'run', + self.exclude_regex, 'failing'], 0) - def test_tempest_run_blackregex_passing(self): - self.assertRunExit(['tempest', 'run', '--black-regex', 'passing'], 1) + def test_tempest_run_exclude_regex_passing(self): + self.assertRunExit(['tempest', 'run', + self.exclude_regex, 'passing'], 1) - def test_tempest_run_blackregex_passing_with_stestr_repository(self): + def test_tempest_run_exclude_regex_passing_with_stestr_repository(self): subprocess.call(['stestr', 'init']) - self.assertRunExit(['tempest', 'run', '--black-regex', 'passing'], 1) + self.assertRunExit(['tempest', 'run', + self.exclude_regex, 'passing'], 1) def test_tempest_run_fails(self): self.assertRunExit(['tempest', 'run'], 1) @@ -149,47 +166,31 @@ class TestRunReturnCode(base.TestCase): self.assertEqual(result, tests) def test_tempest_run_with_worker_file(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - worker_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(worker_file.close) - worker_file.write( - '- worker:\n - passing\n concurrency: 3'.encode('utf-8')) + path = self._get_test_list_file( + '- worker:\n - passing\n concurrency: 3') self.assertRunExit(['tempest', 'run', '--worker-file=%s' % path], 0) - def test_tempest_run_with_whitelist(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - whitelist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(whitelist_file.close) - whitelist_file.write('passing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path], 0) + def test_tempest_run_with_include_list(self): + path = self._get_test_list_file('passing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.include_list, path)], 0) - def test_tempest_run_with_whitelist_regex_include_pass_check_fail(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - whitelist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(whitelist_file.close) - whitelist_file.write('passing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path, + def test_tempest_run_with_include_regex_include_pass_check_fail(self): + path = self._get_test_list_file('passing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.include_list, path), '--regex', 'fail'], 1) - def test_tempest_run_with_whitelist_regex_include_pass_check_pass(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - whitelist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(whitelist_file.close) - whitelist_file.write('passing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path, + def test_tempest_run_with_include_regex_include_pass_check_pass(self): + path = self._get_test_list_file('passing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.include_list, path), '--regex', 'passing'], 0) - def test_tempest_run_with_whitelist_regex_include_fail_check_pass(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - whitelist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(whitelist_file.close) - whitelist_file.write('failing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path, + def test_tempest_run_with_include_regex_include_fail_check_pass(self): + path = self._get_test_list_file('failing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.include_list, path), '--regex', 'pass'], 1) def test_tempest_run_passes_with_config_file(self): @@ -197,50 +198,75 @@ class TestRunReturnCode(base.TestCase): '--config-file', self.stestr_conf_file, '--regex', 'passing'], 0) - def test_tempest_run_with_blacklist_failing(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - blacklist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(blacklist_file.close) - blacklist_file.write('failing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path], 0) + def test_tempest_run_with_exclude_list_failing(self): + path = self._get_test_list_file('failing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.exclude_list, path)], 0) - def test_tempest_run_with_blacklist_passing(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - blacklist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(blacklist_file.close) - blacklist_file.write('passing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path], 1) + def test_tempest_run_with_exclude_list_passing(self): + path = self._get_test_list_file('passing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.exclude_list, path)], 1) - def test_tempest_run_with_blacklist_regex_exclude_fail_check_pass(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - blacklist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(blacklist_file.close) - blacklist_file.write('failing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path, + def test_tempest_run_with_exclude_list_regex_exclude_fail_check_pass(self): + path = self._get_test_list_file('failing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.exclude_list, path), '--regex', 'pass'], 0) - def test_tempest_run_with_blacklist_regex_exclude_pass_check_pass(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - blacklist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(blacklist_file.close) - blacklist_file.write('passing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path, + def test_tempest_run_with_exclude_list_regex_exclude_pass_check_pass(self): + path = self._get_test_list_file('passing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.exclude_list, path), '--regex', 'pass'], 1) - def test_tempest_run_with_blacklist_regex_exclude_pass_check_fail(self): - fd, path = tempfile.mkstemp() - self.addCleanup(os.remove, path) - blacklist_file = os.fdopen(fd, 'wb', 0) - self.addCleanup(blacklist_file.close) - blacklist_file.write('passing'.encode('utf-8')) - self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path, + def test_tempest_run_with_exclude_list_regex_exclude_pass_check_fail(self): + path = self._get_test_list_file('passing') + self.assertRunExit(['tempest', 'run', + '%s=%s' % (self.exclude_list, path), '--regex', 'fail'], 1) +class TestOldArgRunReturnCode(TestRunReturnCode): + """A class for testing deprecated but still supported args. + + This class will be removed once we remove the following arguments: + * --black-regex + * --blacklist-file + * --whitelist-file + """ + exclude_regex = '--black-regex' + exclude_list = '--blacklist-file' + include_list = '--whitelist-file' + + def _test_args_passing(self, args): + self.assertRunExit(['tempest', 'run'] + args, 0) + + def test_tempest_run_new_old_arg_comb(self): + path = self._get_test_list_file('failing') + self._test_args_passing(['--black-regex', 'failing', + '--exclude-regex', 'failing']) + self._test_args_passing(['--blacklist-file=' + path, + '--exclude-list=' + path]) + path = self._get_test_list_file('passing') + self._test_args_passing(['--whitelist-file=' + path, + '--include-list=' + path]) + + def _test_args_passing_with_stestr_repository(self, args): + subprocess.call(['stestr', 'init']) + self.assertRunExit(['tempest', 'run'] + args, 0) + + def test_tempest_run_new_old_arg_comb_with_stestr_repository(self): + path = self._get_test_list_file('failing') + self._test_args_passing_with_stestr_repository( + ['--black-regex', 'failing', '--exclude-regex', 'failing']) + self._test_args_passing_with_stestr_repository( + ['--blacklist-file=' + path, '--exclude-list=' + path]) + path = self._get_test_list_file('passing') + self._test_args_passing_with_stestr_repository( + ['--whitelist-file=' + path, '--include-list=' + path]) + + class TestConfigPathCheck(base.TestCase): def setUp(self): super(TestConfigPathCheck, self).setUp() diff --git a/tools/check_logs.py b/tools/check_logs.py index de7e41d95..7e191a00e 100755 --- a/tools/check_logs.py +++ b/tools/check_logs.py @@ -56,39 +56,39 @@ allowed_dirty = set([ 's-proxy']) -def process_files(file_specs, url_specs, whitelists): +def process_files(file_specs, url_specs, allow_lists): regexp = re.compile(r"^.* (ERROR|CRITICAL|TRACE) .*\[.*\-.*\]") logs_with_errors = [] for (name, filename) in file_specs: - whitelist = whitelists.get(name, []) + allow_list = allow_lists.get(name, []) with open(filename) as content: - if scan_content(content, regexp, whitelist): + if scan_content(content, regexp, allow_list): logs_with_errors.append(name) for (name, url) in url_specs: - whitelist = whitelists.get(name, []) + allow_list = allow_lists.get(name, []) req = urlreq.Request(url) req.add_header('Accept-Encoding', 'gzip') page = urlreq.urlopen(req) buf = six.StringIO(page.read()) f = gzip.GzipFile(fileobj=buf) - if scan_content(f.read().splitlines(), regexp, whitelist): + if scan_content(f.read().splitlines(), regexp, allow_list): logs_with_errors.append(name) return logs_with_errors -def scan_content(content, regexp, whitelist): +def scan_content(content, regexp, allow_list): had_errors = False for line in content: if not line.startswith("Stderr:") and regexp.match(line): - whitelisted = False - for w in whitelist: + allowed = False + for w in allow_list: pat = ".*%s.*%s.*" % (w['module'].replace('.', '\\.'), w['message']) if re.match(pat, line): - whitelisted = True + allowed = True break - if not whitelisted or dump_all_errors: - if not whitelisted: + if not allowed or dump_all_errors: + if not allowed: had_errors = True return had_errors @@ -105,9 +105,9 @@ def main(opts): print("Must provide exactly one of -d or -u") return 1 print("Checking logs...") - WHITELIST_FILE = os.path.join( + ALLOW_LIST_FILE = os.path.join( os.path.abspath(os.path.dirname(os.path.dirname(__file__))), - "etc", "whitelist.yaml") + "etc", "allow-list.yaml") file_matcher = re.compile(r".*screen-([\w-]+)\.log") files = [] @@ -132,17 +132,17 @@ def main(opts): if m: urls_to_process.append((m.group(1), u)) - whitelists = {} - with open(WHITELIST_FILE) as stream: + allow_lists = {} + with open(ALLOW_LIST_FILE) as stream: loaded = yaml.safe_load(stream) if loaded: for (name, l) in six.iteritems(loaded): for w in l: assert 'module' in w, 'no module in %s' % name assert 'message' in w, 'no message in %s' % name - whitelists = loaded + allow_lists = loaded logs_with_errors = process_files(files_to_process, urls_to_process, - whitelists) + allow_lists) failed = False if logs_with_errors: @@ -164,14 +164,14 @@ def main(opts): usage = """ -Find non-white-listed log errors in log files from a devstack-gate run. +Find non-allow-listed log errors in log files from a devstack-gate run. Log files will be searched for ERROR or CRITICAL messages. If any -error messages do not match any of the whitelist entries contained in -etc/whitelist.yaml, those messages will be printed to the console and +error messages do not match any of the allow-list entries contained in +etc/allow-list.yaml, those messages will be printed to the console and failure will be returned. A file directory containing logs or a url to the log files of an OpenStack gate job can be provided. -The whitelist yaml looks like: +The allow-list yaml looks like: log-name: - module: "a.b.c" @@ -179,7 +179,7 @@ log-name: - module: "a.b.c" message: "regexp" -repeated for each log file with a whitelist. +repeated for each log file with an allow-list. """ parser = argparse.ArgumentParser(description=usage) diff --git a/tools/generate-tempest-plugins-list.py b/tools/generate-tempest-plugins-list.py index 618c388d4..415ad057b 100644 --- a/tools/generate-tempest-plugins-list.py +++ b/tools/generate-tempest-plugins-list.py @@ -32,9 +32,9 @@ from urllib3.util import retry # List of projects having tempest plugin stale or unmaintained for a long time # (6 months or more) -# TODO(masayukig): Some of these can be removed from BLACKLIST in the future -# when the patches are merged. -BLACKLIST = [ +# TODO(masayukig): Some of these can be removed from NON_ACTIVE_LIST in the +# future when the patches are merged. +NON_ACTIVE_LIST = [ 'x/gce-api', # It looks gce-api doesn't support python3 yet. 'x/glare', # To avoid sanity-job failure 'x/group-based-policy', # It looks this doesn't support python3 yet. @@ -52,7 +52,7 @@ BLACKLIST = [ 'x/tap-as-a-service', # To avoid sanity-job failure 'x/valet', # https://review.opendev.org/#/c/638339/ 'x/kingbird', # https://bugs.launchpad.net/kingbird/+bug/1869722 - # vmware-nsx is blacklisted since https://review.opendev.org/#/c/736952 + # vmware-nsx is excluded since https://review.opendev.org/#/c/736952 'x/vmware-nsx-tempest-plugin', ] @@ -86,10 +86,10 @@ def has_tempest_plugin(proj): False -if len(sys.argv) > 1 and sys.argv[1] == 'blacklist': - for black_plugin in BLACKLIST: - print(black_plugin) - # We just need BLACKLIST when we use this `blacklist` option. +if len(sys.argv) > 1 and sys.argv[1] == 'nonactivelist': + for non_active_plugin in NON_ACTIVE_LIST: + print(non_active_plugin) + # We just need NON_ACTIVE_LIST when we use this `nonactivelist` option. # So, this exits here. sys.exit() diff --git a/tools/generate-tempest-plugins-list.sh b/tools/generate-tempest-plugins-list.sh index 33675eddc..4430bbf69 100755 --- a/tools/generate-tempest-plugins-list.sh +++ b/tools/generate-tempest-plugins-list.sh @@ -81,17 +81,17 @@ print_plugin_table "${sorted_plugins}" printf "\n\n" -# Print BLACKLIST -if [[ -r doc/source/data/tempest-blacklisted-plugins-registry.header ]]; then - cat doc/source/data/tempest-blacklisted-plugins-registry.header +# Print NON_ACTIVE_LIST +if [[ -r doc/source/data/tempest-non-active-plugins-registry.header ]]; then + cat doc/source/data/tempest-non-active-plugins-registry.header fi -blacklist=$(python tools/generate-tempest-plugins-list.py blacklist) -name_col_len=$(echo "${blacklist}" | wc -L) +nonactivelist=$(python tools/generate-tempest-plugins-list.py nonactivelist) +name_col_len=$(echo "${nonactivelist}" | wc -L) name_col_len=$(( name_col_len + 20 )) printf "\n\n" -print_plugin_table "${blacklist}" +print_plugin_table "${nonactivelist}" printf "\n\n" diff --git a/tools/tempest-integrated-gate-compute-blacklist.txt b/tools/tempest-integrated-gate-compute-exclude-list.txt index 229075173..229075173 100644 --- a/tools/tempest-integrated-gate-compute-blacklist.txt +++ b/tools/tempest-integrated-gate-compute-exclude-list.txt diff --git a/tools/tempest-integrated-gate-networking-blacklist.txt b/tools/tempest-integrated-gate-networking-exclude-list.txt index 263b2e406..263b2e406 100644 --- a/tools/tempest-integrated-gate-networking-blacklist.txt +++ b/tools/tempest-integrated-gate-networking-exclude-list.txt diff --git a/tools/tempest-integrated-gate-object-storage-blacklist.txt b/tools/tempest-integrated-gate-object-storage-exclude-list.txt index c16434316..c16434316 100644 --- a/tools/tempest-integrated-gate-object-storage-blacklist.txt +++ b/tools/tempest-integrated-gate-object-storage-exclude-list.txt diff --git a/tools/tempest-integrated-gate-placement-blacklist.txt b/tools/tempest-integrated-gate-placement-exclude-list.txt index efba796e1..efba796e1 100644 --- a/tools/tempest-integrated-gate-placement-blacklist.txt +++ b/tools/tempest-integrated-gate-placement-exclude-list.txt diff --git a/tools/tempest-integrated-gate-storage-blacklist.txt b/tools/tempest-integrated-gate-storage-blacklist.txt index 1ef6bb5e3..2d691f82a 100644..120000 --- a/tools/tempest-integrated-gate-storage-blacklist.txt +++ b/tools/tempest-integrated-gate-storage-blacklist.txt @@ -1,14 +1 @@ -# This file includes the backlist of tests which need to be -# skipped for Integrated-gate-storage template. Integrated-gate-storage template -# needs to run only Cinder, Glance, Swift and Nova related tests and rest all -# tests will be skipped by below list. - -# Skip network, keystone API tests. -tempest.api.network -tempest.api.identity - -# Skip network only scenario tests. -tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps -tempest.scenario.test_network_basic_ops.TestNetworkBasicOps -tempest.scenario.test_network_v6.TestGettingAddress -tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps +tempest-integrated-gate-storage-exclude-list.txt
\ No newline at end of file diff --git a/tools/tempest-integrated-gate-storage-exclude-list.txt b/tools/tempest-integrated-gate-storage-exclude-list.txt new file mode 100644 index 000000000..1ef6bb5e3 --- /dev/null +++ b/tools/tempest-integrated-gate-storage-exclude-list.txt @@ -0,0 +1,14 @@ +# This file includes the backlist of tests which need to be +# skipped for Integrated-gate-storage template. Integrated-gate-storage template +# needs to run only Cinder, Glance, Swift and Nova related tests and rest all +# tests will be skipped by below list. + +# Skip network, keystone API tests. +tempest.api.network +tempest.api.identity + +# Skip network only scenario tests. +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps +tempest.scenario.test_network_v6.TestGettingAddress +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps diff --git a/tools/tempest-plugin-sanity.sh b/tools/tempest-plugin-sanity.sh index c983da9cd..106a9c6a8 100644 --- a/tools/tempest-plugin-sanity.sh +++ b/tools/tempest-plugin-sanity.sh @@ -44,7 +44,7 @@ set -ex # retrieve a list of projects having tempest plugins PROJECT_LIST="$(python tools/generate-tempest-plugins-list.py)" -BLACKLIST="$(python tools/generate-tempest-plugins-list.py blacklist)" +NON_ACTIVE_LIST="$(python tools/generate-tempest-plugins-list.py nonactivelist)" # Function to clone project using zuul-cloner or from git function clone_project { @@ -117,8 +117,8 @@ passed_plugin='' failed_plugin='' # Perform sanity on all tempest plugin projects for project in $PROJECT_LIST; do - # Remove blacklisted tempest plugins - if ! [[ `echo $BLACKLIST | grep -c $project ` -gt 0 ]]; then + # Remove non-active tempest plugins + if ! [[ `echo $NON_ACTIVE_LIST | grep -c $project ` -gt 0 ]]; then plugin_sanity_check $project && passed_plugin+=", $project" || \ failed_plugin+="$project, " > $SANITY_DIR/$project.txt fi @@ -1,6 +1,6 @@ [tox] envlist = pep8,py36,py38,bashate,pip-check-reqs -minversion = 3.1.1 +minversion = 3.18.0 skipsdist = True ignore_basepython_conflict = True @@ -26,7 +26,7 @@ setenv = passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH TEMPEST_CONFIG TEMPEST_CONFIG_DIR http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY ZUUL_CACHE_DIR REQUIREMENTS_PIP_LOCATION GENERATE_TEMPEST_PLUGIN_LIST usedevelop = True install_command = pip install {opts} {packages} -whitelist_externals = * +allowlist_externals = * deps = -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/requirements.txt @@ -108,7 +108,7 @@ setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} # The regex below is used to select which tests to run and exclude the slow tag: # See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610 -# FIXME: We can replace it with the `--black-regex` option to exclude tests now. +# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now. commands = find . -type f -name "*.pyc" -delete tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs} @@ -132,11 +132,11 @@ basepython = {[tempestenv]basepython} setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} # The regex below is used to select which tests to run and exclude the slow tag and -# tests listed in blacklist file: +# tests listed in exclude-list file: commands = find . -type f -name "*.pyc" -delete - tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-networking-blacklist.txt {posargs} - tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-networking-blacklist.txt {posargs} + tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs} + tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exlude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs} [testenv:integrated-compute] envdir = .tox/tempest @@ -145,11 +145,11 @@ basepython = {[tempestenv]basepython} setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} # The regex below is used to select which tests to run and exclude the slow tag and -# tests listed in blacklist file: +# tests listed in exclude-list file: commands = find . -type f -name "*.pyc" -delete - tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-compute-blacklist.txt {posargs} - tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-compute-blacklist.txt {posargs} + tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs} + tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs} [testenv:integrated-placement] envdir = .tox/tempest @@ -158,11 +158,11 @@ basepython = {[tempestenv]basepython} setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} # The regex below is used to select which tests to run and exclude the slow tag and -# tests listed in blacklist file: +# tests listed in exclude-list file: commands = find . -type f -name "*.pyc" -delete - tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-placement-blacklist.txt {posargs} - tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-placement-blacklist.txt {posargs} + tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs} + tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs} [testenv:integrated-storage] envdir = .tox/tempest @@ -171,11 +171,11 @@ basepython = {[tempestenv]basepython} setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} # The regex below is used to select which tests to run and exclude the slow tag and -# tests listed in blacklist file: +# tests listed in exclude-list file: commands = find . -type f -name "*.pyc" -delete - tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-storage-blacklist.txt {posargs} - tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-storage-blacklist.txt {posargs} + tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs} + tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs} [testenv:integrated-object-storage] envdir = .tox/tempest @@ -184,11 +184,11 @@ basepython = {[tempestenv]basepython} setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} # The regex below is used to select which tests to run and exclude the slow tag and -# tests listed in blacklist file: +# tests listed in exclude-list file: commands = find . -type f -name "*.pyc" -delete - tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-object-storage-blacklist.txt {posargs} - tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-object-storage-blacklist.txt {posargs} + tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs} + tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs} [testenv:full-serial] envdir = .tox/tempest @@ -198,7 +198,7 @@ setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} # The regex below is used to select which tests to run and exclude the slow tag: # See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610 -# FIXME: We can replace it with the `--black-regex` option to exclude tests now. +# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now. commands = find . -type f -name "*.pyc" -delete tempest run --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario))' {posargs} @@ -290,12 +290,12 @@ commands = sphinx-apidoc -f -o doc/source/tests/volume tempest/api/volume rm -rf doc/build sphinx-build -W -b html doc/source doc/build/html -whitelist_externals = +allowlist_externals = rm [testenv:pdf-docs] deps = {[testenv:docs]deps} -whitelist_externals = +allowlist_externals = rm make commands = @@ -369,7 +369,7 @@ commands = rm -rf releasenotes/build sphinx-build -a -E -W -d releasenotes/build/doctrees \ -b html releasenotes/source releasenotes/build/html -whitelist_externals = rm +allowlist_externals = rm [testenv:bashate] # if you want to test out some changes you have made to bashate @@ -377,7 +377,7 @@ whitelist_externals = rm # modified bashate tree deps = {env:BASHATE_INSTALL_PATH:bashate} -whitelist_externals = bash +allowlist_externals = bash commands = bash -c "find {toxinidir}/tools \ -not \( -type d -name .?\* -prune \) \ -type f \ @@ -406,6 +406,6 @@ commands = bindep test [testenv:plugin-sanity-check] # perform tempest plugin sanity -whitelist_externals = bash +allowlist_externals = bash commands = bash tools/tempest-plugin-sanity.sh |