diff options
author | Matt Clay <matt@mystile.com> | 2021-05-05 16:52:32 -0700 |
---|---|---|
committer | Matt Clay <matt@mystile.com> | 2021-05-05 17:21:14 -0700 |
commit | c48f80d0629e3469f3535bba76de1242a5455ff9 (patch) | |
tree | 11b21eab7fbcb3ef1422cd8754656ad98bd6bed3 | |
parent | df2ae06d46f34623b79357b1fa35ff52c16b785b (diff) | |
download | ansible-c48f80d0629e3469f3535bba76de1242a5455ff9.tar.gz |
Rename hacking/shippable to hacking/azp.
References to Shippable were changed to Azure Pipelines.
Also remove rebalance.py as it does not work with Azure Pipelines due to the required data not being present.
-rw-r--r-- | hacking/azp/README.md (renamed from hacking/shippable/README.md) | 17 | ||||
-rwxr-xr-x | hacking/azp/download.py (renamed from hacking/shippable/download.py) | 2 | ||||
-rwxr-xr-x | hacking/azp/get_recent_coverage_runs.py (renamed from hacking/shippable/get_recent_coverage_runs.py) | 0 | ||||
-rwxr-xr-x | hacking/azp/incidental.py (renamed from hacking/shippable/incidental.py) | 6 | ||||
-rwxr-xr-x | hacking/azp/run.py (renamed from hacking/shippable/run.py) | 0 | ||||
-rw-r--r-- | hacking/shippable/rebalance.py | 190 | ||||
-rwxr-xr-x | test/sanity/code-smell/package-data.py | 2 |
7 files changed, 13 insertions, 204 deletions
diff --git a/hacking/shippable/README.md b/hacking/azp/README.md index 8ef94e4158..5784848228 100644 --- a/hacking/shippable/README.md +++ b/hacking/azp/README.md @@ -1,4 +1,4 @@ -# Shippable Scripts +# Azure Pipelines Scripts ## Scripts @@ -8,7 +8,6 @@ This directory contains the following scripts: - get_recent_coverage_runs.py - Retrieve CI URLs of recent coverage test runs. - incidental.py - Report on incidental code coverage using data from CI. - run.py - Start new runs on CI. -- rebalance.py - Re-balance CI group(s) from a downloaded results directory. ## Incidental Code Coverage @@ -33,18 +32,18 @@ As additional intentional tests are added, the exclusive coverage provided by in Reducing incidental test coverage, and eventually removing incidental tests involves the following process: 1. Run the entire test suite with code coverage enabled. - This is done automatically each day on Shippable. + This is done automatically each day on Azure Pipelines. The URLs and statuses of the most recent such test runs can be found with: ```shell - hacking/shippable/get_recent_coverage_runs.py <optional branch name> + hacking/azp/get_recent_coverage_runs.py <optional branch name> ``` The branch name defaults to `devel`. -2. Download code coverage data from Shippable for local analysis. +2. Download code coverage data from Azure Pipelines for local analysis. Example: ```shell # download results to ansible/ansible directory under cwd - # substitute the correct run number for the Shippable coverage run you want to download - hacking/shippable/download.py https://app.shippable.com/github/ansible/ansible/runs/162160 --test-results --run-metadata -v + # substitute the correct run number for the Azure Pipelines coverage run you want to download + hacking/azp/download.py 14075 --artifacts --run-metadata -v ``` 3. Analyze code coverage data to see which portions of the code are covered by each test. Example: @@ -52,7 +51,7 @@ Reducing incidental test coverage, and eventually removing incidental tests invo # make sure ansible-test is in $PATH source hacking/env-setup # run the script using whichever directory results were downloaded into - hacking/shippable/incidental.py ansible/ansible/162160 + hacking/azp/incidental.py 14075/ ``` 4. Create new intentional tests, or extend existing ones, to cover code that is currently covered by incidental tests. Reports are created by default in a ``test/results/.tmp/incidental/{hash}/reports/`` directory. @@ -76,7 +75,7 @@ Repeat step 3 for as many plugins as desired. To report on multiple plugins at once, such as all ``filter`` plugins, the following command can be used: ```shell -find lib/ansible/plugins/filter -name '*.py' -not -name __init__.py -exec hacking/shippable/incidental.py ansible/ansible/162160 --plugin-path '{}' ';' +find lib/ansible/plugins/filter -name '*.py' -not -name __init__.py -exec hacking/azp/incidental.py 14075/ --plugin-path '{}' ';' ``` Each report will show the incidental code coverage missing from the plugin's own tests. diff --git a/hacking/shippable/download.py b/hacking/azp/download.py index 7ac90b55be..c573e0a7e1 100755 --- a/hacking/shippable/download.py +++ b/hacking/azp/download.py @@ -17,7 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see <http://www.gnu.org/licenses/>. -"""CLI tool for downloading results from Shippable CI runs.""" +"""CLI tool for downloading results from Azure Pipelines CI runs.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/hacking/shippable/get_recent_coverage_runs.py b/hacking/azp/get_recent_coverage_runs.py index 6a7fdae71f..6a7fdae71f 100755 --- a/hacking/shippable/get_recent_coverage_runs.py +++ b/hacking/azp/get_recent_coverage_runs.py diff --git a/hacking/shippable/incidental.py b/hacking/azp/incidental.py index 911127abee..10729299dd 100755 --- a/hacking/shippable/incidental.py +++ b/hacking/azp/incidental.py @@ -57,11 +57,11 @@ def parse_args(): """Parse and return args.""" source = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - parser = argparse.ArgumentParser(description='Report on incidental test coverage downloaded from Shippable.') + parser = argparse.ArgumentParser(description='Report on incidental test coverage downloaded from Azure Pipelines.') parser.add_argument('result', type=directory, - help='path to directory containing test results downloaded from Shippable') + help='path to directory containing test results downloaded from Azure Pipelines') parser.add_argument('--output', type=optional_directory, @@ -143,7 +143,7 @@ def incidental_report(args): if not coverage_data.paths: raise ApplicationError('no coverage data found\n' - 'make sure the downloaded results are from a code coverage run on Shippable') + 'make sure the downloaded results are from a code coverage run on Azure Pipelines') # generate a unique subdirectory in the output directory based on the input files being used path_hash = hashlib.sha256(b'\n'.join(p.encode() for p in coverage_data.paths)).hexdigest() diff --git a/hacking/shippable/run.py b/hacking/azp/run.py index 00a177944f..00a177944f 100755 --- a/hacking/shippable/run.py +++ b/hacking/azp/run.py diff --git a/hacking/shippable/rebalance.py b/hacking/shippable/rebalance.py deleted file mode 100644 index 484e23116b..0000000000 --- a/hacking/shippable/rebalance.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python -# PYTHON_ARGCOMPLETE_OK - -# (c) 2020 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -""" -CLI tool that analyses a Shippable run's test result and re-balances the test targets into new groups. - -Before running this script you must run download.py like: - - ./download.py https://app.shippable.com/github/<team>/<repo>/runs/<run_num> --test-results --job-number x --job-number y - -Or to get all job results from a run: - - ./download.py https://app.shippable.com/github/<team>/<repo>/runs/<run_num> --test-results --all - - -Set the dir <team>/<repo>/<run_num> as the value of '-p/--test-path' for this script. -""" - -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import argparse -import json -import operator -import os -import re - -from glob import glob - -try: - import argcomplete -except ImportError: - argcomplete = None - - -def main(): - """Main program body.""" - args = parse_args() - rebalance(args) - - -def parse_args(): - """Parse and return args.""" - parser = argparse.ArgumentParser(description='Re-balance Shippable group(s) from a downloaded results directory.') - - parser.add_argument('group_count', - metavar='group_count', - help='The number of groups to re-balance the tests to.') - - parser.add_argument('-v', '--verbose', - dest='verbose', - action='store_true', - help='Display more detailed info about files being read and edited.') - - parser.add_argument('-p', '--test-results-path', - dest='test_results_path', - required=True, - help='The directory where the downloaded Shippable job test results are.') - - parser.add_argument('-t', '--target-path', - dest='target_path', - required=False, - help='The directory where the test targets are located. If set the aliases will automatically ' - 'by rewritten with the new proposed group.') - - if argcomplete: - argcomplete.autocomplete(parser) - - args = parser.parse_args() - - return args - - -def get_raw_test_targets(args, test_path): - """Scans the test directory for all the test targets that was run and get's the max runtime for each target.""" - target_times = {} - - for job_id in os.listdir(test_path): - json_path = os.path.join(test_path, job_id, 'test', 'testresults', 'data') - - # Some tests to do not have a data directory - if not os.path.exists(json_path): - continue - - json_file = glob(os.path.join(json_path, '*integration-*.json'))[0] - if not os.path.isfile(json_file): - if args.verbose: - print("The test json file '%s' does not exist or is not a file, skipping test job run" % json_file) - continue - - with open(json_file, mode='rb') as fd: - test_info = json.loads(fd.read().decode('utf-8')) - - targets = test_info.get('targets', {}) - - for target_name, target_info in targets.items(): - target_runtime = int(target_info.get('run_time_seconds', 0)) - - # If that target already is found and has a higher runtime than the current one, ignore this entry. - if target_times.get(target_name, 0) > target_runtime: - continue - - target_times[target_name] = target_runtime - - return dict(sorted(target_times.items(), key=lambda i: i[1], reverse=True)) - - -def print_test_runtime(target_times): - """Prints a nice summary of a dict containing test target names and their runtime.""" - target_name_max_len = 0 - for target_name in target_times.keys(): - target_name_max_len = max(target_name_max_len, len(target_name)) - - print("%s | Seconds |" % ("Target Name".ljust(target_name_max_len),)) - print("%s | ------- |" % ("-" * target_name_max_len,)) - for target_name, target_time in target_times.items(): - print("%s | %s |" % (target_name.ljust(target_name_max_len), str(target_time).ljust(7))) - - -def rebalance(args): - """Prints a nice summary of a proposed rebalanced configuration based on the downloaded Shippable result.""" - test_path = os.path.expanduser(os.path.expandvars(args.test_results_path)) - target_times = get_raw_test_targets(args, test_path) - - group_info = dict([(i, {'targets': [], 'total_time': 0}) for i in range(1, int(args.group_count) + 1)]) - - # Now add each test to the group with the lowest running time. - for target_name, target_time in target_times.items(): - index, total_time = min(enumerate([g['total_time'] for g in group_info.values()]), key=operator.itemgetter(1)) - group_info[index + 1]['targets'].append(target_name) - group_info[index + 1]['total_time'] = total_time + target_time - - # Print a summary of the proposed test split. - for group_number, test_info in group_info.items(): - print("Group %d - Total Runtime (s): %d" % (group_number, test_info['total_time'])) - print_test_runtime(dict([(n, target_times[n]) for n in test_info['targets']])) - print() - - if args.target_path: - target_path = os.path.expanduser(os.path.expandvars(args.target_path)) - - for test_root in ['test', 'tests']: # ansible/ansible uses 'test' but collections use 'tests'. - integration_root = os.path.join(target_path, test_root, 'integration', 'targets') - if os.path.isdir(integration_root): - if args.verbose: - print("Found test integration target dir at '%s'" % integration_root) - break - - else: - # Failed to find test integration target folder - raise ValueError("Failed to find the test target folder on test/integration/targets or " - "tests/integration/targets under '%s'." % target_path) - - for group_number, test_info in group_info.items(): - for test_target in test_info['targets']: - test_target_aliases = os.path.join(integration_root, test_target, 'aliases') - if not os.path.isfile(test_target_aliases): - if args.verbose: - print("Cannot find test target alias file at '%s', skipping." % test_target_aliases) - continue - - with open(test_target_aliases, mode='r') as fd: - test_aliases = fd.readlines() - - changed = False - for idx, line in enumerate(test_aliases): - group_match = re.match(r'shippable/(.*)/group(\d+)', line) - if group_match: - if int(group_match.group(2)) != group_number: - new_group = 'shippable/%s/group%d\n' % (group_match.group(1), group_number) - if args.verbose: - print("Changing %s group from '%s' to '%s'" % (test_target, group_match.group(0), - new_group.rstrip())) - test_aliases[idx] = new_group - changed = True - break - else: - if args.verbose: - print("Test target %s matches proposed group number, no changed required" % test_target) - - if changed: - with open(test_target_aliases, mode='w') as fd: - fd.writelines(test_aliases) - - -if __name__ == '__main__': - main() diff --git a/test/sanity/code-smell/package-data.py b/test/sanity/code-smell/package-data.py index ca5f5ef5db..d655b923d3 100755 --- a/test/sanity/code-smell/package-data.py +++ b/test/sanity/code-smell/package-data.py @@ -26,7 +26,7 @@ def assemble_files_to_ship(complete_file_list): '.github/*/*', 'changelogs/fragments/*', 'hacking/backport/*', - 'hacking/shippable/*', + 'hacking/azp/*', 'hacking/tests/*', 'hacking/ticket_stubs/*', 'test/sanity/code-smell/botmeta.*', |