diff options
| -rw-r--r-- | docs/docsite/rst/dev_guide/testing/sanity/update-bundled.rst | 31 | ||||
| -rwxr-xr-x | hacking/update_bundled.py | 34 | ||||
| -rw-r--r-- | lib/ansible/compat/selectors/__init__.py | 3 | ||||
| -rw-r--r-- | lib/ansible/module_utils/compat/ipaddress.py | 5 | ||||
| -rw-r--r-- | lib/ansible/module_utils/six/__init__.py | 4 | ||||
| -rw-r--r-- | lib/ansible/module_utils/urls.py | 4 | ||||
| -rw-r--r-- | test/sanity/code-smell/update-bundled.json | 7 | ||||
| -rwxr-xr-x | test/sanity/code-smell/update-bundled.py | 131 |
8 files changed, 184 insertions, 35 deletions
diff --git a/docs/docsite/rst/dev_guide/testing/sanity/update-bundled.rst b/docs/docsite/rst/dev_guide/testing/sanity/update-bundled.rst new file mode 100644 index 0000000000..f227691f1c --- /dev/null +++ b/docs/docsite/rst/dev_guide/testing/sanity/update-bundled.rst @@ -0,0 +1,31 @@ +:orphan: + +Sanity Tests ยป update-bundled +============================= + +Check whether any of our known bundled code needs to be updated for a new upstream release. + +This test can error in the following ways: + +* The bundled code is out of date with regard to the latest release on pypi. Update the code + to the new version and update the version in _BUNDLED_METADATA to solve this. + +* The code is lacking a _BUNDLED_METADATA variable. This typically happens when a bundled version + is updated and we forget to add a _BUNDLED_METADATA variable to the updated file. Once that is + added, this error should go away. + +* A file has a _BUNDLED_METADATA variable but the file isn't specified in + :file:`test/sanity/code-smell/update-bundled.py`. This typically happens when a new bundled + library is added. Add the file to the `get_bundled_libs()` function in the `update-bundled.py` + test script to solve this error. + +_BUNDLED_METADATA has the following fields: + +:pypi_name: Name of the bundled package on pypi + +:version: Version of the package that we are including here + +:version_constraints: Optional PEP440 specifier for the version range that we are bundling. + Currently, the only valid use of this is to follow a version that is + compatible with the Python stdlib when newer versions of the pypi package + implement a new API. diff --git a/hacking/update_bundled.py b/hacking/update_bundled.py deleted file mode 100755 index 02bf6c118d..0000000000 --- a/hacking/update_bundled.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -import glob -import json -import os.path -from distutils.version import LooseVersion - -from ansible.module_utils.urls import open_url - -basedir = os.path.dirname(__file__) - -for filename in glob.glob(os.path.join(basedir, '../lib/ansible/compat/*/__init__.py')): - if 'compat/tests' in filename: - # compat/tests doesn't bundle any code - continue - - filename = os.path.normpath(filename) - with open(filename, 'r') as module: - for line in module: - if line.strip().startswith('_BUNDLED_METADATA'): - data = line[line.index('{'):].strip() - break - else: - print('WARNING: {0} contained no metadata. Could not check for updates'.format(filename)) - continue - metadata = json.loads(data) - pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])) - pypi_data = json.loads(pypi_fh.read().decode('utf-8')) - if LooseVersion(metadata['version']) < LooseVersion(pypi_data['info']['version']): - print('UPDATE: {0} from {1} to {2} {3}'.format( - metadata['pypi_name'], - metadata['version'], - pypi_data['info']['version'], - 'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))) diff --git a/lib/ansible/compat/selectors/__init__.py b/lib/ansible/compat/selectors/__init__.py index 8d45ea5fbf..2f73c02280 100644 --- a/lib/ansible/compat/selectors/__init__.py +++ b/lib/ansible/compat/selectors/__init__.py @@ -24,7 +24,8 @@ Compat selectors library. Python-3.5 has this builtin. The selectors2 package exists on pypi to backport the functionality as far as python-2.6. ''' # The following makes it easier for us to script updates of the bundled code -_BUNDLED_METADATA = {"pypi_name": "selectors2", "version": "1.1.0"} +_BUNDLED_METADATA = {"pypi_name": "selectors2", "version": "1.1.0", "version_constraints": ">1.0,<2.0"} + # Added these bugfix commits from 2.1.0: # * https://github.com/SethMichaelLarson/selectors2/commit/3bd74f2033363b606e1e849528ccaa76f5067590 # Wrap kqueue.control so that timeout is a keyword arg diff --git a/lib/ansible/module_utils/compat/ipaddress.py b/lib/ansible/module_utils/compat/ipaddress.py index db4f5ac62f..c46ad72a09 100644 --- a/lib/ansible/module_utils/compat/ipaddress.py +++ b/lib/ansible/module_utils/compat/ipaddress.py @@ -66,6 +66,11 @@ from __future__ import unicode_literals import itertools import struct + +# The following makes it easier for us to script updates of the bundled code and is not part of +# upstream +_BUNDLED_METADATA = {"pypi_name": "ipaddress", "version": "1.0.22"} + __version__ = '1.0.22' # Compatibility functions diff --git a/lib/ansible/module_utils/six/__init__.py b/lib/ansible/module_utils/six/__init__.py index 4bc670a645..0f122c3d47 100644 --- a/lib/ansible/module_utils/six/__init__.py +++ b/lib/ansible/module_utils/six/__init__.py @@ -33,6 +33,10 @@ import operator import sys import types +# The following makes it easier for us to script updates of the bundled code. It is not part of +# upstream six +_BUNDLED_METADATA = {"pypi_name": "six", "version": "1.11.0"} + __author__ = "Benjamin Peterson <benjamin@python.org>" __version__ = "1.11.0" diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 0d48eba2b1..a286fa5be1 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -136,6 +136,10 @@ if not HAS_SSLCONTEXT and HAS_SSL: del libssl +# The following makes it easier for us to script updates of the bundled backports.ssl_match_hostname +# The bundled backports.ssl_match_hostname should really be moved into its own file for processing +_BUNDLED_METADATA = {"pypi_name": "backports.ssl_match_hostname", "version": "3.5.0.1"} + LOADED_VERIFY_LOCATIONS = set() HAS_MATCH_HOSTNAME = True diff --git a/test/sanity/code-smell/update-bundled.json b/test/sanity/code-smell/update-bundled.json new file mode 100644 index 0000000000..0f30a44953 --- /dev/null +++ b/test/sanity/code-smell/update-bundled.json @@ -0,0 +1,7 @@ +{ + "ignore_changes": true, + "extensions": [ + ".py" + ], + "output": "path-message" +} diff --git a/test/sanity/code-smell/update-bundled.py b/test/sanity/code-smell/update-bundled.py new file mode 100755 index 0000000000..34daef42a5 --- /dev/null +++ b/test/sanity/code-smell/update-bundled.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# (c) 2018, Ansible Project +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +import fnmatch +import json +import os +import os.path +import re +import sys +from distutils.version import LooseVersion + +import packaging.specifiers + +from ansible.module_utils.urls import open_url + + +BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b') + + +def get_bundled_libs(paths): + bundled_libs = set() + for filename in fnmatch.filter(paths, 'lib/ansible/compat/*/__init__.py'): + bundled_libs.add(filename) + + bundled_libs.add('lib/ansible/module_utils/distro/__init__.py') + bundled_libs.add('lib/ansible/module_utils/six/__init__.py') + bundled_libs.add('lib/ansible/module_utils/compat/ipaddress.py') + # backports.ssl_match_hostname should be moved to its own file in the future + bundled_libs.add('lib/ansible/module_utils/urls.py') + + return bundled_libs + + +def get_files_with_bundled_metadata(paths): + with_metadata = set() + for path in paths: + if path == 'test/sanity/code-smell/update-bundled.py': + continue + + with open(path, 'rb') as f: + body = f.read() + + if BUNDLED_RE.search(body): + with_metadata.add(path) + + return with_metadata + + +def get_bundled_metadata(filename): + with open(filename, 'r') as module: + for line in module: + if line.strip().startswith('_BUNDLED_METADATA'): + data = line[line.index('{'):].strip() + break + else: + raise ValueError('Unable to check bundled library for update. Please add' + ' _BUNDLED_METADATA dictionary to the library file with' + ' information on pypi name and bundled version.') + metadata = json.loads(data) + return metadata + + +def get_latest_applicable_version(pypi_data, constraints=None): + latest_version = "0" + if 'version_constraints' in metadata: + version_specification = packaging.specifiers.SpecifierSet(metadata['version_constraints']) + for version in pypi_data['releases']: + if version in version_specification: + if LooseVersion(version) > LooseVersion(latest_version): + latest_version = version + else: + latest_version = pypi_data['info']['version'] + + return latest_version + + +if __name__ == '__main__': + paths = sys.argv[1:] or sys.stdin.read().splitlines() + + bundled_libs = get_bundled_libs(paths) + files_with_bundled_metadata = get_files_with_bundled_metadata(paths) + + for filename in files_with_bundled_metadata.difference(bundled_libs): + print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to' + ' test/sanity/code-smell/update-bundled.py'.format(filename)) + + for filename in bundled_libs: + try: + metadata = get_bundled_metadata(filename) + except ValueError as e: + print('{0}: ERROR: {1}'.format(filename, e)) + continue + except (IOError, OSError) as e: + if e.errno == 2: + print('{0}: ERROR: {1}. Perhaps the bundled library has been removed' + ' or moved and the bundled library test needs to be modified as' + ' well?'.format(filename, e)) + + pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])) + pypi_data = json.loads(pypi_fh.read().decode('utf-8')) + + constraints = metadata.get('version_constraints', None) + latest_version = get_latest_applicable_version(pypi_data, constraints) + + if LooseVersion(metadata['version']) < LooseVersion(latest_version): + print('{0}: UPDATE {1} from {2} to {3} {4}'.format( + filename, + metadata['pypi_name'], + metadata['version'], + latest_version, + 'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))) |