diff options
Diffstat (limited to 'pbr/packaging.py')
| -rw-r--r-- | pbr/packaging.py | 412 |
1 files changed, 299 insertions, 113 deletions
diff --git a/pbr/packaging.py b/pbr/packaging.py index 44c14cc..18d18e6 100644 --- a/pbr/packaging.py +++ b/pbr/packaging.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack LLC. # Copyright 2012-2013 Hewlett-Packard Development Company, L.P. # All Rights Reserved. @@ -22,16 +20,23 @@ Utilities with minimum-depends for use in setup.py from __future__ import unicode_literals +from distutils.command import install as du_install +import distutils.errors +from distutils import log import email +import functools import io +import itertools import os +import platform import re import subprocess import sys +try: + import cStringIO +except ImportError: + import io as cStringIO -from distutils.command import install as du_install -import distutils.errors -from distutils import log import pkg_resources from setuptools.command import easy_install from setuptools.command import egg_info @@ -39,12 +44,8 @@ from setuptools.command import install from setuptools.command import install_scripts from setuptools.command import sdist -try: - import cStringIO -except ImportError: - import io as cStringIO - from pbr import extra_files +from pbr import version TRUE_VALUES = ('true', '1', 'yes') REQUIREMENTS_FILES = ('requirements.txt', 'tools/pip-requires') @@ -58,10 +59,26 @@ def get_requirements_files(): # Returns a list composed of: # - REQUIREMENTS_FILES with -py2 or -py3 in the name # (e.g. requirements-py3.txt) + # - REQUIREMENTS_FILES with -{platform.system} in the name + # (e.g. requirements-windows.txt) + # - REQUIREMENTS_FILES with both Python version and platform's + # system in the name + # (e.g. requirements-freebsd-py2.txt) # - REQUIREMENTS_FILES - return (list(map(('-py' + str(sys.version_info[0])).join, - map(os.path.splitext, REQUIREMENTS_FILES))) - + list(REQUIREMENTS_FILES)) + pyversion = sys.version_info[0] + system = platform.system().lower() + parts = list(map(os.path.splitext, REQUIREMENTS_FILES)) + + version_cb = functools.partial("{1}-py{0}{2}".format, pyversion) + platform_cb = functools.partial("{1}-{0}{2}".format, system) + both_cb = functools.partial("{2}-{0}-py{1}{3}".format, system, pyversion) + + return list(itertools.chain( + itertools.starmap(both_cb, parts), + itertools.starmap(platform_cb, parts), + itertools.starmap(version_cb, parts), + REQUIREMENTS_FILES, + )) def append_text_list(config, key, text_list): @@ -241,63 +258,108 @@ def get_boolean_option(option_dict, option_name, env_name): str(os.getenv(env_name)).lower() in TRUE_VALUES) -def write_git_changelog(git_dir=None, dest_dir=os.path.curdir, - option_dict=dict()): - """Write a changelog based on the git changelog.""" +def _iter_changelog(changelog): + """Convert a oneline log iterator to formatted strings. + + :param changelog: An iterator of one line log entries like + that given by _iter_log_oneline. + :return: An iterator over (release, formatted changelog) tuples. + """ + first_line = True + current_release = None + yield current_release, "CHANGES\n=======\n\n" + for hash, tags, msg in changelog: + if tags: + current_release = _get_highest_tag(tags) + underline = len(current_release) * '-' + if not first_line: + yield current_release, '\n' + yield current_release, ( + "%(tag)s\n%(underline)s\n\n" % + dict(tag=current_release, underline=underline)) + + if not msg.startswith("Merge "): + if msg.endswith("."): + msg = msg[:-1] + yield current_release, "* %(msg)s\n" % dict(msg=msg) + first_line = False + + +def _iter_log_oneline(git_dir=None, option_dict=None): + """Iterate over --oneline log entries if possible. + + This parses the output into a structured form but does not apply + presentation logic to the output - making it suitable for different + uses. + + :return: An iterator of (hash, tags_set, 1st_line) tuples, or None if + changelog generation is disabled / not available. + """ + if not option_dict: + option_dict = {} should_skip = get_boolean_option(option_dict, 'skip_changelog', 'SKIP_WRITE_GIT_CHANGELOG') if should_skip: return - - new_changelog = os.path.join(dest_dir, 'ChangeLog') - # If there's already a ChangeLog and it's not writable, just use it - if (os.path.exists(new_changelog) - and not os.access(new_changelog, os.W_OK)): - return - log.info('[pbr] Writing ChangeLog') if git_dir is None: git_dir = _get_git_directory() if not git_dir: return + return _iter_log_inner(git_dir) + + +def _iter_log_inner(git_dir): + """Iterate over --oneline log entries. + This parses the output intro a structured form but does not apply + presentation logic to the output - making it suitable for different + uses. + + :return: An iterator of (hash, tags_set, 1st_line) tuples. + """ + log.info('[pbr] Generating ChangeLog') log_cmd = ['log', '--oneline', '--decorate'] changelog = _run_git_command(log_cmd, git_dir) - first_line = True - with io.open(new_changelog, "w", - encoding="utf-8") as changelog_file: - changelog_file.write("CHANGES\n=======\n\n") - for line in changelog.split('\n'): - line_parts = line.split() - if len(line_parts) < 2: - continue - # Tags are in a list contained in ()'s. If a commit - # subject that is tagged happens to have ()'s in it - # this will fail - if line_parts[1].startswith('(') and ')' in line: - msg = line.split(')')[1].strip() - else: - msg = " ".join(line_parts[1:]) - - if "tag:" in line: - tags = [ - tag.split(",")[0] - for tag in line.split(")")[0].split("tag: ")[1:]] - tag = _get_highest_tag(tags) - - underline = len(tag) * '-' - if not first_line: - changelog_file.write('\n') - changelog_file.write( - ("%(tag)s\n%(underline)s\n\n" % - dict(tag=tag, - underline=underline))) - - if not msg.startswith("Merge "): - if msg.endswith("."): - msg = msg[:-1] - changelog_file.write( - ("* %(msg)s\n" % dict(msg=msg))) - first_line = False + for line in changelog.split('\n'): + line_parts = line.split() + if len(line_parts) < 2: + continue + # Tags are in a list contained in ()'s. If a commit + # subject that is tagged happens to have ()'s in it + # this will fail + if line_parts[1].startswith('(') and ')' in line: + msg = line.split(')')[1].strip() + else: + msg = " ".join(line_parts[1:]) + + if "tag:" in line: + tags = set([ + tag.split(",")[0] + for tag in line.split(")")[0].split("tag: ")[1:]]) + else: + tags = set() + + yield line_parts[0], tags, msg + + +def write_git_changelog(git_dir=None, dest_dir=os.path.curdir, + option_dict=dict(), changelog=None): + """Write a changelog based on the git changelog.""" + if not changelog: + changelog = _iter_log_oneline(git_dir=git_dir, option_dict=option_dict) + if changelog: + changelog = _iter_changelog(changelog) + if not changelog: + return + log.info('[pbr] Writing ChangeLog') + new_changelog = os.path.join(dest_dir, 'ChangeLog') + # If there's already a ChangeLog and it's not writable, just use it + if (os.path.exists(new_changelog) + and not os.access(new_changelog, os.W_OK)): + return + with io.open(new_changelog, "w", encoding="utf-8") as changelog_file: + for release, content in changelog: + changelog_file.write(content) def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()): @@ -306,7 +368,6 @@ def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()): 'SKIP_GENERATE_AUTHORS') if should_skip: return - old_authors = os.path.join(dest_dir, 'AUTHORS.in') new_authors = os.path.join(dest_dir, 'AUTHORS') # If there's already an AUTHORS file and it's not writable, just use it @@ -628,7 +689,10 @@ class LocalSDist(sdist.sdist): def run(self): option_dict = self.distribution.get_option_dict('pbr') - write_git_changelog(option_dict=option_dict) + changelog = _iter_log_oneline(option_dict=option_dict) + if changelog: + changelog = _iter_changelog(changelog) + write_git_changelog(option_dict=option_dict, changelog=changelog) generate_authors(option_dict=option_dict) # sdist.sdist is an old style class, can't use super() sdist.sdist.run(self) @@ -654,7 +718,7 @@ try: os.makedirs(source_dir) return source_dir - def generate_autoindex(self): + def generate_autoindex(self, excluded_modules=None): log.info("[pbr] Autodocumenting from %s" % os.path.abspath(os.curdir)) modules = {} @@ -663,8 +727,10 @@ try: if '.' not in pkg: for dirpath, dirnames, files in os.walk(pkg): _find_modules(modules, dirpath, files) - module_list = list(modules.keys()) - module_list.sort() + module_list = set(modules.keys()) + if excluded_modules is not None: + module_list -= set(excluded_modules) + module_list = sorted(module_list) autoindex_filename = os.path.join(source_dir, 'autoindex.rst') with open(autoindex_filename, 'w') as autoindex: autoindex.write(""".. toctree:: @@ -687,7 +753,8 @@ try: def _sphinx_tree(self): source_dir = self._get_source_dir() - apidoc.main(['apidoc', '.', '-H', 'Modules', '-o', source_dir]) + cmd = ['apidoc', '.', '-H', 'Modules', '-o', source_dir] + apidoc.main(cmd + self.autodoc_tree_excludes) def _sphinx_run(self): if not self.verbose: @@ -730,6 +797,9 @@ try: def run(self): option_dict = self.distribution.get_option_dict('pbr') + if _git_is_installed(): + write_git_changelog(option_dict=option_dict) + generate_authors(option_dict=option_dict) tree_index = get_boolean_option(option_dict, 'autodoc_tree_index_modules', 'AUTODOC_TREE_INDEX_MODULES') @@ -737,12 +807,15 @@ try: 'autodoc_index_modules', 'AUTODOC_INDEX_MODULES') if not os.getenv('SPHINX_DEBUG'): - #NOTE(afazekas): These options can be used together, - # but they do a very similar thing in a difffernet way + # NOTE(afazekas): These options can be used together, + # but they do a very similar thing in a different way if tree_index: self._sphinx_tree() if auto_index: - self.generate_autoindex() + self.generate_autoindex( + option_dict.get( + "autodoc_exclude_modules", + [None, ""])[1].split()) for builder in self.builders: self.builder = builder @@ -750,11 +823,19 @@ try: self.project = self.distribution.get_name() self.version = self.distribution.get_version() self.release = self.distribution.get_version() - if 'warnerrors' in option_dict: + if get_boolean_option(option_dict, 'warnerrors', 'WARNERRORS'): self._sphinx_run() else: setup_command.BuildDoc.run(self) + def initialize_options(self): + # Not a new style class, super keyword does not work. + setup_command.BuildDoc.initialize_options(self) + + # NOTE(dstanek): exclude setup.py from the autodoc tree index + # builds because all projects will have an issue with it + self.autodoc_tree_excludes = ['setup.py'] + def finalize_options(self): # Not a new style class, super keyword does not work. setup_command.BuildDoc.finalize_options(self) @@ -762,6 +843,14 @@ try: if not isinstance(self.builders, list) and self.builders: self.builders = self.builders.split(',') + # NOTE(dstanek): check for autodoc tree exclusion overrides + # in the setup.cfg + opt = 'autodoc_tree_excludes' + option_dict = self.distribution.get_option_dict('pbr') + if opt in option_dict: + self.autodoc_tree_excludes = option_dict[opt][1] + self.ensure_string_list(opt) + class LocalBuildLatex(LocalBuildDoc): builders = ['latex'] command_name = 'build_sphinx_latex' @@ -776,44 +865,130 @@ def have_sphinx(): return _have_sphinx -def _get_revno(git_dir): - """Return the number of commits since the most recent tag. +def _get_increment_kwargs(git_dir, tag): + """Calculate the sort of semver increment needed from git history. + + Every commit from HEAD to tag is consider for Sem-Ver metadata lines. + See the pbr docs for their syntax. + + :return: a dict of kwargs for passing into SemanticVersion.increment. + """ + result = {} + if tag: + version_spec = tag + "..HEAD" + else: + version_spec = "HEAD" + changelog = _run_git_command(['log', version_spec], git_dir) + header_len = len(' sem-ver:') + commands = [line[header_len:].strip() for line in changelog.split('\n') + if line.lower().startswith(' sem-ver:')] + symbols = set() + for command in commands: + symbols.update([symbol.strip() for symbol in command.split(',')]) + + def _handle_symbol(symbol, symbols, impact): + if symbol in symbols: + result[impact] = True + symbols.discard(symbol) + _handle_symbol('bugfix', symbols, 'patch') + _handle_symbol('feature', symbols, 'minor') + _handle_symbol('deprecation', symbols, 'minor') + _handle_symbol('api-break', symbols, 'major') + for symbol in symbols: + log.info('[pbr] Unknown Sem-Ver symbol %r' % symbol) + # We don't want patch in the kwargs since it is not a keyword argument - + # its the default minimum increment. + result.pop('patch', None) + return result + + +def _get_revno_and_last_tag(git_dir): + """Return the commit data about the most recent tag. We use git-describe to find this out, but if there are no tags then we fall back to counting commits since the beginning of time. """ - describe = _run_git_command(['describe', '--always'], git_dir) - if "-" in describe: - return describe.rsplit("-", 2)[-2] + changelog = _iter_log_oneline(git_dir=git_dir) + row_count = 0 + for row_count, (ignored, tag_set, ignored) in enumerate(changelog): + version_tags = set() + for tag in list(tag_set): + try: + version_tags.add(version.SemanticVersion.from_pip_string(tag)) + except Exception: + pass + if version_tags: + return max(version_tags).release_string(), row_count + return "", row_count + + +def _get_version_from_git_target(git_dir, target_version): + """Calculate a version from a target version in git_dir. + + This is used for untagged versions only. A new version is calculated as + necessary based on git metadata - distance to tags, current hash, contents + of commit messages. + + :param git_dir: The git directory we're working from. + :param target_version: If None, the last tagged version (or 0 if there are + no tags yet) is incremented as needed to produce an appropriate target + version following semver rules. Otherwise target_version is used as a + constraint - if semver rules would result in a newer version then an + exception is raised. + :return: A semver version object. + """ + sha = _run_git_command( + ['log', '-n1', '--pretty=format:%h'], git_dir) + tag, distance = _get_revno_and_last_tag(git_dir) + last_semver = version.SemanticVersion.from_pip_string(tag or '0') + if distance == 0: + new_version = last_semver + else: + new_version = last_semver.increment( + **_get_increment_kwargs(git_dir, tag)) + if target_version is not None and new_version > target_version: + raise ValueError( + "git history requires a target version of %(new)s, but target " + "version is %(target)s" % + dict(new=new_version, target=target_version)) + if distance == 0: + return last_semver + if target_version is not None: + return target_version.to_dev(distance, sha) + else: + return new_version.to_dev(distance, sha) - # no tags found - revlist = _run_git_command( - ['rev-list', '--abbrev-commit', 'HEAD'], git_dir) - return len(revlist.splitlines()) +def _get_version_from_git(pre_version=None): + """Calculate a version string from git. -def _get_version_from_git(pre_version): - """Return a version which is equal to the tag that's on the current - revision if there is one, or tag plus number of additional revisions - if the current revision has no tag. - """ + If the revision is tagged, return that. Otherwise calculate a semantic + version description of the tree. + The number of revisions since the last tag is included in the dev counter + in the version for untagged versions. + + :param pre_version: If supplied use this as the target version rather than + inferring one from the last tag + commit messages. + """ git_dir = _get_git_directory() if git_dir and _git_is_installed(): - if pre_version: - try: - return _run_git_command( - ['describe', '--exact-match'], git_dir, - throw_on_error=True).replace('-', '.') - except Exception: - sha = _run_git_command( - ['log', '-n1', '--pretty=format:%h'], git_dir) - return "%s.dev%s+g%s" % (pre_version, _get_revno(git_dir), sha) - else: - return _run_git_command( - ['describe', '--always'], - git_dir).replace('-', '.').replace('.g', '+g') + try: + tagged = _run_git_command( + ['describe', '--exact-match'], git_dir, + throw_on_error=True).replace('-', '.') + target_version = version.SemanticVersion.from_pip_string(tagged) + except Exception: + if pre_version: + # not released yet - use pre_version as the target + target_version = version.SemanticVersion.from_pip_string( + pre_version) + else: + # not released yet - just calculate from git history + target_version = None + result = _get_version_from_git_target(git_dir, target_version) + return result.release_string() # If we don't know the version, return an empty string so at least # the downstream users of the value always have the same type of # object to work with. @@ -823,40 +998,51 @@ def _get_version_from_git(pre_version): return '' -def _get_version_from_pkg_info(package_name): - """Get the version from PKG-INFO file if we can.""" - try: - pkg_info_file = open('PKG-INFO', 'r') - except (IOError, OSError): - return None - try: - pkg_info = email.message_from_file(pkg_info_file) - except email.MessageError: - return None +def _get_version_from_pkg_metadata(package_name): + """Get the version from package metadata if present. + + This looks for PKG-INFO if present (for sdists), and if not looks + for METADATA (for wheels) and failing that will return None. + """ + pkg_metadata_filenames = ['PKG-INFO', 'METADATA'] + pkg_metadata = {} + for filename in pkg_metadata_filenames: + try: + pkg_metadata_file = open(filename, 'r') + except (IOError, OSError): + continue + try: + pkg_metadata = email.message_from_file(pkg_metadata_file) + except email.MessageError: + continue + # Check to make sure we're in our own dir - if pkg_info.get('Name', None) != package_name: + if pkg_metadata.get('Name', None) != package_name: return None - return pkg_info.get('Version', None) + return pkg_metadata.get('Version', None) def get_version(package_name, pre_version=None): - """Get the version of the project. First, try getting it from PKG-INFO, if - it exists. If it does, that means we're in a distribution tarball or that - install has happened. Otherwise, if there is no PKG-INFO file, pull the - version from git. + """Get the version of the project. First, try getting it from PKG-INFO or + METADATA, if it exists. If it does, that means we're in a distribution + tarball or that install has happened. Otherwise, if there is no PKG-INFO + or METADATA file, pull the version from git. We do not support setup.py version sanity in git archive tarballs, nor do we support packagers directly sucking our git repo into theirs. We expect that a source tarball be made from our git repo - or that if someone wants to make a source tarball from a fork of our repo with additional tags in it that they understand and desire the results of doing that. + + :param pre_version: The version field from setup.cfg - if set then this + version will be the next release. """ version = os.environ.get( "PBR_VERSION", os.environ.get("OSLO_PACKAGE_VERSION", None)) if version: return version - version = _get_version_from_pkg_info(package_name) + version = _get_version_from_pkg_metadata(package_name) if version: return version version = _get_version_from_git(pre_version) |