Dynamic RPM/DEB versioning

* still partially WIP

6 files changed, 249 insertions, 21 deletions

diff --git a/Makefile b/Makefile
index d8bae62728..c96c4fcd09 100644
--- a/Makefile
+++ b/Makefile
@@ -33,9 +33,13 @@ GENERATE_CLI = docs/bin/generate_man.py
 PYTHON=python
 SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")
 
-# VERSION file provides one place to update the software version
-VERSION := $(shell cat VERSION | cut -f1 -d' ')
-RELEASE := $(shell cat VERSION | cut -f2 -d' ')
+# fetch version from project release.py as single source-of-truth
+VERSION := $(shell $(PYTHON) packaging/release/versionhelper/version_helper.py --raw || echo error)
+ifeq ($(findstring error,$(VERSION)), error)
+$(error "version_helper failed")
+endif
+# if a specific release was not requested, set to 0 (RPMs have "fancier" logic for this further down)
+RELEASE ?= 1
 
 # Get the branch information from git
 ifneq ($(shell which git),)
@@ -61,8 +65,9 @@ DEBUILD_OPTS = --source-option="-I"
 DPUT_BIN ?= dput
 DPUT_OPTS ?=
 DEB_DATE := $(shell LC_TIME=C date +"%a, %d %b %Y %T %z")
+DEB_VERSION := $(shell $(PYTHON) packaging/release/versionhelper/version_helper.py --debversion)
 ifeq ($(OFFICIAL),yes)
-    DEB_RELEASE = $(RELEASE)ppa
+    DEB_RELEASE := $(shell $(PYTHON) packaging/release/versionhelper/version_helper.py --debrelease)ppa
     # Sign OFFICIAL builds using 'DEBSIGN_KEYID'
     # DEBSIGN_KEYID is required when signing
     ifneq ($(DEBSIGN_KEYID),)
@@ -89,7 +94,7 @@ PBUILDER_OPTS ?= --debootstrapopts --variant=buildd --architecture $(PBUILDER_AR
 RPMSPECDIR= packaging/rpm
 RPMSPEC = $(RPMSPECDIR)/ansible.spec
 RPMDIST = $(shell rpm --eval '%{?dist}')
-RPMRELEASE = $(RELEASE)
+
 ifneq ($(OFFICIAL),yes)
     RPMRELEASE = 100.git$(DATE)$(GITINFO)
 endif
@@ -97,7 +102,10 @@ ifeq ($(PUBLISH),nightly)
     # https://fedoraproject.org/wiki/Packaging:Versioning#Snapshots
     RPMRELEASE = $(RELEASE).$(DATE)git.$(GIT_HASH)
 endif
-RPMNVR = "$(NAME)-$(VERSION)-$(RPMRELEASE)$(RPMDIST)"
+
+RPMVERSION ?= $(shell $(PYTHON) packaging/release/versionhelper/version_helper.py --baseversion)
+RPMRELEASE ?= $(shell $(PYTHON) packaging/release/versionhelper/version_helper.py --rpmrelease)
+RPMNVR = "$(NAME)-$(RPMVERSION)-$(RPMRELEASE)$(RPMDIST)"
 
 # MOCK build parameters
 MOCK_BIN ?= mock
@@ -213,7 +221,7 @@ install_manpages:
 	cp $(wildcard ./docs/man/man1/ansible*.1.gz) $(PREFIX)/man/man1/
 
 .PHONY: sdist
-sdist: clean docs
+sdist: clean docs changelog_unified
 	$(PYTHON) setup.py sdist
 
 .PHONY: sdist_upload
@@ -225,15 +233,22 @@ sdist_upload: clean docs
 changelog_reno:
 	reno -d changelogs/ report --title 'Ansible 2.5 "Kashmir" Release Notes' --no-collapse-pre-release --no-show-source --earliest-version v2.5.0b1 --output changelogs/CHANGELOG-v2.5.rst
 
+.PHONY: changelog_unified
+changelog_aggregate:
+	echo "TODO: unified changelog" > changelogs/CHANGELOG.rst
+
 .PHONY: rpmcommon
 rpmcommon: sdist
 	@mkdir -p rpm-build
 	@cp dist/*.gz rpm-build/
-	@sed -e 's#^Version:.*#Version: $(VERSION)#' -e 's#^Release:.*#Release: $(RPMRELEASE)%{?dist}$(REPOTAG)#' $(RPMSPEC) >rpm-build/$(NAME).spec
+	@cp $(RPMSPEC) rpm-build/$(NAME).spec
 
 .PHONY: mock-srpm
 mock-srpm: /etc/mock/$(MOCK_CFG).cfg rpmcommon
-	$(MOCK_BIN) -r $(MOCK_CFG) $(MOCK_ARGS) --resultdir rpm-build/  --buildsrpm --spec rpm-build/$(NAME).spec --sources rpm-build/
+	$(MOCK_BIN) -r $(MOCK_CFG) $(MOCK_ARGS) --resultdir rpm-build/ --bootstrap-chroot --old-chroot --buildsrpm --spec rpm-build/$(NAME).spec --sources rpm-build/ \
+    --define "rpmversion $(RPMVERSION)" \
+	--define "upstream_version $(VERSION)" \
+	--define "rpmrelease $(RPMRELEASE)"
 	@echo "#############################################"
 	@echo "Ansible SRPM is built:"
 	@echo rpm-build/*.src.rpm
@@ -241,7 +256,10 @@ mock-srpm: /etc/mock/$(MOCK_CFG).cfg rpmcommon
 
 .PHONY: mock-rpm
 mock-rpm: /etc/mock/$(MOCK_CFG).cfg mock-srpm
-	$(MOCK_BIN) -r $(MOCK_CFG) $(MOCK_ARGS) --resultdir rpm-build/ --rebuild rpm-build/$(NAME)-*.src.rpm
+	$(MOCK_BIN) -r $(MOCK_CFG) $(MOCK_ARGS) --resultdir rpm-build/ --bootstrap-chroot --old-chroot --rebuild rpm-build/$(NAME)-*.src.rpm \
+    --define "rpmversion $(RPMVERSION)" \
+	--define "upstream_version $(VERSION)" \
+	--define "rpmrelease $(RPMRELEASE)"
 	@echo "#############################################"
 	@echo "Ansible RPM is built:"
 	@echo rpm-build/*.noarch.rpm
@@ -255,6 +273,9 @@ srpm: rpmcommon
 	--define "_srcrpmdir %{_topdir}" \
 	--define "_specdir $(RPMSPECDIR)" \
 	--define "_sourcedir %{_topdir}" \
+	--define "upstream_version $(VERSION)" \
+	--define "rpmversion $(RPMVERSION)" \
+	--define "rpmrelease $(RPMRELEASE)" \
 	-bs rpm-build/$(NAME).spec
 	@rm -f rpm-build/$(NAME).spec
 	@echo "#############################################"
@@ -270,8 +291,11 @@ rpm: rpmcommon
 	--define "_srcrpmdir %{_topdir}" \
 	--define "_specdir $(RPMSPECDIR)" \
 	--define "_sourcedir %{_topdir}" \
-	--define "_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" \
+	--define "_rpmfilename $(RPMNVR).%%{ARCH}.rpm" \
 	--define "__python `which $(PYTHON)`" \
+	--define "upstream_version $(VERSION)" \
+	--define "rpmversion $(RPMVERSION)" \
+	--define "rpmrelease $(RPMRELEASE)" \
 	-ba rpm-build/$(NAME).spec
 	@rm -f rpm-build/$(NAME).spec
 	@echo "#############################################"
@@ -285,7 +309,7 @@ debian: sdist
 	    mkdir -p deb-build/$${DIST} ; \
 	    tar -C deb-build/$${DIST} -xvf dist/$(NAME)-$(VERSION).tar.gz ; \
 	    cp -a packaging/debian deb-build/$${DIST}/$(NAME)-$(VERSION)/ ; \
-        sed -ie "s|%VERSION%|$(VERSION)|g;s|%RELEASE%|$(DEB_RELEASE)|;s|%DIST%|$${DIST}|g;s|%DATE%|$(DEB_DATE)|g" deb-build/$${DIST}/$(NAME)-$(VERSION)/debian/changelog ; \
+        sed -ie "s|%VERSION%|$(DEB_VERSION)|g;s|%RELEASE%|$(DEB_RELEASE)|;s|%DIST%|$${DIST}|g;s|%DATE%|$(DEB_DATE)|g" deb-build/$${DIST}/$(NAME)-$(VERSION)/debian/changelog ; \
 	done
 
 .PHONY: deb
@@ -294,12 +318,12 @@ deb: deb-src
 	    PBUILDER_OPTS="$(PBUILDER_OPTS) --distribution $${DIST} --basetgz $(PBUILDER_CACHE_DIR)/$${DIST}-$(PBUILDER_ARCH)-base.tgz --buildresult $(CURDIR)/deb-build/$${DIST}" ; \
 	    $(PBUILDER_BIN) create $${PBUILDER_OPTS} --othermirror "deb http://archive.ubuntu.com/ubuntu $${DIST} universe" ; \
 	    $(PBUILDER_BIN) update $${PBUILDER_OPTS} ; \
-	    $(PBUILDER_BIN) build $${PBUILDER_OPTS} deb-build/$${DIST}/$(NAME)_$(VERSION)-$(DEB_RELEASE)~$${DIST}.dsc ; \
+	    $(PBUILDER_BIN) build $${PBUILDER_OPTS} deb-build/$${DIST}/$(NAME)_$(DEB_VERSION)-$(DEB_RELEASE)~$${DIST}.dsc ; \
 	done
 	@echo "#############################################"
 	@echo "Ansible DEB artifacts:"
 	@for DIST in $(DEB_DIST) ; do \
-	    echo deb-build/$${DIST}/$(NAME)_$(VERSION)-$(DEB_RELEASE)~$${DIST}_amd64.changes ; \
+	    echo deb-build/$${DIST}/$(NAME)_$(DEB_VERSION)-$(DEB_RELEASE)~$${DIST}_amd64.changes ; \
 	done
 	@echo "#############################################"
 
@@ -313,7 +337,7 @@ local_deb: debian
 	@echo "#############################################"
 	@echo "Ansible DEB artifacts:"
 	@for DIST in $(DEB_DIST) ; do \
-	    echo deb-build/$${DIST}/$(NAME)_$(VERSION)-$(DEB_RELEASE)~$${DIST}_amd64.changes ; \
+	    echo deb-build/$${DIST}/$(NAME)_$(DEB_VERSION)-$(DEB_RELEASE)~$${DIST}_amd64.changes ; \
 	done
 	@echo "#############################################"
 
diff --git a/packaging/release/tests/__init__.py b/packaging/release/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/packaging/release/tests/__init__.py
diff --git a/packaging/release/tests/version_helper_test.py b/packaging/release/tests/version_helper_test.py
new file mode 100644
index 0000000000..f928d4b1ee
--- /dev/null
+++ b/packaging/release/tests/version_helper_test.py
@@ -0,0 +1,36 @@
+import pytest
+
+from packaging.version import InvalidVersion
+from versionhelper.version_helper import AnsibleVersionMunger
+
+@pytest.mark.parametrize('version,revision,output_propname,expected', [
+    ('2.5.0dev1', None, 'raw', '2.5.0dev1'),
+    ('2.5.0', None, 'raw', '2.5.0'),
+    ('2.5.0dev1', None, 'major_version', '2.5'),
+    ('2.5.0', None, 'major_version', '2.5'),
+    ('2.5.0dev1', None, 'base_version', '2.5.0'),
+    ('2.5.0', None, 'base_version', '2.5.0'),
+    ('2.5.0dev1', None, 'deb_version', '2.5.0~dev1'),
+    ('2.5.0b1', None, 'deb_version', '2.5.0~b1'),
+    ('2.5.0', None, 'deb_version', '2.5.0'),
+    ('2.5.0dev1', None, 'deb_release', '1'),
+    ('2.5.0b1', 2, 'deb_release', '2'),
+    ('2.5.0dev1', None, 'rpm_release', '0.1.dev1'),
+    ('2.5.0a1', None, 'rpm_release', '0.101.a1'),
+    ('2.5.0b1', None, 'rpm_release', '0.201.b1'),
+    ('2.5.0rc1', None, 'rpm_release', '0.1001.rc1'),
+    ('2.5.0rc1', '0.99', 'rpm_release', '0.99.rc1'),
+    ('2.5.0.rc.1', None, 'rpm_release', '0.1001.rc.1'),
+    ('2.5.0', None, 'rpm_release', '1'),
+    ('2.5.0', 2, 'rpm_release', '2'),
+    ('2.5.0x1', None, None, InvalidVersion)
+])
+def test_output_values(version, revision, output_propname, expected):
+    try:
+        v = AnsibleVersionMunger(version, revision)
+        assert getattr(v, output_propname) == expected
+    except Exception as ex:
+        if isinstance(expected, type):
+            assert isinstance(ex, expected)
+        else:
+            raise
diff --git a/packaging/release/versionhelper/__init__.py b/packaging/release/versionhelper/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/packaging/release/versionhelper/__init__.py
diff --git a/packaging/release/versionhelper/version_helper.py b/packaging/release/versionhelper/version_helper.py
new file mode 100644
index 0000000000..e3e4513e69
--- /dev/null
+++ b/packaging/release/versionhelper/version_helper.py
@@ -0,0 +1,169 @@
+import argparse
+import os
+import re
+import sys
+
+from packaging.version import Version, VERSION_PATTERN
+
+class AnsibleVersionMunger(object):
+    tag_offsets = dict(
+        dev=0,
+        a=100,
+        b=200,
+        rc=1000
+    )
+
+    # TODO: allow overrides here for packaging bump etc
+    def __init__(self, raw_version, revision=None):
+        self._raw_version = raw_version
+        self._revision = revision
+        self._parsed_version = Version(raw_version)
+        self._parsed_regex_match = re.match(VERSION_PATTERN, raw_version, re.VERBOSE | re.IGNORECASE)
+
+    @property
+    def deb_version(self):
+        v = self._parsed_version
+
+        match = self._parsed_regex_match
+
+        if v.is_prerelease:
+            if match.group('pre'):
+                tag_value = match.group('pre')
+                tag_type = match.group('pre_l')
+                tag_ver = match.group('pre_n')
+            elif match.group('dev'):
+                tag_type = "dev"
+                tag_value = match.group('dev')
+                tag_ver = match.group('dev_n')
+            else:
+                raise Exception("unknown prerelease type for version {0}".format(self._raw_version))
+
+        elif v.is_postrelease:
+            raise Exception("post-release identifiers are not supported")
+        else:
+            tag_type = None
+            tag_value = ''
+            tag_ver = 0
+
+        # not a pre/post/dev release, just return base version
+        if not tag_type:
+            return '{base_version}'.format(base_version=self.base_version)
+
+        # it is a pre/dev release, include the tag value with a ~
+        return '{base_version}~{tag_value}'.format(base_version=self.base_version, tag_value=tag_value)
+
+
+    @property
+    def deb_release(self):
+        return '1' if self._revision is None else str(self._revision)
+
+    @property
+    def rpm_release(self):
+        v = self._parsed_version
+        match = self._parsed_regex_match
+
+        if v.is_prerelease:
+            if match.group('pre'):
+                tag_value = match.group('pre')
+                tag_type = match.group('pre_l')
+                tag_ver = match.group('pre_n')
+            elif match.group('dev'):
+                tag_type = "dev"
+                tag_value = match.group('dev')
+                tag_ver = match.group('dev_n')
+            else:
+                raise Exception("unknown prerelease type for version {0}".format(self._raw_version))
+
+        elif v.is_postrelease:
+            raise Exception("post-release identifiers are not supported")
+        else:
+            tag_type = None
+            tag_value = ''
+            tag_ver = 0
+
+        # not a pre/post/dev release, just append revision (default 1)
+        if not tag_type:
+            if self._revision is None:
+                self._revision = 1
+            return '{revision}'.format(base_version=self.base_version, revision=self._revision)
+
+        # cleanse tag value in case it starts with .
+        tag_value = tag_value.strip('.')
+
+        # coerce to int and None == 0
+        tag_ver = int(tag_ver if tag_ver else 0)
+
+        if self._revision is None:
+            tag_offset = self.tag_offsets.get(tag_type)
+            if tag_offset is None:
+                raise Exception('no tag offset defined for tag {0}'.format(tag_type))
+            pkgrel = '0.{0}'.format(tag_offset + tag_ver)
+        else:
+            pkgrel = self._revision
+
+        return '{pkgrel}.{tag_value}'.format(pkgrel=pkgrel, tag_value=tag_value, base_version=self.base_version)
+
+    @property
+    def raw(self):
+        return self._raw_version
+
+    # return the x.y.z version without any other modifiers present
+    @property
+    def base_version(self):
+        return self._parsed_version.base_version
+
+    # return the x.y version without any other modifiers present
+    @property
+    def major_version(self):
+        return re.match('^(\d+.\d+)', self._raw_version).group(1)
+
+def main():
+    parser = argparse.ArgumentParser(description='Extract/transform Ansible versions to various packaging formats')
+
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument('--raw', action='store_true')
+    group.add_argument('--majorversion', action='store_true')
+    group.add_argument('--baseversion', action='store_true')
+    group.add_argument('--debversion', action='store_true')
+    group.add_argument('--debrelease', action='store_true')
+    group.add_argument('--rpmrelease', action='store_true')
+    group.add_argument('--all', action='store_true')
+
+    parser.add_argument('--revision', action='store', default='auto')
+
+    args = parser.parse_args()
+
+    mydir = os.path.dirname(__file__)
+    release_loc = os.path.normpath(mydir + '/../../../lib')
+
+    sys.path.insert(0, release_loc)
+
+    from ansible import release
+
+    rev = None
+    if args.revision != 'auto':
+        rev = args.revision
+
+    v_raw = release.__version__
+    v = AnsibleVersionMunger(v_raw, revision=rev)
+
+    if args.raw:
+        print(v.raw)
+    elif args.baseversion:
+        print(v.base_version)
+    elif args.majorversion:
+        print(v.major_version)
+    elif args.debversion:
+        print(v.deb_version)
+    elif args.debrelease:
+        print(v.deb_release)
+    elif args.rpmrelease:
+        print(v.rpm_release)
+    elif args.all:
+        props = [name for (name,impl) in vars(AnsibleVersionMunger).items() if isinstance(impl, property)]
+
+        for propname in props:
+            print('{0}: {1}'.format(propname, getattr(v, propname)))
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/packaging/rpm/ansible.spec b/packaging/rpm/ansible.spec
index f0d14192a9..8d941edec8 100644
--- a/packaging/rpm/ansible.spec
+++ b/packaging/rpm/ansible.spec
@@ -1,18 +1,17 @@
 %define name ansible
-%define ansible_version $VERSION
 
 %if 0%{?rhel} == 5
 %define __python2 /usr/bin/python26
 %endif
 
 Name:      %{name}
-Version:   %{ansible_version}
-Release:   1%{?dist}
+Version:   %{rpmversion}
+Release:   %{rpmrelease}
 Url:       https://www.ansible.com
 Summary:   SSH-based application deployment, configuration management, and IT orchestration platform
 License:   GPLv3+
 Group:     Development/Libraries
-Source:    http://releases.ansible.com/ansible/%{name}-%{version}.tar.gz
+Source:    http://releases.ansible.com/ansible/%{name}-%{upstream_version}.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 %{!?__python2: %global __python2 /usr/bin/python2.6}
 %{!?python_sitelib: %global python_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
@@ -90,7 +89,7 @@ on remote nodes. Extension modules can be written in any language and
 are transferred to managed machines automatically.
 
 %prep
-%setup -q
+%setup -q -n %{name}-%{upstream_version}
 
 %build
 %{__python2} setup.py build
@@ -131,7 +130,7 @@ rm -rf %{buildroot}
 %{_bindir}/ansible*
 %dir %{_datadir}/ansible
 %config(noreplace) %{_sysconfdir}/ansible
-%doc README.md PKG-INFO COPYING CHANGELOG.md
+%doc README.md PKG-INFO COPYING changelogs/CHANGELOG.rst
 %doc %{_mandir}/man1/ansible*
 
 %changelog