path: root/tools/dist/release.py

#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#


# About this script:
#   This script is intended to simplify creating Subversion releases, by
#   automating as much as is possible.  It works well with our Apache
#   infrastructure, and should make rolling, posting, and announcing
#   releases dirt simple.
#
#   This script may be run on a number of platforms, but it is intended to
#   be run on people.apache.org.  As such, it may have dependencies (such
#   as Python version) which may not be common, but are guaranteed to be
#   available on people.apache.org.

# It'd be kind of nice to use the Subversion python bindings in this script,
# but people.apache.org doesn't currently have them installed

# Stuff we need
import os
import re
import sys
import glob
import shutil
import urllib2
import hashlib
import tarfile
import logging
import datetime
import operator
import itertools
import subprocess
import argparse       # standard in Python 2.7

# Find ezt, using Subversion's copy, if there isn't one on the system.
try:
    import ezt
except ImportError:
    ezt_path = os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0])))
    ezt_path = os.path.join(ezt_path, 'build', 'generator')
    sys.path.append(ezt_path)

    import ezt


# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'

# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'
people_host = 'minotaur.apache.org'
people_dist_dir = '/www/www.apache.org/dist/subversion'


#----------------------------------------------------------------------
# Utility functions

class Version(object):
    regex = re.compile('(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')

    def __init__(self, ver_str):
        match = self.regex.search(ver_str)

        if not match:
            raise RuntimeError("Bad version string '%s'" % ver_str)

        self.major = int(match.group(1))
        self.minor = int(match.group(2))
        self.patch = int(match.group(3))

        if match.group(4):
            self.pre = match.group(4)
            self.pre_num = int(match.group(5))
        else:
            self.pre = None
            self.pre_num = None

        self.base = '%d.%d.%d' % (self.major, self.minor, self.patch)

    def is_prerelease(self):
        return self.pre != None

    def __lt__(self, that):
        if self.major < that.major: return True
        if self.major > that.major: return False

        if self.minor < that.minor: return True
        if self.minor > that.minor: return False

        if self.patch < that.patch: return True
        if self.patch > that.patch: return False

        if not self.pre and not that.pre: return False
        if not self.pre and that.pre: return False
        if self.pre and not that.pre: return True

        # We are both pre-releases
        if self.pre != that.pre:
            return self.pre < that.pre
        else:
            return self.pre_num < that.pre_num

    def __str(self):
        if self.pre:
            extra = '-%s%d' % (self.pre, self.pre_num)
        else:
            extra = ''

        return self.base + extra

    def __repr__(self):

        return "Version('%s')" % self.__str()

    def __str__(self):
        return self.__str()


def get_prefix(base_dir):
    return os.path.join(base_dir, 'prefix')

def get_tempdir(base_dir):
    return os.path.join(base_dir, 'tempdir')

def get_deploydir(base_dir):
    return os.path.join(base_dir, 'deploy')

def get_tmpldir():
    return os.path.join(os.path.abspath(sys.path[0]), 'templates')

def get_tmplfile(filename):
    try:
        return open(os.path.join(get_tmpldir(), filename))
    except IOError:
        # Hmm, we had a problem with the local version, let's try the repo
        return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename)

def get_nullfile():
    # This is certainly not cross platform
    return open('/dev/null', 'w')

def run_script(verbose, script):
    if verbose:
        stdout = None
        stderr = None
    else:
        stdout = get_nullfile()
        stderr = subprocess.STDOUT

    for l in script.split('\n'):
        subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)

def download_file(url, target):
    response = urllib2.urlopen(url)
    target_file = open(target, 'w')
    target_file.write(response.read())

def assert_people():
    if os.uname()[1] != people_host:
        raise RuntimeError('Not running on expected host "%s"' % people_host)

#----------------------------------------------------------------------
# Cleaning up the environment

def cleanup(args):
    'Remove generated files and folders.'
    logging.info('Cleaning')

    shutil.rmtree(get_prefix(args.base_dir), True)
    shutil.rmtree(get_tempdir(args.base_dir), True)
    shutil.rmtree(get_deploydir(args.base_dir), True)


#----------------------------------------------------------------------
# Creating an environment to roll the release

class RollDep(object):
    'The super class for each of the build dependencies.'
    def __init__(self, base_dir, use_existing, verbose):
        self._base_dir = base_dir
        self._use_existing = use_existing
        self._verbose = verbose

    def _test_version(self, cmd):
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        rc = proc.wait()
        if rc: return ''

        return stdout.split('\n')

    def build(self):
        if not hasattr(self, '_extra_configure_flags'):
            self._extra_configure_flags = ''
        cwd = os.getcwd()
        tempdir = get_tempdir(self._base_dir)
        tarball = os.path.join(tempdir, self._filebase + '.tar.gz')

        if os.path.exists(tarball):
            if not self._use_existing:
                raise RuntimeError('autoconf tarball "%s" already exists'
                                                                    % tarball)
            logging.info('Using existing %s.tar.gz' % self._filebase)
        else:
            logging.info('Fetching %s' % self._filebase)
            download_file(self._url, tarball)

        # Extract tarball
        tarfile.open(tarball).extractall(tempdir)

        logging.info('Building ' + self.label)
        os.chdir(os.path.join(tempdir, self._filebase))
        run_script(self._verbose,
                   '''./configure --prefix=%s %s
                      make
                      make install''' % (get_prefix(self._base_dir),
                                         self._extra_configure_flags))

        os.chdir(cwd)


class AutoconfDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'autoconf'
        self._filebase = 'autoconf-' + autoconf_ver
        self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['autoconf', '-V'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == autoconf_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()


class LibtoolDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'libtool'
        self._filebase = 'libtool-' + libtool_ver
        self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['libtool', '--version'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == libtool_ver

    def use_system(self):
        # We unconditionally return False here, to avoid using a borked
        # system libtool (I'm looking at you, Debian).
        return False


class SwigDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose, sf_mirror):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'swig'
        self._filebase = 'swig-' + swig_ver
        self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
            { 'swig' : self._filebase,
              'sf_mirror' : sf_mirror }
        self._extra_configure_flags = '--without-pcre'

    def have_usable(self):
        output = self._test_version(['swig', '-version'])
        if not output: return False

        version = output[1].split()[-1:][0]
        return version == swig_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()


def build_env(args):
    'Download prerequisites for a release and prepare the environment.'
    logging.info('Creating release environment')

    try:
        os.mkdir(get_prefix(args.base_dir))
        os.mkdir(get_tempdir(args.base_dir))
    except OSError:
        if not args.use_existing:
            raise

    autoconf = AutoconfDep(args.base_dir, args.use_existing, args.verbose)
    libtool = LibtoolDep(args.base_dir, args.use_existing, args.verbose)
    swig = SwigDep(args.base_dir, args.use_existing, args.verbose,
                   args.sf_mirror)

    # iterate over our rolling deps, and build them if needed
    for dep in [autoconf, libtool, swig]:
        if dep.use_system():
            logging.info('Using system %s' % dep.label)
        else:
            dep.build()


#----------------------------------------------------------------------
# Create release artifacts

def roll_tarballs(args):
    'Create the release artifacts.'
    extns = ['zip', 'tar.gz', 'tar.bz2']

    if args.branch:
        branch = args.branch
    else:
        branch = args.version.base[:-1] + 'x'

    logging.info('Rolling release %s from branch %s@%d' % (args.version,
                                                           branch, args.revnum))

    # Ensure we've got the appropriate rolling dependencies available
    autoconf = AutoconfDep(args.base_dir, False, args.verbose)
    libtool = LibtoolDep(args.base_dir, False, args.verbose)
    swig = SwigDep(args.base_dir, False, args.verbose, None)

    for dep in [autoconf, libtool, swig]:
        if not dep.have_usable():
           raise RuntimeError('Cannot find usable %s' % dep.label)

    # Make sure CHANGES is sync'd
    if branch != 'trunk':
        trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
        branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
                                                        args.revnum)
        proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
                                   trunk_CHANGES],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        proc.wait()

        if stdout:
            raise RuntimeError('CHANGES not synced between trunk and branch')

    # Create the output directory
    if not os.path.exists(get_deploydir(args.base_dir)):
        os.mkdir(get_deploydir(args.base_dir))

    # For now, just delegate to dist.sh to create the actual artifacts
    extra_args = ''
    if args.version.is_prerelease():
        extra_args = '-%s %d' % (args.version.pre, args.version.pre_num)
    logging.info('Building UNIX tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
                     % (sys.path[0], args.version.base, branch, args.revnum,
                        extra_args) )
    logging.info('Buildling Windows tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
                     % (sys.path[0], args.version.base, branch, args.revnum,
                        extra_args) )

    # Move the results to the deploy directory
    logging.info('Moving artifacts and calculating checksums')
    for e in extns:
        if args.version.pre == 'nightly':
            filename = 'subversion-trunk.%s' % e
        else:
            filename = 'subversion-%s.%s' % (args.version, e)

        shutil.move(filename, get_deploydir(args.base_dir))
        filename = os.path.join(get_deploydir(args.base_dir), filename)
        m = hashlib.sha1()
        m.update(open(filename, 'r').read())
        open(filename + '.sha1', 'w').write(m.hexdigest())

    shutil.move('svn_version.h.dist', get_deploydir(args.base_dir))

    # And we're done!


#----------------------------------------------------------------------
# Post the candidate release artifacts

def post_candidates(args):
    'Post the generated tarballs to web-accessible directory.'
    if args.target:
        target = args.target
    else:
        target = os.path.join(os.getenv('HOME'), 'public_html', 'svn',
                              args.version)

    if args.code_name:
        dirname = args.code_name
    else:
        dirname = 'deploy'

    if not os.path.exists(target):
        os.makedirs(target)

    data = { 'version'      : args.version,
             'revnum'       : args.revnum,
             'dirname'      : dirname,
           }

    # Choose the right template text
    if args.version.is_prerelease():
        if args.version.pre == 'nightly':
            template_filename = 'nightly-candidates.ezt'
        else:
            template_filename = 'rc-candidates.ezt'
    else:
        template_filename = 'stable-candidates.ezt'

    template = ezt.Template()
    template.parse(get_tmplfile(template_filename).read())
    template.generate(open(os.path.join(target, 'index.html'), 'w'), data)

    logging.info('Moving tarballs to %s' % os.path.join(target, dirname))
    if os.path.exists(os.path.join(target, dirname)):
        shutil.rmtree(os.path.join(target, dirname))
    shutil.copytree(get_deploydir(args.base_dir), os.path.join(target, dirname))


#----------------------------------------------------------------------
# Clean dist

def clean_dist(args):
    'Clean the distribution directory of all but the most recent artifacts.'

    regex = re.compile('subversion-(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')

    if not args.dist_dir:
        assert_people()
        args.dist_dir = people_dist_dir

    logging.info('Cleaning dist dir \'%s\'' % args.dist_dir)

    filenames = glob.glob(os.path.join(args.dist_dir, 'subversion-*.tar.gz'))
    versions = []
    for filename in filenames:
        versions.append(Version(filename))

    for k, g in itertools.groupby(sorted(versions),
                                  lambda x: (x.major, x.minor)):
        releases = list(g)
        logging.info("Saving release '%s'", releases[-1])

        for r in releases[:-1]:
            for filename in glob.glob(os.path.join(args.dist_dir,
                                                   'subversion-%s.*' % r)):
                logging.info("Removing '%s'" % filename)
                os.remove(filename)


#----------------------------------------------------------------------
# Write announcements

def write_news(args):
    'Write text for the Subversion website.'
    data = { 'date' : datetime.date.today().strftime('%Y%m%d'),
             'date_pres' : datetime.date.today().strftime('%Y-%m-%d'),
             'version' : str(args.version),
             'version_base' : args.version.base,
           }

    if args.version.is_prerelease():
        template_filename = 'rc-news.ezt'
    else:
        template_filename = 'stable-news.ezt'

    template = ezt.Template()
    template.parse(get_tmplfile(template_filename).read())
    template.generate(sys.stdout, data)


def get_sha1info(args):
    'Return a list of sha1 info for the release'
    sha1s = glob.glob(os.path.join(get_deploydir(args.base_dir), '*.sha1'))

    class info(object):
        pass

    sha1info = []
    for s in sha1s:
        i = info()
        i.filename = os.path.basename(s)[:-5]
        i.sha1 = open(s, 'r').read()
        sha1info.append(i)

    return sha1info


def write_announcement(args):
    'Write the release announcement.'
    sha1info = get_sha1info(args)

    data = { 'version'              : args.version,
             'sha1info'             : sha1info,
             'siginfo'              : open('getsigs-output', 'r').read(),
             'major-minor'          : args.version.base[:3],
             'major-minor-patch'    : args.version.base,
           }

    if args.version.is_prerelease():
        template_filename = 'rc-release-ann.ezt'
    else:
        template_filename = 'stable-release-ann.ezt'

    template = ezt.Template(compress_whitespace = False)
    template.parse(get_tmplfile(template_filename).read())
    template.generate(sys.stdout, data)


#----------------------------------------------------------------------
# Main entry point for argument parsing and handling

def main():
    'Parse arguments, and drive the appropriate subcommand.'

    # Setup our main parser
    parser = argparse.ArgumentParser(
                            description='Create an Apache Subversion release.')
    parser.add_argument('--clean', action='store_true', default=False,
                   help='Remove any directories previously created by %(prog)s')
    parser.add_argument('--verbose', action='store_true', default=False,
                   help='Increase output verbosity')
    parser.add_argument('--base-dir', default=os.getcwd(),
                   help='''The directory in which to create needed files and
                           folders.  The default is the current working
                           directory.''')
    subparsers = parser.add_subparsers(title='subcommands')

    # Setup the parser for the build-env subcommand
    subparser = subparsers.add_parser('build-env',
                    help='''Download release prerequisistes, including autoconf,
                            libtool, and swig.''')
    subparser.set_defaults(func=build_env)
    subparser.add_argument('--sf-mirror', default='softlayer',
                    help='''The mirror to use for downloading files from
                            SourceForge.  If in the EU, you may want to use
                            'kent' for this value.''')
    subparser.add_argument('--use-existing', action='store_true', default=False,
                    help='''Attempt to use existing build dependencies before
                            downloading and building a private set.''')

    # Setup the parser for the roll subcommand
    subparser = subparsers.add_parser('roll',
                    help='''Create the release artifacts.''')
    subparser.set_defaults(func=roll_tarballs)
    subparser.add_argument('version', type=Version,
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--branch',
                    help='''The branch to base the release on.''')

    # Setup the parser for the post-candidates subcommand
    subparser = subparsers.add_parser('post-candidates',
                    help='''Build the website to host the candidate tarballs.
                            The default location is somewhere in ~/public_html.
                            ''')
    subparser.set_defaults(func=post_candidates)
    subparser.add_argument('version', type=Version,
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--target',
                    help='''The full path to the destination.''')
    subparser.add_argument('--code-name',
                    help='''A whimsical name for the release, used only for
                            naming the download directory.''')

    # The clean-dist subcommand
    subparser = subparsers.add_parser('clean-dist',
                    help='''Clean the distribution directory (and mirrors) of
                            all but the most recent MAJOR.MINOR release.  If no
                            dist-dir is given, this command will assume it is
                            running on people.apache.org.''')
    subparser.set_defaults(func=clean_dist)
    subparser.add_argument('--dist-dir',
                    help='''The directory to clean.''')

    # The write-news subcommand
    subparser = subparsers.add_parser('write-news',
                    help='''Output to stdout template text for use in the news
                            section of the Subversion website.''')
    subparser.set_defaults(func=write_news)
    subparser.add_argument('version', type=Version,
                    help='''The release label, such as '1.7.0-alpha1'.''')

    subparser = subparsers.add_parser('write-announcement',
                    help='''Output to stdout template text for the emailed
                            release announcement.''')
    subparser.set_defaults(func=write_announcement)
    subparser.add_argument('version', type=Version,
                    help='''The release label, such as '1.7.0-alpha1'.''')

    # A meta-target
    subparser = subparsers.add_parser('clean',
                    help='''The same as the '--clean' switch, but as a
                            separate subcommand.''')
    subparser.set_defaults(func=cleanup)

    # Parse the arguments
    args = parser.parse_args()

    # first, process any global operations
    if args.clean:
        cleanup(args)

    # Set up logging
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Fix up our path so we can use our installed versions
    os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
                                                            + os.environ['PATH']

    # finally, run the subcommand, and give it the parsed arguments
    args.func(args)


if __name__ == '__main__':
    main()