diff options
Diffstat (limited to 'buildscripts/packager_enterprise.py')
| -rwxr-xr-x | buildscripts/packager_enterprise.py | 383 |
1 files changed, 383 insertions, 0 deletions
diff --git a/buildscripts/packager_enterprise.py b/buildscripts/packager_enterprise.py new file mode 100755 index 00000000000..0ef782862d9 --- /dev/null +++ b/buildscripts/packager_enterprise.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python +"""Packager Enterprise module.""" + +# This program makes Debian and RPM repositories for MongoDB, by +# downloading our tarballs of statically linked executables and +# insinuating them into Linux packages. It must be run on a +# Debianoid, since Debian provides tools to make RPMs, but RPM-based +# systems don't provide debian packaging crud. + +# Notes: +# +# * Almost anything that you want to be able to influence about how a +# package construction must be embedded in some file that the +# packaging tool uses for input (e.g., debian/rules, debian/control, +# debian/changelog; or the RPM specfile), and the precise details are +# arbitrary and silly. So this program generates all the relevant +# inputs to the packaging tools. +# +# * Once a .deb or .rpm package is made, there's a separate layer of +# tools that makes a "repository" for use by the apt/yum layers of +# package tools. The layouts of these repositories are arbitrary and +# silly, too. +# +# * Before you run the program on a new host, these are the +# prerequisites: +# +# apt-get install dpkg-dev rpm debhelper fakeroot ia32-libs createrepo git-core libsnmp15 +# echo "Now put the dist gnupg signing keys in ~root/.gnupg" + +import errno +from glob import glob +import os +import re +import shutil +import sys +import tempfile +import time + +import packager # pylint: disable=relative-import + +# The MongoDB names for the architectures we support. +ARCH_CHOICES = ["x86_64", "ppc64le", "s390x", "arm64"] + +# Made up names for the flavors of distribution we package for. +DISTROS = ["suse", "debian", "redhat", "ubuntu", "amazon", "amazon2"] + + +class EnterpriseSpec(packager.Spec): + """EnterpriseSpec class.""" + + def suffix(self): + """Suffix.""" + return "-enterprise" if int(self.ver.split(".")[1]) % 2 == 0 else "-enterprise-unstable" + + +class EnterpriseDistro(packager.Distro): + """EnterpriseDistro class.""" + + def repodir(self, arch, build_os, spec): # noqa: D406,D407,D412,D413 + """Return the directory where we'll place the package files. + + This is for (distro, distro_version) in that distro's preferred repository + layout (as distinct from where that distro's packaging building + tools place the package files). + + Packages will go into repos corresponding to the major release + series (2.5, 2.6, 2.7, 2.8, etc.) except for RC's and nightlies + which will go into special separate "testing" directories + + Examples: + + repo/apt/ubuntu/dists/precise/mongodb-enterprise/testing/multiverse/binary-amd64 + repo/apt/ubuntu/dists/precise/mongodb-enterprise/testing/multiverse/binary-i386 + + repo/apt/ubuntu/dists/precise/mongodb-enterprise/2.5/multiverse/binary-amd64 + repo/apt/ubuntu/dists/precise/mongodb-enterprise/2.5/multiverse/binary-i386 + + repo/apt/ubuntu/dists/trusty/mongodb-enterprise/2.5/multiverse/binary-amd64 + repo/apt/ubuntu/dists/trusty/mongodb-enterprise/2.5/multiverse/binary-i386 + + repo/apt/debian/dists/wheezy/mongodb-enterprise/2.5/main/binary-amd64 + repo/apt/debian/dists/wheezy/mongodb-enterprise/2.5/main/binary-i386 + + repo/yum/redhat/6/mongodb-enterprise/2.5/x86_64 + repo/yum/redhat/6/mongodb-enterprise/2.5/i386 + + repo/zypper/suse/11/mongodb-enterprise/2.5/x86_64 + repo/zypper/suse/11/mongodb-enterprise/2.5/i386 + + repo/zypper/suse/11/mongodb-enterprise/testing/x86_64 + repo/zypper/suse/11/mongodb-enterprise/testing/i386 + """ + + repo_directory = "" + + if spec.is_pre_release(): + repo_directory = "testing" + else: + repo_directory = spec.branch() + + if re.search("^(debian|ubuntu)", self.dname): + return "repo/apt/%s/dists/%s/mongodb-enterprise/%s/%s/binary-%s/" % ( + self.dname, self.repo_os_version(build_os), repo_directory, self.repo_component(), + self.archname(arch)) + elif re.search("(redhat|fedora|centos|amazon)", self.dname): + return "repo/yum/%s/%s/mongodb-enterprise/%s/%s/RPMS/" % ( + self.dname, self.repo_os_version(build_os), repo_directory, self.archname(arch)) + elif re.search("(suse)", self.dname): + return "repo/zypper/%s/%s/mongodb-enterprise/%s/%s/RPMS/" % ( + self.dname, self.repo_os_version(build_os), repo_directory, self.archname(arch)) + else: + raise Exception("BUG: unsupported platform?") + + def build_os(self, arch): # pylint: disable=too-many-branches + """Return the build os label in the binary package to download. + + The labels "rhel57", "rhel62", "rhel67" and "rhel70" are for redhat, + the others are delegated to the super class. + """ + # pylint: disable=too-many-return-statements + if arch == "ppc64le": + if self.dname == 'ubuntu': + return ["ubuntu1604"] + if self.dname == 'redhat': + return ["rhel71"] + return [] + if arch == "s390x": + if self.dname == 'redhat': + return ["rhel67", "rhel72"] + if self.dname == 'suse': + return ["suse11", "suse12"] + if self.dname == 'ubuntu': + return ["ubuntu1604"] + return [] + if arch == "arm64": + if self.dname == 'ubuntu': + return ["ubuntu1604"] + return [] + + if re.search("(redhat|fedora|centos)", self.dname): + return ["rhel70", "rhel62", "rhel57"] + return super(EnterpriseDistro, self).build_os(arch) + # pylint: enable=too-many-return-statements + + +def main(): + """Execute Main program.""" + + distros = [EnterpriseDistro(distro) for distro in DISTROS] + + args = packager.get_args(distros, ARCH_CHOICES) + + spec = EnterpriseSpec(args.server_version, args.metadata_gitspec, args.release_number) + + oldcwd = os.getcwd() + srcdir = oldcwd + "/../" + + # Where to do all of our work. Use a randomly-created directory if one + # is not passed in. + prefix = args.prefix + if prefix is None: + prefix = tempfile.mkdtemp() + + print "Working in directory %s" % prefix + + os.chdir(prefix) + try: + made_pkg = False + # Build a package for each distro/spec/arch tuple, and + # accumulate the repository-layout directories. + for (distro, arch) in packager.crossproduct(distros, args.arches): + + for build_os in distro.build_os(arch): + if build_os in args.distros or not args.distros: + + filename = tarfile(build_os, arch, spec) + packager.ensure_dir(filename) + shutil.copyfile(args.tarball, filename) + + repo = make_package(distro, build_os, arch, spec, srcdir) + make_repo(repo, distro, build_os) + + made_pkg = True + + if not made_pkg: + raise Exception("No valid combination of distro and arch selected") + + finally: + os.chdir(oldcwd) + + +def tarfile(build_os, arch, spec): + """Return the location where we store the downloaded tarball for this package.""" + return "dl/mongodb-linux-%s-enterprise-%s-%s.tar.gz" % (spec.version(), build_os, arch) + + +def setupdir(distro, build_os, arch, spec): + """Return the setup directory name.""" + # The setupdir will be a directory containing all inputs to the + # distro's packaging tools (e.g., package metadata files, init + # scripts, etc, along with the already-built binaries). In case + # the following format string is unclear, an example setupdir + # would be dst/x86_64/debian-sysvinit/wheezy/mongodb-org-unstable/ + # or dst/x86_64/redhat/rhel57/mongodb-org-unstable/ + return "dst/%s/%s/%s/%s%s-%s/" % (arch, distro.name(), build_os, distro.pkgbase(), + spec.suffix(), spec.pversion(distro)) + + +def unpack_binaries_into(build_os, arch, spec, where): + """Unpack the tarfile for (build_os, arch, spec) into directory where.""" + rootdir = os.getcwd() + packager.ensure_dir(where) + # Note: POSIX tar doesn't require support for gtar's "-C" option, + # and Python's tarfile module prior to Python 2.7 doesn't have the + # features to make this detail easy. So we'll just do the dumb + # thing and chdir into where and run tar there. + os.chdir(where) + try: + packager.sysassert(["tar", "xvzf", rootdir + "/" + tarfile(build_os, arch, spec)]) + release_dir = glob('mongodb-linux-*')[0] + for releasefile in "bin", "snmp", "LICENSE.txt", "README", "THIRD-PARTY-NOTICES", "MPL-2": + os.rename("%s/%s" % (release_dir, releasefile), releasefile) + os.rmdir(release_dir) + except Exception: + exc = sys.exc_value + os.chdir(rootdir) + raise exc + os.chdir(rootdir) + + +def make_package(distro, build_os, arch, spec, srcdir): + """Construct the package for (arch, distro, spec). + + Get the packaging files from srcdir and any user-specified suffix from suffixes. + """ + + sdir = setupdir(distro, build_os, arch, spec) + packager.ensure_dir(sdir) + # Note that the RPM packages get their man pages from the debian + # directory, so the debian directory is needed in all cases (and + # innocuous in the debianoids' sdirs). + for pkgdir in ["debian", "rpm"]: + print "Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir) + # FIXME: sh-dash-cee is bad. See if tarfile can do this. + packager.sysassert([ + "sh", "-c", + "(cd \"%s\" && git archive %s %s/ ) | (cd \"%s\" && tar xvf -)" % + (srcdir, spec.metadata_gitspec(), pkgdir, sdir) + ]) + # Splat the binaries and snmp files under sdir. The "build" stages of the + # packaging infrastructure will move the files to wherever they + # need to go. + unpack_binaries_into(build_os, arch, spec, sdir) + # Remove the mongoreplay binary due to libpcap dynamic + # linkage. + if os.path.exists(sdir + "bin/mongoreplay"): + os.unlink(sdir + "bin/mongoreplay") + return distro.make_pkg(build_os, arch, spec, srcdir) + + +def make_repo(repodir, distro, build_os): + """Make the repo.""" + if re.search("(debian|ubuntu)", repodir): + make_deb_repo(repodir, distro, build_os) + elif re.search("(suse|centos|redhat|fedora|amazon)", repodir): + packager.make_rpm_repo(repodir) + else: + raise Exception("BUG: unsupported platform?") + + +def make_deb_repo(repo, distro, build_os): + """Make the Debian repo.""" + # Note: the Debian repository Packages files must be generated + # very carefully in order to be usable. + oldpwd = os.getcwd() + os.chdir(repo + "../../../../../../") + try: + dirs = set([ + os.path.dirname(deb)[2:] + for deb in packager.backtick(["find", ".", "-name", "*.deb"]).split() + ]) + for directory in dirs: + st = packager.backtick(["dpkg-scanpackages", directory, "/dev/null"]) + with open(directory + "/Packages", "w") as fh: + fh.wmake_deb_reporite(st) + bt = packager.backtick(["gzip", "-9c", directory + "/Packages"]) + with open(directory + "/Packages.gz", "wb") as fh: + fh.write(bt) + finally: + os.chdir(oldpwd) + # Notes: the Release{,.gpg} files must live in a special place, + # and must be created after all the Packages.gz files have been + # done. + s1 = """Origin: mongodb +Label: mongodb +Suite: %s +Codename: %s/mongodb-enterprise +Architectures: amd64 ppc64el s390x arm64 +Components: %s +Description: MongoDB packages +""" % (distro.repo_os_version(build_os), distro.repo_os_version(build_os), distro.repo_component()) + if os.path.exists(repo + "../../Release"): + os.unlink(repo + "../../Release") + if os.path.exists(repo + "../../Release.gpg"): + os.unlink(repo + "../../Release.gpg") + oldpwd = os.getcwd() + os.chdir(repo + "../../") + s2 = packager.backtick(["apt-ftparchive", "release", "."]) + try: + with open("Release", 'w') as fh: + fh.write(s1) + fh.write(s2) + finally: + os.chdir(oldpwd) + + +def move_repos_into_place(src, dst): # pylint: disable=too-many-branches + """Move the repos into place.""" + # Find all the stuff in src/*, move it to a freshly-created + # directory beside dst, then play some games with symlinks so that + # dst is a name the new stuff and dst+".old" names the previous + # one. This feels like a lot of hooey for something so trivial. + + # First, make a crispy fresh new directory to put the stuff in. + idx = 0 + while True: + date_suffix = time.strftime("%Y-%m-%d") + dname = dst + ".%s.%d" % (date_suffix, idx) + try: + os.mkdir(dname) + break + except OSError: + exc = sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + idx = idx + 1 + + # Put the stuff in our new directory. + for src_file in os.listdir(src): + packager.sysassert(["cp", "-rv", src + "/" + src_file, dname]) + + # Make a symlink to the new directory; the symlink will be renamed + # to dst shortly. + idx = 0 + while True: + tmpnam = dst + ".TMP.%d" % idx + try: + os.symlink(dname, tmpnam) + break + except OSError: # as exc: # Python >2.5 + exc = sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + idx = idx + 1 + + # Make a symlink to the old directory; this symlink will be + # renamed shortly, too. + oldnam = None + if os.path.exists(dst): + idx = 0 + while True: + oldnam = dst + ".old.%d" % idx + try: + os.symlink(os.readlink(dst), oldnam) + break + except OSError: # as exc: # Python >2.5 + exc = sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + + os.rename(tmpnam, dst) + if oldnam: + os.rename(oldnam, dst + ".old") + + +if __name__ == "__main__": + main() |