SERVER-963 Split and externalize Linux packages

This includes splitting rpm and .deb packages into sub-packages and
externalizing the spec and control files to be maintained as static
files.

1 files changed, 21 insertions, 391 deletions

diff --git a/buildscripts/packager.py b/buildscripts/packager.py
index dcddf0ab91b..e3ff008d883 100644
--- a/buildscripts/packager.py
+++ b/buildscripts/packager.py
@@ -28,7 +28,7 @@
 
 import errno
 import getopt
-import httplib
+import httplib2
 import os
 import re
 import stat
@@ -281,20 +281,17 @@ def httpget(url, filename):
     u=urlparse.urlparse(url)
     assert(u.scheme=='http')
     try:
-        conn = httplib.HTTPConnection(u.hostname)
-        conn.request("GET", u.path)
+        h = httplib2.Http(cache = os.environ["HOME"] + "/.cache")
+        resp, content = h.request(url, "GET")
         t=filename+'.TMP'
-        res = conn.getresponse()
-        # FIXME: follow redirects
-        if res.status==200:
+        if resp.status==200:
             f = open(t, 'w')
             try:
-                f.write(res.read())
+                f.write(content)
             finally:
                 f.close()
-                
         else:
-            raise Exception("HTTP error %d" % res.status)
+            raise Exception("HTTP error %d" % resp.status)
         os.rename(t, filename)
     finally:
         if conn:
@@ -367,11 +364,19 @@ def make_deb(distro, arch, spec, srcdir):
         os.unlink(sdir+"debian/init.d")
     else:
         raise Exception("unknown debianoid flavor: not sysvinit or upstart?")
-    # Rewrite the control and rules files
-    write_debian_control_file(sdir+"debian/control", spec)
-    write_debian_rules_file(sdir+"debian/rules", spec)
     write_debian_changelog(sdir+"debian/changelog", spec, srcdir)
     distro_arch=distro.archname(arch)
+    sysassert(["cp", "-v", srcdir+"debian/control", sdir+"debian/"])
+    sysassert(["cp", "-v", srcdir+"debian/rules", sdir+"debian/"])
+    sysassert(["cp", "-v", srcdir+"debian/rules", sdir+"debian/"])
+    # old non-server-package postinst will be hanging around for old versions
+    #
+    os.unlink(sdir+"debian/postinst")
+
+    # copy our postinst files
+    #
+    sysassert(["sh", "-c", "cp -v \"%sdebian/\"*.postinst \"%sdebian/\""%(srcdir, sdir)])
+
     # Do the packaging.
     oldcwd=os.getcwd()
     try:
@@ -383,7 +388,9 @@ def make_deb(distro, arch, spec, srcdir):
     ensure_dir(r)
     # FIXME: see if shutil.copyfile or something can do this without
     # much pain.
-    sysassert(["cp", "-v", sdir+"../%s%s_%s%s_%s.deb"%(distro.pkgbase(), suffix, spec.pversion(distro), "-"+spec.param("revision") if spec.param("revision") else"", distro_arch), r])
+    # sysassert(["cp", "-v", sdir+"../%s%s_%s%s_%s.deb"%(distro.pkgbase(), suffix, spec.pversion(distro), "-"+spec.param("revision") if spec.param("revision") else"", distro_arch), r])
+    # sysassert(["cp", "-v", sdir+"../*.deb", r])
+    sysassert(["sh", "-c", "cp -v \"%s/../\"*.deb \"%s\""%(sdir, r)])
     return r
 
 def make_deb_repo(repo):
@@ -542,181 +549,11 @@ def write_debian_changelog(path, spec, srcdir):
         f.write(s)
     finally:
         f.close()
-
-def write_debian_control_file(path, spec):
-    s="""Source: @@PACKAGE_BASENAME@@
-Section: devel
-Priority: optional
-Maintainer: Richard Kreuter <richard@10gen.com>
-Build-Depends: 
-Standards-Version: 3.8.0
-Homepage: http://www.mongodb.org
-
-Package: @@PACKAGE_BASENAME@@
-Conflicts: @@PACKAGE_CONFLICTS@@
-Architecture: any
-Depends: libc6 (>= 2.3.2), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.1.1)
-Description: An object/document-oriented database
- MongoDB is a high-performance, open source, schema-free 
- document-oriented  data store that's easy to deploy, manage
- and use. It's network accessible, written in C++ and offers
- the following features :
- .
-    * Collection oriented storage - easy storage of object-
-      style data
-    * Full index support, including on inner objects
-    * Query profiling
-    * Replication and fail-over support
-    * Efficient storage of binary data including large 
-      objects (e.g. videos)
-    * Auto-sharding for cloud-level scalability (Q209)
- .
- High performance, scalability, and reasonable depth of
- functionality are the goals for the project.
-"""
-    s=re.sub("@@PACKAGE_BASENAME@@", "mongodb%s" % spec.suffix(), s)
-    conflict_suffixes=["", "-stable", "-unstable", "-nightly", "-10gen", "-10gen-unstable"]
-    conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()]
-    s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongodb"+suffix for suffix in conflict_suffixes]), s)
-    f=open(path, 'w')
-    try:
-        f.write(s)
-    finally:
-        f.close()
-
-def write_debian_rules_file(path, spec):
-    # Note debian/rules is a makefile, so for visual disambiguation we
-    # make all tabs here \t.
-    s="""#!/usr/bin/make -f
-# -*- makefile -*-
-# Sample debian/rules that uses debhelper.
-# This file was originally written by Joey Hess and Craig Small.
-# As a special exception, when this file is copied by dh-make into a
-# dh-make output file, you may use that output file without restriction.
-# This special exception was added by Craig Small in version 0.37 of dh-make.
-
-# Uncomment this to turn on verbose mode.
-#export DH_VERBOSE=1
-
-
-configure: configure-stamp
-configure-stamp:
-\tdh_testdir
-        # Add here commands to configure the package.
-
-\ttouch configure-stamp
-
-
-build: build-stamp
-
-build-stamp: configure-stamp  
-\tdh_testdir
-
-        # Add here commands to compile the package.
-# THE FOLLOWING LINE IS INTENTIONALLY COMMENTED. 
-\t# scons 
-        #docbook-to-man debian/mongodb.sgml > mongodb.1
-\tls debian/*.1 > debian/@@PACKAGE_NAME@@.manpages
-
-\ttouch $@
-
-clean: 
-\tdh_testdir
-\tdh_testroot
-\trm -f build-stamp configure-stamp
-
-\t# FIXME: scons freaks out at the presence of target files
-\t# under debian/mongodb.
-\t#scons -c
-\trm -rf $(CURDIR)/debian/@@PACKAGE_NAME@@
-\trm -f config.log
-\trm -f mongo
-\trm -f mongod
-\trm -f mongoimportjson
-\trm -f mongoexport
-\trm -f mongorestore
-\trm -f mongodump
-\trm -f mongofiles
-\trm -f .sconsign.dblite
-\trm -f libmongoclient.a
-\trm -rf client/*.o
-\trm -rf tools/*.o
-\trm -rf shell/*.o
-\trm -rf .sconf_temp
-\trm -f buildscripts/*.pyc 
-\trm -f *.pyc
-\trm -f buildinfo.cpp
-\tdh_clean debian/files
-
-install: build
-\tdh_testdir
-\tdh_testroot
-\tdh_prep
-\tdh_installdirs
-
-# THE FOLLOWING LINE IS INTENTIONALLY COMMENTED.
-\t# scons --prefix=$(CURDIR)/debian/mongodb/usr install
-\tcp -v $(CURDIR)/@@BINARYDIR@@/usr/bin/* $(CURDIR)/debian/@@PACKAGE_NAME@@/usr/bin
-\tmkdir -p $(CURDIR)/debian/@@PACKAGE_NAME@@/etc
-\tcp $(CURDIR)/debian/mongodb.conf $(CURDIR)/debian/@@PACKAGE_NAME@@/etc/mongodb.conf 
-
-\tmkdir -p $(CURDIR)/debian/@@PACKAGE_NAME@@/usr/share/lintian/overrides/
-\tinstall -m 644 $(CURDIR)/debian/lintian-overrides \
-\t\t$(CURDIR)/debian/@@PACKAGE_NAME@@/usr/share/lintian/overrides/@@PACKAGE_NAME@@
-
-# Build architecture-independent files here.
-binary-indep: build install
-# We have nothing to do by default.
-
-# Build architecture-dependent files here.
-binary-arch: build install
-\tdh_testdir
-\tdh_testroot
-\tdh_installchangelogs 
-\tdh_installdocs
-\tdh_installexamples
-#\tdh_install
-#\tdh_installmenu
-#\tdh_installdebconf\t
-#\tdh_installlogrotate
-#\tdh_installemacsen
-#\tdh_installpam
-#\tdh_installmime
-\tdh_installinit --name=@@PACKAGE_BASENAME@@
-#\tdh_installinfo
-\tdh_installman
-\tdh_link
-# Appears to be broken on Ubuntu 11.10...?
-#\tdh_strip
-\tdh_compress
-\tdh_fixperms
-\tdh_installdeb
-\tdh_shlibdeps
-\tdh_gencontrol
-\tdh_md5sums
-\tdh_builddeb
-
-binary: binary-indep binary-arch
-.PHONY: build clean binary-indep binary-arch binary install configure
-"""
-    s=re.sub("@@PACKAGE_NAME@@", "mongodb%s" % spec.suffix(), s)
-    s=re.sub("@@PACKAGE_BASENAME@@", "mongodb", s)
-    s=re.sub("@@BINARYDIR@@", BINARYDIR, s)
-    f=open(path, 'w')
-    try:
-        f.write(s)
-    finally:
-        f.close()
-    # FIXME: some versions of debianoids seem to
-    # need the rules file to be 755?
-    os.chmod(path, stat.S_IXUSR|stat.S_IWUSR|stat.S_IRUSR|stat.S_IXGRP|stat.S_IRGRP|stat.S_IXOTH|stat.S_IWOTH)
-
 def make_rpm(distro, arch, spec, srcdir):
     # Create the specfile.
     suffix=spec.suffix()
     sdir=setupdir(distro, arch, spec)
-    specfile=sdir+"rpm/mongo%s.spec" % suffix
-    write_rpm_spec_file(specfile, spec)
+    specfile=srcdir+"rpm/mongo%s.spec" % suffix
     topdir=ensure_dir(os.getcwd()+'/rpmbuild/')
     for subdir in ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]:
         ensure_dir("%s/%s/" % (topdir, subdir))
@@ -793,212 +630,5 @@ def write_rpm_macros_file(path, topdir):
     finally:
         f.close()
 
-def write_rpm_spec_file(path, spec):
-    s="""Name: @@PACKAGE_BASENAME@@
-Conflicts: @@PACKAGE_CONFLICTS@@
-Obsoletes: @@PACKAGE_OBSOLETES@@
-Version: @@PACKAGE_VERSION@@
-Release: mongodb_@@PACKAGE_REVISION@@%{?dist}
-Summary: mongo client shell and tools
-License: AGPL 3.0
-URL: http://www.mongodb.org
-Group: Applications/Databases
-
-Source0: %{name}-%{version}.tar.gz
-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
-
-%description
-Mongo (from "huMONGOus") is a schema-free document-oriented database.
-It features dynamic profileable queries, full indexing, replication
-and fail-over support, efficient storage of large binary data objects,
-and auto-sharding.
-
-This package provides the mongo shell, import/export tools, and other
-client utilities.
-
-%package server
-Summary: mongo server, sharding server, and support scripts
-Group: Applications/Databases
-Requires: @@PACKAGE_BASENAME@@
-
-%description server
-Mongo (from "huMONGOus") is a schema-free document-oriented database.
-
-This package provides the mongo server software, mongo sharding server
-softwware, default configuration files, and init.d scripts.
-
-%package devel
-Summary: Headers and libraries for mongo development. 
-Group: Applications/Databases
-
-%description devel
-Mongo (from "huMONGOus") is a schema-free document-oriented database.
-
-This package provides the mongo static library and header files needed
-to develop mongo client software.
-
-%prep
-%setup
-
-%build
-#scons --prefix=$RPM_BUILD_ROOT/usr all
-# XXX really should have shared library here
-
-%install
-#scons --prefix=$RPM_BUILD_ROOT/usr install
-mkdir -p $RPM_BUILD_ROOT/usr
-cp -rv @@BINARYDIR@@/usr/bin $RPM_BUILD_ROOT/usr
-mkdir -p $RPM_BUILD_ROOT/usr/share/man/man1
-cp debian/*.1 $RPM_BUILD_ROOT/usr/share/man/man1/
-# FIXME: remove this rm when mongosniff is back in the package
-rm -v $RPM_BUILD_ROOT/usr/share/man/man1/mongosniff.1*
-mkdir -p $RPM_BUILD_ROOT/etc/rc.d/init.d
-cp -v rpm/init.d-mongod $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod
-chmod a+x $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod
-mkdir -p $RPM_BUILD_ROOT/etc
-cp -v rpm/mongod.conf $RPM_BUILD_ROOT/etc/mongod.conf
-mkdir -p $RPM_BUILD_ROOT/etc/sysconfig
-cp -v rpm/mongod.sysconfig $RPM_BUILD_ROOT/etc/sysconfig/mongod
-mkdir -p $RPM_BUILD_ROOT/var/lib/mongo
-mkdir -p $RPM_BUILD_ROOT/var/log/mongo
-touch $RPM_BUILD_ROOT/var/log/mongo/mongod.log
-
-%clean
-#scons -c
-rm -rf $RPM_BUILD_ROOT
-
-%pre server
-if ! /usr/bin/id -g mongod &>/dev/null; then
-    /usr/sbin/groupadd -r mongod
-fi
-if ! /usr/bin/id mongod &>/dev/null; then
-    /usr/sbin/useradd -M -r -g mongod -d /var/lib/mongo -s /bin/false \
-	-c mongod mongod > /dev/null 2>&1
-fi
-
-%post server
-if test $1 = 1
-then
-  /sbin/chkconfig --add mongod
-fi
-
-%preun server
-if test $1 = 0
-then
-  /sbin/chkconfig --del mongod
-fi
-
-%postun server
-if test $1 -ge 1
-then
-  /sbin/service mongod condrestart >/dev/null 2>&1 || :
-fi
-
-%files
-%defattr(-,root,root,-)
-#%doc README GNU-AGPL-3.0.txt
-
-%{_bindir}/bsondump
-%{_bindir}/mongo
-%{_bindir}/mongodump
-%{_bindir}/mongoexport
-#@@VERSION!=2.1.0@@%{_bindir}/mongofiles
-%{_bindir}/mongoimport
-#@@VERSION>=2.1.0@@%{_bindir}/mongooplog
-#@@VERSION>=2.1.0@@%{_bindir}/mongoperf
-%{_bindir}/mongorestore
-#@@VERSION>1.9@@%{_bindir}/mongotop
-%{_bindir}/mongostat
-# FIXME: uncomment when mongosniff is back in the package
-#%{_bindir}/mongosniff
-
-# FIXME: uncomment this when there's a stable release whose source
-# tree contains a bsondump man page.
-#@@VERSION>1.9@@%{_mandir}/man1/bsondump.1*
-%{_mandir}/man1/mongo.1*
-%{_mandir}/man1/mongodump.1*
-%{_mandir}/man1/mongoexport.1*
-%{_mandir}/man1/mongofiles.1*
-%{_mandir}/man1/mongoimport.1*
-%{_mandir}/man1/mongorestore.1*
-%{_mandir}/man1/mongostat.1*
-# FIXME: uncomment when mongosniff is back in the package
-#%{_mandir}/man1/mongosniff.1*
-#@@VERSION>=2.4.0@@%{_mandir}/man1/mongotop.1*
-#@@VERSION>=2.4.0@@%{_mandir}/man1/mongoperf.1*
-#@@VERSION>=2.4.0@@%{_mandir}/man1/mongooplog.1*
-
-%files server
-%defattr(-,root,root,-)
-%config(noreplace) /etc/mongod.conf
-%{_bindir}/mongod
-%{_bindir}/mongos
-%{_mandir}/man1/mongod.1*
-%{_mandir}/man1/mongos.1*
-/etc/rc.d/init.d/mongod
-/etc/sysconfig/mongod
-#/etc/rc.d/init.d/mongos
-%attr(0755,mongod,mongod) %dir /var/lib/mongo
-%attr(0755,mongod,mongod) %dir /var/log/mongo
-%attr(0640,mongod,mongod) %config(noreplace) %verify(not md5 size mtime) /var/log/mongo/mongod.log
-
-%changelog
-* Thu Jan 28 2010 Richard M Kreuter <richard@10gen.com>
-- Minor fixes.
-
-* Sat Oct 24 2009 Joe Miklojcik <jmiklojcik@shopwiki.com> - 
-- Wrote mongo.spec.
-"""
-    suffix=spec.suffix()
-    s=re.sub("@@PACKAGE_BASENAME@@", "mongo%s" % suffix, s)
-    s=re.sub("@@PACKAGE_VERSION@@", spec.pversion(Distro("redhat")), s)
-    # FIXME, maybe: the RPM guide says that Release numbers ought to
-    # be integers starting at 1, but we use "mongodb_1{%dist}",
-    # whatever the hell that means.
-    s=re.sub("@@PACKAGE_REVISION@@", str(int(spec.param("revision"))+1) if spec.param("revision") else "1", s)
-    s=re.sub("@@BINARYDIR@@", BINARYDIR, s)
-    conflict_suffixes=["", "-10gen", "-10gen-unstable"]
-    conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()]
-    s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongo"+_ for _ in conflict_suffixes]), s)
-    if suffix.endswith("-10gen"):
-        s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-stable", s)
-        s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-stable", s)
-    elif suffix == "-10gen-unstable":
-        s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-unstable", s)
-        s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-unstable", s)
-    else:
-        raise Exception("BUG: unknown suffix %s" % suffix)
-
-    lines=[]
-    for line in s.split("\n"):
-        m = re.search("@@VERSION(>|>=|!=)(\d.*)@@(.*)", line)
-        if m:
-          op = m.group(1)
-          ver = m.group(2)
-          fn = m.group(3)
-          if op == '>':
-            if spec.version_better_than(ver):
-              lines.append(fn)
-          elif op == '>=':
-            if spec.version() == ver or spec.version_better_than(ver):
-              lines.append(fn)
-          elif op == '!=':
-            if spec.version() != ver:
-              lines.append(fn)
-          else:
-            # Since we're inventing our own template system for RPM
-            # specfiles here, we oughtn't use template syntax we don't
-            # support.
-            raise Exception("BUG: probable bug in packager script: %s, %s, %s" % (m.group(1), m.group(2), m.group(3)))
-        else:
-            lines.append(line)
-    s="\n".join(lines)
-
-    f=open(path, 'w')
-    try:
-        f.write(s)
-    finally:
-        f.close()
-
 if __name__ == "__main__":
     main(sys.argv)