Checkpoint changes to packaging scripts.

3 files changed, 149 insertions, 46 deletions

diff --git a/buildscripts/makealldists.py b/buildscripts/makealldists.py
index e96289210a3..cbf6a0c9b9f 100644
--- a/buildscripts/makealldists.py
+++ b/buildscripts/makealldists.py
@@ -188,7 +188,9 @@ def __main__():
              ("ubuntu", "9.4"),
              ("ubuntu", "8.10"),
              ("debian", "5.0"),
-             ("centos", "5.4"),)
+             ("centos", "5.4"),
+             ("fedora", "11"),
+             ("fedora", "12"))
     arches = ("x86", "x86_64")
 #    mongos = branches.split(',')
     # Run a makedist for each distro/version/architecture tuple above.
@@ -199,6 +201,9 @@ def __main__():
     procs = []
     count = 0
     for ((distro, distro_version), arch, spec) in gen([dists, arches, [branches]]):
+        # FIXME: now x86 fedoras on RackSpace circa 04/10.
+        if distro == "fedora" and arch == "x86":
+            continue
         count+=1
         opts = makedistopts
         if distro in ["debian", "ubuntu"]:
@@ -258,9 +263,9 @@ def __main__():
         if r != 0:
             raise Exception("mergerepositories.py exited %d" % r)
         print repodir
-    pushrepo(repodir)
-    shutil.rmtree(outputroot)
-    shutil.rmtree(repodir)
+#    pushrepo(repodir)
+#    shutil.rmtree(outputroot)
+#    shutil.rmtree(repodir)
 
     return 0
 
diff --git a/buildscripts/makedist.py b/buildscripts/makedist.py
index 422e7f890ce..5481d6b3373 100644
--- a/buildscripts/makedist.py
+++ b/buildscripts/makedist.py
@@ -50,6 +50,7 @@ from libcloud.types import Provider
 from libcloud.providers import get_driver
 from libcloud.drivers.ec2 import EC2NodeDriver, NodeImage
 from libcloud.base import Node, NodeImage, NodeSize, NodeState
+from libcloud.ssh import ParamikoSSHClient
 EC2 = get_driver(Provider.EC2)
 EC2Driver=EC2NodeDriver(settings.id, settings.key)
 
@@ -226,6 +227,9 @@ class EC2Instance (object):
     def __exit__(self, type, value, traceback):
         self.stop()
 
+    def setup(self):
+        pass
+    
     def getHostname(self): 
         return self.node.public_ip[0] # FIXME private_ip?
 #        return self.ec2_internal_hostname if self.use_internal_name else self.ec2_external_hostname
@@ -242,7 +246,7 @@ class SshConnectionConfigurator (BaseConfigurator):
                                  (("ubuntu", "9.4", "*"), "root"),
                                  (("ubuntu", "8.10", "*"), "root"),
                                  (("ubuntu", "8.4", "*"), "ubuntu"),
-                                 (("fedora", "8", "*"), "root"),
+                                 (("fedora", "*", "*"), "root"),
                                  (("centos", "*", "*"), "root"))),
                                ]
 
@@ -416,10 +420,10 @@ rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/{distro_arch}/epel-release
 yum -y install {pkg_prereq_str}
 """
     rpm_build_commands="""
-for d in BUILD  BUILDROOT  RPMS  SOURCES  SPECS  SRPMS; do mkdir -p /usr/src/redhat/$d; done
-cp -v "{pkg_name}{pkg_name_suffix}-{pkg_version}/rpm/mongo.spec" /usr/src/redhat/SPECS/{pkg_name}{pkg_name_suffix}.spec
-tar -cpzf /usr/src/redhat/SOURCES/"{pkg_name}{pkg_name_suffix}-{pkg_version}".tar.gz "{pkg_name}{pkg_name_suffix}-{pkg_version}"
-rpmbuild -ba /usr/src/redhat/SPECS/{pkg_name}{pkg_name_suffix}.spec
+for d in BUILD  BUILDROOT  RPMS  SOURCES  SPECS  SRPMS; do mkdir -p {rpmbuild_dir}/$d; done
+cp -v "{pkg_name}{pkg_name_suffix}-{pkg_version}/rpm/mongo.spec" {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec
+tar -cpzf {rpmbuild_dir}/SOURCES/"{pkg_name}{pkg_name_suffix}-{pkg_version}".tar.gz "{pkg_name}{pkg_name_suffix}-{pkg_version}"
+rpmbuild -ba --target={distro_arch} {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec
 # FIXME: should install the rpms, check if mongod is running.
 """ 
     # FIXME: this is clean, but adds 40 minutes or so to the build process.
@@ -495,8 +499,8 @@ git clone git://github.com/mongodb/mongo.git
         self.configuration += [("pkg_product_dir",
                                 ((("ubuntu", "*", "*"), self.deb_productdir),
                                  (("debian", "*", "*"), self.deb_productdir),
-                                 (("fedora", "*", "*"), self.rpm_productdir),
-                                 (("centos", "*", "*"), self.rpm_productdir))),
+                                 (("fedora", "*", "*"), "~/rpmbuild/RPMS"),
+                                 (("centos", "*", "*"), "/usr/src/redhat/RPMS"))),
                                ("pkg_prereqs",
                                 ((("ubuntu", "9.4", "*"),
                                   self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
@@ -510,7 +514,7 @@ git clone git://github.com/mongodb/mongo.git
                                   self.unversioned_deb_boost_prereqs + self.old_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
                                  (("debian", "5.0", "*"),
                                   self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
-                                 (("fedora", "8", "*"),
+                                 (("fedora", "*", "*"),
                                   self.fedora_prereqs),
                                  (("centos", "5.4", "*"),
                                   self.centos_preqres))),
@@ -573,6 +577,11 @@ git clone git://github.com/mongodb/mongo.git
                                ("pkg_name_conflicts",
                                 ((("*", "*", "*"),  ["", "-stable", "-unstable", "-snapshot", "-oldstable"]),
                                  )),
+                               ("rpmbuild_dir",
+                                ((("fedora", "*", "*"), "~/rpmbuild"),
+                                 (("centos", "*", "*"), "/usr/src/redhat"),
+                                 (("*", "*","*"), ''),
+                                 )),
                                 ]
 
 
@@ -668,8 +677,8 @@ class ScriptFile(object):
                              # suppose this works elsewhere
                              pkg_name_conflicts = ", ".join([self.pkg_name+conflict for conflict in pkg_name_conflicts]),
                              mongo_arch=self.mongo_arch,
-                             mongo_pub_version=mongo_pub_version
-                             )
+                             mongo_pub_version=mongo_pub_version,
+                             rpmbuild_dir=self.configurator.default('rpmbuild_dir'))
             script+='rm -rf mongo'
         return script
 
@@ -693,7 +702,95 @@ class ScriptFile(object):
 class Configurator(SshConnectionConfigurator, EC2InstanceConfigurator, ScriptFileConfigurator, BaseHostConfigurator):
     def __init__(self, **kwargs):
         super(Configurator, self).__init__(**kwargs)
+
+class rackspaceNode(object):
+    def __init__(self, configurator, **kwargs):
+        driver = get_driver(Provider.RACKSPACE)
+        self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
+        name=self.imgname+'-'+str(os.getpid())
+        images=filter(lambda x: (x.name.find(self.imgname) > -1), self.conn.list_images())
+        sizes=self.conn.list_sizes()
+        sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
+        node = None
+        if len(images) != 1:
+            raise Exception("too many images with \"%s\" in the name" % self.imgname)
+        image = images[0]
+        self.node = self.conn.create_node(image=image, name=name, size=sizes[0])
+        print self.node
+        self.password = self.node.extra['password']
+
+    def initwait(self):
+        while 1:
+            n=None
+            # EC2 sometimes takes a while to report a node.
+            for i in range(6):
+                nodes = [n for n in self.conn.list_nodes() if (n.id==self.node.id)]
+                if len(nodes)>0:
+                    n=nodes[0]
+                    break
+                else:
+                    time.sleep(10)
+            if not n:
+                raise Exception("couldn't find node with id %s" % self.node.id)
+            if n.state == NodeState.PENDING: 
+                time.sleep(10)
+            else:
+                self.node = n
+                break
+        print "ok"
+        # Now wait for the node's sshd to be accepting connections.
+        print "waiting for ssh"
+        sshwait = True
+        if sshwait == False:
+            return
+        while sshwait:
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            try:
+                try:
+                    s.connect((self.node.public_ip[0], 22))
+                    sshwait = False
+                    print "connected on port 22 (ssh)"
+                    time.sleep(15) # arbitrary timeout, in case the
+                    # remote sshd is slow.
+                except socket.error, err:
+                    pass
+            finally:
+                s.close()
+                time.sleep(3) # arbitrary timeout
+        print "ok"
+        
+    def getHostname(self):
+        return self.node.public_ip[0]
+
+    def setup(self):
+        self.putSshKey()
+
+    def putSshKey(self):
+        keyfile=settings.makedist['ssh_keyfile']
+        ssh = ParamikoSSHClient(hostname = self.node.public_ip[0], password = self.password)
+        ssh.connect()
+        print "putting ssh public key"
+        ssh.put(".ssh/authorized_keys", contents=open(keyfile+'.pub').read(), chmod=0600)
+        print "ok"
+        
+    def __enter__(self):
+        return self
+        
+    def __exit__(self, arg0, arg1, arg2):
+        if arg1:
+            print ">>>>>>>>"
+            print arg1
+            print arg2
+            print "<<<<<<<<"
+        print "shutting down node %s" % self.node
+        self.node.destroy()
+
+class fedoraNode(rackspaceNode):
+    def __init__(self, configurator, **kwargs):
+        self.imgname='Fedora '+kwargs['distro_version']
+        super(fedoraNode, self).__init__(configurator, **kwargs)
 
+
 def parse_mongo_version_spec (spec):
     foo = spec.split(":")
     mongo_version = foo[0] # this can be a commit id, a
@@ -759,9 +856,12 @@ def main():
             print """# Going to run the following on a fresh AMI:"""
             print f.read()
             time.sleep(10)
-        with EC2Instance(configurator, **kwargs) as ec2:
-            ec2.initwait()
-            kwargs["ssh_host"] = ec2.getHostname()
+        # FIXME: it's not the best to have two different pathways for
+        # the different hosting services, but...
+        with EC2Instance(configurator, **kwargs) if kwargs['distro_name'] != 'fedora' else fedoraNode(configurator, **kwargs) as host:
+            host.initwait()
+            host.setup()
+            kwargs["ssh_host"] = host.getHostname()
             with SshConnection(configurator, **kwargs) as ssh:
                 ssh.runRemotely(["uname -a; ls /"])
                 ssh.runRemotely(["mkdir", "pkg"])
diff --git a/buildscripts/mergerepositories.py b/buildscripts/mergerepositories.py
index 3ae5cac7b97..bc50d080173 100644
--- a/buildscripts/mergerepositories.py
+++ b/buildscripts/mergerepositories.py
@@ -39,18 +39,13 @@ def tryEC2():
             node.destroy()
 
 
-# I don't think libcloud's Nodes implement __enter__ and __exit__, and
-# I like the with statement for ensuring that we don't leak nodes when
-# we don't have to.
-class ec2node(object):
+class node(object):
     def initWait(self):
-        print "waiting for node to spin up"
-        # Wait for EC2 to tell us the node is running.
         while 1:
             n=None
             # EC2 sometimes takes a while to report a node.
             for i in range(6):
-                nodes = [n for n in EC2Driver.list_nodes() if (n.id==self.node.id)]
+                nodes = [n for n in self.list_nodes() if (n.id==self.node.id)]
                 if len(nodes)>0:
                     n=nodes[0]
                     break
@@ -77,7 +72,7 @@ class ec2node(object):
                     sshwait = False
                     print "connected on port 22 (ssh)"
                     time.sleep(15) # arbitrary timeout, in case the
-                                  # remote sshd is slow.
+                    # remote sshd is slow.
                 except socket.error, err:
                     pass
             finally:
@@ -85,7 +80,6 @@ class ec2node(object):
                 time.sleep(3) # arbitrary timeout
         print "ok"
 
-
     def __enter__(self):
         return self
         
@@ -93,6 +87,13 @@ class ec2node(object):
         print "shutting down node %s" % self.node
         self.node.destroy()
 
+# I don't think libcloud's Nodes implement __enter__ and __exit__, and
+# I like the with statement for ensuring that we don't leak nodes when
+# we don't have to.
+class ec2node(node):
+    def list_nodes(self):
+        return EC2Driver.list_nodes()
+
 class ubuntuNode(ec2node):
     def __init__(self):
         image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
@@ -107,28 +108,25 @@ class centosNode(ec2node):
 
         self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
 
+class rackspaceNode(node):
+    def list_nodes(self):
+        self.conn.list_nodes()
 
-        
-def tryRackSpace():
-    driver=get_driver(Provider.RACKSPACE)
-    conn = driver(settings.rackspace_account, settings.rackspace_api_key)
-    string='Fedora 11'
-    images=filter(lambda x: (x.name.find(string) > -1), conn.list_images())
-    sizes=conn.list_sizes()
-    sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
-    node = None
-    if len(images) != 1:
-        raise "too many images with \"%s\" in the name" % string
-    try:
+class fedora11Node(rackspaceNode):
+    def __init__(self):
+        driver = get_driver(Provider.RACKSPACE)
+        self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
+        string='Fedora 11'
+        images=filter(lambda x: (x.name.find(string) > -1), self.conn.list_images())
+        sizes=self.conn.list_sizes()
+        sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
+        node = None
+        if len(images) != 1:
+            raise "too many images with \"%s\" in the name" % string
         image = images[0]
-        node = conn.create_node(image=image, name=string, size=sizes[0])
-        print node
-        print node.extras['password']
-        while node.state == NodeState.PENDING: 
-            time.sleep(10)
-    finally:
-        if node:
-            node.destroy()
+        self.node = self.conn.create_node(image=image, name=string, size=sizes[0])
+        print self.node
+        self.password = self.node.extra['password']
 
 class Err(Exception):
     pass