1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
#!/usr/bin/python
from __future__ import with_statement
from libcloud.types import Provider
from libcloud.providers import get_driver
from libcloud.drivers.ec2 import EC2NodeDriver, NodeImage
from libcloud.base import Node, NodeImage, NodeSize, NodeState
# libcloud's SSH client seems to be one of those pointless wrappers
# that (at the moment) both doesn't add anything to the thing it wraps
# (Paramiko) and also fails to expose the underlying thing's features.
# What's wrong with people?
#from libcloud.ssh import SSHClient
import time
import sys
import settings
import subprocess
import os
import socket
EC2 = get_driver(Provider.EC2)
EC2Driver=EC2NodeDriver(settings.id, settings.key)
def tryEC2():
image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
node = None
try:
node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, keyname="kp1", securitygroup=['default', 'dist-slave', 'buildbot-slave'])
print node
print node.id
while node.state == NodeState.PENDING:
time.sleep(3)
finally:
if node:
node.destroy()
class node(object):
def initWait(self):
while 1:
n=None
# EC2 sometimes takes a while to report a node.
for i in range(6):
nodes = [n for n in self.list_nodes() if (n.id==self.node.id)]
if len(nodes)>0:
n=nodes[0]
break
else:
time.sleep(10)
if not n:
raise Exception("couldn't find node with id %s" % self.node.id)
if n.state == NodeState.PENDING:
time.sleep(10)
else:
self.node = n
break
print "ok"
# Now wait for the node's sshd to be accepting connections.
print "waiting for ssh"
sshwait = True
if sshwait == False:
return
while sshwait:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
try:
s.connect((self.node.public_ip[0], 22))
sshwait = False
print "connected on port 22 (ssh)"
time.sleep(15) # arbitrary timeout, in case the
# remote sshd is slow.
except socket.error, err:
pass
finally:
s.close()
time.sleep(3) # arbitrary timeout
print "ok"
def __enter__(self):
return self
def __exit__(self, arg0, arg1, arg2):
print "shutting down node %s" % self.node
self.node.destroy()
# I don't think libcloud's Nodes implement __enter__ and __exit__, and
# I like the with statement for ensuring that we don't leak nodes when
# we don't have to.
class ec2node(node):
def list_nodes(self):
return EC2Driver.list_nodes()
class ubuntuNode(ec2node):
def __init__(self):
image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
class centosNode(ec2node):
def __init__(self):
image=NodeImage('ami-ccb35ea5', 'ubuntu 10.4', EC2)
size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
class rackspaceNode(node):
def list_nodes(self):
self.conn.list_nodes()
class fedora11Node(rackspaceNode):
def __init__(self):
driver = get_driver(Provider.RACKSPACE)
self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
string='Fedora 11'
images=filter(lambda x: (x.name.find(string) > -1), self.conn.list_images())
sizes=self.conn.list_sizes()
sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
node = None
if len(images) != 1:
raise "too many images with \"%s\" in the name" % string
image = images[0]
self.node = self.conn.create_node(image=image, name=string, size=sizes[0])
print self.node
self.password = self.node.extra['password']
class Err(Exception):
pass
def merge_yum_repo(dir, outdir):
dirtail=dir.rstrip('\/').split('/')[-1]
keyfile=settings.makedist['ssh_keyfile']
makeyumrepo="""find . -name RPMS | while read dir; do (cd $dir/.. && createrepo .); done"""
with centosNode() as centos:
centos.initWait()
print centos.node
run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "root@"+centos.node.public_ip[0]+":"])
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "root@"+centos.node.public_ip[0], "cd ./" + dirtail + " && " + makeyumrepo])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no", "-i", keyfile, "-r", "root@"+centos.node.public_ip[0]+":./"+dirtail +'/*', outdir])
def merge_apt_repo(dir, outdir):
dirtail=dir.rstrip('\/').split('/')[-1]
gpgdir=settings.makedist['gpg_homedir']
keyfile=settings.makedist['ssh_keyfile']
makeaptrepo="""for x in debian ubuntu; do (cd $x; for d in `find . -name *.deb | sed 's|^./||; s|/[^/]*$||' | sort -u`; do dpkg-scanpackages $d > $d/Packages; gzip -9c $d/Packages > $d/Packages.gz; done) ; done"""
makereleaseprologue="""Origin: 10gen
Label: 10gen
Suite: 10gen
Codename: VVVVVV
Version: VVVVVV
Architectures: i386 amd64
Components: 10gen
Description: 10gen packages"""
makeaptrelease="""find . -maxdepth 3 -mindepth 3 | while read d; do ( cd $d && (echo '%s' | sed s/VVVVVV/$(basename $(pwd))/; apt-ftparchive release .) > /tmp/Release && mv /tmp/Release . && gpg -r `gpg --list-keys | grep uid | awk '{print $(NF)}'` --no-secmem-warning --no-tty -abs --output Release.gpg Release ); done""" % makereleaseprologue
with ubuntuNode() as ubuntu:
ubuntu.initWait()
print ubuntu.node
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sudo", "sh", "-c", "\"export DEBIAN_FRONTEND=noninteractive; apt-get update; apt-get -y install debhelper\""])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "ubuntu@"+ubuntu.node.public_ip[0]+":"])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", gpgdir, "ubuntu@"+ubuntu.node.public_ip[0]+":.gnupg"])
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sh", "-c", "\"ls -lR ./" + dirtail + "\""])
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "cd ./"+dirtail + " && " + makeaptrepo])
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "cd ./"+dirtail + " && " + makeaptrelease])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no", "-i", keyfile, "-r", "ubuntu@"+ubuntu.node.public_ip[0]+":./"+dirtail +'/*', outdir])
def run_for_effect(argv):
print " ".join(argv)
r=subprocess.Popen(argv).wait()
if r!=0:
raise Err("subprocess %s exited %d" % (argv, r))
if __name__ == "__main__":
(flavor, dir, outdir) = sys.argv[-3:]
if flavor == "deb":
merge_apt_repo(dir, outdir)
elif flavor == "rpm":
merge_yum_repo(dir, outdir)
else:
Err("unknown pkg flavor %s" % flavor)
# TODO: yum repositories
#main()
#tryRackSpace()
|
