1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
|
#!/usr/bin/python
# Copyright (C) 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
'''A Morph deployment write extension for deploying to Docker hosts'''
# bgtunnel: From https://github.com/jmagnusson/bgtunnel
# (using Paramiko would be better, but it's not in Baserock yet. Its
# demos/forward.py demonstrates what we need).
import bgtunnel
# From https://github.com/dotcloud/docker-py
import docker
import cliapp
import contextlib
import gzip
import logging
import os
import sys
import tarfile
import urlparse
import morphlib.writeexts
class DockerWriteExtension(morphlib.writeexts.WriteExtension):
'''Create a Docker image or container from a Morph deployment.
THIS IS A PROTOTYPE!!!
This extension assumes you are accessing a remote Docker service. It uses
the Docker remote API. The Docker remote API cannot be exposed over TCP
directly in a secure way, so instead you should set the Docker daemon on
the server listening on a local-only TCP socket. Morph will then use SSH
to forward this port securely while the write extention runs.
Docker doesn't listen on a TCP socket by default. Run the Docker service
as follows (2375 is an arbitrary number):
docker -d -H='tcp://127.0.0.1:2375"
The location command line argument is a network location that should be
accessible over SSH, followed by the name of the image to be created.
docker+ssh://[USER@]HOST:PORT/IMAGE
Where
* USER is your username on the remote Docker server
* HOST is the hostname of the remote Docker server
* PORT is the local-only TCP port on which Docker is listening (2375 in
the above example)
* IMAGE is the name of the image to create.
Docker image names commonly containly follow the form 'owner/name'. If
a VERSION_LABEL setting is supplied, this will be used to tag the image.
See also:
http://blog.tutum.co/2013/11/21/remote-and-secure-use-of-docker-api-with-python-part-1/
http://coreos.com/docs/launching-containers/building/customizing-docker/
'''
def process_args(self, args):
if len(args) != 2:
raise cliapp.AppException('Wrong number of command line args')
temp_root, location = args
if not location.startswith('docker+ssh://'):
raise cliapp.AppException(
'Sorry, currently this extension only supports remote '
'access to Docker using a port forwarded by SSH.')
user, host, port, image_name = self.parse_location(location)
# FIXME: is the tunnel cleaned up? do we need a 'with' ?
self.status(msg='Connecting to Docker service at %s:%s' % (host, port))
docker_client = self.create_docker_client_with_remote_ssh_tunnel(
user, host, port)
tar_read_fd, tar_write_fd = os.pipe()
tar_read_fileobj = os.fdopen(tar_read_fd, 'r')
print docker_client.info()
# FIXME: hack! The docker-py library should let us put in a fileobj and
# have it handle buffering automatically ... I.E. this hack should be
# sent upstream as an improvement, instead. Still, it's kind of cool
# that Python enables such easy workarounds!
#
# For reference, the Ruby client can already do this:
# https://github.com/swipely/docker-api/blob/master/lib/docker/image.rb
import_url = docker_client._url('/images/create')
logging.debug('Open tar write FD')
tar_write_fileobj = os.fdopen(tar_write_fd, 'w')
logging.debug('Create tar thread')
tar_bytes = 0
import threading
tar_thread = threading.Thread(
target=self.write_system_as_tar, args=[temp_root, tar_write_fileobj])
tar_thread.start()
print tar_thread
print tar_thread.is_alive()
import select
def batch_fileobj(fileobj, batch_size):
'''Split an fileobj up into batches of 'batch_size' items.'''
i = 0
# This is hard, we need to signal end ...
while True:
data = fileobj.read(batch_size)
yield data
print "End of fileobj"
yield []
print "Yielded None, called again ..."
#logging.debug('Prepare request...')
#import_request_prepped = docker_client.prepare_request(import_request)
logging.debug('Send request...')
# FOR SOME REASON THIS SEEMS NEVER TO EXIT!
#docker_client.send(import_request_prepped)
docker_client.post(
import_url,
data=batch_fileobj(tar_read_fileobj, 10240),
params={
'fromSrc': '-',
'repo': image_name
},
headers = {
'Content-Type': 'application/tar',
'Transfer-Encoding': 'chunked',
}
)
print "OK! Wow, that surely didn't actually work."
###
autostart = self.get_environment_boolean('AUTOSTART')
self.status(
msg='Docker image %(image_name)s has been created',
image_name=image_name)
def parse_location(self, location):
'''Parse the location argument to get relevant data.'''
x = urlparse.urlparse(location)
return x.username, x.hostname, x.port, x.path[1:]
def create_docker_client_with_remote_ssh_tunnel(self, user, host, port):
# Taken from: https://gist.github.com/hamiltont/10950399
# Local bind port is randomly chosen.
#tunnel = bgtunnel.open(
# ssh_user=user,
# ssh_address=host,
# host_port=port,
# expect_hello=False,
# # Block for 5 seconds then fail
# timeout=5,
# # Work around 'TypeError: must be encoded string without NULL
# # bytes, not str'. This is due to a bug in bgtunnel where it
# # fetches the SSH path as a Unicode string, then passes it to
# # shlex.split() which returns something horrid. Should be
# # fixed and the patch sent upstream.
# ssh_path=str('/usr/bin/ssh'))
#docker_client = docker.Client(
# base_url='http://127.0.0.1:%d' % tunnel.bind_port)
# FIXME: bgtunnel seems broken, do this manually for now in a separate
# terminal:
# /usr/bin/ssh -T -p 22 -L 127.0.0.1:57714:127.0.0.1:2375 sam@droopy
docker_client = docker.Client(
base_url='http://127.0.0.1:57714')
return docker_client
def write_system_as_tar(self, fs_root, fileobj):
# Using tarfile.TarFile.gzopen() and passing compresslevel=1
# seems to result in compresslevel=9 anyway. That's completely
# unusable on ARM CPUs so it's important to force
# compresslevel=1 or something low.
logging.debug('Writing system as a tar!')
#gzip_stream = gzip.GzipFile(
# mode='wb',
# compresslevel=1,
# fileobj=fileobj)
tar_stream = tarfile.TarFile.gzopen(
name='docker.write-temp',
mode='w',
compresslevel=1,
fileobj=fileobj)#gzip_stream)
logging.debug("Creating tar of rootfs")
tar_stream.add(fs_root, recursive=True)
tar_stream.close()
logging.debug('Tar complete')
tar_finished = True
DockerWriteExtension().run()
|
