#!/usr/bin/env python # # Copyright 2015 Codethink Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. '''openstack/tester This script tests an image on openstack. ''' import os import pipes import socket import time import uuid import yaml import paramiko import cliapp from novaclient import client class TimeoutError(cliapp.AppException): """Error to be raised when a connection waits too long""" def __init__(self, msg): super(TimeoutError, self).__init__(msg) def delete_image(hostname): # TODO: Do all this stuff properly with python novaclient # Remove an image from the openstack tenancy print "Deleting %s test disc image" % (hostname) try: cliapp.runcmd(['nova', 'image-delete', hostname]) except cliapp.AppException as e: # TODO: Stop assuming that image-delete failed because it was # already removed print "- Failed" pass def delete_instance_and_image(hostname): # TODO: Do all this stuff properly with python novaclient # Stop and remove VM, and its image print "Deleting %s test instance" % (hostname) try: cliapp.runcmd(['nova', 'delete', hostname]) except cliapp.AppException as e: # TODO: Stop assuming that delete failed because the instance # wasn't running print "- Failed" pass # Sleep for a bit, or nova silently fails to delete the image # TODO: Test whether the image has been deleted in a retry loop. time.sleep(20) delete_image(hostname) class DeployedSystemInstance(object): def __init__(self, deployment, ip_addr, hostname): self.deployment = deployment self.ip_address = ip_addr self.hostname = hostname self.connection = None def runcmd(self, cmd): stdin, stdout, stderr = self.connection.exec_command(cmd) while not stdout.channel.exit_status_ready(): if stdout.channel.recv_ready(): rl, wl, xl = select.select([stdout.channel], [], [], 0.0) if len(rl) > 0: # Print data from stdout print stdout.channel.recv(1024), print stderr.read(), def _wait_for_dhcp(self, timeout): '''Block until given hostname resolves successfully. Raises TimeoutError if the hostname has not appeared in 'timeout' seconds. ''' start_time = time.time() while True: try: socket.gethostbyname(self.ip_address) return except socket.gaierror: pass if time.time() > start_time + timeout: raise TimeoutError("Host %s did not appear after %i seconds" % (self.ip_address, timeout)) time.sleep(0.5) def _wait_for_ssh(self, timeout): """Wait until the deployed VM is responding via SSH""" print('Attempting to connect to test instance via SSH:') start_time = time.time() while True: try: # TODO: Stop assuming we ssh into test instances as root ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(self.ip_address, username='root') self.connection = ssh print('SSH connection established.') return except Exception as e: print(e) # TODO: Stop assuming the ssh part of the command is what failed if time.time() > start_time + timeout: raise TimeoutError("%s sshd did not start after %i seconds" % (self.ip_address, timeout)) time.sleep(0.5) def _wait_for_cloud_init(self, timeout): """Wait until cloud init has resized the disc""" start_time = time.time() while True: try: out = self.runcmd(['sh', '-c', 'test -e "$1" && echo exists || echo does not exist', '-', '/root/cloud-init-finished']) except: import traceback traceback.print_exc() raise if out.strip() == 'exists': return if time.time() > start_time + timeout: raise TimeoutError("Disc size not increased after %i seconds" % (timeout)) time.sleep(3) def wait_until_online(self, timeout=180): self._wait_for_dhcp(timeout) self._wait_for_ssh(timeout) #self._wait_for_cloud_init(timeout) print "Test system %s ready to run tests." % (self.hostname) def delete(self): if self.connection != None: self.connection.close() delete_instance_and_image(self.hostname) class Deployment(object): def __init__(self, net_id, image_file, flavour): self.net_id = net_id self.image_file = image_file self.flavour = flavour def deploy(self): hostname = str(uuid.uuid4()) # Deploy the image to openstack # TODO: Ensure this is cleaned up when something raises an exception print('Uploading image to openstack host') args = ['glance', 'image-create', '--progress', '--name', hostname, '--disk-format', 'raw', '--container-format', 'bare', '--file', self.image_file] output = cliapp.runcmd(args) print(output) # Get a novaclient object nc = client.Client(2, os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'], os.environ['OS_TENANT_NAME'], os.environ['OS_AUTH_URL']) # Boot an instance from the image # TODO: Ensure this is cleaned up when something raises an exception # TODO: use python-novaclient print('Booting an instance from the image') args = ['nova', 'boot', '--flavor', self.flavour, '--image', hostname, #'--user-data', '/usr/lib/mason/os-init-script', '--nic', "net-id=%s" % (self.net_id), hostname] output = cliapp.runcmd(args) # Print nova boot output, with adminPass line removed output_lines = output.split('\n') for line in output_lines: if line.find('adminPass') != -1: password_line = line output_lines.remove(password_line) output = '\n'.join(output_lines) print output # Sleep for a bit, or nova explodes when trying to assign IP address time.sleep(20) # Assign a floating IP address print('Requesting a floating IP address') for _ in xrange(5): ip_list = nc.floating_ips.list() free_ip = None for ip in ip_list: if ip.instance_id == None: free_ip = ip break else: free_ip = nc.floating_ips.create( nc.floating_ip_pools.list()[0].name) if free_ip != None: instance = nc.servers.find(name=hostname) # TODO: switch back to cli tool, as python # approach gave error. instance.add_floating_ip(free_ip) ip_addr = free_ip.ip break else: delete_instance_and_image(hostname) raise cliapp.AppException('Could not get a floating IP') # Print the IP address print "IP address for instance %s: %s" % (hostname, ip_addr) return DeployedSystemInstance(self, ip_addr, hostname) class ReleaseApp(cliapp.Application): """Cliapp application which handles automatic builds and tests""" def add_settings(self): """Add the command line options needed""" group_main = 'Program Options' self.settings.string(['net-id'], 'Openstack network ID', default=None, group=group_main) self.settings.string(['image-file'], 'Path to system image to test', default=None, group=group_main) self.settings.string(['flavour'], 'Name of flavour to use for instance', default=None, group=group_main) def run_tests(self, instance, tests): instance.wait_until_online() for test in tests: with open(test, 'r') as stream: data = yaml.load(stream) if data == None: continue if 'name' not in data: print('Bad test: no name') continue if 'commands' not in data: print('Bad test: no commands') continue print('Running test: ' + data['name']) for cmd in data['commands']: print('$ ' + cmd) instance.runcmd(cmd) def deploy_and_test_systems(self, tests): """Run the deployments and tests""" deployment = Deployment(self.settings['net-id'], self.settings['image-file'], self.settings['flavour']) instance = deployment.deploy() try: self.run_tests(instance, tests) finally: instance.delete() def process_args(self, args): """Process the command line args and kick off the builds/tests""" for setting in ('net-id', 'image-file', 'flavour'): self.settings.require(setting) if len(args) == 0: print('Warning: No tests specified.') self.deploy_and_test_systems(args) if __name__ == '__main__': ReleaseApp().run()