diff options
author | Max Illfelder <illfelder@google.com> | 2016-08-19 15:10:30 -0700 |
---|---|---|
committer | Max Illfelder <illfelder@google.com> | 2016-08-19 15:10:30 -0700 |
commit | 71d11d878bfefc4007798c1d96df74d70395daf3 (patch) | |
tree | 1cfa257ebbe761c6e2ce2bd8a4053d5a8218d21c | |
parent | 2fe6d5ec45a1fc2598f3272dd1ae33cbd2c06e8a (diff) | |
parent | 04bb5dbd5e284770c3aa7ddefacd6a489cebfef8 (diff) | |
download | google-compute-image-packages-71d11d878bfefc4007798c1d96df74d70395daf3.tar.gz |
Merge branch 'development'20160819
46 files changed, 608 insertions, 200 deletions
diff --git a/.travis.yml b/.travis.yml index 97954b6..ed5d273 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ sudo: true python: - 2.6 - 2.7 -- 3.2 - 3.3 - 3.4 - 3.5 @@ -268,7 +268,7 @@ We build the following packages for the Linux guest environment. for the `google-compute-engine` Python package. Installing this package will configure the `google-compute-engine` package to run on system startup on sysvinit, upstart, or systemd init systems. -* `google-configs` is a package containing non-Python scripts and guest +* `google-config` is a package containing non-Python scripts and guest configuration. * Sets up udev rules and sysctl rules. * Configures the SysLog output that gets sent to serial port output. @@ -336,6 +336,17 @@ yum install -y google-compute-engine google-compute-engine-init google-config ## Troubleshooting +**An old CentOS 6 image fails to install the packages with an error on SCL** + +CentOS 6 images prior to `v20160526` may fail to install the package with +the error: +``` +http://mirror.centos.org/centos/6/SCL/x86_64/repodata/repomd.xml: [Errno 14] PYCURL ERROR 22 - "The requested URL returned error: 404 Not Found" +``` + +Remove the stale repository file: +`sudo rm -f /etc/yum.repos.d/CentOS-SCL.repo` + **Using boto with virtualenv** Specific to running `boto` inside of a Python diff --git a/google_compute_engine/accounts/accounts_daemon.py b/google_compute_engine/accounts/accounts_daemon.py index 7ef82a0..5b9779d 100755 --- a/google_compute_engine/accounts/accounts_daemon.py +++ b/google_compute_engine/accounts/accounts_daemon.py @@ -204,8 +204,9 @@ class AccountsDaemon(object): def main(): parser = optparse.OptionParser() - parser.add_option('-d', '--debug', action='store_true', dest='debug', - help='print debug output to the console.') + parser.add_option( + '-d', '--debug', action='store_true', dest='debug', + help='print debug output to the console.') (options, _) = parser.parse_args() instance_config = config_manager.ConfigManager() if instance_config.GetOptionBool('Daemons', 'accounts_daemon'): diff --git a/google_compute_engine/accounts/tests/accounts_daemon_test.py b/google_compute_engine/accounts/tests/accounts_daemon_test.py index 775ae10..3fd830d 100644 --- a/google_compute_engine/accounts/tests/accounts_daemon_test.py +++ b/google_compute_engine/accounts/tests/accounts_daemon_test.py @@ -38,8 +38,8 @@ class AccountsDaemonTest(unittest.TestCase): @mock.patch('google_compute_engine.accounts.accounts_daemon.metadata_watcher') @mock.patch('google_compute_engine.accounts.accounts_daemon.logger') @mock.patch('google_compute_engine.accounts.accounts_daemon.file_utils') - def testAccountsDaemon(self, mock_lock, mock_logger, mock_watcher, - mock_utils): + def testAccountsDaemon( + self, mock_lock, mock_logger, mock_watcher, mock_utils): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance mocks = mock.Mock() @@ -68,8 +68,8 @@ class AccountsDaemonTest(unittest.TestCase): @mock.patch('google_compute_engine.accounts.accounts_daemon.metadata_watcher') @mock.patch('google_compute_engine.accounts.accounts_daemon.logger') @mock.patch('google_compute_engine.accounts.accounts_daemon.file_utils') - def testAccountsDaemonError(self, mock_lock, mock_logger, mock_watcher, - mock_utils): + def testAccountsDaemonError( + self, mock_lock, mock_logger, mock_watcher, mock_utils): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance mocks = mock.Mock() diff --git a/google_compute_engine/accounts/tests/accounts_utils_test.py b/google_compute_engine/accounts/tests/accounts_utils_test.py index 20ea62d..c5670a6 100644 --- a/google_compute_engine/accounts/tests/accounts_utils_test.py +++ b/google_compute_engine/accounts/tests/accounts_utils_test.py @@ -99,8 +99,8 @@ class AccountsUtilsTest(unittest.TestCase): @mock.patch('google_compute_engine.accounts.accounts_utils.file_utils.SetPermissions') @mock.patch('google_compute_engine.accounts.accounts_utils.subprocess.check_call') @mock.patch('google_compute_engine.accounts.accounts_utils.os.path.exists') - def testCreateSudoersGroupSkip(self, mock_exists, mock_call, - mock_permissions): + def testCreateSudoersGroupSkip( + self, mock_exists, mock_call, mock_permissions): mock_open = mock.mock_open() mocks = mock.Mock() mocks.attach_mock(mock_exists, 'exists') @@ -125,8 +125,8 @@ class AccountsUtilsTest(unittest.TestCase): @mock.patch('google_compute_engine.accounts.accounts_utils.file_utils.SetPermissions') @mock.patch('google_compute_engine.accounts.accounts_utils.subprocess.check_call') @mock.patch('google_compute_engine.accounts.accounts_utils.os.path.exists') - def testCreateSudoersGroupError(self, mock_exists, mock_call, - mock_permissions): + def testCreateSudoersGroupError( + self, mock_exists, mock_call, mock_permissions): mocks = mock.Mock() mocks.attach_mock(mock_exists, 'exists') mocks.attach_mock(mock_call, 'call') @@ -227,8 +227,8 @@ class AccountsUtilsTest(unittest.TestCase): @mock.patch('google_compute_engine.accounts.accounts_utils.shutil.copy') @mock.patch('google_compute_engine.accounts.accounts_utils.tempfile.NamedTemporaryFile') @mock.patch('google_compute_engine.accounts.accounts_utils.os.path.exists') - def testUpdateAuthorizedKeys(self, mock_exists, mock_tempfile, mock_copy, - mock_permissions): + def testUpdateAuthorizedKeys( + self, mock_exists, mock_tempfile, mock_copy, mock_permissions): mock_open = mock.mock_open() user = 'user' ssh_keys = ['Google key 1', 'Google key 2'] @@ -287,8 +287,8 @@ class AccountsUtilsTest(unittest.TestCase): @mock.patch('google_compute_engine.accounts.accounts_utils.shutil.copy') @mock.patch('google_compute_engine.accounts.accounts_utils.tempfile.NamedTemporaryFile') @mock.patch('google_compute_engine.accounts.accounts_utils.os.path.exists') - def testUpdateAuthorizedKeysNoKeys(self, mock_exists, mock_tempfile, - mock_copy, mock_permissions): + def testUpdateAuthorizedKeysNoKeys( + self, mock_exists, mock_tempfile, mock_copy, mock_permissions): user = 'user' ssh_keys = ['Google key 1'] temp_dest = '/tmp/dest' @@ -418,8 +418,9 @@ class AccountsUtilsTest(unittest.TestCase): @mock.patch('google_compute_engine.accounts.accounts_utils.file_utils.SetPermissions') @mock.patch('google_compute_engine.accounts.accounts_utils.shutil.copy') @mock.patch('google_compute_engine.accounts.accounts_utils.tempfile.NamedTemporaryFile') - def testSetConfiguredUsers(self, mock_tempfile, mock_copy, mock_permissions, - mock_exists, mock_makedirs): + def testSetConfiguredUsers( + self, mock_tempfile, mock_copy, mock_permissions, mock_exists, + mock_makedirs): temp_dest = '/temp/dest' users = ['a', 'b', 'c'] mock_tempfile.return_value = mock_tempfile diff --git a/google_compute_engine/boto/boto_config.py b/google_compute_engine/boto/boto_config.py index fe68159..80bf8f8 100644 --- a/google_compute_engine/boto/boto_config.py +++ b/google_compute_engine/boto/boto_config.py @@ -42,13 +42,14 @@ class BotoConfig(object): 'not edit this file directly. If you need to add items to this file, ' 'create or edit %s instead and then re-run the script.') - def __init__(self, project_id=None): + def __init__(self, project_id=None, debug=False): """Constructor. Args: project_id: string, the project ID to use in the config file. + debug: bool, True if debug output should write to the console. """ - self.logger = logger.Logger(name='boto-setup') + self.logger = logger.Logger(name='boto-setup', debug=debug) self.watcher = metadata_watcher.MetadataWatcher(logger=self.logger) self._CreateConfig(project_id) diff --git a/google_compute_engine/boto/tests/boto_config_test.py b/google_compute_engine/boto/tests/boto_config_test.py index 1ee7e52..3ba4168 100644 --- a/google_compute_engine/boto/tests/boto_config_test.py +++ b/google_compute_engine/boto/tests/boto_config_test.py @@ -44,9 +44,9 @@ class BotoConfigTest(unittest.TestCase): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance - boto_config.BotoConfig(self.project_id) + boto_config.BotoConfig(self.project_id, debug=True) expected_calls = [ - mock.call.logger.Logger(name=mock.ANY), + mock.call.logger.Logger(name=mock.ANY, debug=True), mock.call.watcher.MetadataWatcher(logger=mock_logger_instance), mock.call.config( config_file='template', config_header='/tmp/test.py template'), diff --git a/google_compute_engine/clock_skew/clock_skew_daemon.py b/google_compute_engine/clock_skew/clock_skew_daemon.py index d228795..6c5c2e9 100755 --- a/google_compute_engine/clock_skew/clock_skew_daemon.py +++ b/google_compute_engine/clock_skew/clock_skew_daemon.py @@ -69,8 +69,9 @@ class ClockSkewDaemon(object): def main(): parser = optparse.OptionParser() - parser.add_option('-d', '--debug', action='store_true', dest='debug', - help='print debug output to the console.') + parser.add_option( + '-d', '--debug', action='store_true', dest='debug', + help='print debug output to the console.') (options, _) = parser.parse_args() instance_config = config_manager.ConfigManager() if instance_config.GetOptionBool('Daemons', 'clock_skew_daemon'): diff --git a/google_compute_engine/instance_setup/instance_config.py b/google_compute_engine/instance_setup/instance_config.py index ac1ee6c..9d37875 100644 --- a/google_compute_engine/instance_setup/instance_config.py +++ b/google_compute_engine/instance_setup/instance_config.py @@ -65,6 +65,10 @@ class InstanceConfig(config_manager.ConfigManager): 'startup': 'true', 'shutdown': 'true', }, + 'NetworkInterfaces': { + 'setup': 'true', + 'dhcp_binary': 'dhclient', + }, } def __init__(self): @@ -84,24 +88,21 @@ class InstanceConfig(config_manager.ConfigManager): # Use the settings in an instance config file if one exists. If a config # file does not already exist, try to use the distro provided defaults. If # no file exists, use the default configuration settings. - if os.path.exists(self.instance_config): - instance_config = self.instance_config - elif os.path.exists(self.instance_config_distro): - instance_config = self.instance_config_distro - else: - instance_config = None - - if instance_config: - config = parser.SafeConfigParser() - config.read(instance_config) - defaults = dict((s, dict(config.items(s))) for s in config.sections()) - else: - defaults = self.instance_config_options + config_files = [self.instance_config, self.instance_config_distro] + config_defaults = [] + for config_file in config_files: + if os.path.exists(config_file): + config = parser.SafeConfigParser() + config.read(config_file) + config_defaults.append( + dict((s, dict(config.items(s))) for s in config.sections())) + config_defaults.append(self.instance_config_options) - for section, options in sorted(defaults.items()): - for option, value in sorted(options.items()): - super(InstanceConfig, self).SetOption( - section, option, value, overwrite=False) + for defaults in config_defaults: + for section, options in sorted(defaults.items()): + for option, value in sorted(options.items()): + super(InstanceConfig, self).SetOption( + section, option, value, overwrite=False) def WriteConfig(self): """Write the config values to the instance defaults file.""" diff --git a/google_compute_engine/instance_setup/instance_setup.py b/google_compute_engine/instance_setup/instance_setup.py index 82a95f8..612b46e 100755 --- a/google_compute_engine/instance_setup/instance_setup.py +++ b/google_compute_engine/instance_setup/instance_setup.py @@ -40,9 +40,10 @@ class InstanceSetup(object): Args: debug: bool, True if debug output should write to the console. """ + self.debug = debug facility = logging.handlers.SysLogHandler.LOG_DAEMON self.logger = logger.Logger( - name='instance-setup', debug=debug, facility=facility) + name='instance-setup', debug=self.debug, facility=facility) self.watcher = metadata_watcher.MetadataWatcher(logger=self.logger) self.metadata_dict = None self.instance_config = instance_config.InstanceConfig() @@ -168,15 +169,16 @@ class InstanceSetup(object): """Set the boto config so GSUtil works with provisioned service accounts.""" project_id = self._GetNumericProjectId() try: - boto_config.BotoConfig(project_id) + boto_config.BotoConfig(project_id, debug=self.debug) except (IOError, OSError) as e: self.logger.warning(str(e)) def main(): parser = optparse.OptionParser() - parser.add_option('-d', '--debug', action='store_true', dest='debug', - help='print debug output to the console.') + parser.add_option( + '-d', '--debug', action='store_true', dest='debug', + help='print debug output to the console.') (options, _) = parser.parse_args() InstanceSetup(debug=bool(options.debug)) diff --git a/google_compute_engine/instance_setup/tests/instance_config_test.py b/google_compute_engine/instance_setup/tests/instance_config_test.py index 342d733..8f7f595 100644 --- a/google_compute_engine/instance_setup/tests/instance_config_test.py +++ b/google_compute_engine/instance_setup/tests/instance_config_test.py @@ -86,47 +86,24 @@ class InstanceConfigTest(unittest.TestCase): instance_config.InstanceConfig() expected_calls = [ - mock.call.init( - config_file='template', config_header='/tmp/test.py template'), + mock.call.init(config_file='template', config_header='/tmp/test.py template'), mock.call.exists('config'), mock.call.parser.SafeConfigParser(), mock.call.parser.SafeConfigParser().read('config'), mock.call.parser.SafeConfigParser().sections(), - mock.call.set('a', 'key: a', 'value: a', overwrite=False), - mock.call.set('b', 'key: b', 'value: b', overwrite=False), - ] - self.assertEqual(mocks.mock_calls, expected_calls) - - @mock.patch('google_compute_engine.instance_setup.instance_config.os.path.exists') - @mock.patch('google_compute_engine.instance_setup.instance_config.parser') - @mock.patch('google_compute_engine.instance_setup.instance_config.config_manager.ConfigManager.SetOption') - @mock.patch('google_compute_engine.instance_setup.instance_config.config_manager.ConfigManager.__init__') - def testInstanceConfigDistroExists(self, mock_init, mock_set, mock_parser, - mock_exists): - mock_config = mock.create_autospec(instance_config.parser.SafeConfigParser) - mock_config.read = mock.Mock() - mock_config.sections = mock.Mock() - mock_config.sections.return_value = ['a', 'b'] - mock_config.items = lambda key: {'key: %s' % key: 'value: %s' % key} - mock_parser.SafeConfigParser.return_value = mock_config - mocks = mock.Mock() - mocks.attach_mock(mock_init, 'init') - mocks.attach_mock(mock_set, 'set') - mocks.attach_mock(mock_parser, 'parser') - mocks.attach_mock(mock_exists, 'exists') - mock_exists.side_effect = [False, True] - - instance_config.InstanceConfig() - expected_calls = [ - mock.call.init( - config_file='template', config_header='/tmp/test.py template'), - mock.call.exists('config'), mock.call.exists('distro'), mock.call.parser.SafeConfigParser(), mock.call.parser.SafeConfigParser().read('distro'), mock.call.parser.SafeConfigParser().sections(), mock.call.set('a', 'key: a', 'value: a', overwrite=False), mock.call.set('b', 'key: b', 'value: b', overwrite=False), + mock.call.set('a', 'key: a', 'value: a', overwrite=False), + mock.call.set('b', 'key: b', 'value: b', overwrite=False), + mock.call.set('first', 'a', 'false', overwrite=False), + mock.call.set('second', 'b', 'true', overwrite=False), + mock.call.set('third', 'c', '1', overwrite=False), + mock.call.set('third', 'd', '2', overwrite=False), + mock.call.set('third', 'e', '3', overwrite=False) ] self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/instance_setup/tests/instance_setup_test.py b/google_compute_engine/instance_setup/tests/instance_setup_test.py index a6eb159..ecc1de9 100644 --- a/google_compute_engine/instance_setup/tests/instance_setup_test.py +++ b/google_compute_engine/instance_setup/tests/instance_setup_test.py @@ -28,6 +28,7 @@ class InstanceSetupTest(unittest.TestCase): self.mock_instance_config = mock.Mock() self.mock_logger = mock.Mock() self.mock_setup = mock.create_autospec(instance_setup.InstanceSetup) + self.mock_setup.debug = False self.mock_setup.instance_config = self.mock_instance_config self.mock_setup.logger = self.mock_logger @@ -154,8 +155,8 @@ class InstanceSetupTest(unittest.TestCase): @mock.patch('google_compute_engine.instance_setup.instance_setup.shutil.move') @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess.check_call') @mock.patch('google_compute_engine.instance_setup.instance_setup.tempfile.NamedTemporaryFile') - def testGenerateSshKey(self, mock_tempfile, mock_call, mock_move, - mock_permissions): + def testGenerateSshKey( + self, mock_tempfile, mock_call, mock_move, mock_permissions): mocks = mock.Mock() mocks.attach_mock(mock_tempfile, 'tempfile') mocks.attach_mock(mock_call, 'call') @@ -306,7 +307,7 @@ class InstanceSetupTest(unittest.TestCase): mock_project_id.return_value = '123' self.mock_setup._GetNumericProjectId = mock_project_id instance_setup.InstanceSetup._SetupBotoConfig(self.mock_setup) - mock_boto.assert_called_once_with('123') + mock_boto.assert_called_once_with('123', debug=False) @mock.patch('google_compute_engine.instance_setup.instance_setup.boto_config.BotoConfig') def testSetupBotoConfigLocked(self, mock_boto): diff --git a/google_compute_engine/ip_forwarding/ip_forwarding_daemon.py b/google_compute_engine/ip_forwarding/ip_forwarding_daemon.py index b56ffa9..04c0e09 100755 --- a/google_compute_engine/ip_forwarding/ip_forwarding_daemon.py +++ b/google_compute_engine/ip_forwarding/ip_forwarding_daemon.py @@ -32,6 +32,7 @@ from google_compute_engine import config_manager from google_compute_engine import file_utils from google_compute_engine import logger from google_compute_engine import metadata_watcher +from google_compute_engine import network_utils from google_compute_engine.ip_forwarding import ip_forwarding_utils @@ -54,7 +55,8 @@ class IpForwardingDaemon(object): self.logger = logger.Logger( name='google-ip-forwarding', debug=debug, facility=facility) self.watcher = metadata_watcher.MetadataWatcher(logger=self.logger) - self.utils = ip_forwarding_utils.IpForwardingUtils( + self.network_utils = network_utils.NetworkUtils(logger=self.logger) + self.ip_forwarding_utils = ip_forwarding_utils.IpForwardingUtils( logger=self.logger, proto_id=proto_id) try: with file_utils.LockFile(LOCKFILE): @@ -65,7 +67,8 @@ class IpForwardingDaemon(object): except (IOError, OSError) as e: self.logger.warning(str(e)) - def _LogForwardedIpChanges(self, configured, desired, to_add, to_remove, interface): + def _LogForwardedIpChanges( + self, configured, desired, to_add, to_remove, interface): """Log the planned IP address changes. Args: @@ -90,7 +93,7 @@ class IpForwardingDaemon(object): interface: string, the output device to use. """ for address in forwarded_ips: - self.utils.AddForwardedIp(address, interface) + self.ip_forwarding_utils.AddForwardedIp(address, interface) def _RemoveForwardedIps(self, forwarded_ips, interface): """Remove the forwarded IP addresses from the network interface. @@ -100,7 +103,7 @@ class IpForwardingDaemon(object): interface: string, the output device to use. """ for address in forwarded_ips: - self.utils.RemoveForwardedIp(address, interface) + self.ip_forwarding_utils.RemoveForwardedIp(address, interface) def _HandleForwardedIps(self, forwarded_ips, interface): """Handle changes to the forwarded IPs on a network interface. @@ -109,8 +112,8 @@ class IpForwardingDaemon(object): forwarded_ips: list, the forwarded IP address strings desired. interface: string, the output device to configure. """ - desired = self.utils.ParseForwardedIps(forwarded_ips) - configured = self.utils.GetForwardedIps(interface) + desired = self.ip_forwarding_utils.ParseForwardedIps(forwarded_ips) + configured = self.ip_forwarding_utils.GetForwardedIps(interface) to_add = sorted(set(desired) - set(configured)) to_remove = sorted(set(configured) - set(desired)) self._LogForwardedIpChanges( @@ -126,7 +129,7 @@ class IpForwardingDaemon(object): """ for network_interface in result: mac_address = network_interface.get('mac') - interface = self.utils.GetNetworkInterface(mac_address) + interface = self.network_utils.GetNetworkInterface(mac_address) if interface: forwarded_ips = network_interface.get('forwardedIps') self._HandleForwardedIps(forwarded_ips, interface) @@ -137,8 +140,9 @@ class IpForwardingDaemon(object): def main(): parser = optparse.OptionParser() - parser.add_option('-d', '--debug', action='store_true', dest='debug', - help='print debug output to the console.') + parser.add_option( + '-d', '--debug', action='store_true', dest='debug', + help='print debug output to the console.') (options, _) = parser.parse_args() instance_config = config_manager.ConfigManager() if instance_config.GetOptionBool('Daemons', 'ip_forwarding_daemon'): diff --git a/google_compute_engine/ip_forwarding/ip_forwarding_utils.py b/google_compute_engine/ip_forwarding/ip_forwarding_utils.py index 913954b..1045a4b 100644 --- a/google_compute_engine/ip_forwarding/ip_forwarding_utils.py +++ b/google_compute_engine/ip_forwarding/ip_forwarding_utils.py @@ -15,7 +15,6 @@ """Utilities for configuring IP address forwarding.""" -import os import re import subprocess @@ -34,24 +33,6 @@ class IpForwardingUtils(object): """ self.logger = logger self.proto_id = proto_id or '66' - self.interfaces = self._CreateInterfaceMap() - - def _CreateInterfaceMap(self): - """Generate a dictionary mapping MAC address to ethernet interfaces. - - Returns: - dict, string MAC addresses mapped to the string network interface name. - """ - interfaces = {} - for interface in os.listdir('/sys/class/net'): - try: - mac_address = open('/sys/class/net/%s/address' % interface).read().strip() - except (IOError, OSError) as e: - message = 'Unable to determine MAC address for %s. %s.' - self.logger.warning(message, interface, str(e)) - else: - interfaces[mac_address] = interface - return interfaces def _CreateRouteOptions(self, **kwargs): """Create a dictionary of parameters to append to the ip route command. @@ -99,17 +80,6 @@ class IpForwardingUtils(object): return stdout return '' - def GetNetworkInterface(self, mac_address): - """Get the name of the network interface associated with a MAC address. - - Args: - mac_address: string, the hardware address of the network interface. - - Returns: - string, the network interface associated with a MAC address or None. - """ - return self.interfaces.get(mac_address) - def ParseForwardedIps(self, forwarded_ips): """Parse and validate forwarded IP addresses. diff --git a/google_compute_engine/ip_forwarding/tests/ip_forwarding_daemon_test.py b/google_compute_engine/ip_forwarding/tests/ip_forwarding_daemon_test.py index 0b06b36..71dee68 100644 --- a/google_compute_engine/ip_forwarding/tests/ip_forwarding_daemon_test.py +++ b/google_compute_engine/ip_forwarding/tests/ip_forwarding_daemon_test.py @@ -25,26 +25,31 @@ class IpForwardingDaemonTest(unittest.TestCase): def setUp(self): self.mock_logger = mock.Mock() self.mock_watcher = mock.Mock() - self.mock_utils = mock.Mock() + self.mock_ip_forwarding_utils = mock.Mock() + self.mock_network_utils = mock.Mock() self.mock_setup = mock.create_autospec( ip_forwarding_daemon.IpForwardingDaemon) self.mock_setup.logger = self.mock_logger self.mock_setup.watcher = self.mock_watcher - self.mock_setup.utils = self.mock_utils + self.mock_setup.ip_forwarding_utils = self.mock_ip_forwarding_utils + self.mock_setup.network_utils = self.mock_network_utils @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.ip_forwarding_utils') + @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.network_utils') @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.metadata_watcher') @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.logger') @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.file_utils') - def testIpForwardingDaemon(self, mock_lock, mock_logger, mock_watcher, - mock_utils): + def testIpForwardingDaemon( + self, mock_lock, mock_logger, mock_watcher, mock_network_utils, + mock_ip_forwarding_utils): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance mocks = mock.Mock() mocks.attach_mock(mock_lock, 'lock') mocks.attach_mock(mock_logger, 'logger') - mocks.attach_mock(mock_utils, 'utils') + mocks.attach_mock(mock_network_utils, 'network') + mocks.attach_mock(mock_ip_forwarding_utils, 'forwarding') mocks.attach_mock(mock_watcher, 'watcher') metadata_key = ip_forwarding_daemon.IpForwardingDaemon.network_interfaces with mock.patch.object( @@ -54,7 +59,8 @@ class IpForwardingDaemonTest(unittest.TestCase): expected_calls = [ mock.call.logger.Logger(name=mock.ANY, debug=True, facility=mock.ANY), mock.call.watcher.MetadataWatcher(logger=mock_logger_instance), - mock.call.utils.IpForwardingUtils( + mock.call.network.NetworkUtils(logger=mock_logger_instance), + mock.call.forwarding.IpForwardingUtils( logger=mock_logger_instance, proto_id='66'), mock.call.lock.LockFile(ip_forwarding_daemon.LOCKFILE), mock.call.lock.LockFile().__enter__(), @@ -66,17 +72,20 @@ class IpForwardingDaemonTest(unittest.TestCase): self.assertEqual(mocks.mock_calls, expected_calls) @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.ip_forwarding_utils') + @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.network_utils') @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.metadata_watcher') @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.logger') @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_daemon.file_utils') - def testIpForwardingDaemonError(self, mock_lock, mock_logger, mock_watcher, - mock_utils): + def testIpForwardingDaemonError( + self, mock_lock, mock_logger, mock_watcher, mock_network_utils, + mock_ip_forwarding_utils): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance mocks = mock.Mock() mocks.attach_mock(mock_lock, 'lock') mocks.attach_mock(mock_logger, 'logger') - mocks.attach_mock(mock_utils, 'utils') + mocks.attach_mock(mock_network_utils, 'network') + mocks.attach_mock(mock_ip_forwarding_utils, 'forwarding') mocks.attach_mock(mock_watcher, 'watcher') mock_lock.LockFile.side_effect = IOError('Test Error') with mock.patch.object( @@ -86,7 +95,8 @@ class IpForwardingDaemonTest(unittest.TestCase): mock.call.logger.Logger( name=mock.ANY, debug=False, facility=mock.ANY), mock.call.watcher.MetadataWatcher(logger=mock_logger_instance), - mock.call.utils.IpForwardingUtils( + mock.call.network.NetworkUtils(logger=mock_logger_instance), + mock.call.forwarding.IpForwardingUtils( logger=mock_logger_instance, proto_id=None), mock.call.lock.LockFile(ip_forwarding_daemon.LOCKFILE), mock.call.logger.Logger().warning('Test Error'), @@ -114,7 +124,7 @@ class IpForwardingDaemonTest(unittest.TestCase): def testAddForwardedIp(self): ip_forwarding_daemon.IpForwardingDaemon._AddForwardedIps( self.mock_setup, [], 'interface') - self.assertEqual(self.mock_utils.mock_calls, []) + self.assertEqual(self.mock_ip_forwarding_utils.mock_calls, []) ip_forwarding_daemon.IpForwardingDaemon._AddForwardedIps( self.mock_setup, ['a', 'b', 'c'], 'interface') @@ -123,12 +133,12 @@ class IpForwardingDaemonTest(unittest.TestCase): mock.call.AddForwardedIp('b', 'interface'), mock.call.AddForwardedIp('c', 'interface'), ] - self.assertEqual(self.mock_utils.mock_calls, expected_calls) + self.assertEqual(self.mock_ip_forwarding_utils.mock_calls, expected_calls) def testRemoveForwardedIp(self): ip_forwarding_daemon.IpForwardingDaemon._RemoveForwardedIps( self.mock_setup, [], 'interface') - self.assertEqual(self.mock_utils.mock_calls, []) + self.assertEqual(self.mock_ip_forwarding_utils.mock_calls, []) ip_forwarding_daemon.IpForwardingDaemon._RemoveForwardedIps( self.mock_setup, ['a', 'b', 'c'], 'interface') @@ -137,16 +147,16 @@ class IpForwardingDaemonTest(unittest.TestCase): mock.call.RemoveForwardedIp('b', 'interface'), mock.call.RemoveForwardedIp('c', 'interface'), ] - self.assertEqual(self.mock_utils.mock_calls, expected_calls) + self.assertEqual(self.mock_ip_forwarding_utils.mock_calls, expected_calls) def testHandleForwardedIps(self): configured = ['c', 'c', 'b', 'b', 'a', 'a'] desired = ['d', 'd', 'c'] mocks = mock.Mock() - mocks.attach_mock(self.mock_utils, 'utils') + mocks.attach_mock(self.mock_ip_forwarding_utils, 'forwarding') mocks.attach_mock(self.mock_setup, 'setup') - self.mock_utils.ParseForwardedIps.return_value = desired - self.mock_utils.GetForwardedIps.return_value = configured + self.mock_ip_forwarding_utils.ParseForwardedIps.return_value = desired + self.mock_ip_forwarding_utils.GetForwardedIps.return_value = configured forwarded_ips = 'forwarded ips' interface = 'interface' expected_add = ['d'] @@ -155,8 +165,8 @@ class IpForwardingDaemonTest(unittest.TestCase): ip_forwarding_daemon.IpForwardingDaemon._HandleForwardedIps( self.mock_setup, forwarded_ips, interface) expected_calls = [ - mock.call.utils.ParseForwardedIps(forwarded_ips), - mock.call.utils.GetForwardedIps(interface), + mock.call.forwarding.ParseForwardedIps(forwarded_ips), + mock.call.forwarding.GetForwardedIps(interface), mock.call.setup._LogForwardedIpChanges( configured, desired, expected_add, expected_remove, interface), mock.call.setup._AddForwardedIps(expected_add, interface), @@ -166,9 +176,9 @@ class IpForwardingDaemonTest(unittest.TestCase): def testHandleNetworkInterfaces(self): mocks = mock.Mock() - mocks.attach_mock(self.mock_utils, 'utils') + mocks.attach_mock(self.mock_network_utils, 'network') mocks.attach_mock(self.mock_setup, 'setup') - self.mock_utils.GetNetworkInterface.side_effect = [ + self.mock_network_utils.GetNetworkInterface.side_effect = [ 'eth0', 'eth1', 'eth2', None] result = [ {'mac': '1', 'forwardedIps': 'a'}, @@ -180,13 +190,13 @@ class IpForwardingDaemonTest(unittest.TestCase): ip_forwarding_daemon.IpForwardingDaemon.HandleNetworkInterfaces( self.mock_setup, result) expected_calls = [ - mock.call.utils.GetNetworkInterface('1'), + mock.call.network.GetNetworkInterface('1'), mock.call.setup._HandleForwardedIps('a', 'eth0'), - mock.call.utils.GetNetworkInterface('2'), + mock.call.network.GetNetworkInterface('2'), mock.call.setup._HandleForwardedIps('b', 'eth1'), - mock.call.utils.GetNetworkInterface('3'), + mock.call.network.GetNetworkInterface('3'), mock.call.setup._HandleForwardedIps(None, 'eth2'), - mock.call.utils.GetNetworkInterface(None), + mock.call.network.GetNetworkInterface(None), mock.call.setup.logger.warning(mock.ANY, None), ] self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/ip_forwarding/tests/ip_forwarding_utils_test.py b/google_compute_engine/ip_forwarding/tests/ip_forwarding_utils_test.py index 7c82dfb..a6e37a8 100644 --- a/google_compute_engine/ip_forwarding/tests/ip_forwarding_utils_test.py +++ b/google_compute_engine/ip_forwarding/tests/ip_forwarding_utils_test.py @@ -25,43 +25,10 @@ class IpForwardingUtilsTest(unittest.TestCase): def setUp(self): self.mock_logger = mock.Mock() - self.interfaces = {'address': 'interface'} self.options = {'hello': 'world'} self.mock_utils = ip_forwarding_utils.IpForwardingUtils(self.mock_logger) - self.mock_utils.interfaces = self.interfaces self.mock_utils.proto_id = 'proto' - @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_utils.os.listdir') - def testCreateInterfaceMap(self, mock_listdir): - mock_open = mock.mock_open() - interface_map = { - '1': 'a', - '2': 'b', - '3': 'c', - } - mock_listdir.return_value = interface_map.values() - - with mock.patch('%s.open' % builtin, mock_open, create=False): - addresses = interface_map.keys() - addresses = ['%s\n' % address for address in addresses] - mock_open().read.side_effect = interface_map.keys() - self.assertEqual(self.mock_utils._CreateInterfaceMap(), interface_map) - - @mock.patch('google_compute_engine.ip_forwarding.ip_forwarding_utils.os.listdir') - def testCreateInterfaceMapError(self, mock_listdir): - mock_open = mock.mock_open() - mock_listdir.return_value = ['a', 'b', 'c'] - - with mock.patch('%s.open' % builtin, mock_open, create=False): - mock_open().read.side_effect = [ - '1', OSError('OSError'), IOError('IOError')] - self.assertEqual(self.mock_utils._CreateInterfaceMap(), {'1': 'a'}) - expected_calls = [ - mock.call.warning(mock.ANY, 'b', 'OSError'), - mock.call.warning(mock.ANY, 'c', 'IOError'), - ] - self.assertEqual(self.mock_logger.mock_calls, expected_calls) - def testCreateRouteOptions(self): # Default options. expected_options = { @@ -131,11 +98,6 @@ class IpForwardingUtilsTest(unittest.TestCase): self.mock_logger.warning.assert_called_once_with( mock.ANY, command, 'Test Error') - def testGetNetworkInterface(self): - self.assertIsNone(self.mock_utils.GetNetworkInterface('invalid')) - self.assertEqual( - self.mock_utils.GetNetworkInterface('address'), 'interface') - def testParseForwardedIps(self): self.assertEqual(self.mock_utils.ParseForwardedIps(None), []) self.assertEqual(self.mock_utils.ParseForwardedIps([]), []) diff --git a/google_compute_engine/metadata_scripts/script_executor.py b/google_compute_engine/metadata_scripts/script_executor.py index d152d60..19aa288 100644 --- a/google_compute_engine/metadata_scripts/script_executor.py +++ b/google_compute_engine/metadata_scripts/script_executor.py @@ -54,7 +54,7 @@ class ScriptExecutor(object): stderr=subprocess.STDOUT, stdout=subprocess.PIPE) while True: for line in iter(process.stdout.readline, b''): - message = line.decode('utf-8').rstrip('\n') + message = line.decode('utf-8', 'replace').rstrip('\n') if message: self.logger.info('%s: %s', metadata_key, message) if process.poll() is not None: diff --git a/google_compute_engine/metadata_scripts/script_manager.py b/google_compute_engine/metadata_scripts/script_manager.py index 511cb6b..2993c36 100755 --- a/google_compute_engine/metadata_scripts/script_manager.py +++ b/google_compute_engine/metadata_scripts/script_manager.py @@ -64,6 +64,7 @@ class ScriptManager(object): self._RunScripts() def _RunScripts(self): + """Retrieve metadata scripts and execute them.""" with _CreateTempDir(self.script_type) as dest_dir: try: self.logger.info('Starting %s scripts.', self.script_type) @@ -76,10 +77,11 @@ class ScriptManager(object): def main(): script_types = ('startup', 'shutdown') parser = optparse.OptionParser() - parser.add_option('-d', '--debug', action='store_true', dest='debug', - help='print debug output to the console.') - parser.add_option('--script-type', dest='script_type', - help='metadata script type.') + parser.add_option( + '-d', '--debug', action='store_true', dest='debug', + help='print debug output to the console.') + parser.add_option( + '--script-type', dest='script_type', help='metadata script type.') (options, _) = parser.parse_args() if options.script_type and options.script_type.lower() in script_types: script_type = options.script_type.lower() diff --git a/google_compute_engine/metadata_scripts/tests/script_manager_test.py b/google_compute_engine/metadata_scripts/tests/script_manager_test.py index acc6ae1..83fa879 100644 --- a/google_compute_engine/metadata_scripts/tests/script_manager_test.py +++ b/google_compute_engine/metadata_scripts/tests/script_manager_test.py @@ -27,8 +27,9 @@ class ScriptManagerTest(unittest.TestCase): @mock.patch('google_compute_engine.metadata_scripts.script_manager.script_executor') @mock.patch('google_compute_engine.metadata_scripts.script_manager.shutil.rmtree') @mock.patch('google_compute_engine.metadata_scripts.script_manager.tempfile.mkdtemp') - def testRunScripts(self, mock_mkdir, mock_rmtree, mock_executor, mock_logger, - mock_retriever): + def testRunScripts( + self, mock_mkdir, mock_rmtree, mock_executor, mock_logger, + mock_retriever): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance mock_retriever_instance = mock.Mock() diff --git a/google_compute_engine/network_setup/__init__.py b/google_compute_engine/network_setup/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/google_compute_engine/network_setup/__init__.py diff --git a/google_compute_engine/network_setup/network_setup.py b/google_compute_engine/network_setup/network_setup.py new file mode 100755 index 0000000..d528ed4 --- /dev/null +++ b/google_compute_engine/network_setup/network_setup.py @@ -0,0 +1,94 @@ +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Enables the network interfaces provided in metadata.""" + +import logging.handlers +import optparse +import subprocess + +from google_compute_engine import config_manager +from google_compute_engine import logger +from google_compute_engine import metadata_watcher +from google_compute_engine import network_utils + + +class NetworkSetup(object): + """Enable network interfaces specified by metadata.""" + + network_interfaces = 'instance/network-interfaces' + + def __init__(self, dhcp_binary=None, debug=False): + """Constructor. + + Args: + dhcp_binary: string, an executable to enable an ethernet interface. + debug: bool, True if debug output should write to the console. + """ + self.dhcp_binary = dhcp_binary or 'dhclient' + facility = logging.handlers.SysLogHandler.LOG_DAEMON + self.logger = logger.Logger( + name='network-setup', debug=debug, facility=facility) + self.watcher = metadata_watcher.MetadataWatcher(logger=self.logger) + self.network_utils = network_utils.NetworkUtils(logger=self.logger) + self._SetupNetworkInterfaces() + + def _EnableNetworkInterface(self, interface): + """Enable the network interface. + + Args: + interface: string, the output device to enable. + """ + if self.network_utils.IsEnabled(interface): + return + + command = [self.dhcp_binary, interface] + try: + self.logger.info('Enabling the ethernet interface %s.', interface) + subprocess.check_call(command) + except subprocess.CalledProcessError: + self.logger.warning('Could not enable the interface %s.', interface) + + def _SetupNetworkInterfaces(self): + """Get network interfaces metadata and enable each ethernet interface.""" + result = self.watcher.GetMetadata( + metadata_key=self.network_interfaces, recursive=True) + + for network_interface in result: + mac_address = network_interface.get('mac') + interface = self.network_utils.GetNetworkInterface(mac_address) + if interface: + self._EnableNetworkInterface(interface) + else: + message = 'Network interface not found for MAC address: %s.' + self.logger.warning(message, mac_address) + + +def main(): + parser = optparse.OptionParser() + parser.add_option( + '-d', '--debug', action='store_true', dest='debug', + help='print debug output to the console.') + (options, _) = parser.parse_args() + instance_config = config_manager.ConfigManager() + if instance_config.GetOptionBool('NetworkInterfaces', 'setup'): + NetworkSetup( + dhcp_binary=instance_config.GetOptionString( + 'NetworkInterfaces', 'dhcp_binary'), + debug=bool(options.debug)) + + +if __name__ == '__main__': + main() diff --git a/google_compute_engine/network_setup/tests/network_setup_test.py b/google_compute_engine/network_setup/tests/network_setup_test.py new file mode 100644 index 0000000..12b2588 --- /dev/null +++ b/google_compute_engine/network_setup/tests/network_setup_test.py @@ -0,0 +1,114 @@ +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittest for network_setup.py module.""" + +import subprocess + +from google_compute_engine.network_setup import network_setup +from google_compute_engine.test_compat import mock +from google_compute_engine.test_compat import unittest + + +class NetworkSetupTest(unittest.TestCase): + + def setUp(self): + self.mock_logger = mock.Mock() + self.mock_watcher = mock.Mock() + self.mock_ip_forwarding_utils = mock.Mock() + self.mock_network_utils = mock.Mock() + self.metadata_key = 'metadata_key' + self.dhcp_binary = 'binary' + + self.mock_setup = mock.create_autospec(network_setup.NetworkSetup) + self.mock_setup.logger = self.mock_logger + self.mock_setup.watcher = self.mock_watcher + self.mock_setup.network_utils = self.mock_network_utils + self.mock_setup.network_interfaces = self.metadata_key + self.mock_setup.dhcp_binary = self.dhcp_binary + + @mock.patch('google_compute_engine.network_setup.network_setup.network_utils') + @mock.patch('google_compute_engine.network_setup.network_setup.metadata_watcher') + @mock.patch('google_compute_engine.network_setup.network_setup.logger') + def testNetworkSetup(self, mock_logger, mock_watcher, mock_network_utils): + mock_logger_instance = mock.Mock() + mock_logger.Logger.return_value = mock_logger_instance + mocks = mock.Mock() + mocks.attach_mock(mock_logger, 'logger') + mocks.attach_mock(mock_watcher, 'watcher') + mocks.attach_mock(mock_network_utils, 'network') + with mock.patch.object( + network_setup.NetworkSetup, '_SetupNetworkInterfaces'): + network_setup.NetworkSetup(dhcp_binary='binary', debug=True) + expected_calls = [ + mock.call.logger.Logger(name=mock.ANY, debug=True, facility=mock.ANY), + mock.call.watcher.MetadataWatcher(logger=mock_logger_instance), + mock.call.network.NetworkUtils(logger=mock_logger_instance), + ] + self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.network_setup.network_setup.subprocess.check_call') + def testEnableNetworkInterface(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + mocks.attach_mock(self.mock_logger, 'logger') + mocks.attach_mock(self.mock_watcher, 'watcher') + mocks.attach_mock(self.mock_network_utils, 'network') + mock_call.side_effect = [None, subprocess.CalledProcessError(1, 'Test')] + self.mock_network_utils.IsEnabled.side_effect = [True, False, False] + + network_setup.NetworkSetup._EnableNetworkInterface(self.mock_setup, 'a') + network_setup.NetworkSetup._EnableNetworkInterface(self.mock_setup, 'b') + network_setup.NetworkSetup._EnableNetworkInterface(self.mock_setup, 'c') + expected_calls = [ + # The network interface is already enabled. + mock.call.network.IsEnabled('a'), + # Successfully enable the network interface. + mock.call.network.IsEnabled('b'), + mock.call.logger.info(mock.ANY, 'b'), + mock.call.call([self.dhcp_binary, 'b']), + # Exception while enabling the network interface. + mock.call.network.IsEnabled('c'), + mock.call.logger.info(mock.ANY, 'c'), + mock.call.call([self.dhcp_binary, 'c']), + mock.call.logger.warning(mock.ANY, 'c'), + ] + self.assertEqual(mocks.mock_calls, expected_calls) + + def testSetupNetworkInterfaces(self): + mocks = mock.Mock() + mocks.attach_mock(self.mock_logger, 'logger') + mocks.attach_mock(self.mock_watcher, 'watcher') + mocks.attach_mock(self.mock_network_utils, 'network') + mocks.attach_mock(self.mock_setup, 'setup') + self.mock_watcher.GetMetadata.return_value = [ + {'mac': '1'}, {'mac': '2'}, {}] + self.mock_network_utils.GetNetworkInterface.side_effect = [ + 'eth0', None, None] + + with mock.patch.object( + network_setup.NetworkSetup, '_EnableNetworkInterface'): + network_setup.NetworkSetup._SetupNetworkInterfaces(self.mock_setup) + expected_calls = [ + mock.call.watcher.GetMetadata( + metadata_key=self.metadata_key, recursive=True), + mock.call.network.GetNetworkInterface('1'), + mock.call.setup._EnableNetworkInterface('eth0'), + mock.call.network.GetNetworkInterface('2'), + mock.call.logger.warning(mock.ANY, '2'), + mock.call.network.GetNetworkInterface(None), + mock.call.logger.warning(mock.ANY, None), + ] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/network_utils.py b/google_compute_engine/network_utils.py new file mode 100644 index 0000000..635bb4b --- /dev/null +++ b/google_compute_engine/network_utils.py @@ -0,0 +1,79 @@ +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for configuring IP address forwarding.""" + +import logging +import os + + +class NetworkUtils(object): + """System network ethernet interface utilities.""" + + def __init__(self, logger=logging): + """Constructor. + + Args: + logger: logger object, used to write to SysLog and serial port. + """ + self.logger = logger + self.interfaces = self._CreateInterfaceMap() + + def _CreateInterfaceMap(self): + """Generate a dictionary mapping MAC address to ethernet interfaces. + + Returns: + dict, string MAC addresses mapped to the string network interface name. + """ + interfaces = {} + for interface in os.listdir('/sys/class/net'): + try: + mac_address = open( + '/sys/class/net/%s/address' % interface).read().strip() + except (IOError, OSError) as e: + message = 'Unable to determine MAC address for %s. %s.' + self.logger.warning(message, interface, str(e)) + else: + interfaces[mac_address] = interface + return interfaces + + def GetNetworkInterface(self, mac_address): + """Get the name of the network interface associated with a MAC address. + + Args: + mac_address: string, the hardware address of the network interface. + + Returns: + string, the network interface associated with a MAC address or None. + """ + return self.interfaces.get(mac_address) + + def IsEnabled(self, interface): + """Check whether a network interface is enabled. + + Args: + interface: string, the network interface. + + Returns: + bool, True if the network interface is enabled. + """ + try: + state = open('/sys/class/net/%s/operstate' % interface).read().strip() + except (IOError, OSError) as e: + message = 'Unable to determine the state for %s. %s.' + self.logger.warning(message, interface, str(e)) + return False + else: + return 'up' == state diff --git a/google_compute_engine/tests/file_utils_test.py b/google_compute_engine/tests/file_utils_test.py index 21d9784..0d67807 100644 --- a/google_compute_engine/tests/file_utils_test.py +++ b/google_compute_engine/tests/file_utils_test.py @@ -55,8 +55,8 @@ class FileUtilsTest(unittest.TestCase): @mock.patch('google_compute_engine.file_utils.os.mkdir') @mock.patch('google_compute_engine.file_utils.os.chown') @mock.patch('google_compute_engine.file_utils.os.chmod') - def testSetPermissions(self, mock_chmod, mock_chown, mock_mkdir, mock_exists, - mock_context): + def testSetPermissions( + self, mock_chmod, mock_chown, mock_mkdir, mock_exists, mock_context): mocks = mock.Mock() mocks.attach_mock(mock_chmod, 'chmod') mocks.attach_mock(mock_chown, 'chown') diff --git a/google_compute_engine/tests/metadata_watcher_test.py b/google_compute_engine/tests/metadata_watcher_test.py index 8b2678b..a1e5240 100644 --- a/google_compute_engine/tests/metadata_watcher_test.py +++ b/google_compute_engine/tests/metadata_watcher_test.py @@ -76,8 +76,8 @@ class MetadataWatcherTest(unittest.TestCase): @mock.patch('google_compute_engine.metadata_watcher.urlrequest.build_opener') @mock.patch('google_compute_engine.metadata_watcher.urlrequest.ProxyHandler') @mock.patch('google_compute_engine.metadata_watcher.urlrequest.Request') - def testGetMetadataRequestRetry(self, mock_request, mock_proxy, mock_opener, - mock_time): + def testGetMetadataRequestRetry( + self, mock_request, mock_proxy, mock_opener, mock_time): mock_open = mock.Mock() mock_handler = mock.Mock() mocks = mock.Mock() @@ -123,8 +123,8 @@ class MetadataWatcherTest(unittest.TestCase): @mock.patch('google_compute_engine.metadata_watcher.urlrequest.build_opener') @mock.patch('google_compute_engine.metadata_watcher.urlrequest.ProxyHandler') @mock.patch('google_compute_engine.metadata_watcher.urlrequest.Request') - def testGetMetadataRequestHttpException(self, mock_request, mock_proxy, - mock_opener): + def testGetMetadataRequestHttpException( + self, mock_request, mock_proxy, mock_opener): mock_open = mock.Mock() mock_handler = mock.Mock() mock_response = mock.Mock() @@ -145,8 +145,8 @@ class MetadataWatcherTest(unittest.TestCase): @mock.patch('google_compute_engine.metadata_watcher.urlrequest.build_opener') @mock.patch('google_compute_engine.metadata_watcher.urlrequest.ProxyHandler') @mock.patch('google_compute_engine.metadata_watcher.urlrequest.Request') - def testGetMetadataRequestException(self, mock_request, mock_proxy, - mock_opener): + def testGetMetadataRequestException( + self, mock_request, mock_proxy, mock_opener): mock_open = mock.Mock() mock_handler = mock.Mock() mock_response = mock.Mock() diff --git a/google_compute_engine/tests/network_utils_test.py b/google_compute_engine/tests/network_utils_test.py new file mode 100644 index 0000000..1c0cb7c --- /dev/null +++ b/google_compute_engine/tests/network_utils_test.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittest for network_utils.py module.""" + +from google_compute_engine import network_utils +from google_compute_engine.test_compat import builtin +from google_compute_engine.test_compat import mock +from google_compute_engine.test_compat import unittest + + +class NetworkUtilsTest(unittest.TestCase): + + def setUp(self): + self.mock_logger = mock.Mock() + self.interfaces = {'address': 'interface'} + self.mock_utils = network_utils.NetworkUtils(self.mock_logger) + self.mock_utils.interfaces = self.interfaces + + @mock.patch('google_compute_engine.network_utils.os.listdir') + def testCreateInterfaceMap(self, mock_listdir): + mock_open = mock.mock_open() + interface_map = { + '1': 'a', + '2': 'b', + '3': 'c', + } + mock_listdir.return_value = interface_map.values() + + with mock.patch('%s.open' % builtin, mock_open, create=False): + addresses = interface_map.keys() + addresses = ['%s\n' % address for address in addresses] + mock_open().read.side_effect = interface_map.keys() + self.assertEqual(self.mock_utils._CreateInterfaceMap(), interface_map) + + @mock.patch('google_compute_engine.network_utils.os.listdir') + def testCreateInterfaceMapError(self, mock_listdir): + mock_open = mock.mock_open() + mock_listdir.return_value = ['a', 'b', 'c'] + + with mock.patch('%s.open' % builtin, mock_open, create=False): + mock_open().read.side_effect = [ + '1', OSError('OSError'), IOError('IOError')] + self.assertEqual(self.mock_utils._CreateInterfaceMap(), {'1': 'a'}) + expected_calls = [ + mock.call.warning(mock.ANY, 'b', 'OSError'), + mock.call.warning(mock.ANY, 'c', 'IOError'), + ] + self.assertEqual(self.mock_logger.mock_calls, expected_calls) + + def testGetNetworkInterface(self): + self.assertIsNone(self.mock_utils.GetNetworkInterface('invalid')) + self.assertEqual( + self.mock_utils.GetNetworkInterface('address'), 'interface') + + def testIsEnabled(self): + mock_open = mock.mock_open() + + with mock.patch('%s.open' % builtin, mock_open, create=False): + mock_open().read.side_effect = ['up', 'down', 'up\n', '', 'Garbage'] + self.assertEqual(self.mock_utils.IsEnabled('a'), True) + self.assertEqual(self.mock_utils.IsEnabled('a'), False) + self.assertEqual(self.mock_utils.IsEnabled('a'), True) + self.assertEqual(self.mock_utils.IsEnabled('a'), False) + self.assertEqual(self.mock_utils.IsEnabled('a'), False) + + def testIsEnabledError(self): + mock_open = mock.mock_open() + + with mock.patch('%s.open' % builtin, mock_open, create=False): + mock_open().read.side_effect = [OSError('OSError')] + self.assertEqual(self.mock_utils.IsEnabled('a'), False) + self.mock_logger.warning.assert_called_once_with( + mock.ANY, 'a', 'OSError'), diff --git a/google_compute_engine_init/build_packages.sh b/google_compute_engine_init/build_packages.sh index 5882721..ce119ad 100755 --- a/google_compute_engine_init/build_packages.sh +++ b/google_compute_engine_init/build_packages.sh @@ -59,7 +59,7 @@ function build_distro() { "google-compute-daemon: ${init_config}/rpm_replace" \ --url 'https://github.com/GoogleCloudPlatform/compute-image-packages' \ --vendor 'Google Compute Engine Team' \ - --version '2.0.2' \ + --version '2.1.0' \ "${init_files[@]}" } diff --git a/google_compute_engine_init/systemd/google-network-setup.service b/google_compute_engine_init/systemd/google-network-setup.service new file mode 100644 index 0000000..4400391 --- /dev/null +++ b/google_compute_engine_init/systemd/google-network-setup.service @@ -0,0 +1,13 @@ +[Unit] +Description=Google Compute Engine Network Setup +After=local-fs.target network-online.target network.target rsyslog.service +After=google-instance-setup.service +Wants=local-fs.target network-online.target network.target + +[Service] +ExecStart=/usr/bin/google_network_setup +KillMode=process +Type=oneshot + +[Install] +WantedBy=multi-user.target diff --git a/google_compute_engine_init/systemd/postinst.sh b/google_compute_engine_init/systemd/postinst.sh index 485e27e..eae4a87 100755 --- a/google_compute_engine_init/systemd/postinst.sh +++ b/google_compute_engine_init/systemd/postinst.sh @@ -23,12 +23,16 @@ systemctl enable google-accounts-daemon.service systemctl enable google-clock-skew-daemon.service systemctl enable google-instance-setup.service systemctl enable google-ip-forwarding-daemon.service +systemctl enable google-network-setup.service systemctl enable google-shutdown-scripts.service systemctl enable google-startup-scripts.service # Run instance setup manually to prevent startup script execution. /usr/bin/google_instance_setup +# Enable network interfaces. +/usr/bin/google_network_setup + # Start daemons. systemctl start --no-block google-accounts-daemon systemctl start --no-block google-clock-skew-daemon diff --git a/google_compute_engine_init/systemd/prerm.sh b/google_compute_engine_init/systemd/prerm.sh index 16419b8..d746b6f 100755 --- a/google_compute_engine_init/systemd/prerm.sh +++ b/google_compute_engine_init/systemd/prerm.sh @@ -22,6 +22,7 @@ if [ "$1" = purge ]; then systemctl --no-reload disable google-clock-skew-daemon.service systemctl --no-reload disable google-instance-setup.service systemctl --no-reload disable google-ip-forwarding-daemon.service + systemctl --no-reload disable google-network-setup.service systemctl --no-reload disable google-shutdown-scripts.service systemctl --no-reload disable google-startup-scripts.service fi diff --git a/google_compute_engine_init/systemd/rpm_replace b/google_compute_engine_init/systemd/rpm_replace index 2e7d562..35366e8 100755 --- a/google_compute_engine_init/systemd/rpm_replace +++ b/google_compute_engine_init/systemd/rpm_replace @@ -5,12 +5,16 @@ systemctl enable google-accounts-daemon.service systemctl enable google-clock-skew-daemon.service systemctl enable google-instance-setup.service systemctl enable google-ip-forwarding-daemon.service +systemctl enable google-network-setup.service systemctl enable google-shutdown-scripts.service systemctl enable google-startup-scripts.service # Run instance setup manually. /usr/bin/google_instance_setup +# Enable network interfaces. +/usr/bin/google_network_setup + # Start daemons. systemctl start --no-block google-accounts-daemon systemctl start --no-block google-clock-skew-daemon diff --git a/google_compute_engine_init/sysvinit/google-network-setup b/google_compute_engine_init/sysvinit/google-network-setup new file mode 100755 index 0000000..41b800c --- /dev/null +++ b/google_compute_engine_init/sysvinit/google-network-setup @@ -0,0 +1,50 @@ +#!/bin/sh +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +### BEGIN INIT INFO +# Provides: google_network_setup +# Required-Start: $all $google_instance_setup +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Google Compute Engine Network Setup +# Description: Enables network interfaces on boot. +### END INIT INFO + +NAME=google-network-setup +SCRIPTNAME=/etc/init.d/$NAME + +# Load the rcS variables. +. /lib/init/vars.sh + +# +# Function that starts the daemon/service. +# +do_start() +{ + /usr/bin/google_network_setup > /dev/null +} + +case "$1" in + start) + do_start + ;; + *) + echo "Usage: $SCRIPTNAME start" >&2 + exit 1 + ;; +esac + +: diff --git a/google_compute_engine_init/sysvinit/postinst.sh b/google_compute_engine_init/sysvinit/postinst.sh index e1cd77b..96676d9 100755 --- a/google_compute_engine_init/sysvinit/postinst.sh +++ b/google_compute_engine_init/sysvinit/postinst.sh @@ -17,8 +17,12 @@ update-rc.d google-accounts-daemon defaults update-rc.d google-clock-skew-daemon defaults update-rc.d google-instance-setup defaults update-rc.d google-ip-forwarding-daemon defaults +update-rc.d google-network-setup defaults update-rc.d google-shutdown-scripts defaults update-rc.d google-startup-scripts defaults # Run instance setup. /etc/init.d/google-instance-setup start + +# Enable network interfaces. +/etc/init.d/google-network-setup start diff --git a/google_compute_engine_init/sysvinit/prerm.sh b/google_compute_engine_init/sysvinit/prerm.sh index abc306d..466a357 100755 --- a/google_compute_engine_init/sysvinit/prerm.sh +++ b/google_compute_engine_init/sysvinit/prerm.sh @@ -18,6 +18,7 @@ if [ "$1" = purge ]; then update-rc.d google-clock-skew-daemon remove update-rc.d google-instance-setup remove update-rc.d google-ip-forwarding-daemon remove + update-rc.d google-network-setup remove update-rc.d google-shutdown-scripts defaults update-rc.d google-startup-scripts defaults fi diff --git a/google_compute_engine_init/sysvinit/rpm_replace b/google_compute_engine_init/sysvinit/rpm_replace index 85aee43..782941b 100755 --- a/google_compute_engine_init/sysvinit/rpm_replace +++ b/google_compute_engine_init/sysvinit/rpm_replace @@ -2,3 +2,6 @@ # Run instance setup. /usr/bin/google_instance_setup + +# Enable network interfaces. +/usr/bin/google_network_setup diff --git a/google_compute_engine_init/upstart/google-network-setup.conf b/google_compute_engine_init/upstart/google-network-setup.conf new file mode 100644 index 0000000..2255544 --- /dev/null +++ b/google_compute_engine_init/upstart/google-network-setup.conf @@ -0,0 +1,4 @@ +# Enables network interfaces on boot. +start on stopped google-instance-setup + +exec /usr/bin/google_network_setup diff --git a/google_compute_engine_init/upstart/postinst.sh b/google_compute_engine_init/upstart/postinst.sh index 5904b22..8214102 100755 --- a/google_compute_engine_init/upstart/postinst.sh +++ b/google_compute_engine_init/upstart/postinst.sh @@ -21,6 +21,9 @@ stop --no-wait google-ip-forwarding-daemon # Run instance setup manually to prevent startup script execution. /usr/bin/google_instance_setup +# Enable network interfaces. +/usr/bin/google_network_setup + # Start daemons start --no-wait google-accounts-daemon start --no-wait google-clock-skew-daemon diff --git a/google_compute_engine_init/upstart/rpm_replace b/google_compute_engine_init/upstart/rpm_replace index 80f3a5a..0663e78 100755 --- a/google_compute_engine_init/upstart/rpm_replace +++ b/google_compute_engine_init/upstart/rpm_replace @@ -1,9 +1,12 @@ # Replace existing guest in EL6. -# Run instance setup +# Run instance setup. /usr/bin/google_instance_setup -# Manually start daemon's +# Enable network interfaces. +/usr/bin/google_network_setup + +# Manually start daemons. start --no-wait google-accounts-daemon start --no-wait google-clock-skew-daemon start --no-wait google-ip-forwarding-daemon diff --git a/google_configs/bin/set_hostname b/google_config/bin/set_hostname index f7d0b10..f7d0b10 100755 --- a/google_configs/bin/set_hostname +++ b/google_config/bin/set_hostname diff --git a/google_configs/build_packages.sh b/google_config/build_packages.sh index af35e7d..af35e7d 100755 --- a/google_configs/build_packages.sh +++ b/google_config/build_packages.sh diff --git a/google_configs/dhcp/google_hostname.sh b/google_config/dhcp/google_hostname.sh index 67231e0..67231e0 100755 --- a/google_configs/dhcp/google_hostname.sh +++ b/google_config/dhcp/google_hostname.sh diff --git a/google_configs/rsyslog/90-google.conf b/google_config/rsyslog/90-google.conf index 81b2ed7..81b2ed7 100644 --- a/google_configs/rsyslog/90-google.conf +++ b/google_config/rsyslog/90-google.conf diff --git a/google_configs/sysctl/11-gce-network-security.conf b/google_config/sysctl/11-gce-network-security.conf index 0e4db8c..0e4db8c 100644 --- a/google_configs/sysctl/11-gce-network-security.conf +++ b/google_config/sysctl/11-gce-network-security.conf diff --git a/google_configs/udev/64-gce-disk-removal.rules b/google_config/udev/64-gce-disk-removal.rules index 4ff1f99..4ff1f99 100644 --- a/google_configs/udev/64-gce-disk-removal.rules +++ b/google_config/udev/64-gce-disk-removal.rules diff --git a/google_configs/udev/65-gce-disk-naming.rules b/google_config/udev/65-gce-disk-naming.rules index c686837..c686837 100644 --- a/google_configs/udev/65-gce-disk-naming.rules +++ b/google_config/udev/65-gce-disk-naming.rules @@ -32,7 +32,7 @@ setuptools.setup( packages=setuptools.find_packages(), scripts=glob.glob('scripts/*'), url='https://github.com/GoogleCloudPlatform/compute-image-packages', - version='2.1.3', + version='2.2.0', # Entry points create scripts in /usr/bin that call a function. entry_points={ 'console_scripts': [ @@ -40,6 +40,7 @@ setuptools.setup( 'google_clock_skew_daemon=google_compute_engine.clock_skew.clock_skew_daemon:main', 'google_ip_forwarding_daemon=google_compute_engine.ip_forwarding.ip_forwarding_daemon:main', 'google_instance_setup=google_compute_engine.instance_setup.instance_setup:main', + 'google_network_setup=google_compute_engine.network_setup.network_setup:main', 'google_metadata_script_runner=google_compute_engine.metadata_scripts.script_manager:main', ], }, |