diff options
| author | Zach Marano <zmarano@google.com> | 2018-09-05 14:51:39 -0700 |
|---|---|---|
| committer | Zach Marano <zmarano@google.com> | 2018-09-05 14:51:39 -0700 |
| commit | e725e854cfeb6ede8470692dca082b5549bae2b3 (patch) | |
| tree | b7b24ec0150b6bdc3c2f304574774ef957660302 | |
| parent | 4f276598fc18da3fbbccf8abf30c905a5f91de06 (diff) | |
| parent | c3ad62ccd8b318938a3e2e374562454532497fd3 (diff) | |
| download | google-compute-image-packages-20180905.tar.gz | |
Merge branch 'development'20180905
59 files changed, 1102 insertions, 411 deletions
diff --git a/.travis.yml b/.travis.yml index 1f78c4a..fbf57a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,14 @@ language: python sudo: true python: - 2.7 -- 3.3 - 3.4 - 3.5 - 3.6 - pypy +matrix: + include: + - python: 3.7 + dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) os: - linux install: @@ -269,7 +269,7 @@ script execution. ## Packaging The guest Python code is packaged as a -[compliant PyPI Python package](http://python-packaging-user-guide.readthedocs.io/en/latest/) +[compliant PyPI Python package](https://packaging.python.org/) that can be used as a library or run independently. In addition to the Python package, deb and rpm packages are created with appropriate init configuration for supported GCE distros. The packages are targeted towards distribution diff --git a/debian/changelog b/debian/changelog index 4b41bbf..bb15cd0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +google-compute-image-packages (2.8.4-1) stable; urgency=low + + * Remove ntp dependency. + * Support Debian 10 Buster. + * Restart the network daemon if networking is restarted. + * Prevent setup of the default ethernet interface. + * Accounts daemon can now verify username is 32 characters or less. + + -- Google Cloud Team <gc-team@google.com> Wed, 05 Sep 2018 12:00:00 -0700 + google-compute-image-packages (2.8.3-1) stable; urgency=low * Prevent IP forwarding daemon log spam. diff --git a/debian/control b/debian/control index 5bd9fc4..922aee7 100644 --- a/debian/control +++ b/debian/control @@ -28,7 +28,6 @@ Depends: google-compute-engine-oslogin, ${misc:Depends}, python-google-compute-engine (= ${source:Version}), python3-google-compute-engine (= ${source:Version}), - chrony | ntp | time-daemon | systemd, system-log-daemon, systemd Recommends: google-cloud-sdk, rsyslog diff --git a/google_compute_engine/accounts/accounts_utils.py b/google_compute_engine/accounts/accounts_utils.py index 3a7c6d9..dac5cf5 100644 --- a/google_compute_engine/accounts/accounts_utils.py +++ b/google_compute_engine/accounts/accounts_utils.py @@ -26,7 +26,7 @@ import tempfile from google_compute_engine import constants from google_compute_engine import file_utils -USER_REGEX = re.compile(r'\A[A-Za-z0-9._][A-Za-z0-9._-]*\Z') +USER_REGEX = re.compile(r'\A[A-Za-z0-9._][A-Za-z0-9._-]{0,31}\Z') DEFAULT_GROUPADD_CMD = 'groupadd {group}' DEFAULT_USERADD_CMD = 'useradd -m -s /bin/bash -p * {user}' DEFAULT_USERDEL_CMD = 'userdel -r {user}' diff --git a/google_compute_engine/accounts/oslogin_utils.py b/google_compute_engine/accounts/oslogin_utils.py index 1a1d2b6..e8cf520 100644 --- a/google_compute_engine/accounts/oslogin_utils.py +++ b/google_compute_engine/accounts/oslogin_utils.py @@ -15,6 +15,7 @@ """Utilities for provisioning or deprovisioning a Linux user account.""" +import errno import os import subprocess import time @@ -50,7 +51,7 @@ class OsLoginUtils(object): try: return subprocess.call([constants.OSLOGIN_CONTROL_SCRIPT, action]) except OSError as e: - if e.errno == os.errno.ENOENT: + if e.errno == errno.ENOENT: return None else: raise @@ -83,7 +84,7 @@ class OsLoginUtils(object): try: return subprocess.call([constants.OSLOGIN_NSS_CACHE_SCRIPT]) except OSError as e: - if e.errno == os.errno.ENOENT: + if e.errno == errno.ENOENT: return None else: raise @@ -94,7 +95,7 @@ class OsLoginUtils(object): try: os.remove(constants.OSLOGIN_NSS_CACHE) except OSError as e: - if e.errno != os.errno.ENOENT: + if e.errno != errno.ENOENT: raise def UpdateOsLogin(self, enable, duration=NSS_CACHE_DURATION_SEC): diff --git a/google_compute_engine/accounts/tests/accounts_utils_test.py b/google_compute_engine/accounts/tests/accounts_utils_test.py index c021896..5509dba 100644 --- a/google_compute_engine/accounts/tests/accounts_utils_test.py +++ b/google_compute_engine/accounts/tests/accounts_utils_test.py @@ -572,6 +572,7 @@ class AccountsUtilsTest(unittest.TestCase): 'abc xyz', 'xyz*', 'xyz$', + 'areallylongusernamethatexceedsthethirtytwocharacterlimit' ] for user in invalid_users: self.assertFalse( diff --git a/google_compute_engine/constants.py b/google_compute_engine/constants.py index 7afb57b..d908275 100644 --- a/google_compute_engine/constants.py +++ b/google_compute_engine/constants.py @@ -18,7 +18,6 @@ import platform OSLOGIN_CONTROL_SCRIPT = 'google_oslogin_control' -OSLOGIN_NSS_CACHE = '/etc/oslogin_passwd.cache' OSLOGIN_NSS_CACHE_SCRIPT = 'google_oslogin_nss_cache' if platform.system() == 'FreeBSD': @@ -26,13 +25,16 @@ if platform.system() == 'FreeBSD': BOTOCONFDIR = '/usr/local' SYSCONFDIR = '/usr/local/etc' LOCALSTATEDIR = '/var/spool' + OSLOGIN_NSS_CACHE = '/usr/local/etc/oslogin_passwd.cache' elif platform.system() == 'OpenBSD': LOCALBASE = '/usr/local' BOTOCONFDIR = '' SYSCONFDIR = '/usr/local/etc' LOCALSTATEDIR = '/var/spool' + OSLOGIN_NSS_CACHE = '/usr/local/etc/oslogin_passwd.cache' else: LOCALBASE = '' BOTOCONFDIR = '' SYSCONFDIR = '/etc/default' LOCALSTATEDIR = '/var' + OSLOGIN_NSS_CACHE = '/etc/oslogin_passwd.cache' diff --git a/google_compute_engine/distro_lib/debian_8/tests/utils_test.py b/google_compute_engine/distro_lib/debian_8/tests/utils_test.py index e529a26..6ff9f9c 100644 --- a/google_compute_engine/distro_lib/debian_8/tests/utils_test.py +++ b/google_compute_engine/distro_lib/debian_8/tests/utils_test.py @@ -44,3 +44,12 @@ class UtilsTest(unittest.TestCase): utils.Utils.HandleClockSync(self.mock_setup, self.mock_logger) expected_calls = [mock.call.call(mock.ANY)] self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.IpForwardingUtilsIproute') + def testIpForwardingUtils(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + + utils.Utils.IpForwardingUtils(self.mock_setup, self.mock_logger, '66') + expected_calls = [mock.call.call(mock.ANY, '66')] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/distro_lib/debian_8/utils.py b/google_compute_engine/distro_lib/debian_8/utils.py index 6666269..9bdccb8 100644 --- a/google_compute_engine/distro_lib/debian_8/utils.py +++ b/google_compute_engine/distro_lib/debian_8/utils.py @@ -16,6 +16,7 @@ """Utilities that are distro specific for use on Debian 8.""" from google_compute_engine.distro_lib import helpers +from google_compute_engine.distro_lib import ip_forwarding_utils from google_compute_engine.distro_lib import utils @@ -40,3 +41,12 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ helpers.CallHwclock(logger) + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + return ip_forwarding_utils.IpForwardingUtilsIproute(logger, proto_id) diff --git a/google_compute_engine/distro_lib/debian_9/tests/utils_test.py b/google_compute_engine/distro_lib/debian_9/tests/utils_test.py index ebfde1a..2717f00 100644 --- a/google_compute_engine/distro_lib/debian_9/tests/utils_test.py +++ b/google_compute_engine/distro_lib/debian_9/tests/utils_test.py @@ -44,3 +44,12 @@ class UtilsTest(unittest.TestCase): utils.Utils.HandleClockSync(self.mock_setup, self.mock_logger) expected_calls = [mock.call.call(mock.ANY)] self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.IpForwardingUtilsIproute') + def testIpForwardingUtils(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + + utils.Utils.IpForwardingUtils(self.mock_setup, self.mock_logger, '66') + expected_calls = [mock.call.call(mock.ANY, '66')] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/distro_lib/debian_9/utils.py b/google_compute_engine/distro_lib/debian_9/utils.py index ad56ea1..452e02f 100644 --- a/google_compute_engine/distro_lib/debian_9/utils.py +++ b/google_compute_engine/distro_lib/debian_9/utils.py @@ -16,6 +16,7 @@ """Utilities that are distro specific for use on Debian 9.""" from google_compute_engine.distro_lib import helpers +from google_compute_engine.distro_lib import ip_forwarding_utils from google_compute_engine.distro_lib import utils @@ -40,3 +41,12 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ helpers.CallHwclock(logger) + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + return ip_forwarding_utils.IpForwardingUtilsIproute(logger, proto_id) diff --git a/google_compute_engine/distro_lib/el_6/tests/utils_test.py b/google_compute_engine/distro_lib/el_6/tests/utils_test.py index dcfde8d..539b059 100644 --- a/google_compute_engine/distro_lib/el_6/tests/utils_test.py +++ b/google_compute_engine/distro_lib/el_6/tests/utils_test.py @@ -50,3 +50,12 @@ class UtilsTest(unittest.TestCase): utils.Utils.HandleClockSync(self.mock_setup, self.mock_logger) expected_calls = [mock.call.call(mock.ANY)] self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.IpForwardingUtilsIproute') + def testIpForwardingUtils(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + + utils.Utils.IpForwardingUtils(self.mock_setup, self.mock_logger, '66') + expected_calls = [mock.call.call(mock.ANY, '66')] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/distro_lib/el_6/utils.py b/google_compute_engine/distro_lib/el_6/utils.py index f3ea305..20b8e2f 100644 --- a/google_compute_engine/distro_lib/el_6/utils.py +++ b/google_compute_engine/distro_lib/el_6/utils.py @@ -16,6 +16,7 @@ """Utilities that are distro specific for use on EL 6.""" from google_compute_engine.distro_lib import helpers +from google_compute_engine.distro_lib import ip_forwarding_utils from google_compute_engine.distro_lib import utils @@ -40,3 +41,12 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ helpers.CallHwclock(logger) + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + return ip_forwarding_utils.IpForwardingUtilsIproute(logger, proto_id) diff --git a/google_compute_engine/distro_lib/el_7/tests/utils_test.py b/google_compute_engine/distro_lib/el_7/tests/utils_test.py index e9803b7..17f81d0 100644 --- a/google_compute_engine/distro_lib/el_7/tests/utils_test.py +++ b/google_compute_engine/distro_lib/el_7/tests/utils_test.py @@ -129,3 +129,12 @@ class UtilsTest(unittest.TestCase): utils.Utils.HandleClockSync(self.mock_setup, self.mock_logger) expected_calls = [mock.call.call(mock.ANY)] self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.IpForwardingUtilsIproute') + def testIpForwardingUtils(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + + utils.Utils.IpForwardingUtils(self.mock_setup, self.mock_logger, '66') + expected_calls = [mock.call.call(mock.ANY, '66')] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/distro_lib/el_7/utils.py b/google_compute_engine/distro_lib/el_7/utils.py index 11f8462..b11a98f 100644 --- a/google_compute_engine/distro_lib/el_7/utils.py +++ b/google_compute_engine/distro_lib/el_7/utils.py @@ -21,6 +21,7 @@ import re from google_compute_engine import constants from google_compute_engine.distro_lib import helpers +from google_compute_engine.distro_lib import ip_forwarding_utils from google_compute_engine.distro_lib import utils @@ -51,7 +52,8 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ for interface in interfaces: - interface_config = os.path.join(self.network_path, 'ifcfg-%s' % interface) + interface_config = os.path.join( + self.network_path, 'ifcfg-%s' % interface) if os.path.exists(interface_config): self._ModifyInterface( interface_config, 'DEVICE', interface, replace=False) @@ -97,3 +99,12 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ helpers.CallHwclock(logger) + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + return ip_forwarding_utils.IpForwardingUtilsIproute(logger, proto_id) diff --git a/google_compute_engine/distro_lib/freebsd_11/tests/utils_test.py b/google_compute_engine/distro_lib/freebsd_11/tests/utils_test.py index e35a576..ead9fa4 100644 --- a/google_compute_engine/distro_lib/freebsd_11/tests/utils_test.py +++ b/google_compute_engine/distro_lib/freebsd_11/tests/utils_test.py @@ -44,3 +44,12 @@ class UtilsTest(unittest.TestCase): utils.Utils.HandleClockSync(self.mock_setup, self.mock_logger) expected_calls = [mock.call.call(mock.ANY)] self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.IpForwardingUtilsIfconfig') + def testIpForwardingUtils(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + + utils.Utils.IpForwardingUtils(self.mock_setup, self.mock_logger, 66) + expected_calls = [mock.call.call(mock.ANY)] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/distro_lib/freebsd_11/utils.py b/google_compute_engine/distro_lib/freebsd_11/utils.py index d9a39ec..4a9e706 100644 --- a/google_compute_engine/distro_lib/freebsd_11/utils.py +++ b/google_compute_engine/distro_lib/freebsd_11/utils.py @@ -16,6 +16,7 @@ """Utilities that are distro specific for use on FreeBSD 11.""" from google_compute_engine.distro_lib import helpers +from google_compute_engine.distro_lib import ip_forwarding_utils from google_compute_engine.distro_lib import utils @@ -40,3 +41,12 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ helpers.CallNtpdate(logger) + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + return ip_forwarding_utils.IpForwardingUtilsIfconfig(logger) diff --git a/google_compute_engine/distro_lib/ip_forwarding_utils.py b/google_compute_engine/distro_lib/ip_forwarding_utils.py new file mode 100644 index 0000000..63532e2 --- /dev/null +++ b/google_compute_engine/distro_lib/ip_forwarding_utils.py @@ -0,0 +1,302 @@ +#!/usr/bin/python +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for configuring IP address forwarding.""" + +import re +import subprocess +try: + # The following modules are required by IpForwardingUtilsIfconfig. + import netifaces + import netaddr +except ImportError: + netifaces = None + netaddr = None + + +IP_REGEX = re.compile(r'\A(\d{1,3}\.){3}\d{1,3}\Z') +IP_ALIAS_REGEX = re.compile(r'\A(\d{1,3}\.){3}\d{1,3}/\d{1,2}\Z') + + +class IpForwardingUtilsBase(object): + """System IP address configuration utilities.""" + + def ParseForwardedIps(self, forwarded_ips): + """Parse and validate forwarded IP addresses. + + Args: + forwarded_ips: list, the IP address strings to parse. + + Returns: + list, the valid IP address strings. + """ + pass + + def GetForwardedIps(self, interface, interface_ip=None): + """Retrieve the list of configured forwarded IP addresses. + + Args: + interface: string, the output device to query. + interface_ip: string, current interface ip address. + + Returns: + list, the IP address strings. + """ + pass + + def AddForwardedIp(self, address, interface): + """Configure a new IP address on the network interface. + + Args: + address: string, the IP address to configure. + interface: string, the output device to use. + """ + pass + + def RemoveForwardedIp(self, address, interface): + """Delete an IP address on the network interface. + + Args: + address: string, the IP address to configure. + interface: string, the output device to use. + """ + pass + + +class IpForwardingUtilsIproute(IpForwardingUtilsBase): + """System IP address configuration utilities. + + Command used to add IPs: + ip route add to local $IP/32 dev eth0 proto 66 + Command used to fetch list of configured IPs: + ip route ls table local type local dev eth0 scope host proto 66 + """ + + def __init__(self, logger, proto_id=None): + """Constructor. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + self.logger = logger + self.proto_id = proto_id or '66' + + def _CreateRouteOptions(self, **kwargs): + """Create a dictionary of parameters to append to the ip route command. + + Args: + **kwargs: dict, the string parameters to update in the ip route command. + + Returns: + dict, the string parameters to append to the ip route command. + """ + options = { + 'proto': self.proto_id, + 'scope': 'host', + } + options.update(kwargs) + return options + + def _RunIpRoute(self, args=None, options=None): + """Run a command with ip route and return the response. + + Args: + args: list, the string ip route command args to execute. + options: dict, the string parameters to append to the ip route command. + + Returns: + string, the standard output from the ip route command execution. + """ + args = args or [] + options = options or {} + command = ['ip', 'route'] + command.extend(args) + for item in options.items(): + command.extend(item) + try: + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + except OSError as e: + self.logger.warning('Exception running %s. %s.', command, str(e)) + else: + if process.returncode: + message = 'Non-zero exit status running %s. %s.' + self.logger.warning(message, command, stderr.strip()) + else: + return stdout.decode('utf-8', 'replace') + return '' + + def ParseForwardedIps(self, forwarded_ips): + """Parse and validate forwarded IP addresses. + + Args: + forwarded_ips: list, the IP address strings to parse. + + Returns: + list, the valid IP address strings. + """ + addresses = [] + forwarded_ips = forwarded_ips or [] + for ip in forwarded_ips: + if ip and (IP_REGEX.match(ip) or IP_ALIAS_REGEX.match(ip)): + addresses.append(ip[:-3] if ip.endswith('/32') else ip) + else: + self.logger.warning('Could not parse IP address: "%s".', ip) + return addresses + + def GetForwardedIps(self, interface, interface_ip=None): + """Retrieve the list of configured forwarded IP addresses. + + Args: + interface: string, the output device to query. + interface_ip: string, current interface ip address. + + Returns: + list, the IP address strings. + """ + args = ['ls', 'table', 'local', 'type', 'local'] + options = self._CreateRouteOptions(dev=interface) + result = self._RunIpRoute(args=args, options=options) + result = re.sub(r'local\s', r'', result) + return self.ParseForwardedIps(result.split()) + + def AddForwardedIp(self, address, interface): + """Configure a new IP address on the network interface. + + Args: + address: string, the IP address to configure. + interface: string, the output device to use. + """ + address = address if IP_ALIAS_REGEX.match(address) else '%s/32' % address + args = ['add', 'to', 'local', address] + options = self._CreateRouteOptions(dev=interface) + self._RunIpRoute(args=args, options=options) + + def RemoveForwardedIp(self, address, interface): + """Delete an IP address on the network interface. + + Args: + address: string, the IP address to configure. + interface: string, the output device to use. + """ + address = address if IP_ALIAS_REGEX.match(address) else '%s/32' % address + args = ['delete', 'to', 'local', address] + options = self._CreateRouteOptions(dev=interface) + self._RunIpRoute(args=args, options=options) + + +class IpForwardingUtilsIfconfig(IpForwardingUtilsBase): + """System IP address configuration utilities.""" + + def __init__(self, logger): + """Constructor. + + Args: + logger: logger object, used to write to SysLog and serial port. + """ + + self.logger = logger + + def _RunIfconfig(self, args=None, options=None): + """Run a command with ifconfig and return the response. + + Args: + args: list, the string ip route command args to execute. + options: dict, the string parameters to append to the ip route command. + + Returns: + string, the standard output from the ip route command execution. + """ + args = args or [] + options = options or {} + command = ['ifconfig'] + command.extend(args) + for item in options.items(): + command.extend(item) + try: + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + except OSError as e: + self.logger.warning('Exception running %s. %s.', command, str(e)) + else: + if process.returncode: + message = 'Non-zero exit status running %s. %s.' + self.logger.warning(message, command, stderr.strip()) + else: + return stdout.decode('utf-8', 'replace') + return '' + + def ParseForwardedIps(self, forwarded_ips): + """Parse and validate forwarded IP addresses. + + Args: + forwarded_ips: list, the IP address strings to parse. + + Returns: + list, the valid IP address strings. + """ + addresses = [] + forwarded_ips = forwarded_ips or [] + for ip in forwarded_ips: + if ip and (IP_REGEX.match(ip) or IP_ALIAS_REGEX.match(ip)): + addresses.extend([str(addr) for addr in list(netaddr.IPNetwork(ip))]) + else: + self.logger.warning('Could not parse IP address: "%s".', ip) + return addresses + + def GetForwardedIps(self, interface, interface_ip=None): + """Retrieve the list of configured forwarded IP addresses. + + Args: + interface: string, the output device to query. + interface_ip: string, current interface ip address. + + Returns: + list, the IP address strings. + """ + try: + ips = netifaces.ifaddresses(interface) + ips = ips[netifaces.AF_INET] + except (ValueError, IndexError): + return [] + forwarded_ips = [] + for ip in ips: + if ip['addr'] != interface_ip: + full_addr = '%s/%d' % (ip['addr'], netaddr.IPAddress(ip['netmask']).netmask_bits()) + forwarded_ips.append(full_addr) + return self.ParseForwardedIps(forwarded_ips) + + def AddForwardedIp(self, address, interface): + """Configure a new IP address on the network interface. + + Args: + address: string, the IP address to configure. + interface: string, the output device to use. + """ + for ip in list(netaddr.IPNetwork(address)): + self._RunIfconfig(args=[interface, 'alias', '%s/32' % str(ip)]) + + def RemoveForwardedIp(self, address, interface): + """Delete an IP address on the network interface. + + Args: + address: string, the IP address to configure. + interface: string, the output device to use. + """ + ip = netaddr.IPNetwork(address) + self._RunIfconfig(args=[interface, '-alias', str(ip.ip)]) diff --git a/google_compute_engine/distro_lib/sles_11/tests/utils_test.py b/google_compute_engine/distro_lib/sles_11/tests/utils_test.py index 78110b6..22d54de 100644 --- a/google_compute_engine/distro_lib/sles_11/tests/utils_test.py +++ b/google_compute_engine/distro_lib/sles_11/tests/utils_test.py @@ -80,3 +80,12 @@ class UtilsTest(unittest.TestCase): utils.Utils.HandleClockSync(self.mock_setup, self.mock_logger) expected_calls = [mock.call.call(mock.ANY)] self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.IpForwardingUtilsIproute') + def testIpForwardingUtils(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + + utils.Utils.IpForwardingUtils(self.mock_setup, self.mock_logger, '66') + expected_calls = [mock.call.call(mock.ANY, '66')] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/distro_lib/sles_11/utils.py b/google_compute_engine/distro_lib/sles_11/utils.py index 448f662..6d1d0f2 100644 --- a/google_compute_engine/distro_lib/sles_11/utils.py +++ b/google_compute_engine/distro_lib/sles_11/utils.py @@ -20,6 +20,7 @@ import subprocess from google_compute_engine import constants from google_compute_engine.distro_lib import helpers +from google_compute_engine.distro_lib import ip_forwarding_utils from google_compute_engine.distro_lib import utils @@ -67,3 +68,12 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ helpers.CallHwclock(logger) + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + return ip_forwarding_utils.IpForwardingUtilsIproute(logger, proto_id) diff --git a/google_compute_engine/distro_lib/sles_12/tests/utils_test.py b/google_compute_engine/distro_lib/sles_12/tests/utils_test.py index d6f8e25..1f89c2e 100644 --- a/google_compute_engine/distro_lib/sles_12/tests/utils_test.py +++ b/google_compute_engine/distro_lib/sles_12/tests/utils_test.py @@ -100,3 +100,12 @@ class UtilsTest(unittest.TestCase): utils.Utils.HandleClockSync(self.mock_setup, self.mock_logger) expected_calls = [mock.call.call(mock.ANY)] self.assertEqual(mocks.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.IpForwardingUtilsIproute') + def testIpForwardingUtils(self, mock_call): + mocks = mock.Mock() + mocks.attach_mock(mock_call, 'call') + + utils.Utils.IpForwardingUtils(self.mock_setup, self.mock_logger, '66') + expected_calls = [mock.call.call(mock.ANY, '66')] + self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine/distro_lib/sles_12/utils.py b/google_compute_engine/distro_lib/sles_12/utils.py index fb15830..380df7b 100644 --- a/google_compute_engine/distro_lib/sles_12/utils.py +++ b/google_compute_engine/distro_lib/sles_12/utils.py @@ -20,6 +20,7 @@ import subprocess from google_compute_engine import constants from google_compute_engine.distro_lib import helpers +from google_compute_engine.distro_lib import ip_forwarding_utils from google_compute_engine.distro_lib import utils @@ -89,3 +90,12 @@ class Utils(utils.Utils): logger: logger object, used to write to SysLog and serial port. """ helpers.CallHwclock(logger) + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + return ip_forwarding_utils.IpForwardingUtilsIproute(logger, proto_id) diff --git a/google_compute_engine/distro_lib/tests/ip_forwarding_utils_test.py b/google_compute_engine/distro_lib/tests/ip_forwarding_utils_test.py new file mode 100644 index 0000000..e3f4bbc --- /dev/null +++ b/google_compute_engine/distro_lib/tests/ip_forwarding_utils_test.py @@ -0,0 +1,435 @@ +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittest for ip_forwarding_utils.py module.""" + +from google_compute_engine.distro_lib import ip_forwarding_utils +from google_compute_engine.test_compat import mock +from google_compute_engine.test_compat import unittest + + +def _CreateMockProcess(returncode, stdout, stderr): + mock_process = mock.Mock() + mock_process.returncode = returncode + mock_process.communicate.return_value = (stdout, stderr) + return mock_process + + +class IpForwardingUtilsIprouteTest(unittest.TestCase): + + def setUp(self): + self.mock_logger = mock.Mock() + self.options = {'hello': 'world'} + with mock.patch( + 'google_compute_engine.distro_lib.ip_forwarding_utils' + '.subprocess') as mock_subprocess: + mock_subprocess.Popen.return_value = _CreateMockProcess( + 0, b'out', b'') + self.mock_utils = ip_forwarding_utils.IpForwardingUtilsIproute( + self.mock_logger) + self.mock_utils.proto_id = 'proto' + + def testCreateRouteOptions(self): + # Default options. + expected_options = { + 'proto': 'proto', + 'scope': 'host', + } + self.assertEqual(self.mock_utils._CreateRouteOptions(), expected_options) + + # Update dictionary when arguments are specified. + expected_options = { + 'proto': 'proto', + 'scope': 'host', + 'num': 1, + 'string': 'hello world', + } + self.assertEqual( + self.mock_utils._CreateRouteOptions(num=1, string='hello world'), + expected_options) + + # Update the default options. + expected_options = { + 'proto': 'test 1', + 'scope': 'test 2', + } + self.assertEqual( + self.mock_utils._CreateRouteOptions(proto='test 1', scope='test 2'), + expected_options) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.subprocess') + def testRunIpRoute(self, mock_subprocess): + mock_process = _CreateMockProcess(0, b'out', b'') + mock_subprocess.Popen.return_value = mock_process + args = ['foo', 'bar'] + options = {'one': 'two'} + + self.assertEqual( + self.mock_utils._RunIpRoute(args=args, options=options), 'out') + command = ['ip', 'route', 'foo', 'bar', 'one', 'two'] + mock_subprocess.Popen.assert_called_once_with( + command, stdout=mock_subprocess.PIPE, stderr=mock_subprocess.PIPE) + mock_process.communicate.assert_called_once_with() + self.mock_logger.warning.assert_not_called() + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.subprocess') + def testRunIpRouteReturnCode(self, mock_subprocess): + mock_process = _CreateMockProcess(1, b'out', b'error\n') + mock_subprocess.Popen.return_value = mock_process + + self.assertEqual( + self.mock_utils._RunIpRoute(args=['foo', 'bar'], options=self.options), + '') + command = ['ip', 'route', 'foo', 'bar', 'hello', 'world'] + self.mock_logger.warning.assert_called_once_with( + mock.ANY, command, b'error') + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.subprocess') + def testRunIpRouteException(self, mock_subprocess): + mock_subprocess.Popen.side_effect = OSError('Test Error') + + self.assertEqual( + self.mock_utils._RunIpRoute(args=['foo', 'bar'], options=self.options), + '') + command = ['ip', 'route', 'foo', 'bar', 'hello', 'world'] + self.mock_logger.warning.assert_called_once_with( + mock.ANY, command, 'Test Error') + + def testParseForwardedIps(self): + self.assertEqual(self.mock_utils.ParseForwardedIps(None), []) + self.assertEqual(self.mock_utils.ParseForwardedIps([]), []) + self.assertEqual(self.mock_utils.ParseForwardedIps([None]), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['invalid']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1a1a1a1']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1.1.1.1.1']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1111.1.1.1']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1.1.1.1111']), []) + expected_calls = [ + mock.call.warning(mock.ANY, None), + mock.call.warning(mock.ANY, 'invalid'), + mock.call.warning(mock.ANY, '1a1a1a1'), + mock.call.warning(mock.ANY, '1.1.1.1.1'), + mock.call.warning(mock.ANY, '1111.1.1.1'), + mock.call.warning(mock.ANY, '1.1.1.1111'), + ] + self.assertEqual(self.mock_logger.mock_calls, expected_calls) + + def testParseForwardedIpsComplex(self): + forwarded_ips = { + '{{}}\n\"hello\"\n!@#$%^&*()\n\n': False, + '1111.1.1.1': False, + '1.1.1.1': True, + 'hello': False, + '123.123.123.123': True, + '1.1.1.': False, + '1.1.1.a': False, + None: False, + '1.0.0.0': True, + '1.1.1.1/1': True, + '1.1.1.1/11': True, + '123.123.123.123/1': True, + '123.123.123.123/123': False, + '123.123.123.123/a': False, + '123.123.123.123/': False, + } + input_ips = forwarded_ips.keys() + valid_ips = [ip for ip, valid in forwarded_ips.items() if valid] + invalid_ips = [ip for ip, valid in forwarded_ips.items() if not valid] + + self.assertEqual(self.mock_utils.ParseForwardedIps(input_ips), valid_ips) + expected_calls = [mock.call.warning(mock.ANY, ip) for ip in invalid_ips] + self.assertEqual(self.mock_logger.mock_calls, expected_calls) + + def testParseForwardedIpsSubnet(self): + forwarded_ips = { + '1.1.1.1': '1.1.1.1', + '1.1.1.1/32': '1.1.1.1', + '1.1.1.1/1': '1.1.1.1/1', + '1.1.1.1/10': '1.1.1.1/10', + '1.1.1.1/24': '1.1.1.1/24', + } + for ip, value in forwarded_ips.items(): + self.assertEqual(self.mock_utils.ParseForwardedIps([ip]), [value]) + + def testGetForwardedIps(self): + mock_options = mock.Mock() + mock_options.return_value = self.options + mock_run = mock.Mock() + mock_run.return_value = 'a\n b \nlocal c' + mock_parse = mock.Mock() + mock_parse.return_value = ['Test'] + self.mock_utils._CreateRouteOptions = mock_options + self.mock_utils._RunIpRoute = mock_run + self.mock_utils.ParseForwardedIps = mock_parse + + self.assertEqual( + self.mock_utils.GetForwardedIps('interface', 'ip'), ['Test']) + mock_options.assert_called_once_with(dev='interface') + mock_run.assert_called_once_with( + args=['ls', 'table', 'local', 'type', 'local'], options=self.options) + mock_parse.assert_called_once_with(['a', 'b', 'c']) + + def testAddForwardedIp(self): + mock_options = mock.Mock() + mock_options.return_value = self.options + mock_run = mock.Mock() + self.mock_utils._CreateRouteOptions = mock_options + self.mock_utils._RunIpRoute = mock_run + + self.mock_utils.AddForwardedIp('1.1.1.1', 'interface') + mock_options.assert_called_once_with(dev='interface') + mock_run.assert_called_once_with( + args=['add', 'to', 'local', '1.1.1.1/32'], options=self.options) + + def testAddIpAlias(self): + mock_options = mock.Mock() + mock_options.return_value = self.options + mock_run = mock.Mock() + self.mock_utils._CreateRouteOptions = mock_options + self.mock_utils._RunIpRoute = mock_run + + self.mock_utils.AddForwardedIp('1.1.1.1/24', 'interface') + mock_options.assert_called_once_with(dev='interface') + mock_run.assert_called_once_with( + args=['add', 'to', 'local', '1.1.1.1/24'], options=self.options) + + def testRemoveForwardedIp(self): + mock_options = mock.Mock() + mock_options.return_value = self.options + mock_run = mock.Mock() + self.mock_utils._CreateRouteOptions = mock_options + self.mock_utils._RunIpRoute = mock_run + + self.mock_utils.RemoveForwardedIp('1.1.1.1', 'interface') + mock_options.assert_called_once_with(dev='interface') + mock_run.assert_called_once_with( + args=['delete', 'to', 'local', '1.1.1.1/32'], options=self.options) + + def testRemoveAliasIp(self): + mock_options = mock.Mock() + mock_options.return_value = self.options + mock_run = mock.Mock() + self.mock_utils._CreateRouteOptions = mock_options + self.mock_utils._RunIpRoute = mock_run + + self.mock_utils.RemoveForwardedIp('1.1.1.1/24', 'interface') + mock_options.assert_called_once_with(dev='interface') + mock_run.assert_called_once_with( + args=['delete', 'to', 'local', '1.1.1.1/24'], options=self.options) + + +class IpForwardingUtilsIfconfigTest(unittest.TestCase): + + def setUp(self): + self.mock_logger = mock.Mock() + with mock.patch( + 'google_compute_engine.distro_lib.ip_forwarding_utils' + '.subprocess') as mock_subprocess: + mock_subprocess.Popen.return_value = _CreateMockProcess( + 0, b'out', b'') + self.mock_utils = ip_forwarding_utils.IpForwardingUtilsIfconfig( + self.mock_logger) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.subprocess') + def testRunIfconfig(self, mock_subprocess): + mock_process = _CreateMockProcess(0, b'out', b'') + mock_subprocess.Popen.return_value = mock_process + args = ['foo', 'bar'] + options = {'one': 'two'} + + self.assertEqual( + self.mock_utils._RunIfconfig(args=args, options=options), 'out') + command = ['ifconfig', 'foo', 'bar', 'one', 'two'] + mock_subprocess.Popen.assert_called_once_with( + command, stdout=mock_subprocess.PIPE, stderr=mock_subprocess.PIPE) + mock_process.communicate.assert_called_once_with() + self.mock_logger.warning.assert_not_called() + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.subprocess') + def testRunIfconfigReturnCode(self, mock_subprocess): + mock_process = _CreateMockProcess(1, b'out', b'error\n') + mock_subprocess.Popen.return_value = mock_process + + self.assertEqual( + self.mock_utils._RunIfconfig(args=['foo', 'bar']), + '') + command = ['ifconfig', 'foo', 'bar'] + self.mock_logger.warning.assert_called_once_with( + mock.ANY, command, b'error') + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.subprocess') + def testRunIfconfigException(self, mock_subprocess): + mock_subprocess.Popen.side_effect = OSError('Test Error') + + self.assertEqual( + self.mock_utils._RunIfconfig(args=['foo', 'bar']), + '') + command = ['ifconfig', 'foo', 'bar'] + self.mock_logger.warning.assert_called_once_with( + mock.ANY, command, 'Test Error') + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testParseForwardedIps(self, mock_netaddr): + self.assertEqual(self.mock_utils.ParseForwardedIps(None), []) + self.assertEqual(self.mock_utils.ParseForwardedIps([]), []) + self.assertEqual(self.mock_utils.ParseForwardedIps([None]), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['invalid']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1a1a1a1']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1.1.1.1.1']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1111.1.1.1']), []) + self.assertEqual(self.mock_utils.ParseForwardedIps(['1.1.1.1111']), []) + expected_calls = [ + mock.call.warning(mock.ANY, None), + mock.call.warning(mock.ANY, 'invalid'), + mock.call.warning(mock.ANY, '1a1a1a1'), + mock.call.warning(mock.ANY, '1.1.1.1.1'), + mock.call.warning(mock.ANY, '1111.1.1.1'), + mock.call.warning(mock.ANY, '1.1.1.1111'), + ] + self.assertEqual(self.mock_logger.mock_calls, expected_calls) + self.assertEqual(mock_netaddr.IPNetwork.mock_calls, []) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testParseForwardedIpsComplex(self, mock_netaddr): + def side_effect(arg): + return [arg] + mock_netaddr.IPNetwork.side_effect = side_effect + forwarded_ips = { + '{{}}\n\"hello\"\n!@#$%^&*()\n\n': False, + '1111.1.1.1': False, + '1.1.1.1': True, + 'hello': False, + '123.123.123.123': True, + '1.1.1.': False, + '1.1.1.a': False, + None: False, + '1.0.0.0': True, + '1.1.1.1/1': True, + '1.1.1.1/11': True, + '123.123.123.123/1': True, + '123.123.123.123/123': False, + '123.123.123.123/a': False, + '123.123.123.123/': False, + } + input_ips = forwarded_ips.keys() + valid_ips = [ip for ip, valid in forwarded_ips.items() if valid] + invalid_ips = [ip for ip, valid in forwarded_ips.items() if not valid] + + self.assertEqual(self.mock_utils.ParseForwardedIps(input_ips), valid_ips) + expected_calls = [mock.call.warning(mock.ANY, ip) for ip in invalid_ips] + self.assertEqual(self.mock_logger.mock_calls, expected_calls) + expected_calls = [mock.call.IPNetwork(ip) for ip in valid_ips] + self.assertEqual(mock_netaddr.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testParseForwardedIpsSubnet(self, mock_netaddr): + mock_netaddr.IPNetwork.return_value = ['1.1.1.1'] + forwarded_ips = [ + '1.1.1.1', + '1.1.1.1/32', + '1.1.1.1/1', + '1.1.1.1/10', + '1.1.1.1/24' + ] + for ip in forwarded_ips: + self.assertEqual(self.mock_utils.ParseForwardedIps([ip]), ['1.1.1.1']) + expected_calls = [mock.call.IPNetwork(ip) for ip in forwarded_ips] + self.assertEqual(mock_netaddr.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netifaces') + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testGetForwardedIps(self, mock_netaddr, mock_netifaces): + mock_netifaces.AF_INET = 0 + mock_netifaces.ifaddresses.return_value = [[ + {'addr': 'a', 'netmask': 'a mask'}, + {'addr': 'b', 'netmask': 'b mask'}, + {'addr': 'c', 'netmask': 'c mask'}, + ]] + mock_netaddr.IPAddress().netmask_bits.return_value = 32 + mock_parse = mock.Mock() + mock_parse.return_value = ['Test'] + self.mock_utils.ParseForwardedIps = mock_parse + + self.assertEqual( + self.mock_utils.GetForwardedIps('interface', 'ip'), ['Test']) + mock_netifaces.ifaddresses.assert_called_once_with('interface') + mock_parse.assert_called_once_with(['a/32', 'b/32', 'c/32']) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netifaces') + def testGetForwardedIpsEmpty(self, mock_netifaces): + mock_netifaces.AF_INET = 0 + mock_netifaces.ifaddresses.return_value = [] + + self.assertEqual( + self.mock_utils.GetForwardedIps('interface', 'ip'), []) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testAddForwardedIp(self, mock_netaddr): + mock_netaddr.IPNetwork.return_value = ['1.1.1.1'] + mock_run = mock.Mock() + self.mock_utils._RunIfconfig = mock_run + + self.mock_utils.AddForwardedIp('1.1.1.1', 'interface') + mock_netaddr.IPNetwork.assert_called_once_with('1.1.1.1') + mock_run.assert_called_once_with( + args=['interface', 'alias', '1.1.1.1/32']) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testAddIpAlias(self, mock_netaddr): + mock_netaddr.IPNetwork.return_value = [ + '1.1.1.0', '1.1.1.1', '1.1.1.2', '1.1.1.3' + ] + mock_run = mock.Mock() + self.mock_utils._RunIfconfig = mock_run + + self.mock_utils.AddForwardedIp('1.1.1.1/30', 'interface') + mock_netaddr.IPNetwork.assert_called_once_with('1.1.1.1/30') + expected_calls = [ + mock.call(args=['interface', 'alias', '1.1.1.0/32']), + mock.call(args=['interface', 'alias', '1.1.1.1/32']), + mock.call(args=['interface', 'alias', '1.1.1.2/32']), + mock.call(args=['interface', 'alias', '1.1.1.3/32']) + ] + self.assertEqual(mock_run.mock_calls, expected_calls) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testRemoveForwardedIp(self, mock_netaddr): + mock_ip = mock.Mock() + mock_ip.ip = '1.1.1.1' + mock_netaddr.IPNetwork.return_value = mock_ip + mock_run = mock.Mock() + self.mock_utils._RunIfconfig = mock_run + + self.mock_utils.RemoveForwardedIp('1.1.1.1', 'interface') + mock_netaddr.IPNetwork.assert_called_once_with('1.1.1.1') + mock_run.assert_called_once_with( + args=['interface', '-alias', '1.1.1.1']) + + @mock.patch('google_compute_engine.distro_lib.ip_forwarding_utils.netaddr') + def testRemoveAliasIp(self, mock_netaddr): + mock_ip = mock.Mock() + mock_ip.ip = '1.1.1.1' + mock_netaddr.IPNetwork.return_value = mock_ip + mock_run = mock.Mock() + self.mock_utils._RunIfconfig = mock_run + + self.mock_utils.RemoveForwardedIp('1.1.1.1/24', 'interface') + mock_netaddr.IPNetwork.assert_called_once_with('1.1.1.1/24') + mock_run.assert_called_once_with( + args=['interface', '-alias', '1.1.1.1']) + + +if __name__ == '__main__': + unittest.main() diff --git a/google_compute_engine/distro_lib/utils.py b/google_compute_engine/distro_lib/utils.py index bb26390..75b656e 100644 --- a/google_compute_engine/distro_lib/utils.py +++ b/google_compute_engine/distro_lib/utils.py @@ -45,3 +45,12 @@ class Utils(object): logger: logger object, used to write to SysLog and serial port. """ pass + + def IpForwardingUtils(self, logger, proto_id=None): + """Get system IP address configuration utilities. + + Args: + logger: logger object, used to write to SysLog and serial port. + proto_id: string, the routing protocol identifier for Google IP changes. + """ + pass diff --git a/google_compute_engine/networking/ip_forwarding/ip_forwarding.py b/google_compute_engine/networking/ip_forwarding/ip_forwarding.py index 3c92294..6a16956 100755 --- a/google_compute_engine/networking/ip_forwarding/ip_forwarding.py +++ b/google_compute_engine/networking/ip_forwarding/ip_forwarding.py @@ -17,12 +17,7 @@ When given a list of public endpoint IPs, compare it with the IPs configured for the associated interfaces, and add or remove addresses from the interfaces -to make them match. Only remove those which match our proto code. - -Command used to add IPs: - ip route add to local $IP/32 dev eth0 proto 66 -Command used to fetch list of configured IPs: - ip route ls table local type local dev eth0 scope host proto 66 +to make them match. """ import logging.handlers @@ -85,15 +80,17 @@ class IpForwarding(object): for address in forwarded_ips: self.ip_forwarding_utils.RemoveForwardedIp(address, interface) - def HandleForwardedIps(self, interface, forwarded_ips): + def HandleForwardedIps(self, interface, forwarded_ips, interface_ip=None): """Handle changes to the forwarded IPs on a network interface. Args: interface: string, the output device to configure. forwarded_ips: list, the forwarded IP address strings desired. + interface_ip: string, current interface ip address. """ desired = self.ip_forwarding_utils.ParseForwardedIps(forwarded_ips) - configured = self.ip_forwarding_utils.GetForwardedIps(interface) + configured = self.ip_forwarding_utils.GetForwardedIps( + interface, interface_ip) to_add = sorted(set(desired) - set(configured)) to_remove = sorted(set(configured) - set(desired)) self._LogForwardedIpChanges( diff --git a/google_compute_engine/networking/ip_forwarding/ip_forwarding_utils.py b/google_compute_engine/networking/ip_forwarding/ip_forwarding_utils.py index b2ddc18..1f0564e 100644 --- a/google_compute_engine/networking/ip_forwarding/ip_forwarding_utils.py +++ b/google_compute_engine/networking/ip_forwarding/ip_forwarding_utils.py @@ -15,125 +15,11 @@ """Utilities for configuring IP address forwarding.""" -import re -import subprocess - -IP_REGEX = re.compile(r'\A(\d{1,3}\.){3}\d{1,3}\Z') -IP_ALIAS_REGEX = re.compile(r'\A(\d{1,3}\.){3}\d{1,3}/\d{1,2}\Z') +from google_compute_engine.compat import distro_utils class IpForwardingUtils(object): - """System IP address configuration utilities.""" - - def __init__(self, logger, proto_id=None): - """Constructor. - - Args: - logger: logger object, used to write to SysLog and serial port. - proto_id: string, the routing protocol identifier for Google IP changes. - """ - self.logger = logger - self.proto_id = proto_id or '66' - - def _CreateRouteOptions(self, **kwargs): - """Create a dictionary of parameters to append to the ip route command. - - Args: - **kwargs: dict, the string parameters to update in the ip route command. - - Returns: - dict, the string parameters to append to the ip route command. - """ - options = { - 'proto': self.proto_id, - 'scope': 'host', - } - options.update(kwargs) - return options - - def _RunIpRoute(self, args=None, options=None): - """Run a command with ip route and return the response. - - Args: - args: list, the string ip route command args to execute. - options: dict, the string parameters to append to the ip route command. - - Returns: - string, the standard output from the ip route command execution. - """ - args = args or [] - options = options or {} - command = ['ip', 'route'] - command.extend(args) - for item in options.items(): - command.extend(item) - try: - process = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - except OSError as e: - self.logger.warning('Exception running %s. %s.', command, str(e)) - else: - if process.returncode: - message = 'Non-zero exit status running %s. %s.' - self.logger.warning(message, command, stderr.strip()) - else: - return stdout.decode('utf-8', 'replace') - return '' - - def ParseForwardedIps(self, forwarded_ips): - """Parse and validate forwarded IP addresses. - - Args: - forwarded_ips: list, the IP address strings to parse. - - Returns: - list, the valid IP address strings. - """ - addresses = [] - forwarded_ips = forwarded_ips or [] - for ip in forwarded_ips: - if ip and (IP_REGEX.match(ip) or IP_ALIAS_REGEX.match(ip)): - addresses.append(ip[:-3] if ip.endswith('/32') else ip) - else: - self.logger.warning('Could not parse IP address: "%s".', ip) - return addresses - - def GetForwardedIps(self, interface): - """Retrieve the list of configured forwarded IP addresses. - - Args: - interface: string, the output device to query. - - Returns: - list, the IP address strings. - """ - args = ['ls', 'table', 'local', 'type', 'local'] - options = self._CreateRouteOptions(dev=interface) - result = self._RunIpRoute(args=args, options=options) - result = re.sub(r'local\s', r'', result) - return self.ParseForwardedIps(result.split()) - - def AddForwardedIp(self, address, interface): - """Configure a new IP address on the network interface. - - Args: - address: string, the IP address to configure. - interface: string, the output device to use. - """ - address = address if IP_ALIAS_REGEX.match(address) else '%s/32' % address - args = ['add', 'to', 'local', address] - options = self._CreateRouteOptions(dev=interface) - self._RunIpRoute(args=args, options=options) - - def RemoveForwardedIp(self, address, interface): - """Delete an IP address on the network interface. + """Deprecated. Overridden for backwards compatibility.""" - Args: - address: string, the IP address to configure. - interface: string, the output device to use. - """ - address = address if IP_ALIAS_REGEX.match(address) else '%s/32' % address - args = ['delete', 'to', 'local', address] - options = self._CreateRouteOptions(dev=interface) - self._RunIpRoute(args=args, options=options) + def __new__(self, logger, proto_id=None): + return distro_utils.Utils().IpForwardingUtils(logger, proto_id) diff --git a/google_compute_engine/networking/ip_forwarding/tests/ip_forwarding_test.py b/google_compute_engine/networking/ip_forwarding/tests/ip_forwarding_test.py index f5a2fc9..986b20a 100644 --- a/google_compute_engine/networking/ip_forwarding/tests/ip_forwarding_test.py +++ b/google_compute_engine/networking/ip_forwarding/tests/ip_forwarding_test.py @@ -103,15 +103,16 @@ class IpForwardingTest(unittest.TestCase): self.mock_ip_forwarding_utils.ParseForwardedIps.return_value = desired self.mock_ip_forwarding_utils.GetForwardedIps.return_value = configured forwarded_ips = 'forwarded ips' + interface_ip = 'interface ip' interface = 'interface' expected_add = ['d'] expected_remove = ['a', 'b'] ip_forwarding.IpForwarding.HandleForwardedIps( - self.mock_setup, interface, forwarded_ips) + self.mock_setup, interface, forwarded_ips, interface_ip) expected_calls = [ mock.call.forwarding.ParseForwardedIps(forwarded_ips), - mock.call.forwarding.GetForwardedIps(interface), + mock.call.forwarding.GetForwardedIps(interface, interface_ip), mock.call.setup._LogForwardedIpChanges( configured, desired, expected_add, expected_remove, interface), mock.call.setup._AddForwardedIps(expected_add, interface), diff --git a/google_compute_engine/networking/ip_forwarding/tests/ip_forwarding_utils_test.py b/google_compute_engine/networking/ip_forwarding/tests/ip_forwarding_utils_test.py deleted file mode 100644 index 8a7f573..0000000 --- a/google_compute_engine/networking/ip_forwarding/tests/ip_forwarding_utils_test.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/python -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unittest for ip_forwarding_utils.py module.""" - -from google_compute_engine.networking.ip_forwarding import ip_forwarding_utils -from google_compute_engine.test_compat import mock -from google_compute_engine.test_compat import unittest - - -def _CreateMockProcess(returncode, stdout, stderr): - mock_process = mock.Mock() - mock_process.returncode = returncode - mock_process.communicate.return_value = (stdout, stderr) - return mock_process - - -class IpForwardingUtilsTest(unittest.TestCase): - - def setUp(self): - self.mock_logger = mock.Mock() - self.options = {'hello': 'world'} - with mock.patch( - 'google_compute_engine.networking.ip_forwarding.ip_forwarding_utils' - '.subprocess') as mock_subprocess: - mock_subprocess.Popen.return_value = _CreateMockProcess( - 0, b'out', b'') - self.mock_utils = ip_forwarding_utils.IpForwardingUtils( - self.mock_logger) - self.mock_utils.proto_id = 'proto' - - def testCreateRouteOptions(self): - # Default options. - expected_options = { - 'proto': 'proto', - 'scope': 'host', - } - self.assertEqual(self.mock_utils._CreateRouteOptions(), expected_options) - - # Update dictionary when arguments are specified. - expected_options = { - 'proto': 'proto', - 'scope': 'host', - 'num': 1, - 'string': 'hello world', - } - self.assertEqual( - self.mock_utils._CreateRouteOptions(num=1, string='hello world'), - expected_options) - - # Update the default options. - expected_options = { - 'proto': 'test 1', - 'scope': 'test 2', - } - self.assertEqual( - self.mock_utils._CreateRouteOptions(proto='test 1', scope='test 2'), - expected_options) - - @mock.patch('google_compute_engine.networking.ip_forwarding.ip_forwarding_utils.subprocess') - def testRunIpRoute(self, mock_subprocess): - mock_process = _CreateMockProcess(0, b'out', b'') - mock_subprocess.Popen.return_value = mock_process - args = ['foo', 'bar'] - options = {'one': 'two'} - - self.assertEqual( - self.mock_utils._RunIpRoute(args=args, options=options), 'out') - command = ['ip', 'route', 'foo', 'bar', 'one', 'two'] - mock_subprocess.Popen.assert_called_once_with( - command, stdout=mock_subprocess.PIPE, stderr=mock_subprocess.PIPE) - mock_process.communicate.assert_called_once_with() - self.mock_logger.warning.assert_not_called() - - @mock.patch('google_compute_engine.networking.ip_forwarding.ip_forwarding_utils.subprocess') - def testRunIpRouteReturnCode(self, mock_subprocess): - mock_process = _CreateMockProcess(1, b'out', b'error\n') - mock_subprocess.Popen.return_value = mock_process - - self.assertEqual( - self.mock_utils._RunIpRoute(args=['foo', 'bar'], options=self.options), - '') - command = ['ip', 'route', 'foo', 'bar', 'hello', 'world'] - self.mock_logger.warning.assert_called_once_with( - mock.ANY, command, b'error') - - @mock.patch('google_compute_engine.networking.ip_forwarding.ip_forwarding_utils.subprocess') - def testRunIpRouteException(self, mock_subprocess): - mock_subprocess.Popen.side_effect = OSError('Test Error') - - self.assertEqual( - self.mock_utils._RunIpRoute(args=['foo', 'bar'], options=self.options), - '') - command = ['ip', 'route', 'foo', 'bar', 'hello', 'world'] - self.mock_logger.warning.assert_called_once_with( - mock.ANY, command, 'Test Error') - - def testParseForwardedIps(self): - self.assertEqual(self.mock_utils.ParseForwardedIps(None), []) - self.assertEqual(self.mock_utils.ParseForwardedIps([]), []) - self.assertEqual(self.mock_utils.ParseForwardedIps([None]), []) - self.assertEqual(self.mock_utils.ParseForwardedIps(['invalid']), []) - self.assertEqual(self.mock_utils.ParseForwardedIps(['1a1a1a1']), []) - self.assertEqual(self.mock_utils.ParseForwardedIps(['1.1.1.1.1']), []) - self.assertEqual(self.mock_utils.ParseForwardedIps(['1111.1.1.1']), []) - self.assertEqual(self.mock_utils.ParseForwardedIps(['1.1.1.1111']), []) - expected_calls = [ - mock.call.warning(mock.ANY, None), - mock.call.warning(mock.ANY, 'invalid'), - mock.call.warning(mock.ANY, '1a1a1a1'), - mock.call.warning(mock.ANY, '1.1.1.1.1'), - mock.call.warning(mock.ANY, '1111.1.1.1'), - mock.call.warning(mock.ANY, '1.1.1.1111'), - ] - self.assertEqual(self.mock_logger.mock_calls, expected_calls) - - def testParseForwardedIpsComplex(self): - forwarded_ips = { - '{{}}\n\"hello\"\n!@#$%^&*()\n\n': False, - '1111.1.1.1': False, - '1.1.1.1': True, - 'hello': False, - '123.123.123.123': True, - '1.1.1.': False, - '1.1.1.a': False, - None: False, - '1.0.0.0': True, - '1.1.1.1/1': True, - '1.1.1.1/11': True, - '123.123.123.123/1': True, - '123.123.123.123/123': False, - '123.123.123.123/a': False, - '123.123.123.123/': False, - } - input_ips = forwarded_ips.keys() - valid_ips = [ip for ip, valid in forwarded_ips.items() if valid] - invalid_ips = [ip for ip, valid in forwarded_ips.items() if not valid] - - self.assertEqual(self.mock_utils.ParseForwardedIps(input_ips), valid_ips) - expected_calls = [mock.call.warning(mock.ANY, ip) for ip in invalid_ips] - self.assertEqual(self.mock_logger.mock_calls, expected_calls) - - def testParseForwardedIpsSubnet(self): - forwarded_ips = { - '1.1.1.1': '1.1.1.1', - '1.1.1.1/32': '1.1.1.1', - '1.1.1.1/1': '1.1.1.1/1', - '1.1.1.1/10': '1.1.1.1/10', - '1.1.1.1/24': '1.1.1.1/24', - } - for ip, value in forwarded_ips.items(): - self.assertEqual(self.mock_utils.ParseForwardedIps([ip]), [value]) - - def testGetForwardedIps(self): - mock_options = mock.Mock() - mock_options.return_value = self.options - mock_run = mock.Mock() - mock_run.return_value = 'a\n b \nlocal c' - mock_parse = mock.Mock() - mock_parse.return_value = ['Test'] - self.mock_utils._CreateRouteOptions = mock_options - self.mock_utils._RunIpRoute = mock_run - self.mock_utils.ParseForwardedIps = mock_parse - - self.assertEqual(self.mock_utils.GetForwardedIps('interface'), ['Test']) - mock_options.assert_called_once_with(dev='interface') - mock_run.assert_called_once_with( - args=['ls', 'table', 'local', 'type', 'local'], options=self.options) - mock_parse.assert_called_once_with(['a', 'b', 'c']) - - def testAddForwardedIp(self): - mock_options = mock.Mock() - mock_options.return_value = self.options - mock_run = mock.Mock() - self.mock_utils._CreateRouteOptions = mock_options - self.mock_utils._RunIpRoute = mock_run - - self.mock_utils.AddForwardedIp('1.1.1.1', 'interface') - mock_options.assert_called_once_with(dev='interface') - mock_run.assert_called_once_with( - args=['add', 'to', 'local', '1.1.1.1/32'], options=self.options) - - def testAddIpAlias(self): - mock_options = mock.Mock() - mock_options.return_value = self.options - mock_run = mock.Mock() - self.mock_utils._CreateRouteOptions = mock_options - self.mock_utils._RunIpRoute = mock_run - - self.mock_utils.AddForwardedIp('1.1.1.1/24', 'interface') - mock_options.assert_called_once_with(dev='interface') - mock_run.assert_called_once_with( - args=['add', 'to', 'local', '1.1.1.1/24'], options=self.options) - - def testRemoveForwardedIp(self): - mock_options = mock.Mock() - mock_options.return_value = self.options - mock_run = mock.Mock() - self.mock_utils._CreateRouteOptions = mock_options - self.mock_utils._RunIpRoute = mock_run - - self.mock_utils.RemoveForwardedIp('1.1.1.1', 'interface') - mock_options.assert_called_once_with(dev='interface') - mock_run.assert_called_once_with( - args=['delete', 'to', 'local', '1.1.1.1/32'], options=self.options) - - def testRemoveAliasIp(self): - mock_options = mock.Mock() - mock_options.return_value = self.options - mock_run = mock.Mock() - self.mock_utils._CreateRouteOptions = mock_options - self.mock_utils._RunIpRoute = mock_run - - self.mock_utils.RemoveForwardedIp('1.1.1.1/24', 'interface') - mock_options.assert_called_once_with(dev='interface') - mock_run.assert_called_once_with( - args=['delete', 'to', 'local', '1.1.1.1/24'], options=self.options) - - -if __name__ == '__main__': - unittest.main() diff --git a/google_compute_engine/networking/network_daemon.py b/google_compute_engine/networking/network_daemon.py index 81b26df..95b190b 100644 --- a/google_compute_engine/networking/network_daemon.py +++ b/google_compute_engine/networking/network_daemon.py @@ -91,12 +91,12 @@ class NetworkDaemon(object): if self.network_setup_enabled: self.network_setup.EnableNetworkInterfaces( - [interface.name for interface in network_interfaces]) + [interface.name for interface in network_interfaces[1:]]) for interface in network_interfaces: if self.ip_forwarding_enabled: self.ip_forwarding.HandleForwardedIps( - interface.name, interface.forwarded_ips) + interface.name, interface.forwarded_ips, interface.ip) def _ExtractInterfaceMetadata(self, metadata): """Extracts network interface metadata. @@ -119,7 +119,7 @@ class NetworkDaemon(object): if self.target_instance_ips: ip_addresses.extend(network_interface.get('targetInstanceIps', [])) interfaces.append(NetworkDaemon.NetworkInterface( - interface, ip_addresses)) + interface, ip_addresses, network_interface.get('ip', []))) else: message = 'Network interface not found for MAC address: %s.' self.logger.warning(message, mac_address) @@ -128,9 +128,10 @@ class NetworkDaemon(object): class NetworkInterface(object): """Network interface information extracted from metadata.""" - def __init__(self, name, forwarded_ips=None): + def __init__(self, name, forwarded_ips=None, ip=None): self.name = name self.forwarded_ips = forwarded_ips + self.ip = ip def main(): diff --git a/google_compute_engine/networking/network_setup/network_setup.py b/google_compute_engine/networking/network_setup/network_setup.py index 1408809..9166722 100755 --- a/google_compute_engine/networking/network_setup/network_setup.py +++ b/google_compute_engine/networking/network_setup/network_setup.py @@ -56,8 +56,6 @@ class NetworkSetup(object): self.logger.info('Ethernet interfaces: %s.', interfaces) self.interfaces = set(interfaces) - if len(interfaces) <= 1: - return if self.dhcp_command: try: diff --git a/google_compute_engine/networking/network_setup/tests/network_setup_test.py b/google_compute_engine/networking/network_setup/tests/network_setup_test.py index a148b78..644510a 100644 --- a/google_compute_engine/networking/network_setup/tests/network_setup_test.py +++ b/google_compute_engine/networking/network_setup/tests/network_setup_test.py @@ -72,29 +72,25 @@ class NetworkSetupTest(unittest.TestCase): network_setup.NetworkSetup.EnableNetworkInterfaces( self.mock_setup, ['A', 'B', 'C']) self.assertEqual(self.mock_setup.interfaces, set(['A', 'B', 'C'])) - # A single interface is enabled by default. - network_setup.NetworkSetup.EnableNetworkInterfaces(self.mock_setup, ['D']) - self.assertEqual(self.mock_setup.interfaces, set(['D'])) # Run a user supplied command successfully. self.mock_setup.dhcp_command = 'success' network_setup.NetworkSetup.EnableNetworkInterfaces( - self.mock_setup, ['E', 'F']) - self.assertEqual(self.mock_setup.interfaces, set(['E', 'F'])) + self.mock_setup, ['D', 'E']) + self.assertEqual(self.mock_setup.interfaces, set(['D', 'E'])) # Run a user supplied command and logger error messages. self.mock_setup.dhcp_command = 'failure' network_setup.NetworkSetup.EnableNetworkInterfaces( - self.mock_setup, ['G', 'H']) - self.assertEqual(self.mock_setup.interfaces, set(['G', 'H'])) + self.mock_setup, ['F', 'G']) + self.assertEqual(self.mock_setup.interfaces, set(['F', 'G'])) expected_calls = [ mock.call.logger.info(mock.ANY, ['A', 'B']), mock.call.enable(['A', 'B'], mock.ANY, dhclient_script='/bin/script'), mock.call.logger.info(mock.ANY, ['A', 'B', 'C']), mock.call.enable( ['A', 'B', 'C'], mock.ANY, dhclient_script='/bin/script'), - mock.call.logger.info(mock.ANY, ['D']), - mock.call.logger.info(mock.ANY, ['E', 'F']), + mock.call.logger.info(mock.ANY, ['D', 'E']), mock.call.call(['success']), - mock.call.logger.info(mock.ANY, ['G', 'H']), + mock.call.logger.info(mock.ANY, ['F', 'G']), mock.call.call(['failure']), mock.call.logger.warning(mock.ANY), ] diff --git a/google_compute_engine/networking/tests/network_daemon_test.py b/google_compute_engine/networking/tests/network_daemon_test.py index 6a6f133..acf0371 100644 --- a/google_compute_engine/networking/tests/network_daemon_test.py +++ b/google_compute_engine/networking/tests/network_daemon_test.py @@ -148,9 +148,9 @@ class NetworkDaemonTest(unittest.TestCase): self.mock_setup, result) expected_calls = [ mock.call.setup._ExtractInterfaceMetadata(result), - mock.call.network_setup.EnableNetworkInterfaces(['a', 'b']), - mock.call.forwarding.HandleForwardedIps('a', None), - mock.call.forwarding.HandleForwardedIps('b', None), + mock.call.network_setup.EnableNetworkInterfaces(['b']), + mock.call.forwarding.HandleForwardedIps('a', None, None), + mock.call.forwarding.HandleForwardedIps('b', None, None), ] self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/google_compute_engine_init/systemd/google-network-daemon.service b/google_compute_engine_init/systemd/google-network-daemon.service index c5da336..74bf82e 100644 --- a/google_compute_engine_init/systemd/google-network-daemon.service +++ b/google_compute_engine_init/systemd/google-network-daemon.service @@ -4,6 +4,7 @@ After=local-fs.target network-online.target network.target rsyslog.service After=google-instance-setup.service Wants=local-fs.target network-online.target network.target Requires=network.target +PartOf=network.service [Service] Type=simple diff --git a/google_compute_engine_oslogin/Makefile b/google_compute_engine_oslogin/Makefile index 5f78099..4af7ed1 100644 --- a/google_compute_engine_oslogin/Makefile +++ b/google_compute_engine_oslogin/Makefile @@ -4,7 +4,7 @@ BASENAME = oslogin NAME = google-compute-engine-$(BASENAME) MAJOR = 1 MINOR = 3 -REVISION = 0 +REVISION = 1 LIBNSS_CACHE_OSLOGIN = libnss_cache_$(BASENAME) LIBNSS_CACHE_OSLOGIN_NAME = libnss_cache_$(NAME)-$(MAJOR).$(MINOR).$(REVISION).so @@ -96,7 +96,7 @@ $(NSS): $(NSS_LIBRARY_SOURCE) $(UTILS) $(NSS_SRC) $(UTILS) $(LIBS) $(NSS_CACHE_BIN): $(NSS_CACHE_SRC) $(UTILS_SRC) - $(CXX) $(LDFLAGS) $(INCLUDE_FLAGS) -o $(NSS_CACHE_BIN) $(NSS_CACHE_SRC) $(UTILS_SRC) $(LIBS) + $(CXX) $(CXXFLAGS) $(LDFLAGS) $(INCLUDE_FLAGS) -o $(NSS_CACHE_BIN) $(NSS_CACHE_SRC) $(UTILS_SRC) $(LIBS) $(LIBNSS_CACHE_OSLOGIN_NAME): $(LIBNSS_CACHE_OBJ) $(LIBNSS_COMPAT_OBJ) $(CXX) $(LIBNSS_SO_FLAGS) -o $(LIBNSS_CACHE_OSLOGIN_NAME) $(LIBNSS_CACHE_OBJ) $(LIBNSS_COMPAT_OBJ) @@ -122,7 +122,7 @@ $(PAM_ADMIN_OBJ): $(PAM_ADMIN_SRC) $(CXX) $(CXXFLAGS) -c $(PAM_ADMIN_SRC) -o $(PAM_ADMIN_OBJ) $(AUTHKEYS_BIN): $(AUTHKEYS_SRC) $(UTILS_SRC) - $(CXX) $(LDFLAGS) $(INCLUDE_FLAGS) -o $(AUTHKEYS_BIN) $(AUTHKEYS_SRC) $(UTILS_SRC) $(LIBS) + $(CXX) $(CXXFLAGS) $(LDFLAGS) $(INCLUDE_FLAGS) -o $(AUTHKEYS_BIN) $(AUTHKEYS_SRC) $(UTILS_SRC) $(LIBS) $(UTILS): $(UTILS_SRC) $(CXX) $(CXXFLAGS) $(INCLUDE_FLAGS) -c $(UTILS_SRC) -o $(UTILS) diff --git a/google_compute_engine_oslogin/README.md b/google_compute_engine_oslogin/README.md index 4e9a19d..ce15018 100644 --- a/google_compute_engine_oslogin/README.md +++ b/google_compute_engine_oslogin/README.md @@ -232,10 +232,13 @@ places: * `Makefile` Update the MAJOR, MINOR, and REVISION variables. * `packaging/debian8/changelog` Add a new entry with the new version. * `packaging/debian9/changelog` Add a new entry with the new version. +* `packaging/debian10/changelog` Add a new entry with the new version. * `packaging/debian8/google-compute-engine-oslogin.links` Update the libnss version string. * `packaging/debian9/google-compute-engine-oslogin.links` Update the libnss version string. +* `packaging/debian10/google-compute-engine-oslogin.links` Update the libnss + version string. * `packaging/rpmbuild/SPECS/google-compute-engine-oslogin.spec` Update the Version field. * `packaging/setup_deb.sh` Update VERSION variable. diff --git a/google_compute_engine_oslogin/packaging/debian10/changelog b/google_compute_engine_oslogin/packaging/debian10/changelog new file mode 100644 index 0000000..775dcba --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/changelog @@ -0,0 +1,12 @@ +google-compute-engine-oslogin (1.3.1-1+deb10) unstable; urgency=low + + * Add user name validation to pam modules. + * Return false on failed final load. + + -- Google Cloud Team <gc-team@google.com> Wed, 05 Sep 2018 12:00:00 -0700 + +google-compute-engine-oslogin (1.3.0-1+deb10) unstable; urgency=low + + * Include libnss cache as part of the OS Login package. + + -- Google Cloud Team <gc-team@google.com> Tue, 01 May 2018 12:00:00 -0700 diff --git a/google_compute_engine_oslogin/packaging/debian10/compat b/google_compute_engine_oslogin/packaging/debian10/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/compat @@ -0,0 +1 @@ +9 diff --git a/google_compute_engine_oslogin/packaging/debian10/control b/google_compute_engine_oslogin/packaging/debian10/control new file mode 100644 index 0000000..e7ecc24 --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/control @@ -0,0 +1,13 @@ +Source: google-compute-engine-oslogin +Maintainer: Google Compute Engine <gc-team@google.com> +Section: misc +Priority: optional +Standards-Version: 3.9.6.1 +Build-Depends: debhelper (>= 9), libcurl4-openssl-dev, libjson-c-dev | libjson0-dev, libpam-dev + +Package: google-compute-engine-oslogin +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends} +Description: Google Compute Engine OS Login + Contains libraries, applications and configurations for using OS Login + on Google Compute Engine Virtual Machine Instances. diff --git a/google_compute_engine_oslogin/packaging/debian10/copyright b/google_compute_engine_oslogin/packaging/debian10/copyright new file mode 100644 index 0000000..f1c5775 --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/copyright @@ -0,0 +1,27 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: google-compute-engine-oslogin +Upstream-Contact: gc-team@google.com + +Files: * +Copyright: Copyright 2017 Google Inc. +License: Apache-2.0 + +Files: debian/* +Copyright: Copyright 2017 Google Inc. +License: Apache-2.0 + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the complete text of the Apache version 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/google_compute_engine_oslogin/packaging/debian10/google-compute-engine-oslogin.links b/google_compute_engine_oslogin/packaging/debian10/google-compute-engine-oslogin.links new file mode 100644 index 0000000..0fb2688 --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/google-compute-engine-oslogin.links @@ -0,0 +1,2 @@ +/lib/libnss_google-compute-engine-oslogin-1.3.1.so /lib/libnss_oslogin.so.2 +/lib/libnss_cache_google-compute-engine-oslogin-1.3.1.so /lib/libnss_cache_oslogin.so.2 diff --git a/google_compute_engine_oslogin/packaging/debian10/rules b/google_compute_engine_oslogin/packaging/debian10/rules new file mode 100755 index 0000000..65b1f4b --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f +%: + dh $@ + +override_dh_auto_install: + dh_auto_install -- PAM_INSTALL_PATH=/lib/x86_64-linux-gnu/security BIN_INSTALL_PATH=/usr/bin diff --git a/google_compute_engine_oslogin/packaging/debian10/shlibs.local b/google_compute_engine_oslogin/packaging/debian10/shlibs.local new file mode 100644 index 0000000..08af74d --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/shlibs.local @@ -0,0 +1 @@ +libcurl 4 libcurl4 diff --git a/google_compute_engine_oslogin/packaging/debian10/source/format b/google_compute_engine_oslogin/packaging/debian10/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/google_compute_engine_oslogin/packaging/debian10/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/google_compute_engine_oslogin/packaging/debian9/changelog b/google_compute_engine_oslogin/packaging/debian9/changelog index cb30765..81ad860 100644 --- a/google_compute_engine_oslogin/packaging/debian9/changelog +++ b/google_compute_engine_oslogin/packaging/debian9/changelog @@ -1,3 +1,10 @@ +google-compute-engine-oslogin (1.3.1-1+deb9) unstable; urgency=low + + * Add user name validation to pam modules. + * Return false on failed final load. + + -- Google Cloud Team <gc-team@google.com> Wed, 05 Sep 2018 12:00:00 -0700 + google-compute-engine-oslogin (1.3.0-1+deb9) unstable; urgency=low * Include libnss cache as part of the OS Login package. diff --git a/google_compute_engine_oslogin/packaging/debian9/google-compute-engine-oslogin.links b/google_compute_engine_oslogin/packaging/debian9/google-compute-engine-oslogin.links index 9eee787..0fb2688 100644 --- a/google_compute_engine_oslogin/packaging/debian9/google-compute-engine-oslogin.links +++ b/google_compute_engine_oslogin/packaging/debian9/google-compute-engine-oslogin.links @@ -1,2 +1,2 @@ -/lib/libnss_google-compute-engine-oslogin-1.3.0.so /lib/libnss_oslogin.so.2 -/lib/libnss_cache_google-compute-engine-oslogin-1.3.0.so /lib/libnss_cache_oslogin.so.2 +/lib/libnss_google-compute-engine-oslogin-1.3.1.so /lib/libnss_oslogin.so.2 +/lib/libnss_cache_google-compute-engine-oslogin-1.3.1.so /lib/libnss_cache_oslogin.so.2 diff --git a/google_compute_engine_oslogin/packaging/rpmbuild/SPECS/google-compute-engine-oslogin.spec b/google_compute_engine_oslogin/packaging/rpmbuild/SPECS/google-compute-engine-oslogin.spec index ba326a2..1edaa1e 100644 --- a/google_compute_engine_oslogin/packaging/rpmbuild/SPECS/google-compute-engine-oslogin.spec +++ b/google_compute_engine_oslogin/packaging/rpmbuild/SPECS/google-compute-engine-oslogin.spec @@ -18,19 +18,21 @@ %endif Name: google-compute-engine-oslogin -Version: 1.3.0 +Version: 1.3.1 Release: 1%{?dist} Summary: OS Login Functionality for Google Compute Engine License: ASL 2.0 Source0: %{name}_%{version}.orig.tar.gz +BuildRequires: boost-devel BuildRequires: gcc-c++ BuildRequires: make BuildRequires: libcurl BuildRequires: json-c BuildRequires: pam-devel BuildRequires: policycoreutils-python +Requires: boost-regex Requires: policycoreutils-python %define pam_install_path /%{_lib}/security @@ -43,7 +45,7 @@ for Google Compute Engine. %setup %build -make %{?_smp_mflags} LIBS="-lcurl -ljson-c" +make %{?_smp_mflags} LIBS="-lcurl -ljson-c -lboost_regex" %install rm -rf %{buildroot} diff --git a/google_compute_engine_oslogin/packaging/setup_deb.sh b/google_compute_engine_oslogin/packaging/setup_deb.sh index 6cdd8d1..4d0a1d0 100755 --- a/google_compute_engine_oslogin/packaging/setup_deb.sh +++ b/google_compute_engine_oslogin/packaging/setup_deb.sh @@ -20,7 +20,7 @@ # Run from the top of the source directory. NAME="google-compute-engine-oslogin" -VERSION="1.3.0" +VERSION="1.3.1" working_dir=${PWD} diff --git a/google_compute_engine_oslogin/packaging/setup_rpm.sh b/google_compute_engine_oslogin/packaging/setup_rpm.sh index b50c3fc..c00ff4c 100755 --- a/google_compute_engine_oslogin/packaging/setup_rpm.sh +++ b/google_compute_engine_oslogin/packaging/setup_rpm.sh @@ -20,13 +20,13 @@ # Run from the top of the source directory. NAME="google-compute-engine-oslogin" -VERSION="1.3.0" +VERSION="1.3.1" working_dir=${PWD} rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} # Build dependencies. -sudo yum -y install make gcc-c++ libcurl-devel json-c json-c-devel pam-devel policycoreutils-python +sudo yum -y install make gcc-c++ libcurl-devel json-c json-c-devel pam-devel policycoreutils-python boost-devel # .rpm creation tools. sudo yum -y install rpmdevtools diff --git a/google_compute_engine_oslogin/pam_module/pam_oslogin_admin.cc b/google_compute_engine_oslogin/pam_module/pam_oslogin_admin.cc index 04d0808..73fe133 100644 --- a/google_compute_engine_oslogin/pam_module/pam_oslogin_admin.cc +++ b/google_compute_engine_oslogin/pam_module/pam_oslogin_admin.cc @@ -34,6 +34,7 @@ using oslogin_utils::HttpGet; using oslogin_utils::ParseJsonToAuthorizeResponse; using oslogin_utils::ParseJsonToEmail; using oslogin_utils::UrlEncode; +using oslogin_utils::ValidateUserName; using oslogin_utils::kMetadataServerUrl; static const char kSudoersDir[] = "/var/google-sudoers.d/"; @@ -51,6 +52,10 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags, int argc, return pam_result; } string str_user_name(user_name); + if (!ValidateUserName(user_name)) { + // If the user name is not a valid oslogin user, don't bother continuing. + return PAM_SUCCESS; + } std::stringstream url; url << kMetadataServerUrl diff --git a/google_compute_engine_oslogin/pam_module/pam_oslogin_login.cc b/google_compute_engine_oslogin/pam_module/pam_oslogin_login.cc index 9e708f4..377ca2d 100644 --- a/google_compute_engine_oslogin/pam_module/pam_oslogin_login.cc +++ b/google_compute_engine_oslogin/pam_module/pam_oslogin_login.cc @@ -34,6 +34,7 @@ using oslogin_utils::HttpGet; using oslogin_utils::ParseJsonToAuthorizeResponse; using oslogin_utils::ParseJsonToEmail; using oslogin_utils::UrlEncode; +using oslogin_utils::ValidateUserName; using oslogin_utils::kMetadataServerUrl; static const char kUsersDir[] = "/var/google-users.d/"; @@ -49,6 +50,10 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags, int argc, return pam_result; } string str_user_name(user_name); + if (!ValidateUserName(user_name)) { + // If the user name is not a valid oslogin user, don't bother continuing. + return PAM_SUCCESS; + } string users_filename = kUsersDir; users_filename.append(user_name); struct stat buffer; diff --git a/google_compute_engine_oslogin/utils/oslogin_utils.cc b/google_compute_engine_oslogin/utils/oslogin_utils.cc index 06e85f0..4e0cb79 100644 --- a/google_compute_engine_oslogin/utils/oslogin_utils.cc +++ b/google_compute_engine_oslogin/utils/oslogin_utils.cc @@ -23,6 +23,19 @@ #include <iostream> #include <sstream> +#ifdef __GNUC__ +#if __GNUC__ > 4 || \ + (__GNUC__ == 4 && (__GNUC_MINOR__ > 9 || \ + (__GNUC_MINOR__ == 9 && \ + __GNUC_PATCHLEVEL__ > 0))) +#include <regex> +#define Regex std +#else +#include <boost/regex.hpp> +#define Regex boost +#endif +#endif + #include "oslogin_utils.h" using std::string; @@ -30,6 +43,9 @@ using std::string; // Maximum number of retries for HTTP requests. const int kMaxRetries = 1; +// Regex for validating user names. +const char kUserNameRegex[] = "^[a-zA-Z0-9._][a-zA-Z0-9._-]{0,31}$"; + namespace oslogin_utils { BufferManager::BufferManager(char* buf, size_t buflen) @@ -154,9 +170,12 @@ bool NssCache::NssGetpwentHelper(BufferManager* buf, struct passwd* result, string response; long http_code = 0; if (!HttpGet(url.str(), &response, &http_code) || http_code != 200 || - response.empty() || - (!LoadJsonArrayToCache(response) && !on_last_page_)) { - *errnop = ENOENT; + response.empty() || !LoadJsonArrayToCache(response)) { + // It is possible this to be true after LoadJsonArrayToCache(), so we + // must check it again. + if(!OnLastPage()) { + *errnop = ENOENT; + } return false; } } @@ -219,6 +238,11 @@ bool HttpGet(const string& url, string* response, long* http_code) { return true; } +bool ValidateUserName(const string& user_name) { + Regex::regex r(kUserNameRegex); + return Regex::regex_match(user_name, r); +} + string UrlEncode(const string& param) { CURL* curl = curl_easy_init(); char* encoded = curl_easy_escape(curl, param.c_str(), param.length()); diff --git a/google_compute_engine_oslogin/utils/oslogin_utils.h b/google_compute_engine_oslogin/utils/oslogin_utils.h index 81c13a6..958edad 100644 --- a/google_compute_engine_oslogin/utils/oslogin_utils.h +++ b/google_compute_engine_oslogin/utils/oslogin_utils.h @@ -139,6 +139,9 @@ OnCurlWrite(void* buf, size_t size, size_t nmemb, void* userp); // stored in response, and the HTTP response code will be stored in http_code. bool HttpGet(const string& url, string* response, long* http_code); +// Returns whether user_name is a valid OsLogin user name. +bool ValidateUserName(const string& user_name); + // URL encodes the given parameter. Returns the encoded parameter. std::string UrlEncode(const string& param); diff --git a/google_compute_engine_oslogin/utils/oslogin_utils_test.cc b/google_compute_engine_oslogin/utils/oslogin_utils_test.cc index 4653639..c1247c8 100644 --- a/google_compute_engine_oslogin/utils/oslogin_utils_test.cc +++ b/google_compute_engine_oslogin/utils/oslogin_utils_test.cc @@ -152,7 +152,7 @@ TEST(NssCacheTest, TestLoadJsonFinalResponse) { string response = "{\"nextPageToken\": \"0\"}"; - ASSERT_TRUE(nss_cache.LoadJsonArrayToCache(response)); + ASSERT_FALSE(nss_cache.LoadJsonArrayToCache(response)); ASSERT_STREQ(nss_cache.GetPageToken().c_str(), ""); size_t buflen = 500; @@ -164,6 +164,7 @@ TEST(NssCacheTest, TestLoadJsonFinalResponse) { // Verify that there are no more users stored. ASSERT_FALSE(nss_cache.HasNextPasswd()); + ASSERT_TRUE(nss_cache.OnLastPage()); ASSERT_FALSE(nss_cache.GetNextPasswd(&buf, &result, &test_errno)); EXPECT_EQ(test_errno, ENOENT); } @@ -428,6 +429,41 @@ TEST(ParseJsonAuthorizeSuccess, SuccessfullyAuthorized) { ASSERT_TRUE(ParseJsonToAuthorizeResponse(response)); } +TEST(ValidateUserNameTest, ValidateValidUserNames) { + string cases[] = { + "user", + "_", + ".", + ".abc_", + "_abc-", + "ABC", + "A_.-", + "ausernamethirtytwocharacterslong" + }; + for (auto test_user : cases) { + ASSERT_TRUE(ValidateUserName(test_user)); + } +} + +TEST(ValidateUserNameTest, ValidateInvalidUserNames) { + string cases[] = { + "", + "!#$%^", + "-abc", + "#abc", + "^abc", + "abc*xyz", + "abc xyz", + "xyz*", + "xyz$", + "usernamethirtythreecharacterslong", + "../../etc/shadow", + }; + for (auto test_user : cases) { + ASSERT_FALSE(ValidateUserName(test_user)); + } +} + } // namespace oslogin_utils int main(int argc, char **argv) { diff --git a/google_compute_engine_oslogin/utils/run_tests.sh b/google_compute_engine_oslogin/utils/run_tests.sh index b68248f..83adcdc 100755 --- a/google_compute_engine_oslogin/utils/run_tests.sh +++ b/google_compute_engine_oslogin/utils/run_tests.sh @@ -14,6 +14,6 @@ # limitations under the License. # Unit tests require gtest to be installed. -g++ -o test_runner oslogin_utils_test.cc oslogin_utils.cc -I/usr/include/json-c -lcurl -ljson -lgtest -lpthread +g++ -o test_runner oslogin_utils_test.cc oslogin_utils.cc -I/usr/include/json-c -lcurl -ljson-c -lgtest -lpthread ./test_runner rm ./test_runner @@ -36,7 +36,7 @@ setuptools.setup( packages=setuptools.find_packages(), scripts=glob.glob('scripts/*'), url='https://github.com/GoogleCloudPlatform/compute-image-packages', - version='2.8.3', + version='2.8.4', # Entry points create scripts in /usr/bin that call a function. entry_points={ 'console_scripts': [ @@ -64,6 +64,7 @@ setuptools.setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Installation/Setup', diff --git a/specs/google-compute-engine.spec b/specs/google-compute-engine.spec index 026c3a2..9d9d6bf 100644 --- a/specs/google-compute-engine.spec +++ b/specs/google-compute-engine.spec @@ -18,7 +18,7 @@ %endif Name: google-compute-engine -Version: 2.8.3 +Version: 2.8.4 Release: 1%{?dist} Summary: Google Compute Engine guest environment. License: ASL 2.0 @@ -33,7 +33,6 @@ BuildRequires: systemd Requires: curl Requires: google-compute-engine-oslogin -Requires: ntp Requires: python-google-compute-engine = %{version} Requires: python-setuptools Requires: rsyslog diff --git a/specs/python-google-compute-engine.spec b/specs/python-google-compute-engine.spec index 6858c6e..d5a6fb8 100644 --- a/specs/python-google-compute-engine.spec +++ b/specs/python-google-compute-engine.spec @@ -18,7 +18,7 @@ %endif Name: python-google-compute-engine -Version: 2.8.3 +Version: 2.8.4 Release: 1%{?dist} Summary: Google Compute Engine python library License: ASL 2.0 @@ -1,9 +1,9 @@ [tox] -envlist = py26,py27,py32,py33,py34,py35,pypy,pypy3 +envlist = py26,py27,py32,py33,py34,py35,py36,py37,pypy,pypy3 [testenv] deps = - py{35,36}: distro + py{35,py36,py37}: distro setuptools>=20 pytest pytest-cov @@ -64,4 +64,4 @@ exclude = # See https://github.com/ryanhiebert/tox-travis#advanced-configuration [travis] python = - 2.7: py27, lint + 3.6: py36, lint |