diff options
author | Zach Marano <zmarano@google.com> | 2019-12-10 15:27:59 -0800 |
---|---|---|
committer | Zach Marano <zmarano@google.com> | 2019-12-10 15:27:59 -0800 |
commit | c553111756bf881e376533e4ff0c062ca76c17e5 (patch) | |
tree | 982ad26c0eddbb01db7a14742e35e188efec2d1b | |
parent | 28f4f9e2df19da49d08f0ee31c2b58587763b3c4 (diff) | |
parent | 1fd4dfba815795fbe49ff15942eff8bfa033a188 (diff) | |
download | google-compute-image-packages-c553111756bf881e376533e4ff0c062ca76c17e5.tar.gz |
Merge branch 'development'v20191210
11 files changed, 91 insertions, 27 deletions
diff --git a/packages/python-google-compute-engine/LICENSE b/packages/python-google-compute-engine/LICENSE index 261eeb9..04cb0d7 100644 --- a/packages/python-google-compute-engine/LICENSE +++ b/packages/python-google-compute-engine/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2013 Google Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py index cb1a2a6..efe100b 100755 --- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py +++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py @@ -28,6 +28,7 @@ from google_compute_engine import file_utils from google_compute_engine import logger from google_compute_engine import metadata_watcher from google_compute_engine.boto import boto_config +from google_compute_engine.compat import distro_name from google_compute_engine.compat import urlerror from google_compute_engine.compat import urlrequest from google_compute_engine.instance_setup import instance_config @@ -65,22 +66,38 @@ class InstanceSetup(object): instance_config_metadata = self._GetInstanceConfig() self.instance_config = instance_config.InstanceConfig( logger=self.logger, instance_config_metadata=instance_config_metadata) + if self.instance_config.GetOptionBool('InstanceSetup', 'set_host_keys'): host_key_types = self.instance_config.GetOptionString( 'InstanceSetup', 'host_key_types') self._SetSshHostKeys(host_key_types=host_key_types) + if self.instance_config.GetOptionBool('InstanceSetup', 'set_boto_config'): self._SetupBotoConfig() + + self._DisableOvercommit() + if self.instance_config.GetOptionBool( 'InstanceSetup', 'optimize_local_ssd'): self._RunScript('google_optimize_local_ssd') + if self.instance_config.GetOptionBool('InstanceSetup', 'set_multiqueue'): self._RunScript('google_set_multiqueue') + try: self.instance_config.WriteConfig() except (IOError, OSError) as e: self.logger.warning(str(e)) + def _DisableOvercommit(self, distro=distro_name): + """Disable overcommit accounting on E2 machine types.""" + + # Expected machine type format: + # 'projects/00000000000/machineTypes/n1-standard-1' + machine_type = self.metadata_dict['instance']['machineType'].split('/')[-1] + if machine_type.startswith('e2-') and 'bsd' not in distro: + subprocess.call(['sysctl', 'vm.overcommit_memory=1']) + def _GetInstanceConfig(self): """Get the instance configuration specified in metadata. diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py index a418219..5c6e513 100644 --- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py @@ -46,7 +46,10 @@ class InstanceSetupTest(unittest.TestCase): mock_logger_instance = mock.Mock() mock_logger.Logger.return_value = mock_logger_instance mock_watcher_instance = mock.Mock() - mock_watcher_instance.GetMetadata.return_value = {'hello': 'world'} + mock_watcher_instance.GetMetadata.return_value = { + 'hello': 'world', + 'instance': {'machineType': 'fake'}, + } mock_watcher.MetadataWatcher.return_value = mock_watcher_instance mock_config_instance = mock.Mock() mock_config_instance.GetOptionBool.return_value = True @@ -79,6 +82,7 @@ class InstanceSetupTest(unittest.TestCase): mock.call.config.InstanceConfig().GetOptionBool( 'InstanceSetup', 'set_boto_config'), mock.call.setup._SetupBotoConfig(), + mock.call.setup._DisableOvercommit(), # Setup for local SSD. mock.call.config.InstanceConfig().GetOptionBool( 'InstanceSetup', 'optimize_local_ssd'), @@ -91,7 +95,10 @@ class InstanceSetupTest(unittest.TestCase): mock.call.config.InstanceConfig().WriteConfig(), ] self.assertEqual(mocks.mock_calls, expected_calls) - self.assertEqual(mock_setup.metadata_dict, {'hello': 'world'}) + self.assertEqual(mock_setup.metadata_dict, { + 'hello': 'world', + 'instance':{'machineType': 'fake'}, + }) @mock.patch('google_compute_engine.instance_setup.instance_setup.instance_config') @mock.patch('google_compute_engine.instance_setup.instance_setup.metadata_watcher') @@ -421,6 +428,37 @@ class InstanceSetupTest(unittest.TestCase): instance_setup.InstanceSetup._SetupBotoConfig(self.mock_setup) self.mock_logger.warning.assert_called_once_with('Test Error') + @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess') + def testDisableOvercommitNonE2(self, mock_subprocess): + self.mock_setup.metadata_dict = { + 'instance': { + 'machineType': 'projects/00000000000/machineTypes/n1-standard-1', + } + } + instance_setup.InstanceSetup._DisableOvercommit(self.mock_setup) + mock_subprocess.call.assert_not_called() + + @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess') + def testDisableOvercommitE2(self, mock_subprocess): + self.mock_setup.metadata_dict = { + 'instance': { + 'machineType': 'projects/00000000000/machineTypes/e2-standard-1', + } + } + instance_setup.InstanceSetup._DisableOvercommit(self.mock_setup) + mock_subprocess.call.assert_called_once_with( + ['sysctl', 'vm.overcommit_memory=1']) + + @mock.patch('google_compute_engine.instance_setup.instance_setup.subprocess') + def testDisableOvercommitBSD(self, mock_subprocess): + self.mock_setup.metadata_dict = { + 'instance': { + 'machineType': 'projects/00000000000/machineTypes/e2-standard-1', + } + } + instance_setup.InstanceSetup._DisableOvercommit(self.mock_setup, 'bsd') + mock_subprocess.call.assert_not_called() + if __name__ == '__main__': unittest.main() diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py index 3de425b..7acd26f 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py @@ -97,7 +97,7 @@ class ScriptRetriever(object): if not self.token: response = self.watcher.GetMetadata( - self.token_metadata_key, recursive=False, retry=False) + self.token_metadata_key, recursive=False, retry_limit=3) if not response: self.logger.info( diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py index 0df8830..558b0b0 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py @@ -89,7 +89,7 @@ class ScriptRetrieverTest(unittest.TestCase): # GetMetadata includes a prefix, so remove it. stripped_url = token_url.replace(metadata_prefix, '') mock_get_metadata.assert_called_once_with( - stripped_url, recursive=False, retry=False) + stripped_url, recursive=False, retry_limit=3) self.assertEqual(self.retriever.token, 'foo bar') @@ -119,7 +119,7 @@ class ScriptRetrieverTest(unittest.TestCase): prefix = 'http://metadata.google.internal/computeMetadata/v1/' stripped_url = token_url.replace(prefix, '') mock_get_metadata.assert_called_once_with( - stripped_url, recursive=False, retry=False) + stripped_url, recursive=False, retry_limit=3) mock_download_url.assert_called_once_with(auth_url, self.dest_dir) self.assertIsNone(self.retriever.token) diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py b/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py index 72a12f9..e399839 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py @@ -154,7 +154,7 @@ class MetadataWatcher(object): def _HandleMetadataUpdate( self, metadata_key='', recursive=True, wait=True, timeout=None, - retry=True): + retry_limit=None): """Wait for a successful metadata response. Args: @@ -162,25 +162,25 @@ class MetadataWatcher(object): recursive: bool, True if we should recursively watch for metadata changes. wait: bool, True if we should wait for a metadata change. timeout: int, timeout in seconds for returning metadata output. - retry: bool, True if we should retry on failure. + retry_limit: int or None, limit for number of times to retry on failure. + Retry indefinitely when set to None. Do not retry when set to zero. Returns: json, the deserialized contents of the metadata server. """ exception = None - while True: + while retry_limit is None or retry_limit >= 0: try: return self._GetMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=wait, timeout=timeout) except (httpclient.HTTPException, socket.error, urlerror.URLError) as e: + if retry_limit is not None: + retry_limit -= 1 if not isinstance(e, type(exception)): exception = e self.logger.error('GET request error retrieving metadata. %s.', e) - if retry: - continue - else: - break + time.sleep(1) def WatchMetadata( self, handler, metadata_key='', recursive=True, timeout=None): @@ -202,18 +202,19 @@ class MetadataWatcher(object): self.logger.exception('Exception calling the response handler. %s.', e) def GetMetadata( - self, metadata_key='', recursive=True, timeout=None, retry=True): + self, metadata_key='', recursive=True, timeout=None, retry_limit=None): """Retrieve the contents of metadata server for a metadata key. Args: metadata_key: string, the metadata key to watch for changes. recursive: bool, True if we should recursively watch for metadata changes. timeout: int, timeout in seconds for returning metadata output. - retry: bool, True if we should retry on failure. + retry_limit: int or None, limit for number of times to retry on failure. + Retry indefinitely when set to None. Do not retry when set to zero. Returns: json, the deserialized contents of the metadata server or None if error. """ return self._HandleMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=False, - timeout=timeout, retry=retry) + timeout=timeout, retry_limit=retry_limit) diff --git a/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py b/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py index 1bce509..fe8aad9 100644 --- a/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py @@ -259,12 +259,12 @@ class MetadataWatcherTest(unittest.TestCase): metadata_key = 'instance/id' recursive = False wait = False - retry = True + retry_limit = None self.assertEqual( self.mock_watcher._HandleMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=wait, - timeout=None, retry=retry), + timeout=None, retry_limit=retry_limit), {}) expected_calls = [ mock.call( @@ -282,12 +282,12 @@ class MetadataWatcherTest(unittest.TestCase): metadata_key = 'instance/id' recursive = False wait = False - retry = False + retry_limit = 0 self.assertIsNone( self.mock_watcher._HandleMetadataUpdate( metadata_key=metadata_key, recursive=recursive, wait=wait, - timeout=None, retry=retry)) + timeout=None, retry_limit=retry_limit)) expected_calls = [ mock.call( metadata_key=metadata_key, recursive=recursive, wait=wait, @@ -333,7 +333,8 @@ class MetadataWatcherTest(unittest.TestCase): self.assertEqual(self.mock_watcher.GetMetadata(), {}) mock_response.assert_called_once_with( - metadata_key='', recursive=True, wait=False, timeout=None, retry=True) + metadata_key='', recursive=True, wait=False, timeout=None, + retry_limit=None) self.mock_watcher.logger.exception.assert_not_called() def testGetMetadataArgs(self): @@ -342,15 +343,15 @@ class MetadataWatcherTest(unittest.TestCase): self.mock_watcher._HandleMetadataUpdate = mock_response metadata_key = 'instance/id' recursive = False - retry = False + retry_limit = None response = self.mock_watcher.GetMetadata( metadata_key=metadata_key, recursive=recursive, timeout=60, - retry=retry) + retry_limit=retry_limit) self.assertEqual(response, {}) mock_response.assert_called_once_with( metadata_key=metadata_key, recursive=False, wait=False, timeout=60, - retry=False) + retry_limit=None) self.mock_watcher.logger.exception.assert_not_called() diff --git a/packages/python-google-compute-engine/packaging/debian/changelog b/packages/python-google-compute-engine/packaging/debian/changelog index 655a82f..b4261c2 100644 --- a/packages/python-google-compute-engine/packaging/debian/changelog +++ b/packages/python-google-compute-engine/packaging/debian/changelog @@ -1,3 +1,10 @@ +python-google-compute-engine (1:20191210.00-g1) stable; urgency=medium + + * Enable sysctl change for E2 platform. + * Support retry limits in metadata retrieval. + + -- Google Cloud Team <gc-team@google.com> Tue, 10 Dec 2019 15:02:59 -0800 + python-google-compute-engine (1:20191120.00-g1) stable; urgency=medium * REVERT: Retry metadata lookups in agent. diff --git a/packages/python-google-compute-engine/packaging/setup_deb.sh b/packages/python-google-compute-engine/packaging/setup_deb.sh index 44664c0..764017e 100755 --- a/packages/python-google-compute-engine/packaging/setup_deb.sh +++ b/packages/python-google-compute-engine/packaging/setup_deb.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="20191120.00" +VERSION="20191210.00" working_dir=${PWD} if [[ $(basename "$working_dir") != $NAME ]]; then diff --git a/packages/python-google-compute-engine/packaging/setup_rpm.sh b/packages/python-google-compute-engine/packaging/setup_rpm.sh index e9a0c4f..7c97455 100755 --- a/packages/python-google-compute-engine/packaging/setup_rpm.sh +++ b/packages/python-google-compute-engine/packaging/setup_rpm.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="20191120.00" +VERSION="20191210.00" rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} working_dir=${PWD} diff --git a/packages/python-google-compute-engine/setup.py b/packages/python-google-compute-engine/setup.py index e248ca8..de05b12 100755 --- a/packages/python-google-compute-engine/setup.py +++ b/packages/python-google-compute-engine/setup.py @@ -37,7 +37,7 @@ setuptools.setup( name='google-compute-engine', packages=setuptools.find_packages(), url='https://github.com/GoogleCloudPlatform/compute-image-packages', - version='20191112.0', + version='20191210.0', # Entry points create scripts in /usr/bin that call a function. entry_points={ 'console_scripts': [ |