diff options
| author | Johnson Shi <Johnson.Shi@microsoft.com> | 2021-03-25 07:20:10 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-03-25 09:20:10 -0500 |
| commit | 6ae1145f5e980a47ac2b1ff3afa228a5da3f6e70 (patch) | |
| tree | 486ac9f4d906f9a14dec06f14e7f1f109f279006 | |
| parent | d4acc0920cdc19e5be3e1054a9a5fbc20e954492 (diff) | |
| download | cloud-init-git-6ae1145f5e980a47ac2b1ff3afa228a5da3f6e70.tar.gz | |
Azure helper: Ensure Azure http handler sleeps between retries (#842)
Ensure that the Azure helper's http handler sleeps a fixed duration
between retry failure attempts. The http handler will sleep a fixed
duration between failed attempts regardless of whether the attempt
failed due to (1) request timing out or (2) instant failure (no
timeout).
Due to certain platform issues, the http request to the Azure endpoint
may instantly fail without reaching the http timeout duration. Without
sleeping a fixed duration in between retry attempts, the http handler
will loop through the max retry attempts quickly. This causes the
communication between cloud-init and the Azure platform to be less
resilient due to the short total duration if there is no sleep in
between retries.
| -rwxr-xr-x | cloudinit/sources/helpers/azure.py | 2 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_azure_helper.py | 11 |
2 files changed, 11 insertions, 2 deletions
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index d3055d08..03e7156b 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str: max_readurl_attempts = 240 default_readurl_timeout = 5 + sleep_duration_between_retries = 5 periodic_logging_attempts = 12 if 'timeout' not in kwargs: @@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str: 'attempt %d with exception: %s' % (url, attempt, e), logger_func=LOG.debug) + time.sleep(sleep_duration_between_retries) raise exc diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index b8899807..63482c6c 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase): max_readurl_attempts = 240 default_readurl_timeout = 5 + sleep_duration_between_retries = 5 periodic_logging_attempts = 12 def setUp(self): @@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase): self.m_readurl = patches.enter_context( mock.patch.object( azure_helper.url_helper, 'readurl', mock.MagicMock())) - patches.enter_context( - mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) + self.m_sleep = patches.enter_context( + mock.patch.object(azure_helper.time, 'sleep', autospec=True)) def test_http_with_retries(self): self.m_readurl.return_value = 'TestResp' @@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase): self.m_readurl.call_count, self.periodic_logging_attempts + 1) + # Ensure that cloud-init did sleep between each failed request + self.assertEqual( + self.m_sleep.call_count, + self.periodic_logging_attempts) + self.m_sleep.assert_called_with(self.sleep_duration_between_retries) + def test_http_with_retries_long_delay_logs_periodic_failure_msg(self): self.m_readurl.side_effect = \ [SentinelException] * self.periodic_logging_attempts + \ |