diff options
| author | Paul Belanger <pabelanger@redhat.com> | 2019-06-13 15:17:49 -0400 |
|---|---|---|
| committer | James E. Blair <jeblair@redhat.com> | 2019-06-13 13:07:04 -0700 |
| commit | 46ac8a6ad25e8f798de025f7755b82fe8875db56 (patch) | |
| tree | d429cb56c502ac2c0c68fc20254a6a0581c1c37c | |
| parent | 7e45f84f056b3fa021aae1eecb0c23d9055656f3 (diff) | |
| download | zuul-46ac8a6ad25e8f798de025f7755b82fe8875db56.tar.gz | |
Improve retry handling for github driver
It seems that a 1 second sleep for our 'retry' logic for github events
might not be long enough. This was raising an abuse expection from
github:
2019-06-13 15:55:34,132 ERROR zuul.GithubEventProcessor: [e: ad668d74-8df3-11e9-93ab-4ff1818b4f8e] Exception when handling event:
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/zuul/driver/github/githubconnection.py", line 265, in _process_event
event = method()
File "/usr/local/lib/python3.5/dist-packages/zuul/driver/github/githubconnection.py", line 427, in _event_status
self.body['sha'], project, self.zuul_event_id)
File "/usr/local/lib/python3.5/dist-packages/zuul/driver/github/githubconnection.py", line 1312, in getPullBySha
repo = github.repository(owner, repository)
File "/usr/local/lib/python3.5/dist-packages/github3/github.py", line 1981, in repository
json = self._json(self._get(url), 200)
File "/usr/local/lib/python3.5/dist-packages/github3/models.py", line 156, in _json
raise exceptions.error_for(response)
github3.exceptions.ForbiddenError: 403 You have triggered an abuse detection mechanism. Please wait a few minutes before you try again.
Read documentation, github _should_ return the 'Retry-After' headers but
to be safe, we now default to 60 seconds if missing. 60 seconds was a
best guess on a default value for github, based on manual testing.
Change-Id: Ie1da348291b48e9f68336b20510830351402a343
Signed-off-by: Paul Belanger <pabelanger@redhat.com>
| -rw-r--r-- | zuul/driver/github/githubconnection.py | 15 |
1 files changed, 11 insertions, 4 deletions
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py index 4ef0283dc..f98ad185d 100644 --- a/zuul/driver/github/githubconnection.py +++ b/zuul/driver/github/githubconnection.py @@ -260,14 +260,21 @@ class GithubEventProcessor(object): self.log.debug("Handling %s event", self.event_type) event = None try: - for retry in range(5): + for retry in range(1, 6): try: event = method() break - except github3.exceptions.ServerError: + except (github3.exceptions.ForbiddenError, + github3.exceptions.ServerError) as e: + # NOTE(pabelanger) Check for 'Retry-After' header, if + # missing default to 60, to try to keep github happy. + retry_after = e.response.headers.get('Retry-After') + retry_delay = float(retry_after or 60) self.log.exception( - "Failed handling %s event; retrying", self.event_type) - time.sleep(1) + "Failed handling %s event; remote said retry after %s," + "will retry attempt %s/5 in %s seconds", + self.event_type, retry_after, retry, retry_delay) + time.sleep(retry_delay) except Exception: # NOTE(pabelanger): We should report back to the PR we could # not process the event, to give the user a chance to |