diff options
| -rw-r--r-- | docs/api-usage.rst | 4 | ||||
| -rw-r--r-- | gitlab/__init__.py | 30 | ||||
| -rw-r--r-- | gitlab/tests/test_gitlab.py | 70 |
3 files changed, 93 insertions, 11 deletions
diff --git a/docs/api-usage.rst b/docs/api-usage.rst index a15aecb..7b7ab78 100644 --- a/docs/api-usage.rst +++ b/docs/api-usage.rst @@ -142,7 +142,9 @@ parameter to get all the items when using listing methods: python-gitlab will iterate over the list by calling the correspnding API multiple times. This might take some time if you have a lot of items to retrieve. This might also consume a lot of memory as all the items will be - stored in RAM. + stored in RAM. If you're encountering the python recursion limit exception, + use ``safe_all=True`` instead to stop pagination automatically if the + recursion limit is hit. Sudo ==== diff --git a/gitlab/__init__.py b/gitlab/__init__.py index 721106c..421b9eb 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -112,8 +112,8 @@ class Gitlab(object): # build the "submanagers" for parent_cls in six.itervalues(globals()): if (not inspect.isclass(parent_cls) - or not issubclass(parent_cls, GitlabObject) - or parent_cls == CurrentUser): + or not issubclass(parent_cls, GitlabObject) + or parent_cls == CurrentUser): continue if not parent_cls.managers: @@ -312,11 +312,13 @@ class Gitlab(object): params = extra_attrs.copy() params.update(kwargs.copy()) - get_all_results = kwargs.get('all', False) + catch_recursion_limit = kwargs.get('safe_all', False) + get_all_results = (kwargs.get('all', False) is True + or catch_recursion_limit) # Remove these keys to avoid breaking the listing (urls will get too # long otherwise) - for key in ['all', 'next_url']: + for key in ['all', 'next_url', 'safe_all']: if key in params: del params[key] @@ -334,12 +336,20 @@ class Gitlab(object): results = [cls(self, item, **params) for item in r.json() if item is not None] - if ('next' in r.links and 'url' in r.links['next'] - and get_all_results is True): - args = kwargs.copy() - args.update(extra_attrs) - args['next_url'] = r.links['next']['url'] - results.extend(self.list(cls, **args)) + try: + if ('next' in r.links and 'url' in r.links['next'] + and get_all_results): + args = kwargs.copy() + args.update(extra_attrs) + args['next_url'] = r.links['next']['url'] + results.extend(self.list(cls, **args)) + except Exception as e: + # Catch the recursion limit exception if the 'safe_all' + # kwarg was provided + if not (catch_recursion_limit and + "maximum recursion depth exceeded" in str(e)): + raise e + return results def _raw_post(self, path_, data=None, content_type=None, **kwargs): diff --git a/gitlab/tests/test_gitlab.py b/gitlab/tests/test_gitlab.py index 4adf07f..4670def 100644 --- a/gitlab/tests/test_gitlab.py +++ b/gitlab/tests/test_gitlab.py @@ -26,6 +26,7 @@ except ImportError: from httmock import HTTMock # noqa from httmock import response # noqa from httmock import urlmatch # noqa +import six import gitlab from gitlab import * # noqa @@ -243,6 +244,75 @@ class TestGitlabMethods(unittest.TestCase): self.assertEqual(data[0].ref, "b") self.assertEqual(len(data), 2) + def test_list_recursion_limit_caught(self): + @urlmatch(scheme="http", netloc="localhost", + path='/api/v3/projects/1/repository/branches', method="get") + def resp_one(url, request): + """First request: + + http://localhost/api/v3/projects/1/repository/branches?per_page=1 + """ + headers = { + 'content-type': 'application/json', + 'link': '<http://localhost/api/v3/projects/1/repository/branc' + 'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3' + '/projects/1/repository/branches?page=2&per_page=0>; rel="las' + 't", <http://localhost/api/v3/projects/1/repository/branches?' + 'page=1&per_page=0>; rel="first"' + } + content = ('[{"branch_name": "otherbranch", ' + '"project_id": 1, "ref": "b"}]').encode("utf-8") + resp = response(200, content, headers, None, 5, request) + return resp + + @urlmatch(scheme="http", netloc="localhost", + path='/api/v3/projects/1/repository/branches', method="get", + query=r'.*page=2.*') + def resp_two(url, request): + # Mock a runtime error + raise RuntimeError("maximum recursion depth exceeded") + + with HTTMock(resp_two, resp_one): + data = self.gl.list(ProjectBranch, project_id=1, per_page=1, + safe_all=True) + self.assertEqual(data[0].branch_name, "otherbranch") + self.assertEqual(data[0].project_id, 1) + self.assertEqual(data[0].ref, "b") + self.assertEqual(len(data), 1) + + def test_list_recursion_limit_not_caught(self): + @urlmatch(scheme="http", netloc="localhost", + path='/api/v3/projects/1/repository/branches', method="get") + def resp_one(url, request): + """First request: + + http://localhost/api/v3/projects/1/repository/branches?per_page=1 + """ + headers = { + 'content-type': 'application/json', + 'link': '<http://localhost/api/v3/projects/1/repository/branc' + 'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3' + '/projects/1/repository/branches?page=2&per_page=0>; rel="las' + 't", <http://localhost/api/v3/projects/1/repository/branches?' + 'page=1&per_page=0>; rel="first"' + } + content = ('[{"branch_name": "otherbranch", ' + '"project_id": 1, "ref": "b"}]').encode("utf-8") + resp = response(200, content, headers, None, 5, request) + return resp + + @urlmatch(scheme="http", netloc="localhost", + path='/api/v3/projects/1/repository/branches', method="get", + query=r'.*page=2.*') + def resp_two(url, request): + # Mock a runtime error + raise RuntimeError("maximum recursion depth exceeded") + + with HTTMock(resp_two, resp_one): + with six.assertRaisesRegex(self, GitlabError, + "(maximum recursion depth exceeded)"): + self.gl.list(ProjectBranch, project_id=1, per_page=1, all=True) + def test_list_401(self): @urlmatch(scheme="http", netloc="localhost", path="/api/v3/projects/1/repository/branches", method="get") |