feat(downloads): allow streaming downloads access to response iterator (#1956)

* feat(downloads): allow streaming downloads access to response iterator

Allow access to the underlying response iterator when downloading in
streaming mode by specifying `iterator=True`.

Update type annotations to support this change.

* docs(api-docs): add iterator example to artifact download

Document the usage of the `iterator=True` option when downloading
artifacts

* test(packages): add tests for streaming downloads

10 files changed, 111 insertions, 36 deletions

diff --git a/gitlab/mixins.py b/gitlab/mixins.py
index f2df27d..3c897d4 100644
--- a/gitlab/mixins.py
+++ b/gitlab/mixins.py
@@ -20,6 +20,7 @@ from typing import (
     Any,
     Callable,
     Dict,
+    Iterator,
     List,
     Optional,
     Tuple,
@@ -612,16 +613,19 @@ class DownloadMixin(_RestObjectBase):
     def download(
         self,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Download the archive of a resource export.
 
         Args:
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -640,7 +644,7 @@ class DownloadMixin(_RestObjectBase):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
 
 class SubscribableMixin(_RestObjectBase):
diff --git a/gitlab/utils.py b/gitlab/utils.py
index bab6705..6acb861 100644
--- a/gitlab/utils.py
+++ b/gitlab/utils.py
@@ -19,7 +19,7 @@ import pathlib
 import traceback
 import urllib.parse
 import warnings
-from typing import Any, Callable, Dict, Optional, Tuple, Type, Union
+from typing import Any, Callable, Dict, Iterator, Optional, Tuple, Type, Union
 
 import requests
 
@@ -34,9 +34,13 @@ class _StdoutStream:
 def response_content(
     response: requests.Response,
     streamed: bool,
+    iterator: bool,
     action: Optional[Callable],
     chunk_size: int,
-) -> Optional[bytes]:
+) -> Optional[Union[bytes, Iterator[Any]]]:
+    if iterator:
+        return response.iter_content(chunk_size=chunk_size)
+
     if streamed is False:
         return response.content
 
diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py
index 2b0d4ce..ba2e788 100644
--- a/gitlab/v4/cli.py
+++ b/gitlab/v4/cli.py
@@ -127,6 +127,7 @@ class GitlabCLI:
             data = export_status.download()
             if TYPE_CHECKING:
                 assert data is not None
+                assert isinstance(data, bytes)
             sys.stdout.buffer.write(data)
 
         except Exception as e:  # pragma: no cover, cli.die is unit-tested
diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py
index 541e5e2..f5f106d 100644
--- a/gitlab/v4/objects/artifacts.py
+++ b/gitlab/v4/objects/artifacts.py
@@ -2,7 +2,7 @@
 GitLab API:
 https://docs.gitlab.com/ee/api/job_artifacts.html
 """
-from typing import Any, Callable, Optional, TYPE_CHECKING
+from typing import Any, Callable, Iterator, Optional, TYPE_CHECKING, Union
 
 import requests
 
@@ -40,10 +40,14 @@ class ProjectArtifactManager(RESTManager):
             ),
             category=DeprecationWarning,
         )
-        return self.download(
+        data = self.download(
             *args,
             **kwargs,
         )
+        if TYPE_CHECKING:
+            assert data is not None
+            assert isinstance(data, bytes)
+        return data
 
     @exc.on_http_error(exc.GitlabDeleteError)
     def delete(self, **kwargs: Any) -> None:
@@ -71,10 +75,11 @@ class ProjectArtifactManager(RESTManager):
         ref_name: str,
         job: str,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Get the job artifacts archive from a specific tag or branch.
 
         Args:
@@ -85,6 +90,8 @@ class ProjectArtifactManager(RESTManager):
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -103,7 +110,7 @@ class ProjectArtifactManager(RESTManager):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
     @cli.register_custom_action(
         "ProjectArtifactManager", ("ref_name", "artifact_path", "job")
@@ -115,10 +122,11 @@ class ProjectArtifactManager(RESTManager):
         artifact_path: str,
         job: str,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Download a single artifact file from a specific tag or branch from
         within the job's artifacts archive.
 
@@ -130,6 +138,8 @@ class ProjectArtifactManager(RESTManager):
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -148,4 +158,4 @@ class ProjectArtifactManager(RESTManager):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py
index aa86704..2fd79fd 100644
--- a/gitlab/v4/objects/files.py
+++ b/gitlab/v4/objects/files.py
@@ -1,5 +1,15 @@
 import base64
-from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING
+from typing import (
+    Any,
+    Callable,
+    cast,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    TYPE_CHECKING,
+    Union,
+)
 
 import requests
 
@@ -220,10 +230,11 @@ class ProjectFileManager(GetMixin, CreateMixin, UpdateMixin, DeleteMixin, RESTMa
         file_path: str,
         ref: str,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Return the content of a file for a commit.
 
         Args:
@@ -232,6 +243,8 @@ class ProjectFileManager(GetMixin, CreateMixin, UpdateMixin, DeleteMixin, RESTMa
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -252,7 +265,7 @@ class ProjectFileManager(GetMixin, CreateMixin, UpdateMixin, DeleteMixin, RESTMa
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
     @cli.register_custom_action("ProjectFileManager", ("file_path", "ref"))
     @exc.on_http_error(exc.GitlabListError)
diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py
index fbcb1fd..8502277 100644
--- a/gitlab/v4/objects/jobs.py
+++ b/gitlab/v4/objects/jobs.py
@@ -1,4 +1,4 @@
-from typing import Any, Callable, cast, Dict, Optional, TYPE_CHECKING, Union
+from typing import Any, Callable, cast, Dict, Iterator, Optional, TYPE_CHECKING, Union
 
 import requests
 
@@ -116,16 +116,19 @@ class ProjectJob(RefreshMixin, RESTObject):
     def artifacts(
         self,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Get the job artifacts.
 
         Args:
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -144,7 +147,7 @@ class ProjectJob(RefreshMixin, RESTObject):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
     @cli.register_custom_action("ProjectJob")
     @exc.on_http_error(exc.GitlabGetError)
@@ -152,10 +155,11 @@ class ProjectJob(RefreshMixin, RESTObject):
         self,
         path: str,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Get a single artifact file from within the job's artifacts archive.
 
         Args:
@@ -163,6 +167,8 @@ class ProjectJob(RefreshMixin, RESTObject):
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -181,13 +187,14 @@ class ProjectJob(RefreshMixin, RESTObject):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
     @cli.register_custom_action("ProjectJob")
     @exc.on_http_error(exc.GitlabGetError)
     def trace(
         self,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
@@ -198,6 +205,8 @@ class ProjectJob(RefreshMixin, RESTObject):
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -216,7 +225,9 @@ class ProjectJob(RefreshMixin, RESTObject):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return_value = utils.response_content(result, streamed, action, chunk_size)
+        return_value = utils.response_content(
+            result, streamed, iterator, action, chunk_size
+        )
         if TYPE_CHECKING:
             assert isinstance(return_value, dict)
         return return_value
diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py
index 882cb1a..a820801 100644
--- a/gitlab/v4/objects/packages.py
+++ b/gitlab/v4/objects/packages.py
@@ -5,7 +5,7 @@ https://docs.gitlab.com/ee/user/packages/generic_packages/
 """
 
 from pathlib import Path
-from typing import Any, Callable, cast, Optional, TYPE_CHECKING, Union
+from typing import Any, Callable, cast, Iterator, Optional, TYPE_CHECKING, Union
 
 import requests
 
@@ -103,10 +103,11 @@ class GenericPackageManager(RESTManager):
         package_version: str,
         file_name: str,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Download a generic package.
 
         Args:
@@ -116,6 +117,8 @@ class GenericPackageManager(RESTManager):
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -132,7 +135,7 @@ class GenericPackageManager(RESTManager):
         result = self.gitlab.http_get(path, streamed=streamed, raw=True, **kwargs)
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
 
 class GroupPackage(RESTObject):
diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py
index 4893788..875fc8b 100644
--- a/gitlab/v4/objects/projects.py
+++ b/gitlab/v4/objects/projects.py
@@ -1,4 +1,14 @@
-from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING, Union
+from typing import (
+    Any,
+    Callable,
+    cast,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    TYPE_CHECKING,
+    Union,
+)
 
 import requests
 
@@ -466,10 +476,11 @@ class Project(RefreshMixin, SaveMixin, ObjectDeleteMixin, RepositoryMixin, RESTO
         self,
         wiki: bool = False,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Return a snapshot of the repository.
 
         Args:
@@ -477,6 +488,8 @@ class Project(RefreshMixin, SaveMixin, ObjectDeleteMixin, RepositoryMixin, RESTO
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment.
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -495,7 +508,7 @@ class Project(RefreshMixin, SaveMixin, ObjectDeleteMixin, RepositoryMixin, RESTO
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
     @cli.register_custom_action("Project", ("scope", "search"))
     @exc.on_http_error(exc.GitlabSearchError)
@@ -579,7 +592,11 @@ class Project(RefreshMixin, SaveMixin, ObjectDeleteMixin, RepositoryMixin, RESTO
             ),
             category=DeprecationWarning,
         )
-        return self.artifacts.raw(*args, **kwargs)
+        data = self.artifacts.raw(*args, **kwargs)
+        if TYPE_CHECKING:
+            assert data is not None
+            assert isinstance(data, bytes)
+        return data
 
 
 class ProjectManager(CRUDMixin, RESTManager):
diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py
index 5826d9d..1f10473 100644
--- a/gitlab/v4/objects/repositories.py
+++ b/gitlab/v4/objects/repositories.py
@@ -3,7 +3,7 @@ GitLab API: https://docs.gitlab.com/ee/api/repositories.html
 
 Currently this module only contains repository-related methods for projects.
 """
-from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING, Union
 
 import requests
 
@@ -107,10 +107,11 @@ class RepositoryMixin(_RestObjectBase):
         self,
         sha: str,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Return the raw file contents for a blob.
 
         Args:
@@ -118,6 +119,8 @@ class RepositoryMixin(_RestObjectBase):
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -136,7 +139,7 @@ class RepositoryMixin(_RestObjectBase):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
     @cli.register_custom_action("Project", ("from_", "to"))
     @exc.on_http_error(exc.GitlabGetError)
@@ -192,11 +195,12 @@ class RepositoryMixin(_RestObjectBase):
         self,
         sha: str = None,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         format: Optional[str] = None,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Return an archive of the repository.
 
         Args:
@@ -204,6 +208,8 @@ class RepositoryMixin(_RestObjectBase):
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -228,7 +234,7 @@ class RepositoryMixin(_RestObjectBase):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
     @cli.register_custom_action("Project")
     @exc.on_http_error(exc.GitlabDeleteError)
diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py
index 597a3aa..aa46c77 100644
--- a/gitlab/v4/objects/snippets.py
+++ b/gitlab/v4/objects/snippets.py
@@ -1,4 +1,4 @@
-from typing import Any, Callable, cast, List, Optional, TYPE_CHECKING, Union
+from typing import Any, Callable, cast, Iterator, List, Optional, TYPE_CHECKING, Union
 
 import requests
 
@@ -29,16 +29,19 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject):
     def content(
         self,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Return the content of a snippet.
 
         Args:
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment.
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -57,7 +60,7 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject):
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
 
 class SnippetManager(CRUDMixin, RESTManager):
@@ -103,16 +106,19 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj
     def content(
         self,
         streamed: bool = False,
+        iterator: bool = False,
         action: Optional[Callable[..., Any]] = None,
         chunk_size: int = 1024,
         **kwargs: Any,
-    ) -> Optional[bytes]:
+    ) -> Optional[Union[bytes, Iterator[Any]]]:
         """Return the content of a snippet.
 
         Args:
             streamed: If True the data will be processed by chunks of
                 `chunk_size` and each chunk is passed to `action` for
                 treatment.
+            iterator: If True directly return the underlying response
+                iterator
             action: Callable responsible of dealing with chunk of
                 data
             chunk_size: Size of each chunk
@@ -131,7 +137,7 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj
         )
         if TYPE_CHECKING:
             assert isinstance(result, requests.Response)
-        return utils.response_content(result, streamed, action, chunk_size)
+        return utils.response_content(result, streamed, iterator, action, chunk_size)
 
 
 class ProjectSnippetManager(CRUDMixin, RESTManager):