From 914b62502cf510a6c67b61453a67f90c86222f6e Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Mon, 8 May 2023 12:27:21 -0500 Subject: Fix loading cached JSON content when decode_content=True and the root element is a list --- HISTORY.md | 1 + requests_cache/models/response.py | 2 +- requests_cache/serializers/cattrs.py | 11 ++++++++--- tests/conftest.py | 8 ++++++++ tests/integration/base_cache_test.py | 19 +++++++++++++++++-- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 968a957..0f703a6 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,7 @@ ## 1.0.2 (2023-TBD) * Revert normalizing `CachedResponse.url` so it matches the original request URL +* Fix loading cached JSON content when `decode_content=True` and the root element is a list ## 1.0.1 (2023-03-24) * Ignore `Cache-Control: must-revalidate` and `no-cache` response headers with `cache_control=False` diff --git a/requests_cache/models/response.py b/requests_cache/models/response.py index 2cd6047..134d04b 100755 --- a/requests_cache/models/response.py +++ b/requests_cache/models/response.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from ..policy.actions import CacheActions DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' # Format used for __str__ only -DecodedContent = Union[Dict, str, None] +DecodedContent = Union[Dict, List, str, None] logger = getLogger(__name__) diff --git a/requests_cache/serializers/cattrs.py b/requests_cache/serializers/cattrs.py index bf45c2d..439463e 100644 --- a/requests_cache/serializers/cattrs.py +++ b/requests_cache/serializers/cattrs.py @@ -109,9 +109,12 @@ def init_converter( converter.register_structure_hook( CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj) ) - # Convert decoded JSON body back to string + + # Convert decoded JSON body back to a string. If the object is a valid JSON root (dict or list), + # that means it was previously saved in human-readable format due to `decode_content=True`. + # After this hook runs, the body will also be re-encoded with `_encode_content()`. converter.register_structure_hook( - DecodedContent, lambda obj, cls: json.dumps(obj) if isinstance(obj, dict) else obj + DecodedContent, lambda obj, cls: json.dumps(obj) if isinstance(obj, (dict, list)) else obj ) # Resolve forward references (required for CachedResponse.history) @@ -157,7 +160,9 @@ def _decode_content(response: CachedResponse, response_dict: Dict) -> Dict: def _encode_content(response: CachedResponse) -> CachedResponse: - """Re-encode response body if saved as JSON or text; has no effect for a binary response body""" + """Re-encode response body if saved as JSON or text (via ``decode_content=True``). + This has no effect for a binary response body. + """ if isinstance(response._decoded_content, str): response._content = response._decoded_content.encode('utf-8') response._decoded_content = None diff --git a/tests/conftest.py b/tests/conftest.py index e4ca8a6..099ddb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -82,6 +82,7 @@ MOCKED_URL = 'http+mock://requests-cache.com/text' MOCKED_URL_ETAG = 'http+mock://requests-cache.com/etag' MOCKED_URL_HTTPS = 'https+mock://requests-cache.com/text' MOCKED_URL_JSON = 'http+mock://requests-cache.com/json' +MOCKED_URL_JSON_LIST = 'http+mock://requests-cache.com/json_list' MOCKED_URL_REDIRECT = 'http+mock://requests-cache.com/redirect' MOCKED_URL_REDIRECT_TARGET = 'http+mock://requests-cache.com/redirect_target' MOCKED_URL_VARY = 'http+mock://requests-cache.com/vary' @@ -206,6 +207,13 @@ def get_mock_adapter() -> Adapter: json={'message': 'mock json response'}, status_code=200, ) + adapter.register_uri( + ANY_METHOD, + MOCKED_URL_JSON_LIST, + headers={'Content-Type': 'application/json'}, + json=['item_1', 'item_2', {'message': 'mock json response'}], + status_code=200, + ) adapter.register_uri( ANY_METHOD, MOCKED_URL_REDIRECT, diff --git a/tests/integration/base_cache_test.py b/tests/integration/base_cache_test.py index 7801570..3e4d230 100644 --- a/tests/integration/base_cache_test.py +++ b/tests/integration/base_cache_test.py @@ -25,12 +25,15 @@ from tests.conftest import ( HTTPBIN_METHODS, HTTPDATE_STR, LAST_MODIFIED, + MOCKED_URL_JSON, + MOCKED_URL_JSON_LIST, N_ITERATIONS, N_REQUESTS_PER_ITERATION, N_WORKERS, USE_PYTEST_HTTPBIN, assert_delta_approx_equal, httpbin, + mount_mock_adapter, skip_pypy, ) @@ -259,9 +262,9 @@ class BaseCacheTest: assert response.is_expired is False @pytest.mark.parametrize('stream', [True, False]) - def test_response_decode(self, stream): + def test_decode_gzip_response(self, stream): """Test that gzip-compressed raw responses (including streamed responses) can be manually - decompressed with decode_content=True + decompressed with `decode_content=True` """ session = self.init_session() response = session.get(httpbin('gzip'), stream=stream) @@ -274,6 +277,18 @@ class BaseCacheTest: assert b'gzipped' in cached_response.content assert b'gzipped' in cached_response.raw.read(None, decode_content=True) + @pytest.mark.parametrize('decode_content', [True, False]) + @pytest.mark.parametrize('url', [MOCKED_URL_JSON, MOCKED_URL_JSON_LIST]) + def test_decode_json_response(self, decode_content, url): + """Test that JSON responses (with both dict and list root) are correctly returned from the + cache, regardless of `decode_content` setting""" + session = self.init_session(decode_content=decode_content) + session = mount_mock_adapter(session) + + r1 = session.get(url) + r2 = session.get(url) + assert r1.json() == r2.json() + def test_multipart_upload(self): session = self.init_session() session.post(httpbin('post'), files={'file1': BytesIO(b'10' * 1024)}) -- cgit v1.2.1