summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan Cook <jordan.cook.git@proton.me>2023-05-08 12:27:21 -0500
committerJordan Cook <jordan.cook.git@proton.me>2023-05-08 12:29:57 -0500
commit914b62502cf510a6c67b61453a67f90c86222f6e (patch)
tree9b664b13956cee79a6f018f6c763420c1359e815
parente5d54b4f930956bbfdaf21d819e269a9857fc28a (diff)
downloadrequests-cache-914b62502cf510a6c67b61453a67f90c86222f6e.tar.gz
Fix loading cached JSON content when decode_content=True and the root element is a list
-rw-r--r--HISTORY.md1
-rwxr-xr-xrequests_cache/models/response.py2
-rw-r--r--requests_cache/serializers/cattrs.py11
-rw-r--r--tests/conftest.py8
-rw-r--r--tests/integration/base_cache_test.py19
5 files changed, 35 insertions, 6 deletions
diff --git a/HISTORY.md b/HISTORY.md
index 968a957..0f703a6 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -5,6 +5,7 @@
## 1.0.2 (2023-TBD)
* Revert normalizing `CachedResponse.url` so it matches the original request URL
+* Fix loading cached JSON content when `decode_content=True` and the root element is a list
## 1.0.1 (2023-03-24)
* Ignore `Cache-Control: must-revalidate` and `no-cache` response headers with `cache_control=False`
diff --git a/requests_cache/models/response.py b/requests_cache/models/response.py
index 2cd6047..134d04b 100755
--- a/requests_cache/models/response.py
+++ b/requests_cache/models/response.py
@@ -18,7 +18,7 @@ if TYPE_CHECKING:
from ..policy.actions import CacheActions
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' # Format used for __str__ only
-DecodedContent = Union[Dict, str, None]
+DecodedContent = Union[Dict, List, str, None]
logger = getLogger(__name__)
diff --git a/requests_cache/serializers/cattrs.py b/requests_cache/serializers/cattrs.py
index bf45c2d..439463e 100644
--- a/requests_cache/serializers/cattrs.py
+++ b/requests_cache/serializers/cattrs.py
@@ -109,9 +109,12 @@ def init_converter(
converter.register_structure_hook(
CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj)
)
- # Convert decoded JSON body back to string
+
+ # Convert decoded JSON body back to a string. If the object is a valid JSON root (dict or list),
+ # that means it was previously saved in human-readable format due to `decode_content=True`.
+ # After this hook runs, the body will also be re-encoded with `_encode_content()`.
converter.register_structure_hook(
- DecodedContent, lambda obj, cls: json.dumps(obj) if isinstance(obj, dict) else obj
+ DecodedContent, lambda obj, cls: json.dumps(obj) if isinstance(obj, (dict, list)) else obj
)
# Resolve forward references (required for CachedResponse.history)
@@ -157,7 +160,9 @@ def _decode_content(response: CachedResponse, response_dict: Dict) -> Dict:
def _encode_content(response: CachedResponse) -> CachedResponse:
- """Re-encode response body if saved as JSON or text; has no effect for a binary response body"""
+ """Re-encode response body if saved as JSON or text (via ``decode_content=True``).
+ This has no effect for a binary response body.
+ """
if isinstance(response._decoded_content, str):
response._content = response._decoded_content.encode('utf-8')
response._decoded_content = None
diff --git a/tests/conftest.py b/tests/conftest.py
index e4ca8a6..099ddb3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -82,6 +82,7 @@ MOCKED_URL = 'http+mock://requests-cache.com/text'
MOCKED_URL_ETAG = 'http+mock://requests-cache.com/etag'
MOCKED_URL_HTTPS = 'https+mock://requests-cache.com/text'
MOCKED_URL_JSON = 'http+mock://requests-cache.com/json'
+MOCKED_URL_JSON_LIST = 'http+mock://requests-cache.com/json_list'
MOCKED_URL_REDIRECT = 'http+mock://requests-cache.com/redirect'
MOCKED_URL_REDIRECT_TARGET = 'http+mock://requests-cache.com/redirect_target'
MOCKED_URL_VARY = 'http+mock://requests-cache.com/vary'
@@ -208,6 +209,13 @@ def get_mock_adapter() -> Adapter:
)
adapter.register_uri(
ANY_METHOD,
+ MOCKED_URL_JSON_LIST,
+ headers={'Content-Type': 'application/json'},
+ json=['item_1', 'item_2', {'message': 'mock json response'}],
+ status_code=200,
+ )
+ adapter.register_uri(
+ ANY_METHOD,
MOCKED_URL_REDIRECT,
headers={'Content-Type': 'text/plain', 'Location': MOCKED_URL_REDIRECT_TARGET},
text='mock redirect response',
diff --git a/tests/integration/base_cache_test.py b/tests/integration/base_cache_test.py
index 7801570..3e4d230 100644
--- a/tests/integration/base_cache_test.py
+++ b/tests/integration/base_cache_test.py
@@ -25,12 +25,15 @@ from tests.conftest import (
HTTPBIN_METHODS,
HTTPDATE_STR,
LAST_MODIFIED,
+ MOCKED_URL_JSON,
+ MOCKED_URL_JSON_LIST,
N_ITERATIONS,
N_REQUESTS_PER_ITERATION,
N_WORKERS,
USE_PYTEST_HTTPBIN,
assert_delta_approx_equal,
httpbin,
+ mount_mock_adapter,
skip_pypy,
)
@@ -259,9 +262,9 @@ class BaseCacheTest:
assert response.is_expired is False
@pytest.mark.parametrize('stream', [True, False])
- def test_response_decode(self, stream):
+ def test_decode_gzip_response(self, stream):
"""Test that gzip-compressed raw responses (including streamed responses) can be manually
- decompressed with decode_content=True
+ decompressed with `decode_content=True`
"""
session = self.init_session()
response = session.get(httpbin('gzip'), stream=stream)
@@ -274,6 +277,18 @@ class BaseCacheTest:
assert b'gzipped' in cached_response.content
assert b'gzipped' in cached_response.raw.read(None, decode_content=True)
+ @pytest.mark.parametrize('decode_content', [True, False])
+ @pytest.mark.parametrize('url', [MOCKED_URL_JSON, MOCKED_URL_JSON_LIST])
+ def test_decode_json_response(self, decode_content, url):
+ """Test that JSON responses (with both dict and list root) are correctly returned from the
+ cache, regardless of `decode_content` setting"""
+ session = self.init_session(decode_content=decode_content)
+ session = mount_mock_adapter(session)
+
+ r1 = session.get(url)
+ r2 = session.get(url)
+ assert r1.json() == r2.json()
+
def test_multipart_upload(self):
session = self.init_session()
session.post(httpbin('post'), files={'file1': BytesIO(b'10' * 1024)})