summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan Cook <jordan.cook@pioneer.com>2021-08-21 18:39:47 -0500
committerJordan Cook <jordan.cook@pioneer.com>2021-12-01 10:21:00 -0600
commit85dce906c7b8d7343098c0b76571a3293c860d8d (patch)
tree9cc125be58d7e1f36f3d66a5c90339f256c2b41d
parent45640082b02d595baa6ca378166fdc2daf2cd025 (diff)
downloadrequests-cache-85dce906c7b8d7343098c0b76571a3293c860d8d.tar.gz
Improve some RedisDict methods and rename to RedisHashDict
-rw-r--r--requests_cache/backends/__init__.py4
-rw-r--r--requests_cache/backends/redis.py68
-rw-r--r--requests_cache/serializers/__init__.py1
-rw-r--r--requests_cache/serializers/cattrs.py4
-rw-r--r--requests_cache/serializers/preconf.py3
-rw-r--r--tests/integration/base_storage_test.py7
-rw-r--r--tests/integration/test_redis.py6
7 files changed, 57 insertions, 36 deletions
diff --git a/requests_cache/backends/__init__.py b/requests_cache/backends/__init__.py
index 0c525fe..27841b7 100644
--- a/requests_cache/backends/__init__.py
+++ b/requests_cache/backends/__init__.py
@@ -42,9 +42,9 @@ try:
except ImportError as e:
MongoCache = MongoDict = MongoPickleDict = get_placeholder_class(e) # type: ignore
try:
- from .redis import RedisCache, RedisDict
+ from .redis import RedisCache, RedisHashDict
except ImportError as e:
- RedisCache = RedisDict = get_placeholder_class(e) # type: ignore
+ RedisCache = RedisHashDict = get_placeholder_class(e) # type: ignore
try:
# Note: Heroku doesn't support SQLite due to ephemeral storage
from .sqlite import SQLiteCache, SQLiteDict, SQLitePickleDict
diff --git a/requests_cache/backends/redis.py b/requests_cache/backends/redis.py
index 41d908a..c460b12 100644
--- a/requests_cache/backends/redis.py
+++ b/requests_cache/backends/redis.py
@@ -31,7 +31,8 @@ API Reference
:classes-only:
:nosignatures:
"""
-from typing import Iterable
+from logging import getLogger
+from typing import Any, Iterable, Iterator, List, Tuple
from redis import Redis, StrictRedis
@@ -39,6 +40,8 @@ from .._utils import get_valid_kwargs
from ..cache_keys import decode, encode
from . import BaseCache, BaseStorage
+logger = getLogger(__name__)
+
class RedisCache(BaseCache):
"""Redis cache backend
@@ -51,51 +54,68 @@ class RedisCache(BaseCache):
def __init__(self, namespace='http_cache', connection: Redis = None, **kwargs):
super().__init__(**kwargs)
- self.responses = RedisDict(namespace, 'responses', connection=connection, **kwargs)
- self.redirects = RedisDict(
+ self.responses = RedisHashDict(namespace, 'responses', connection=connection, **kwargs)
+ self.redirects = RedisHashDict(
namespace, 'redirects', connection=self.responses.connection, **kwargs
)
-class RedisDict(BaseStorage):
- """A dictionary-like interface for Redis operations
+class RedisHashDict(BaseStorage):
+ """A dictionary-like interface for operations on a single Redis hash
**Notes:**
- * In order to deal with how Redis stores data, all keys will be encoded and all values will
- be serialized.
- * The full hash name will be ``namespace:collection_name``
+ * All keys will be encoded and all values will be serialized
+ * Items will be stored in a hash named ``namespace:collection_name``
"""
- def __init__(self, namespace, collection_name='http_cache', connection=None, **kwargs):
+ def __init__(
+ self, namespace: str = 'http_cache', collection_name: str = None, connection=None, **kwargs
+ ):
super().__init__(**kwargs)
connection_kwargs = get_valid_kwargs(Redis, kwargs)
self.connection = connection or StrictRedis(**connection_kwargs)
- self._self_key = f'{namespace}:{collection_name}'
+ self._hash_key = f'{namespace}:{collection_name}'
+
+ def __contains__(self, key: str) -> bool:
+ return self.connection.hexists(self._hash_key, encode(key))
- def __getitem__(self, key):
- result = self.connection.hget(self._self_key, encode(key))
+ def __getitem__(self, key: str):
+ result = self.connection.hget(self._hash_key, encode(key))
if result is None:
raise KeyError
return self.serializer.loads(result)
- def __setitem__(self, key, item):
- self.connection.hset(self._self_key, encode(key), self.serializer.dumps(item))
+ def __setitem__(self, key: str, item):
+ self.connection.hset(self._hash_key, encode(key), self.serializer.dumps(item))
- def __delitem__(self, key):
- if not self.connection.hdel(self._self_key, encode(key)):
+ def __delitem__(self, key: str):
+ if not self.connection.hdel(self._hash_key, encode(key)):
raise KeyError
- def __len__(self):
- return self.connection.hlen(self._self_key)
+ def __iter__(self) -> Iterator[str]:
+ yield from self.keys()
- def __iter__(self):
- for key in self.connection.hkeys(self._self_key):
- yield decode(key)
+ def __len__(self) -> int:
+ return self.connection.hlen(self._hash_key)
def bulk_delete(self, keys: Iterable[str]):
- """Delete multiple keys from the cache. Does not raise errors for missing keys."""
+ """Delete multiple keys from the cache, without raising errors for missing keys"""
if keys:
- self.connection.hdel(self._self_key, *[encode(key) for key in keys])
+ self.connection.hdel(self._hash_key, *[encode(key) for key in keys])
def clear(self):
- self.connection.delete(self._self_key)
+ self.connection.delete(self._hash_key)
+
+ def keys(self) -> List[str]:
+ return [decode(key) for key in self.connection.hkeys(self._hash_key)]
+
+ def items(self) -> List[Tuple[str, Any]]:
+ """Get all ``(key, value)`` pairs in the hash"""
+ return [
+ (decode(k), self.serializer.loads(v))
+ for k, v in self.connection.hgetall(self._hash_key).items()
+ ]
+
+ def values(self) -> List:
+ """Get all values in the hash"""
+ return [self.serializer.loads(v) for v in self.connection.hvals(self._hash_key)]
diff --git a/requests_cache/serializers/__init__.py b/requests_cache/serializers/__init__.py
index 085f5cb..08ac11c 100644
--- a/requests_cache/serializers/__init__.py
+++ b/requests_cache/serializers/__init__.py
@@ -5,6 +5,7 @@ from .cattrs import CattrStage
from .pipeline import SerializerPipeline, Stage
from .preconf import (
bson_serializer,
+ dict_serializer,
json_serializer,
pickle_serializer,
safe_pickle_serializer,
diff --git a/requests_cache/serializers/cattrs.py b/requests_cache/serializers/cattrs.py
index 522f0db..b28acd0 100644
--- a/requests_cache/serializers/cattrs.py
+++ b/requests_cache/serializers/cattrs.py
@@ -1,6 +1,6 @@
"""
-Utilities to break down :py:class:`.CachedResponse` objects into python builtin types using
-`cattrs <https://cattrs.readthedocs.io>`_. This does the majority of the work needed for any
+Utilities to break down :py:class:`.CachedResponse` objects into a dict of python builtin types
+using `cattrs <https://cattrs.readthedocs.io>`_. This does the majority of the work needed for any
serialization format.
.. automodsumm:: requests_cache.serializers.cattrs
diff --git a/requests_cache/serializers/preconf.py b/requests_cache/serializers/preconf.py
index cf9aeee..4c695a5 100644
--- a/requests_cache/serializers/preconf.py
+++ b/requests_cache/serializers/preconf.py
@@ -23,7 +23,8 @@ from .._utils import get_placeholder_class
from .cattrs import CattrStage
from .pipeline import SerializerPipeline, Stage
-base_stage = CattrStage() #: Base stage for all serializer pipelines
+base_stage = CattrStage() #: Base stage for all serializer pipelines (or standalone dict serializer)
+dict_serializer = base_stage #: Partial serializer that converts responses to dicts
bson_preconf_stage = CattrStage(bson_preconf.make_converter) #: Pre-serialization steps for BSON
json_preconf_stage = CattrStage(json_preconf.make_converter) #: Pre-serialization steps for JSON
msgpack_preconf_stage = CattrStage(msgpack.make_converter) #: Pre-serialization steps for msgpack
diff --git a/tests/integration/base_storage_test.py b/tests/integration/base_storage_test.py
index 9a8e3e0..893afdb 100644
--- a/tests/integration/base_storage_test.py
+++ b/tests/integration/base_storage_test.py
@@ -23,10 +23,9 @@ class BaseStorageTest:
cache.clear()
return cache
- def tearDown(self):
- for i in range(self.num_instances):
- self.init_cache(i, clear=True)
- super().tearDown()
+ def teardown_class(cls):
+ for i in range(cls.num_instances):
+ cls().init_cache(i, clear=True)
def test_basic_methods(self):
"""Test basic dict methods with multiple cache instances:
diff --git a/tests/integration/test_redis.py b/tests/integration/test_redis.py
index bdabc26..f9b1f3e 100644
--- a/tests/integration/test_redis.py
+++ b/tests/integration/test_redis.py
@@ -2,7 +2,7 @@ from unittest.mock import patch
import pytest
-from requests_cache.backends.redis import RedisCache, RedisDict
+from requests_cache.backends.redis import RedisCache, RedisHashDict
from tests.conftest import fail_if_no_connection
from tests.integration.base_cache_test import BaseCacheTest
from tests.integration.base_storage_test import BaseStorageTest
@@ -17,8 +17,8 @@ def ensure_connection():
Redis().info()
-class TestRedisDict(BaseStorageTest):
- storage_class = RedisDict
+class TestRedisHashDict(BaseStorageTest):
+ storage_class = RedisHashDict
picklable = True
@patch('requests_cache.backends.redis.StrictRedis')