diff options
author | Jordan Cook <jordan.cook@pioneer.com> | 2021-08-21 18:39:47 -0500 |
---|---|---|
committer | Jordan Cook <jordan.cook@pioneer.com> | 2021-12-01 10:21:00 -0600 |
commit | 85dce906c7b8d7343098c0b76571a3293c860d8d (patch) | |
tree | 9cc125be58d7e1f36f3d66a5c90339f256c2b41d | |
parent | 45640082b02d595baa6ca378166fdc2daf2cd025 (diff) | |
download | requests-cache-85dce906c7b8d7343098c0b76571a3293c860d8d.tar.gz |
Improve some RedisDict methods and rename to RedisHashDict
-rw-r--r-- | requests_cache/backends/__init__.py | 4 | ||||
-rw-r--r-- | requests_cache/backends/redis.py | 68 | ||||
-rw-r--r-- | requests_cache/serializers/__init__.py | 1 | ||||
-rw-r--r-- | requests_cache/serializers/cattrs.py | 4 | ||||
-rw-r--r-- | requests_cache/serializers/preconf.py | 3 | ||||
-rw-r--r-- | tests/integration/base_storage_test.py | 7 | ||||
-rw-r--r-- | tests/integration/test_redis.py | 6 |
7 files changed, 57 insertions, 36 deletions
diff --git a/requests_cache/backends/__init__.py b/requests_cache/backends/__init__.py index 0c525fe..27841b7 100644 --- a/requests_cache/backends/__init__.py +++ b/requests_cache/backends/__init__.py @@ -42,9 +42,9 @@ try: except ImportError as e: MongoCache = MongoDict = MongoPickleDict = get_placeholder_class(e) # type: ignore try: - from .redis import RedisCache, RedisDict + from .redis import RedisCache, RedisHashDict except ImportError as e: - RedisCache = RedisDict = get_placeholder_class(e) # type: ignore + RedisCache = RedisHashDict = get_placeholder_class(e) # type: ignore try: # Note: Heroku doesn't support SQLite due to ephemeral storage from .sqlite import SQLiteCache, SQLiteDict, SQLitePickleDict diff --git a/requests_cache/backends/redis.py b/requests_cache/backends/redis.py index 41d908a..c460b12 100644 --- a/requests_cache/backends/redis.py +++ b/requests_cache/backends/redis.py @@ -31,7 +31,8 @@ API Reference :classes-only: :nosignatures: """ -from typing import Iterable +from logging import getLogger +from typing import Any, Iterable, Iterator, List, Tuple from redis import Redis, StrictRedis @@ -39,6 +40,8 @@ from .._utils import get_valid_kwargs from ..cache_keys import decode, encode from . import BaseCache, BaseStorage +logger = getLogger(__name__) + class RedisCache(BaseCache): """Redis cache backend @@ -51,51 +54,68 @@ class RedisCache(BaseCache): def __init__(self, namespace='http_cache', connection: Redis = None, **kwargs): super().__init__(**kwargs) - self.responses = RedisDict(namespace, 'responses', connection=connection, **kwargs) - self.redirects = RedisDict( + self.responses = RedisHashDict(namespace, 'responses', connection=connection, **kwargs) + self.redirects = RedisHashDict( namespace, 'redirects', connection=self.responses.connection, **kwargs ) -class RedisDict(BaseStorage): - """A dictionary-like interface for Redis operations +class RedisHashDict(BaseStorage): + """A dictionary-like interface for operations on a single Redis hash **Notes:** - * In order to deal with how Redis stores data, all keys will be encoded and all values will - be serialized. - * The full hash name will be ``namespace:collection_name`` + * All keys will be encoded and all values will be serialized + * Items will be stored in a hash named ``namespace:collection_name`` """ - def __init__(self, namespace, collection_name='http_cache', connection=None, **kwargs): + def __init__( + self, namespace: str = 'http_cache', collection_name: str = None, connection=None, **kwargs + ): super().__init__(**kwargs) connection_kwargs = get_valid_kwargs(Redis, kwargs) self.connection = connection or StrictRedis(**connection_kwargs) - self._self_key = f'{namespace}:{collection_name}' + self._hash_key = f'{namespace}:{collection_name}' + + def __contains__(self, key: str) -> bool: + return self.connection.hexists(self._hash_key, encode(key)) - def __getitem__(self, key): - result = self.connection.hget(self._self_key, encode(key)) + def __getitem__(self, key: str): + result = self.connection.hget(self._hash_key, encode(key)) if result is None: raise KeyError return self.serializer.loads(result) - def __setitem__(self, key, item): - self.connection.hset(self._self_key, encode(key), self.serializer.dumps(item)) + def __setitem__(self, key: str, item): + self.connection.hset(self._hash_key, encode(key), self.serializer.dumps(item)) - def __delitem__(self, key): - if not self.connection.hdel(self._self_key, encode(key)): + def __delitem__(self, key: str): + if not self.connection.hdel(self._hash_key, encode(key)): raise KeyError - def __len__(self): - return self.connection.hlen(self._self_key) + def __iter__(self) -> Iterator[str]: + yield from self.keys() - def __iter__(self): - for key in self.connection.hkeys(self._self_key): - yield decode(key) + def __len__(self) -> int: + return self.connection.hlen(self._hash_key) def bulk_delete(self, keys: Iterable[str]): - """Delete multiple keys from the cache. Does not raise errors for missing keys.""" + """Delete multiple keys from the cache, without raising errors for missing keys""" if keys: - self.connection.hdel(self._self_key, *[encode(key) for key in keys]) + self.connection.hdel(self._hash_key, *[encode(key) for key in keys]) def clear(self): - self.connection.delete(self._self_key) + self.connection.delete(self._hash_key) + + def keys(self) -> List[str]: + return [decode(key) for key in self.connection.hkeys(self._hash_key)] + + def items(self) -> List[Tuple[str, Any]]: + """Get all ``(key, value)`` pairs in the hash""" + return [ + (decode(k), self.serializer.loads(v)) + for k, v in self.connection.hgetall(self._hash_key).items() + ] + + def values(self) -> List: + """Get all values in the hash""" + return [self.serializer.loads(v) for v in self.connection.hvals(self._hash_key)] diff --git a/requests_cache/serializers/__init__.py b/requests_cache/serializers/__init__.py index 085f5cb..08ac11c 100644 --- a/requests_cache/serializers/__init__.py +++ b/requests_cache/serializers/__init__.py @@ -5,6 +5,7 @@ from .cattrs import CattrStage from .pipeline import SerializerPipeline, Stage from .preconf import ( bson_serializer, + dict_serializer, json_serializer, pickle_serializer, safe_pickle_serializer, diff --git a/requests_cache/serializers/cattrs.py b/requests_cache/serializers/cattrs.py index 522f0db..b28acd0 100644 --- a/requests_cache/serializers/cattrs.py +++ b/requests_cache/serializers/cattrs.py @@ -1,6 +1,6 @@ """ -Utilities to break down :py:class:`.CachedResponse` objects into python builtin types using -`cattrs <https://cattrs.readthedocs.io>`_. This does the majority of the work needed for any +Utilities to break down :py:class:`.CachedResponse` objects into a dict of python builtin types +using `cattrs <https://cattrs.readthedocs.io>`_. This does the majority of the work needed for any serialization format. .. automodsumm:: requests_cache.serializers.cattrs diff --git a/requests_cache/serializers/preconf.py b/requests_cache/serializers/preconf.py index cf9aeee..4c695a5 100644 --- a/requests_cache/serializers/preconf.py +++ b/requests_cache/serializers/preconf.py @@ -23,7 +23,8 @@ from .._utils import get_placeholder_class from .cattrs import CattrStage from .pipeline import SerializerPipeline, Stage -base_stage = CattrStage() #: Base stage for all serializer pipelines +base_stage = CattrStage() #: Base stage for all serializer pipelines (or standalone dict serializer) +dict_serializer = base_stage #: Partial serializer that converts responses to dicts bson_preconf_stage = CattrStage(bson_preconf.make_converter) #: Pre-serialization steps for BSON json_preconf_stage = CattrStage(json_preconf.make_converter) #: Pre-serialization steps for JSON msgpack_preconf_stage = CattrStage(msgpack.make_converter) #: Pre-serialization steps for msgpack diff --git a/tests/integration/base_storage_test.py b/tests/integration/base_storage_test.py index 9a8e3e0..893afdb 100644 --- a/tests/integration/base_storage_test.py +++ b/tests/integration/base_storage_test.py @@ -23,10 +23,9 @@ class BaseStorageTest: cache.clear() return cache - def tearDown(self): - for i in range(self.num_instances): - self.init_cache(i, clear=True) - super().tearDown() + def teardown_class(cls): + for i in range(cls.num_instances): + cls().init_cache(i, clear=True) def test_basic_methods(self): """Test basic dict methods with multiple cache instances: diff --git a/tests/integration/test_redis.py b/tests/integration/test_redis.py index bdabc26..f9b1f3e 100644 --- a/tests/integration/test_redis.py +++ b/tests/integration/test_redis.py @@ -2,7 +2,7 @@ from unittest.mock import patch import pytest -from requests_cache.backends.redis import RedisCache, RedisDict +from requests_cache.backends.redis import RedisCache, RedisHashDict from tests.conftest import fail_if_no_connection from tests.integration.base_cache_test import BaseCacheTest from tests.integration.base_storage_test import BaseStorageTest @@ -17,8 +17,8 @@ def ensure_connection(): Redis().info() -class TestRedisDict(BaseStorageTest): - storage_class = RedisDict +class TestRedisHashDict(BaseStorageTest): + storage_class = RedisHashDict picklable = True @patch('requests_cache.backends.redis.StrictRedis') |